From 7940dde4b9925abd8638aa2506e6036caef2f19b Mon Sep 17 00:00:00 2001
From: Christoph Niethammer <niethammer@hlrs.de>
Date: Wed, 2 Nov 2022 13:43:52 +0000
Subject: [PATCH] Add simple functionality check of lmod modules

---
 check_lmod_modules.sh | 171 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 171 insertions(+)
 create mode 100755 check_lmod_modules.sh

diff --git a/check_lmod_modules.sh b/check_lmod_modules.sh
new file mode 100755
index 0000000..7e8c876
--- /dev/null
+++ b/check_lmod_modules.sh
@@ -0,0 +1,171 @@
+#!/bin/bash -l
+#
+# Copyright (c) 2022      Christoph Niethammer <niethammer@hlrs.de>
+#
+# Script checking an lmod based module environment for problems during module loading/unloading
+#
+
+function print_usage() {
+echo "Script checking the module environment for problems during module loading/unloading"
+echo "usage: $0 [MODULE_PATTERN] [--exclude PATTERN]"
+}
+
+# Command line options:
+declare -a modulespecs  # check only subset of modules
+declare -a exclude_pattern # exclude modules with given pattern, e.g. nightly builds
+
+while [[ $# -gt 0 ]]
+do
+    key="$1"
+    case $key in
+        --exclude-pattern|--exclude)
+            exclude_patterns+=($2)
+            shift 2
+        ;;
+        -h|--help)
+            print_usage
+            exit 0
+        ;;
+        *)
+            modulespecs+=($1)
+            shift
+        ;;
+    esac
+done
+
+# definitions for color output if outputting to tty
+if [ -t 1 ] ; then
+    Color_Off='\e[0m'       # Text Reset
+    IGreen='\e[0;92m'       # Intense Green
+    IRed='\e[0;91m'         # Intense Red
+    IMagenta='\e[0;95m'     # magenta
+fi
+
+# intermediate files, logfiles
+LOGDIR=${LOGDIR:=$PWD}
+tmpdir=/tmp/check_modules-$USER
+mkdir -p $tmpdir
+logfile="$LOGDIR/check_modules.log"  # logfile with detailed information
+module_load_logfile="$tmpdir/.module_load.log"  # output of 'module load commands
+module_rm_logfile="$tmpdir/.module_rm.log"  # output of 'module rm' commands
+module_clean_env_file="$tmpdir/.module_clean_env"  # original environment
+
+# safe the original environment
+set > $module_clean_env_file
+module save check_modules
+
+# list of all failed modules
+failed_modules=()
+
+
+
+echo "--------------------" | tee $logfile
+echo "Module environment check" | tee -a $logfile
+echo "--------------------" | tee -a $logfile
+echo "Date: $(date)" | tee -a $logfile
+echo "Host: $(/bin/hostname)" | tee -a $logfile
+echo "USER: $USER" | tee -a $logfile
+echo "Logfile: $logfile" | tee -a $logfile
+echo "--------------------" | tee -a $logfile
+echo Modulespecs: ${modulespecs[@]}  | tee -a $logfile
+echo Exclude: ${exclude_patterns[@]}  | tee -a $logfile
+echo "--------------------" | tee -a $logfile
+echo "Environment:" >> $logfile
+cat $module_clean_env_file >> $logfile
+echo "--------------------" >> $logfile
+
+for modulespec in ${modulespecs[@]}
+do
+    for m_original in $(module -t spider $modulespec 2>&1); do
+        match=0
+        for exclude_pattern in ${exclude_patterns[@]}
+        do
+            if [[ "$m_original" =~ "$exclude_pattern" ]]; then
+                echo -e "Skipping $m_original (matches $exclude_pattern) ... ${IMagenta} skipped${Color_Off}"
+                match=1
+                continue
+            fi
+        done
+        if [ $match == 1 ]; then
+            continue
+        fi
+        if [[ $m_original =~ ^[A-Za-z] ]]; then # skip any non module line in output
+            m=$(echo $m_original | sed -e 's/(.*)//') # Remove aliases e.g. (default)
+            echo "Checking $m_original ... "
+            echo "Checking $m_original ... " >> $logfile
+
+	    readarray deps < <( module spider $m_original |& grep '^      ')
+	    for dep in "${deps[@]}" ; do
+		    dep=${dep//[$'\t\r\n']}
+		    echo -n "... with deps $dep ..."
+		    echo "... with deps $dep ..." >> $logfile
+		    module load $dep >>$logfile 2>&1
+
+		    cmd="module load $m"
+		    echo $cmd >> $logfile
+
+		    $cmd > $module_load_logfile 2>&1
+		    cat $module_load_logfile >> $logfile
+		    module li >>$logfile 2>&1
+
+		    # check if module was loaded and did not report errors during loading
+		    if  module -t li 2>&1 | grep $m >/dev/null &&  ! grep ERROR $module_load_logfile >/dev/null  ; then
+
+			    cmd="module rm $m"
+			    echo $cmd >> $logfile
+			    $cmd > $module_rm_logfile 2>&1
+			    cat $module_rm_logfile >> $logfile
+			    module li >>$logfile 2>&1
+
+			    # check if module was unloaded
+			    if module -t li 2>&1 | grep $m > /dev/null; then
+				    echo -e "${IRed}unloading failed${Color_Off}"
+				    echo "ERROR: unloading module '$m' failed" >> $logfile
+				    failed_modules=(${failed_modules[@]} "$m_original [$dep]")
+			    else
+				    echo -e "${IGreen}success${Color_Off}"
+				    echo "SUCCESS" >> $logfile
+			    fi
+
+		    else
+			    echo -e "${IRed}loading failed${Color_Off}"
+			    echo "ERROR: loading module '$m' failed" >> $logfile
+			    failed_modules=(${failed_modules[@]} "$m [$dep]")
+		    fi
+		    module unload $dep >>$logfile 2>&1
+
+		    # clean up module environment
+		    cmd="module purge"
+		    echo $cmd >> $logfile
+		    $cmd >>$logfile 2>&1
+		    module li >>$logfile 2>&1
+		    echo "Resetting environment ..." >>$logfile
+
+		    # Reset the complete environment manually to overcome problems with
+		    # inconsistent internal caches of the module command after module purge.
+		    #source $module_clean_env_file 2>/dev/null
+		    module restore check_modules 2>/dev/null
+
+		    # clean up intermediate files
+		    rm -f $module_load_logfile
+		    rm -f $module_rm_logfile
+
+		    echo >>$logfile 2>&1
+	    done
+        fi
+    done
+done
+
+# clean up file storing the initial environment
+rm -f $module_clean_env_file
+rm -rf $tmpdir
+
+
+echo "----------------------------------------"
+echo "Summary of failed modules (${#failed_modules[@]}):"
+echo "----------------------------------------"
+for m in ${failed_modules[@]}; do
+    echo "$m"
+done
+echo "----------------------------------------"
+