#!/bin/bash -l
#
# Christoph Niethammer <niethammer@hlrs.de>, (c) 2012
#
# Script checking the module environment for problems during module loading/unloading
# 
# Usage:
# check_modules.sh [MODULE]
#
# Options:
#   MODULE   May be any valid module path as used for 'module avail [MODULE]'
#

# Command line options: 
moduleclass=mpi/openmpi  # check only subset of modules 
mpi_test_suite_dir=$HOME/mpi_test_suite.latest

# definitions for esear color output to display 
Color_Off='\e[0m'       # Text Reset
IGreen='\e[0;92m'       # Intense Green
IRed='\e[0;91m'         # Intense Red

# intermediate files, logfiles
LOGDIR=${LOGDIR:=$PWD}
tmpdir=/tmp/check_modules-$USER
mkdir -p $tmpdir
logfile="$LOGDIR/check_modules.log"  # logfile with detailed information
module_load_logfile="$tmpdir/.module_load.log"  # output of 'module load commands 
module_rm_logfile="$tmpdir/.module_rm.log"  # output of 'module rm' commands
module_clean_env_file="$tmpdir/.module_clean_env"  # original environment

# safe the original environment
set > $module_clean_env_file

# list of all failed modules
failed_modules=()



echo "--------------------" | tee $logfile
echo "Module environment check" | tee -a $logfile
echo "--------------------" | tee -a $logfile
echo "Date: $(date)" | tee -a $logfile
echo "Host: $(/bin/hostname)" | tee -a $logfile
echo "USER: $USER" | tee -a $logfile
echo "Logfile: $logfile" | tee -a $logfile
echo "--------------------" | tee -a $logfile
echo "Environment:" >> $logfile
cat $module_clean_env_file >> $logfile
echo "--------------------" >> $logfile

for m_original in $(module av -t $moduleclass 2>&1); do
    if [[ $m_original =~ ^[A-Za-z] ]]; then # skip any non module line in output
        m=$(echo $m_original | sed -e 's/(.*)//') # Remove aliases e.g. (default)
        echo -n "Checking $m_original ... " 
        echo "Checking $m_original ... " >> $logfile

        cmd="module load $m"
        echo $cmd >> $logfile
        $cmd > $module_load_logfile 2>&1 
        cat $module_load_logfile >> $logfile
        module li >>$logfile 2>&1

        # check if module was loaded and did not report errors during loading
        if  module li -t 2>&1 | grep $m >/dev/null &&  ! grep ERROR $module_load_logfile >/dev/null  ; then 
            mpi_test_suite_log=$(echo $m | sed -e's/\//_/g').log
            cd $mpi_test_suite_dir
            make clean >$mpi_test_suite_log 2>&1
            make -j $(grep processor /proc/cpuinfo | wc -l) >>$mpi_test_suite_log 2>&1
            mpirun -np $(grep processor /proc/cpuinfo | wc -l) mpi_test_suite -t All,^io,^one-sided >>$mpi_test_suite_log 2>&1
            if grep "Number of failed tests:0" $mpi_test_suite_log >/dev/null ; then 
                echo -en "${IGreen}passed test suite${Color_Off} ... "
                echo -e "passed test suite" >> $logfile
            else
                echo -en "${IRed}Testsuite failed${Color_Off} "
                echo "ERROR: Testsuite failed for module '$m'" >> $logfile
                failed_modules=(${failed_modules[@]} $m_original)
            fi

            cmd="module rm $m"
            echo $cmd >> $logfile
            $cmd > $module_rm_logfile 2>&1
            cat $module_rm_logfile >> $logfile
            module li >>$logfile 2>&1

            # check if module was unloaded
            if module li -t 2>&1 | grep $m > /dev/null; then
                echo -e "${IRed}unloading failed${Color_Off}"
                echo "ERROR: unloading module '$m' failed" >> $logfile
                failed_modules=(${failed_modules[@]} $m_original)
            else
                echo -e "${IGreen}success${Color_Off}"
                echo "SUCCESS" >> $logfile
            fi

        else 
            echo -e "${IRed}loading failed${Color_Off}"
            echo "ERROR: loading module '$m' failed" >> $logfile
            failed_modules=(${failed_modules[@]} $m)
        fi

        # clean up module environment
        cmd="module purge"
        echo $cmd >> $logfile
        $cmd >>$logfile 2>&1
        module li >>$logfile 2>&1
        echo "Resetting environment ..." >>$logfile

        # Reset the complete environment manually to overcome problems with
        # inconsistent internal caches of the module command after module purge.
        source $module_clean_env_file 2>/dev/null

        # clean up intermediate files
        rm -f $module_load_logfile
        rm -f $module_rm_logfile

        echo >>$logfile 2>&1
    fi
done

# clean up file storing the initial environment 
rm -f $module_clean_env_file
rm -rf $tmpdir


echo "----------------------------------------"
echo "Summary of failed modules (${#failed_modules[@]}):"
echo "----------------------------------------"
for m in ${failed_modules[@]}; do
    echo "$m"
done
echo "----------------------------------------"