sit/module_stats.py

#!/usr/bin/env python2.6
#
# Copyright(c) 2013-2015 Christoph Niethammer <niethammer@hlrs.de>
#

import os
import datetime
from optparse import OptionParser
import re
import subprocess


modules = list()
modulestats = dict()
moduleusers = dict()
total_modules = 0
total_users   = set()

logdir = '/sw/laki/hlrs/system/modules/logs/'
enddate = datetime.date.today()
startdate = datetime.date(enddate.year, enddate.month, 1)
modulepattern=""

parser = OptionParser(usage="%prog [options] [pattern]")
parser.add_option("--logdir", metavar="DIR", dest="logdir")
parser.add_option("--startdate", metavar="YYYY-MM-DD", dest="startdate")
parser.add_option("--enddate", metavar="YYYY-MM-DD", dest="enddate")
parser.add_option("--nohpc", action="store_true", default=False, dest="nohpc", help="Exclude hpc* accounts from stats")
parser.add_option("--printusers", action="store_true", default=False, dest="printusers", help="Print detailed user lists for module usage.")
parser.add_option("--excludeusers", metavar="REGEX", dest="excludeusers", help="Exclude users mathich given regular expression")
parser.add_option("--list-unused", action="store_true", default=False, dest="list_unused", help="List only unused modules.")
(options, args) = parser.parse_args()
if options.logdir :
    logdir = options.logdir
if options.startdate :
    d = options.startdate
    startdate = datetime.date(int(d[0:4]),int(d[5:7]),int(d[8:10]))
if options.enddate :
    d = options.enddate
    enddate = datetime.date(int(d[0:4]),int(d[5:7]),int(d[8:10]))
if len(args) > 0 :
    modulepattern = args[0]
if options.excludeusers :
    excludeusers_re = re.compile(str(options.excludeusers))


for logfilename in os.listdir(logdir) :
    d = datetime.date(int(logfilename[0:4]),int(logfilename[4:6]),int(logfilename[6:8]))
    if d < startdate :
        continue
    if d > enddate :
        continue
    date = d.strftime('%Y-%m-%d')
    logfile = os.path.join(logdir, logfilename)
    f = open(logfile)
    for line in f :
        if ' load' in line and modulepattern in line :
            module = line.split()[-1]
            user = line.split()[-3][5:] # remove user: in front
            if options.nohpc and 'hpc' in user :
                continue
            if options.excludeusers and excludeusers_re.match(user):
                continue
            total_users.add(user)
            total_modules = total_modules + 1
            if module not in modules:
                modules.append(module)
                modulestats[module] = 1
		moduleusers[module] = dict()
		moduleusers[module][user] = 1
            else :
                modulestats[module] = modulestats[module] + 1
                if user not in moduleusers[module] :
                    moduleusers[module][user] = 1
                else :
                    moduleusers[module][user] = moduleusers[module][user] + 1
#        if ' swap' in line :
#            print line.split()

print "-"*78
print "Module statistics for " + startdate.strftime('%Y-%m-%d') + " - " + enddate.strftime('%Y-%m-%d')
print "-"*78
if options.list_unused :
    print "Unused modules:"
    print "-"*78
    avail_modules = []
    r = re.compile("\(.*\)")
    p = subprocess.Popen("module av -t " + modulepattern, shell=True, stderr=subprocess.PIPE).stderr
    for line in p :
        if line[0] == '/' :
            continue
        if modulepattern in line :
            avail_modules.append(r.sub('', line.strip()))
    for module in avail_modules :
        if not module in modulestats :
            print module
else :
    print "{0:60s}{1:>8s}  {2:>8s}".format('module name', '# uses', '# users')
    print "-"*78
    for (v,m) in sorted( ((v,k) for k,v in modulestats.iteritems()), reverse=True) :
        print "{0:60s}{1:>8d}  {2:>8d}".format(m, v, len(moduleusers[m].keys()))
        if options.printusers :
            print moduleusers[m]
    print "-"*78
    print "{0:60s}{1:>8d}  {2:>8d}".format('total', total_modules, len(total_users))
    if options.printusers :
        print sorted([ (u) for u in total_users ])
print "-"*78