hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

qa : flake8 issues

Browse source
This commit is contained in:
alalazo 2016-08-11 09:08:00 +02:00
parent b4b9ebe7d7
commit f5433477b9
5 changed files with 77 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
lib/spack/llnl/util/tty/log.py
View file

@ -24,12 +24,11 @@
############################################################################## ##############################################################################
"""Utility classes for logging the output of blocks of code. """Utility classes for logging the output of blocks of code.
""" """
import sys import multiprocessing
import os import os
import re import re
import select import select
import inspect import sys
import multiprocessing
import llnl.util.tty as tty import llnl.util.tty as tty
import llnl.util.tty.color as color import llnl.util.tty.color as color
@ -117,9 +116,10 @@ class log_output(object):
# do things ... output will be logged # do things ... output will be logged
# and also printed to stdout. # and also printed to stdout.
Opens a stream in 'w' mode at daemon spawning and closes it at daemon joining. Opens a stream in 'w' mode at daemon spawning and closes it at
If echo is True, also prints the output to stdout. daemon joining. If echo is True, also prints the output to stdout.
""" """
def __init__(self, filename, echo=False, force_color=False, debug=False): def __init__(self, filename, echo=False, force_color=False, debug=False):
self.filename = filename self.filename = filename
# Various output options # Various output options
@ -133,7 +133,11 @@ def __init__(self, filename, echo=False, force_color=False, debug=False):
self.read, self.write = os.pipe() self.read, self.write = os.pipe()
# Sets a daemon that writes to file what it reads from a pipe # Sets a daemon that writes to file what it reads from a pipe
self.p = multiprocessing.Process(target=self._spawn_writing_daemon, args=(self.read,), name='logger_daemon') self.p = multiprocessing.Process(
target=self._spawn_writing_daemon,
args=(self.read,),
name='logger_daemon'
)
self.p.daemon = True self.p.daemon = True
# Needed to un-summon the daemon # Needed to un-summon the daemon
self.parent_pipe, self.child_pipe = multiprocessing.Pipe() self.parent_pipe, self.child_pipe = multiprocessing.Pipe()
@ -186,6 +190,7 @@ def __del__(self):
os.close(self.read) os.close(self.read)
class OutputRedirection(object): class OutputRedirection(object):
def __init__(self, other): def __init__(self, other):
self.__dict__.update(other.__dict__) self.__dict__.update(other.__dict__)

15
lib/spack/spack/build_environment.py
View file

@ -51,16 +51,14 @@
Skimming this module is a nice way to get acquainted with the types of Skimming this module is a nice way to get acquainted with the types of
calls you can make from within the install() function. calls you can make from within the install() function.
""" """
import os
import sys
import shutil
import multiprocessing import multiprocessing
import platform import os
import shutil
import sys
import llnl.util.tty as tty import llnl.util.tty as tty
from llnl.util.filesystem import *
import spack import spack
from llnl.util.filesystem import *
from spack.environment import EnvironmentModifications, validate from spack.environment import EnvironmentModifications, validate
from spack.util.environment import * from spack.util.environment import *
from spack.util.executable import Executable, which from spack.util.executable import Executable, which
@ -502,7 +500,10 @@ def child_execution(child_connection):
child_connection.close() child_connection.close()
parent_connection, child_connection = multiprocessing.Pipe() parent_connection, child_connection = multiprocessing.Pipe()
p = multiprocessing.Process(target=child_execution, args=(child_connection,)) p = multiprocessing.Process(
target=child_execution,
args=(child_connection,)
)
p.start() p.start()
exc_type, exception, traceback = parent_connection.recv() exc_type, exception, traceback = parent_connection.recv()
p.join() p.join()

7
lib/spack/spack/cmd/setup.py
View file

@ -58,8 +58,6 @@ def spack_transitive_include_path():
def write_spconfig(package): def write_spconfig(package):
spec = package.spec
prefix = spec.prefix
# Set-up the environment # Set-up the environment
spack.build_environment.setup_package(package) spack.build_environment.setup_package(package)
@ -79,7 +77,7 @@ def write_spconfig(package):
setup_fname = 'spconfig.py' setup_fname = 'spconfig.py'
with open(setup_fname, 'w') as fout: with open(setup_fname, 'w') as fout:
fout.write( fout.write(
r"""#!%s r"""#!%s
# #
import sys import sys
@ -108,8 +106,7 @@ def cmdlist(str):
fout.write(' %s\n' % part) fout.write(' %s\n' % part)
fout.write('"""))\n') fout.write('"""))\n')
fout.write( fout.write("env['CMAKE_TRANSITIVE_INCLUDE_PATH'] = env['SPACK_TRANSITIVE_INCLUDE_PATH'] # Deprecated\n") # NOQA: ignore=E501
"env['CMAKE_TRANSITIVE_INCLUDE_PATH'] = env['SPACK_TRANSITIVE_INCLUDE_PATH'] # Deprecated\n")
fout.write('\ncmd = cmdlist("""\n') fout.write('\ncmd = cmdlist("""\n')
fout.write('%s\n' % cmd[0]) fout.write('%s\n' % cmd[0])
for arg in cmd[1:]: for arg in cmd[1:]:

68
lib/spack/spack/package.py
View file

@ -75,12 +75,13 @@
class InstallPhase(object): class InstallPhase(object):
"""Manages a single phase of the installation """Manages a single phase of the installation
This descriptor stores at creation time the name of the method it should search This descriptor stores at creation time the name of the method it should
for execution. The method is retrieved at __get__ time, so that it can be overridden search for execution. The method is retrieved at __get__ time, so that
by subclasses of whatever class declared the phases. it can be overridden by subclasses of whatever class declared the phases.
It also provides hooks to execute prerequisite and sanity checks. It also provides hooks to execute prerequisite and sanity checks.
""" """
def __init__(self, name): def __init__(self, name):
self.name = name self.name = name
self.preconditions = [] self.preconditions = []
@ -94,6 +95,7 @@ def __get__(self, instance, owner):
# If instance is there the caller wants to execute the # If instance is there the caller wants to execute the
# install phase, thus return a properly set wrapper # install phase, thus return a properly set wrapper
phase = getattr(instance, self.name) phase = getattr(instance, self.name)
@functools.wraps(phase) @functools.wraps(phase)
def phase_wrapper(spec, prefix): def phase_wrapper(spec, prefix):
# Check instance attributes at the beginning of a phase # Check instance attributes at the beginning of a phase
@ -101,7 +103,8 @@ def phase_wrapper(spec, prefix):
# Execute phase pre-conditions, # Execute phase pre-conditions,
# and give them the chance to fail # and give them the chance to fail
for check in self.preconditions: for check in self.preconditions:
check(instance) # Do something sensible at some point # Do something sensible at some point
check(instance)
phase(spec, prefix) phase(spec, prefix)
# Execute phase sanity_checks, # Execute phase sanity_checks,
# and give them the chance to fail # and give them the chance to fail
@ -147,8 +150,8 @@ def __new__(meta, name, bases, attr_dict):
# Check if phases is in attr dict, then set # Check if phases is in attr dict, then set
# install phases wrappers # install phases wrappers
if 'phases' in attr_dict: if 'phases' in attr_dict:
_InstallPhase_phases = [PackageMeta.phase_fmt.format(x) for x in attr_dict['phases']] _InstallPhase_phases = [PackageMeta.phase_fmt.format(x) for x in attr_dict['phases']] # NOQA: ignore=E501
for phase_name, callback_name in zip(_InstallPhase_phases, attr_dict['phases']): for phase_name, callback_name in zip(_InstallPhase_phases, attr_dict['phases']): # NOQA: ignore=E501
attr_dict[phase_name] = InstallPhase(callback_name) attr_dict[phase_name] = InstallPhase(callback_name)
attr_dict['_InstallPhase_phases'] = _InstallPhase_phases attr_dict['_InstallPhase_phases'] = _InstallPhase_phases
@ -160,15 +163,22 @@ def _append_checks(check_name):
for phase_name, funcs in checks.items(): for phase_name, funcs in checks.items():
try: try:
# Search for the phase in the attribute dictionary # Search for the phase in the attribute dictionary
phase = attr_dict[PackageMeta.phase_fmt.format(phase_name)] phase = attr_dict[
PackageMeta.phase_fmt.format(phase_name)]
except KeyError: except KeyError:
# If it is not there it's in the bases # If it is not there it's in the bases
# and we added a check. We need to copy # and we added a check. We need to copy
# and extend # and extend
for base in bases: for base in bases:
phase = getattr(base, PackageMeta.phase_fmt.format(phase_name), None) phase = getattr(
attr_dict[PackageMeta.phase_fmt.format(phase_name)] = phase.copy() base,
phase = attr_dict[PackageMeta.phase_fmt.format(phase_name)] PackageMeta.phase_fmt.format(phase_name),
None
)
attr_dict[PackageMeta.phase_fmt.format(
phase_name)] = phase.copy()
phase = attr_dict[
PackageMeta.phase_fmt.format(phase_name)]
getattr(phase, check_name).extend(funcs) getattr(phase, check_name).extend(funcs)
# Clear the attribute for the next class # Clear the attribute for the next class
setattr(meta, attr_name, {}) setattr(meta, attr_name, {})
@ -190,8 +200,9 @@ def on_package_attributes(**attrs):
def _execute_under_condition(func): def _execute_under_condition(func):
@functools.wraps(func) @functools.wraps(func)
def _wrapper(instance): def _wrapper(instance):
# If all the attributes have the value we require, then execute # If all the attributes have the value we require, then
if all([getattr(instance, key, None) == value for key, value in attrs.items()]): # execute
if all([getattr(instance, key, None) == value for key, value in attrs.items()]): # NOQA: ignore=E501
func(instance) func(instance)
return _wrapper return _wrapper
return _execute_under_condition return _execute_under_condition
@ -1081,7 +1092,8 @@ def build_process():
else: else:
self.do_stage() self.do_stage()
tty.msg("Building {0} [{1}]".format(self.name, type(self).__base__ )) tty.msg("Building {0} [{1}]".format(
self.name, type(self).__base__))
self.stage.keep = keep_stage self.stage.keep = keep_stage
self.build_directory = join_path(self.stage.path, 'spack-build') self.build_directory = join_path(self.stage.path, 'spack-build')
@ -1106,13 +1118,22 @@ def build_process():
self.log_path = log_path self.log_path = log_path
self.env_path = env_path self.env_path = env_path
dump_environment(env_path) dump_environment(env_path)
# Spawn a daemon that reads from a pipe and redirects everything to log_path # Spawn a daemon that reads from a pipe and redirects
with log_output(log_path, verbose, sys.stdout.isatty(), True) as log_redirection: # everything to log_path
for phase_name, phase in zip(self.phases, self._InstallPhase_phases): redirection_context = log_output(
tty.msg('Executing phase : \'{0}\''.format(phase_name)) log_path, verbose,
sys.stdout.isatty(),
True
)
with redirection_context as log_redirection:
for phase_name, phase in zip(self.phases, self._InstallPhase_phases): # NOQA: ignore=E501
tty.msg(
'Executing phase : \'{0}\''.format(phase_name) # NOQA: ignore=E501
)
# Redirect stdout and stderr to daemon pipe # Redirect stdout and stderr to daemon pipe
with log_redirection: with log_redirection:
getattr(self, phase)(self.spec, self.prefix) getattr(self, phase)(
self.spec, self.prefix)
self.log() self.log()
# Run post install hooks before build stage is removed. # Run post install hooks before build stage is removed.
spack.hooks.post_install(self) spack.hooks.post_install(self)
@ -1174,7 +1195,7 @@ def _do_install_pop_kwargs(self, kwargs):
""" """
self.last_phase = kwargs.pop('stop_at', None) self.last_phase = kwargs.pop('stop_at', None)
if self.last_phase is not None and self.last_phase not in self.phases: if self.last_phase is not None and self.last_phase not in self.phases:
tty.die('\'{0.last_phase}\' is not among the allowed phases for package {0.name}'.format(self)) tty.die('\'{0.last_phase}\' is not among the allowed phases for package {0.name}'.format(self)) # NOQA: ignore=E501
def log(self): def log(self):
# Copy provenance into the install directory on success # Copy provenance into the install directory on success
@ -1188,7 +1209,8 @@ def log(self):
# Remove first if we're overwriting another build # Remove first if we're overwriting another build
# (can happen with spack setup) # (can happen with spack setup)
try: try:
shutil.rmtree(packages_dir) # log_install_path and env_install_path are inside this # log_install_path and env_install_path are inside this
shutil.rmtree(packages_dir)
except Exception: except Exception:
# FIXME : this potentially catches too many things... # FIXME : this potentially catches too many things...
pass pass
@ -1609,7 +1631,8 @@ def autoreconf(self, spec, prefix):
@PackageBase.sanity_check('autoreconf') @PackageBase.sanity_check('autoreconf')
def is_configure_or_die(self): def is_configure_or_die(self):
if not os.path.exists('configure'): if not os.path.exists('configure'):
raise RuntimeError('configure script not found in {0}'.format(os.getcwd())) raise RuntimeError(
'configure script not found in {0}'.format(os.getcwd()))
def configure_args(self): def configure_args(self):
return list() return list()
@ -1668,7 +1691,8 @@ def cmake_args(self):
return list() return list()
def cmake(self, spec, prefix): def cmake(self, spec, prefix):
options = [self.root_cmakelists_dir()] + self.std_cmake_args + self.cmake_args() options = [self.root_cmakelists_dir()] + self.std_cmake_args + \
self.cmake_args()
create = not os.path.exists(self.wdir()) create = not os.path.exists(self.wdir())
with working_dir(self.wdir(), create=create): with working_dir(self.wdir(), create=create):
inspect.getmodule(self).cmake(*options) inspect.getmodule(self).cmake(*options)

20
var/spack/repos/builtin/packages/hdf5/package.py
View file

@ -149,7 +149,7 @@ def configure_args(self):
def check_install(self): def check_install(self):
"Build and run a small program to test the installed HDF5 library" "Build and run a small program to test the installed HDF5 library"
spec = self.spec spec = self.spec
print "Checking HDF5 installation..." print("Checking HDF5 installation...")
checkdir = "spack-check" checkdir = "spack-check"
with working_dir(checkdir, create=True): with working_dir(checkdir, create=True):
source = r""" source = r"""
@ -186,15 +186,15 @@ def check_install(self):
output = "" output = ""
success = output == expected success = output == expected
if not success: if not success:
print "Produced output does not match expected output." print("Produced output does not match expected output.")
print "Expected output:" print("Expected output:")
print '-' * 80 print('-' * 80)
print expected print(expected)
print '-' * 80 print('-' * 80)
print "Produced output:" print("Produced output:")
print '-' * 80 print('-' * 80)
print output print(output)
print '-' * 80 print('-' * 80)
raise RuntimeError("HDF5 install check failed") raise RuntimeError("HDF5 install check failed")
shutil.rmtree(checkdir) shutil.rmtree(checkdir)