refactor: convert build_process to use BuildProcessInstaller (#25167)
`build_process` has been around a long time but it's become a very large,
unwieldy method. It's hard to work with because it has a lot of local
variables that need to persist across all of the code.
- [x] To address this, convert it its own `BuildInfoProcess` class.
- [x] Start breaking the method apart by factoring out the main
installation logic into its own function.
This commit is contained in:
Todd Gamblin
GitHub
parent
0a0338ddfa
commit
cf8d1b0387
1 changed files with 208 additions and 148 deletions
|
|
@ -144,14 +144,8 @@ def _handle_external_and_upstream(pkg, explicit):
|
||||||
|
|
||||||
|
|
||||||
def _do_fake_install(pkg):
|
def _do_fake_install(pkg):
|
||||||
|
"""Make a fake install directory with fake executables, headers, and libraries.
|
||||||
"""
|
"""
|
||||||
Make a fake install directory containing fake executables, headers,
|
|
||||||
and libraries.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
pkg (spack.package.PackageBase): the package whose installation is to be faked
|
|
||||||
"""
|
|
||||||
|
|
||||||
command = pkg.name
|
command = pkg.name
|
||||||
header = pkg.name
|
header = pkg.name
|
||||||
library = pkg.name
|
library = pkg.name
|
||||||
|
|
@ -1170,8 +1164,7 @@ def _install_task(self, task):
|
||||||
# stop early from clients, and is not an error at this point
|
# stop early from clients, and is not an error at this point
|
||||||
pid = '{0}: '.format(pkg.pid) if tty.show_pid() else ''
|
pid = '{0}: '.format(pkg.pid) if tty.show_pid() else ''
|
||||||
tty.debug('{0}{1}'.format(pid, str(e)))
|
tty.debug('{0}{1}'.format(pid, str(e)))
|
||||||
tty.debug('Package stage directory: {0}'
|
tty.debug('Package stage directory: {0}' .format(pkg.stage.source_path))
|
||||||
.format(pkg.stage.source_path))
|
|
||||||
|
|
||||||
def _next_is_pri0(self):
|
def _next_is_pri0(self):
|
||||||
"""
|
"""
|
||||||
|
|
@ -1678,152 +1671,219 @@ def install(self):
|
||||||
'reported errors for failing package(s).')
|
'reported errors for failing package(s).')
|
||||||
|
|
||||||
|
|
||||||
def build_process(pkg, kwargs):
|
class BuildProcessInstaller(object):
|
||||||
|
"""This class implements the part installation that happens in the child process."""
|
||||||
|
|
||||||
|
def __init__(self, pkg, install_args):
|
||||||
|
"""Create a new BuildProcessInstaller.
|
||||||
|
|
||||||
|
It is assumed that the lifecycle of this object is the same as the child
|
||||||
|
process in the build.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
pkg (spack.package.PackageBase) the package being installed.
|
||||||
|
install_args (dict) arguments to do_install() from parent process.
|
||||||
|
|
||||||
|
"""
|
||||||
|
self.pkg = pkg
|
||||||
|
|
||||||
|
# whether to do a fake install
|
||||||
|
self.fake = install_args.get('fake', False)
|
||||||
|
|
||||||
|
# whether to install source code with the packag
|
||||||
|
self.install_source = install_args.get('install_source', False)
|
||||||
|
|
||||||
|
# whether to keep the build stage after installation
|
||||||
|
self.keep_stage = install_args.get('keep_stage', False)
|
||||||
|
|
||||||
|
# whether to skip the patch phase
|
||||||
|
self.skip_patch = install_args.get('skip_patch', False)
|
||||||
|
|
||||||
|
# whether to enable echoing of build output initially or not
|
||||||
|
self.verbose = install_args.get('verbose', False)
|
||||||
|
|
||||||
|
# env before starting installation
|
||||||
|
self.unmodified_env = install_args.get('unmodified_env', {})
|
||||||
|
|
||||||
|
# timer for build phases
|
||||||
|
self.timer = Timer()
|
||||||
|
|
||||||
|
# If we are using a padded path, filter the output to compress padded paths
|
||||||
|
# The real log still has full-length paths.
|
||||||
|
filter_padding = spack.config.get("config:install_tree:padded_length", None)
|
||||||
|
self.filter_fn = spack.util.path.padding_filter if filter_padding else None
|
||||||
|
|
||||||
|
# info/debug information
|
||||||
|
pid = '{0}: '.format(pkg.pid) if tty.show_pid() else ''
|
||||||
|
self.pre = '{0}{1}:'.format(pid, pkg.name)
|
||||||
|
self.pkg_id = package_id(pkg)
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
"""Main entry point from ``build_process`` to kick off install in child."""
|
||||||
|
|
||||||
|
if not self.fake:
|
||||||
|
if not self.skip_patch:
|
||||||
|
self.pkg.do_patch()
|
||||||
|
else:
|
||||||
|
self.pkg.do_stage()
|
||||||
|
|
||||||
|
tty.debug(
|
||||||
|
'{0} Building {1} [{2}]' .format(
|
||||||
|
self.pre,
|
||||||
|
self.pkg_id,
|
||||||
|
self.pkg.build_system_class
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# get verbosity from do_install() parameter or saved value
|
||||||
|
self.echo = self.verbose
|
||||||
|
if spack.package.PackageBase._verbose is not None:
|
||||||
|
self.echo = spack.package.PackageBase._verbose
|
||||||
|
|
||||||
|
self.pkg.stage.keep = self.keep_stage
|
||||||
|
|
||||||
|
with self.pkg.stage:
|
||||||
|
# Run the pre-install hook in the child process after
|
||||||
|
# the directory is created.
|
||||||
|
spack.hooks.pre_install(self.pkg.spec)
|
||||||
|
if self.fake:
|
||||||
|
_do_fake_install(self.pkg)
|
||||||
|
else:
|
||||||
|
if self.install_source:
|
||||||
|
self._install_source()
|
||||||
|
|
||||||
|
self._real_install()
|
||||||
|
|
||||||
|
# Stop the timer and save results
|
||||||
|
self.timer.stop()
|
||||||
|
with open(self.pkg.times_log_path, 'w') as timelog:
|
||||||
|
self.timer.write_json(timelog)
|
||||||
|
|
||||||
|
# Run post install hooks before build stage is removed.
|
||||||
|
spack.hooks.post_install(self.pkg.spec)
|
||||||
|
|
||||||
|
build_time = self.timer.total - self.pkg._fetch_time
|
||||||
|
tty.msg('{0} Successfully installed {1}'.format(self.pre, self.pkg_id),
|
||||||
|
'Fetch: {0}. Build: {1}. Total: {2}.'
|
||||||
|
.format(_hms(self.pkg._fetch_time), _hms(build_time),
|
||||||
|
_hms(self.timer.total)))
|
||||||
|
_print_installed_pkg(self.pkg.prefix)
|
||||||
|
|
||||||
|
# Send final status that install is successful
|
||||||
|
spack.hooks.on_install_success(self.pkg.spec)
|
||||||
|
|
||||||
|
# preserve verbosity across runs
|
||||||
|
return self.echo
|
||||||
|
|
||||||
|
def _install_source(self):
|
||||||
|
"""Install source code from stage into share/pkg/src if necessary."""
|
||||||
|
pkg = self.pkg
|
||||||
|
if not os.path.isdir(pkg.stage.source_path):
|
||||||
|
return
|
||||||
|
|
||||||
|
src_target = os.path.join(pkg.spec.prefix, 'share', pkg.name, 'src')
|
||||||
|
tty.debug('{0} Copying source to {1}' .format(self.pre, src_target))
|
||||||
|
|
||||||
|
fs.install_tree(pkg.stage.source_path, src_target)
|
||||||
|
|
||||||
|
def _real_install(self):
|
||||||
|
pkg = self.pkg
|
||||||
|
|
||||||
|
# Do the real install in the source directory.
|
||||||
|
with fs.working_dir(pkg.stage.source_path):
|
||||||
|
# Save the build environment in a file before building.
|
||||||
|
dump_environment(pkg.env_path)
|
||||||
|
|
||||||
|
for attr in ('configure_args', 'cmake_args'):
|
||||||
|
try:
|
||||||
|
configure_args = getattr(pkg, attr)()
|
||||||
|
configure_args = ' '.join(configure_args)
|
||||||
|
|
||||||
|
with open(pkg.configure_args_path, 'w') as \
|
||||||
|
args_file:
|
||||||
|
args_file.write(configure_args)
|
||||||
|
|
||||||
|
break
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# cache debug settings
|
||||||
|
debug_level = tty.debug_level()
|
||||||
|
|
||||||
|
# Spawn a daemon that reads from a pipe and redirects
|
||||||
|
# everything to log_path, and provide the phase for logging
|
||||||
|
for i, (phase_name, phase_attr) in enumerate(zip(
|
||||||
|
pkg.phases, pkg._InstallPhase_phases)):
|
||||||
|
|
||||||
|
# Keep a log file for each phase
|
||||||
|
log_dir = os.path.dirname(pkg.log_path)
|
||||||
|
log_file = "spack-build-%02d-%s-out.txt" % (
|
||||||
|
i + 1, phase_name.lower()
|
||||||
|
)
|
||||||
|
log_file = os.path.join(log_dir, log_file)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# DEBUGGING TIP - to debug this section, insert an IPython
|
||||||
|
# embed here, and run the sections below without log capture
|
||||||
|
log_contextmanager = log_output(
|
||||||
|
log_file,
|
||||||
|
self.echo,
|
||||||
|
True,
|
||||||
|
env=self.unmodified_env,
|
||||||
|
filter_fn=self.filter_fn
|
||||||
|
)
|
||||||
|
|
||||||
|
with log_contextmanager as logger:
|
||||||
|
with logger.force_echo():
|
||||||
|
inner_debug_level = tty.debug_level()
|
||||||
|
tty.set_debug(debug_level)
|
||||||
|
tty.msg(
|
||||||
|
"{0} Executing phase: '{1}'" .format(
|
||||||
|
self.pre,
|
||||||
|
phase_name
|
||||||
|
)
|
||||||
|
)
|
||||||
|
tty.set_debug(inner_debug_level)
|
||||||
|
|
||||||
|
# Redirect stdout and stderr to daemon pipe
|
||||||
|
phase = getattr(pkg, phase_attr)
|
||||||
|
self.timer.phase(phase_name)
|
||||||
|
|
||||||
|
# Catch any errors to report to logging
|
||||||
|
phase(pkg.spec, pkg.prefix)
|
||||||
|
spack.hooks.on_phase_success(pkg, phase_name, log_file)
|
||||||
|
|
||||||
|
except BaseException:
|
||||||
|
combine_phase_logs(pkg.phase_log_files, pkg.log_path)
|
||||||
|
spack.hooks.on_phase_error(pkg, phase_name, log_file)
|
||||||
|
raise
|
||||||
|
|
||||||
|
# We assume loggers share echo True/False
|
||||||
|
self.echo = logger.echo
|
||||||
|
|
||||||
|
# After log, we can get all output/error files from the package stage
|
||||||
|
combine_phase_logs(pkg.phase_log_files, pkg.log_path)
|
||||||
|
log(pkg)
|
||||||
|
|
||||||
|
|
||||||
|
def build_process(pkg, install_args):
|
||||||
"""Perform the installation/build of the package.
|
"""Perform the installation/build of the package.
|
||||||
|
|
||||||
This runs in a separate child process, and has its own process and
|
This runs in a separate child process, and has its own process and
|
||||||
python module space set up by build_environment.start_build_process().
|
python module space set up by build_environment.start_build_process().
|
||||||
|
|
||||||
|
This essentially wraps an instance of ``BuildProcessInstaller`` so that we can
|
||||||
|
more easily create one in a subprocess.
|
||||||
|
|
||||||
This function's return value is returned to the parent process.
|
This function's return value is returned to the parent process.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
pkg (spack.package.PackageBase): the package being installed.
|
||||||
|
install_args (dict): arguments to do_install() from parent process.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
fake = kwargs.get('fake', False)
|
installer = BuildProcessInstaller(pkg, install_args)
|
||||||
install_source = kwargs.get('install_source', False)
|
return installer.run()
|
||||||
keep_stage = kwargs.get('keep_stage', False)
|
|
||||||
skip_patch = kwargs.get('skip_patch', False)
|
|
||||||
unmodified_env = kwargs.get('unmodified_env', {})
|
|
||||||
verbose = kwargs.get('verbose', False)
|
|
||||||
|
|
||||||
timer = Timer()
|
|
||||||
|
|
||||||
# If we are using a padded path, filter the output to compress padded paths
|
|
||||||
# The real log still has full-length paths.
|
|
||||||
filter_padding = spack.config.get("config:install_tree:padded_length", None)
|
|
||||||
filter_fn = spack.util.path.padding_filter if filter_padding else None
|
|
||||||
|
|
||||||
if not fake:
|
|
||||||
if not skip_patch:
|
|
||||||
pkg.do_patch()
|
|
||||||
else:
|
|
||||||
pkg.do_stage()
|
|
||||||
|
|
||||||
pid = '{0}: '.format(pkg.pid) if tty.show_pid() else ''
|
|
||||||
pre = '{0}{1}:'.format(pid, pkg.name)
|
|
||||||
pkg_id = package_id(pkg)
|
|
||||||
|
|
||||||
tty.debug('{0} Building {1} [{2}]'
|
|
||||||
.format(pre, pkg_id, pkg.build_system_class))
|
|
||||||
|
|
||||||
# get verbosity from do_install() parameter or saved value
|
|
||||||
echo = verbose
|
|
||||||
if spack.package.PackageBase._verbose is not None:
|
|
||||||
echo = spack.package.PackageBase._verbose
|
|
||||||
|
|
||||||
pkg.stage.keep = keep_stage
|
|
||||||
|
|
||||||
with pkg.stage:
|
|
||||||
# Run the pre-install hook in the child process after
|
|
||||||
# the directory is created.
|
|
||||||
spack.hooks.pre_install(pkg.spec)
|
|
||||||
if fake:
|
|
||||||
_do_fake_install(pkg)
|
|
||||||
else:
|
|
||||||
source_path = pkg.stage.source_path
|
|
||||||
if install_source and os.path.isdir(source_path):
|
|
||||||
src_target = os.path.join(pkg.spec.prefix, 'share',
|
|
||||||
pkg.name, 'src')
|
|
||||||
tty.debug('{0} Copying source to {1}'
|
|
||||||
.format(pre, src_target))
|
|
||||||
fs.install_tree(pkg.stage.source_path, src_target)
|
|
||||||
|
|
||||||
# Do the real install in the source directory.
|
|
||||||
with fs.working_dir(pkg.stage.source_path):
|
|
||||||
|
|
||||||
# Save the build environment in a file before building.
|
|
||||||
dump_environment(pkg.env_path)
|
|
||||||
|
|
||||||
for attr in ('configure_args', 'cmake_args'):
|
|
||||||
try:
|
|
||||||
configure_args = getattr(pkg, attr)()
|
|
||||||
configure_args = ' '.join(configure_args)
|
|
||||||
|
|
||||||
with open(pkg.configure_args_path, 'w') as \
|
|
||||||
args_file:
|
|
||||||
args_file.write(configure_args)
|
|
||||||
|
|
||||||
break
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# cache debug settings
|
|
||||||
debug_level = tty.debug_level()
|
|
||||||
|
|
||||||
# Spawn a daemon that reads from a pipe and redirects
|
|
||||||
# everything to log_path, and provide the phase for logging
|
|
||||||
for i, (phase_name, phase_attr) in enumerate(zip(
|
|
||||||
pkg.phases, pkg._InstallPhase_phases)):
|
|
||||||
|
|
||||||
# Keep a log file for each phase
|
|
||||||
log_dir = os.path.dirname(pkg.log_path)
|
|
||||||
log_file = "spack-build-%02d-%s-out.txt" % (
|
|
||||||
i + 1, phase_name.lower()
|
|
||||||
)
|
|
||||||
log_file = os.path.join(log_dir, log_file)
|
|
||||||
|
|
||||||
try:
|
|
||||||
# DEBUGGING TIP - to debug this section, insert an IPython
|
|
||||||
# embed here, and run the sections below without log capture
|
|
||||||
with log_output(
|
|
||||||
log_file, echo, True, env=unmodified_env,
|
|
||||||
filter_fn=filter_fn
|
|
||||||
) as logger:
|
|
||||||
|
|
||||||
with logger.force_echo():
|
|
||||||
inner_debug_level = tty.debug_level()
|
|
||||||
tty.set_debug(debug_level)
|
|
||||||
tty.msg("{0} Executing phase: '{1}'"
|
|
||||||
.format(pre, phase_name))
|
|
||||||
tty.set_debug(inner_debug_level)
|
|
||||||
|
|
||||||
# Redirect stdout and stderr to daemon pipe
|
|
||||||
phase = getattr(pkg, phase_attr)
|
|
||||||
timer.phase(phase_name)
|
|
||||||
|
|
||||||
# Catch any errors to report to logging
|
|
||||||
phase(pkg.spec, pkg.prefix)
|
|
||||||
spack.hooks.on_phase_success(pkg, phase_name, log_file)
|
|
||||||
|
|
||||||
except BaseException:
|
|
||||||
combine_phase_logs(pkg.phase_log_files, pkg.log_path)
|
|
||||||
spack.hooks.on_phase_error(pkg, phase_name, log_file)
|
|
||||||
raise
|
|
||||||
|
|
||||||
# We assume loggers share echo True/False
|
|
||||||
echo = logger.echo
|
|
||||||
|
|
||||||
# After log, we can get all output/error files from the package stage
|
|
||||||
combine_phase_logs(pkg.phase_log_files, pkg.log_path)
|
|
||||||
log(pkg)
|
|
||||||
|
|
||||||
# Stop the timer and save results
|
|
||||||
timer.stop()
|
|
||||||
with open(pkg.times_log_path, 'w') as timelog:
|
|
||||||
timer.write_json(timelog)
|
|
||||||
|
|
||||||
# Run post install hooks before build stage is removed.
|
|
||||||
spack.hooks.post_install(pkg.spec)
|
|
||||||
|
|
||||||
build_time = timer.total - pkg._fetch_time
|
|
||||||
tty.msg('{0} Successfully installed {1}'.format(pre, pkg_id),
|
|
||||||
'Fetch: {0}. Build: {1}. Total: {2}.'
|
|
||||||
.format(_hms(pkg._fetch_time), _hms(build_time),
|
|
||||||
_hms(timer.total)))
|
|
||||||
_print_installed_pkg(pkg.prefix)
|
|
||||||
|
|
||||||
# Send final status that install is successful
|
|
||||||
spack.hooks.on_install_success(pkg.spec)
|
|
||||||
|
|
||||||
# preserve verbosity across runs
|
|
||||||
return echo
|
|
||||||
|
|
||||||
|
|
||||||
class BuildTask(object):
|
class BuildTask(object):
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue