hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #1047 from epfl-scitas/source_diffs_in_environment_modifications

Browse source 
feature : construct environment modifications sourcing files
This commit is contained in:
Todd Gamblin 2016-06-20 16:56:30 -07:00 committed by GitHub
commit a3431c5a73
4 changed files with 196 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

135
lib/spack/spack/environment.py
View file

@ -1,4 +1,4 @@
##############################################################################
#
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
@ -21,14 +21,17 @@
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
#
import collections
import inspect
import json
import os
import os.path
import subprocess
class NameModifier(object):
def __init__(self, name, **kwargs):
self.name = name
self.args = {'name': name}
@ -36,6 +39,7 @@ def __init__(self, name, **kwargs):
class NameValueModifier(object):
def __init__(self, name, value, **kwargs):
self.name = name
self.value = value
@ -45,23 +49,27 @@ def __init__(self, name, value, **kwargs):
class SetEnv(NameValueModifier):
def execute(self):
os.environ[self.name] = str(self.value)
class UnsetEnv(NameModifier):
def execute(self):
# Avoid throwing if the variable was not set
os.environ.pop(self.name, None)
class SetPath(NameValueModifier):
def execute(self):
string_path = concatenate_paths(self.value, separator=self.separator)
os.environ[self.name] = string_path
class AppendPath(NameValueModifier):
def execute(self):
environment_value = os.environ.get(self.name, '')
directories = environment_value.split(
@ -71,6 +79,7 @@ def execute(self):
class PrependPath(NameValueModifier):
def execute(self):
environment_value = os.environ.get(self.name, '')
directories = environment_value.split(
@ -80,6 +89,7 @@ def execute(self):
class RemovePath(NameValueModifier):
def execute(self):
environment_value = os.environ.get(self.name, '')
directories = environment_value.split(
@ -90,6 +100,7 @@ def execute(self):
class EnvironmentModifications(object):
"""
Keeps track of requests to modify the current environment.
@ -240,6 +251,126 @@ def apply_modifications(self):
for x in actions:
x.execute()
@staticmethod
def from_sourcing_files(*args, **kwargs):
"""
Creates an instance of EnvironmentModifications that, if executed,
has the same effect on the environment as sourcing the files passed as
parameters
Args:
*args: list of files to be sourced
Returns:
instance of EnvironmentModifications
"""
env = EnvironmentModifications()
# Check if the files are actually there
if not all(os.path.isfile(file) for file in args):
raise RuntimeError('trying to source non-existing files')
# Relevant kwd parameters and formats
info = dict(kwargs)
info.setdefault('shell', '/bin/bash')
info.setdefault('shell_options', '-c')
info.setdefault('source_command', 'source')
info.setdefault('suppress_output', '&> /dev/null')
info.setdefault('concatenate_on_success', '&&')
shell = '{shell}'.format(**info)
shell_options = '{shell_options}'.format(**info)
source_file = '{source_command} {file} {concatenate_on_success}'
dump_environment = 'python -c "import os, json; print json.dumps(dict(os.environ))"' # NOQA: ignore=E501
# Construct the command that will be executed
command = [source_file.format(file=file, **info) for file in args]
command.append(dump_environment)
command = ' '.join(command)
command = [
shell,
shell_options,
command
]
# Try to source all the files,
proc = subprocess.Popen(
command, stdout=subprocess.PIPE, env=os.environ)
proc.wait()
if proc.returncode != 0:
raise RuntimeError('sourcing files returned a non-zero exit code')
output = ''.join([line for line in proc.stdout])
# Construct a dictionary with all the variables in the new environment
after_source_env = dict(json.loads(output))
this_environment = dict(os.environ)
# Filter variables that are not related to sourcing a file
to_be_filtered = 'SHLVL', '_', 'PWD', 'OLDPWD'
for d in after_source_env, this_environment:
for name in to_be_filtered:
d.pop(name, None)
# Fill the EnvironmentModifications instance
# New variables
new_variables = set(after_source_env) - set(this_environment)
for x in new_variables:
env.set(x, after_source_env[x])
# Variables that have been unset
unset_variables = set(this_environment) - set(after_source_env)
for x in unset_variables:
env.unset(x)
# Variables that have been modified
common_variables = set(this_environment).intersection(set(after_source_env)) # NOQA: ignore=E501
modified_variables = [x for x in common_variables if this_environment[x] != after_source_env[x]] # NOQA: ignore=E501
def return_separator_if_any(first_value, second_value):
separators = ':', ';'
for separator in separators:
if separator in first_value and separator in second_value:
return separator
return None
for x in modified_variables:
current = this_environment[x]
modified = after_source_env[x]
sep = return_separator_if_any(current, modified)
if sep is None:
# We just need to set the variable to the new value
env.set(x, after_source_env[x])
else:
current_list = current.split(sep)
modified_list = modified.split(sep)
# Paths that have been removed
remove_list = [
ii for ii in current_list if ii not in modified_list]
# Check that nothing has been added in the middle of vurrent
# list
remaining_list = [
ii for ii in current_list if ii in modified_list]
start = modified_list.index(remaining_list[0])
end = modified_list.index(remaining_list[-1])
search = sep.join(modified_list[start:end + 1])
if search not in current:
# We just need to set the variable to the new value
env.set(x, after_source_env[x])
break
else:
try:
prepend_list = modified_list[:start]
except KeyError:
prepend_list = []
try:
append_list = modified_list[end + 1:]
except KeyError:
append_list = []
for item in remove_list:
env.remove_path(x, item)
for item in append_list:
env.append_path(x, item)
for item in prepend_list:
env.prepend_path(x, item)
return env
def concatenate_paths(paths, separator=':'):
"""

3
lib/spack/spack/test/data/sourceme_first.sh Normal file
View file

@ -0,0 +1,3 @@
#!/usr/bin/env bash
export NEW_VAR='new'
export UNSET_ME='overridden'

3
lib/spack/spack/test/data/sourceme_second.sh Normal file
View file

@ -0,0 +1,3 @@
#!/usr/bin/env bash
export PATH_LIST='/path/first:/path/second:/path/fourth'
unset EMPTY_PATH_LIST

62
lib/spack/spack/test/environment.py
View file

@ -24,16 +24,21 @@
##############################################################################
import unittest
import os
import copy
from spack import spack_root
from llnl.util.filesystem import join_path
from spack.environment import EnvironmentModifications
from spack.environment import SetEnv, UnsetEnv
from spack.environment import RemovePath, PrependPath, AppendPath
class EnvironmentTest(unittest.TestCase):
def setUp(self):
os.environ['UNSET_ME'] = 'foo'
os.environ['EMPTY_PATH_LIST'] = ''
os.environ['PATH_LIST'] = '/path/second:/path/third'
os.environ['REMOVE_PATH_LIST'] = '/a/b:/duplicate:/a/c:/remove/this:/a/d:/duplicate/:/f/g'
os.environ['REMOVE_PATH_LIST'] = '/a/b:/duplicate:/a/c:/remove/this:/a/d:/duplicate/:/f/g' # NOQA: ignore=E501
def tearDown(self):
pass
@ -77,9 +82,18 @@ def test_path_manipulation(self):
env.remove_path('REMOVE_PATH_LIST', '/duplicate/')
env.apply_modifications()
self.assertEqual('/path/first:/path/second:/path/third:/path/last', os.environ['PATH_LIST'])
self.assertEqual('/path/first:/path/middle:/path/last', os.environ['EMPTY_PATH_LIST'])
self.assertEqual('/path/first:/path/middle:/path/last', os.environ['NEWLY_CREATED_PATH_LIST'])
self.assertEqual(
'/path/first:/path/second:/path/third:/path/last',
os.environ['PATH_LIST']
)
self.assertEqual(
'/path/first:/path/middle:/path/last',
os.environ['EMPTY_PATH_LIST']
)
self.assertEqual(
'/path/first:/path/middle:/path/last',
os.environ['NEWLY_CREATED_PATH_LIST']
)
self.assertEqual('/a/b:/a/c:/a/d:/f/g', os.environ['REMOVE_PATH_LIST'])
def test_extra_arguments(self):
@ -98,3 +112,41 @@ def test_extend(self):
self.assertEqual(len(copy_construct), 2)
for x, y in zip(env, copy_construct):
assert x is y
def test_source_files(self):
datadir = join_path(spack_root, 'lib', 'spack',
'spack', 'test', 'data')
files = [
join_path(datadir, 'sourceme_first.sh'),
join_path(datadir, 'sourceme_second.sh')
]
env = EnvironmentModifications.from_sourcing_files(*files)
modifications = env.group_by_name()
self.assertEqual(len(modifications), 4)
# Set new variables
self.assertEqual(len(modifications['NEW_VAR']), 1)
self.assertTrue(isinstance(modifications['NEW_VAR'][0], SetEnv))
self.assertEqual(modifications['NEW_VAR'][0].value, 'new')
# Unset variables
self.assertEqual(len(modifications['EMPTY_PATH_LIST']), 1)
self.assertTrue(isinstance(
modifications['EMPTY_PATH_LIST'][0], UnsetEnv))
# Modified variables
self.assertEqual(len(modifications['UNSET_ME']), 1)
self.assertTrue(isinstance(modifications['UNSET_ME'][0], SetEnv))
self.assertEqual(modifications['UNSET_ME'][0].value, 'overridden')
self.assertEqual(len(modifications['PATH_LIST']), 3)
self.assertTrue(
isinstance(modifications['PATH_LIST'][0], RemovePath)
)
self.assertEqual(modifications['PATH_LIST'][0].value, '/path/third')
self.assertTrue(
isinstance(modifications['PATH_LIST'][1], AppendPath)
)
self.assertEqual(modifications['PATH_LIST'][1].value, '/path/fourth')
self.assertTrue(
isinstance(modifications['PATH_LIST'][2], PrependPath)
)
self.assertEqual(modifications['PATH_LIST'][2].value, '/path/first')