Resolve Python2/Python3 unicode issues by using str()

- Remove ascii encoding assumption from spack_yaml
- proc.communicate() returns bytes; convert to str before adding.
- Fix various byte string/unicode issues for Python 2/3 support
- Need to decode subprocess output as utf-8 in from_sourcing_files.
- Fix comments in strify()
This commit is contained in:
Todd Gamblin 2017-03-07 15:18:48 -08:00
parent 1d1a14dbe9
commit 0cd6555388
7 changed files with 15 additions and 13 deletions

View file

@ -310,7 +310,7 @@ def from_sourcing_files(*args, **kwargs):
proc.wait()
if proc.returncode != 0:
raise RuntimeError('sourcing files returned a non-zero exit code')
output = ''.join([line for line in proc.stdout])
output = ''.join([line.decode('utf-8') for line in proc.stdout])
# Construct a dictionaries of the environment before and after
# sourcing the files, so that we can diff them.

View file

@ -226,7 +226,7 @@ def __init__(
self._lock = None
if lock:
if self.name not in Stage.stage_locks:
sha1 = hashlib.sha1(self.name).digest()
sha1 = hashlib.sha1(self.name.encode('utf-8')).digest()
lock_id = prefix_bits(sha1, bit_length(sys.maxsize))
stage_lock_path = join_path(spack.stage_path, '.lock')

View file

@ -138,8 +138,8 @@ def test_user_defaults(config):
def test_user_input_combination(config):
platform = spack.architecture.platform()
os_list = platform.operating_sys.keys()
target_list = platform.targets.keys()
os_list = list(platform.operating_sys.keys())
target_list = list(platform.targets.keys())
additional = ["fe", "be", "frontend", "backend"]
os_list.extend(additional)

View file

@ -22,6 +22,7 @@
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import sys
import hashlib
"""Set of acceptable hashes that Spack will use."""
@ -104,11 +105,16 @@ def check(self, filename):
def prefix_bits(byte_array, bits):
"""Return the first <bits> bits of a byte array as an integer."""
if sys.version_info < (3,):
b2i = ord # In Python 2, indexing byte_array gives str
else:
b2i = lambda b: b # In Python 3, indexing byte_array gives int
result = 0
n = 0
for i, b in enumerate(byte_array):
n += 8
result = (result << 8) | ord(b)
result = (result << 8) | b2i(b)
if n >= bits:
break

View file

@ -178,9 +178,9 @@ def streamify(arg, mode):
if output is str or error is str:
result = ''
if output is str:
result += out
result += out.decode('utf-8')
if error is str:
result += err
result += err.decode('utf-8')
return result
except OSError as e:

View file

@ -68,6 +68,7 @@ def _byteify(data, ignore_dicts=False):
return dict((_byteify(key, ignore_dicts=True),
_byteify(value, ignore_dicts=True)) for key, value in
iteritems(data))
# if it's anything else, return it in its original form
return data

View file

@ -85,11 +85,6 @@ class OrderedLineLoader(Loader):
def construct_yaml_str(self, node):
value = self.construct_scalar(node)
try:
value = value.encode('ascii')
except UnicodeEncodeError:
pass
value = syaml_str(value)
mark(value, node)
@ -181,7 +176,7 @@ def represent_mapping(self, tag, mapping, flow_style=None):
# if it's a syaml_dict, preserve OrderedDict order.
# Otherwise do the default thing.
sort = not isinstance(mapping, syaml_dict)
mapping = mapping.items()
mapping = list(mapping.items())
if sort:
mapping.sort()