From d861a52ebec2a73451fde5870b92b2ff4fb81620 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 12 Oct 2016 18:25:18 -0700 Subject: [PATCH] Use cYAML if it is available in Python. (#2010) --- bin/spack | 10 ++++++++++ lib/spack/spack/spec.py | 7 +++++-- lib/spack/spack/util/spack_yaml.py | 8 ++++++-- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/bin/spack b/bin/spack index 503ef068bf..29991c070d 100755 --- a/bin/spack +++ b/bin/spack @@ -40,6 +40,16 @@ SPACK_PREFIX = os.path.dirname(os.path.dirname(SPACK_FILE)) # Allow spack libs to be imported in our scripts SPACK_LIB_PATH = os.path.join(SPACK_PREFIX, "lib", "spack") sys.path.insert(0, SPACK_LIB_PATH) + +# Try to use system YAML if it is available, as it might have libyaml +# support (for faster loading via C). Load it before anything in +# lib/spack/external so it will take precedence over Spack's PyYAML. +try: + import yaml +except ImportError: + pass # ignore and use slow yaml + +# Add external libs SPACK_EXTERNAL_LIBS = os.path.join(SPACK_LIB_PATH, "external") sys.path.insert(0, SPACK_EXTERNAL_LIBS) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index fc4bf41e34..37a3cf4d7e 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -98,7 +98,7 @@ import base64 import hashlib import imp -import sys +import ctypes from StringIO import StringIO from operator import attrgetter @@ -203,6 +203,9 @@ legal_deps = tuple(special_types) + alldeps +"""Max integer helps avoid passing too large a value to cyaml.""" +maxint = 2 ** (ctypes.sizeof(ctypes.c_int) * 8 - 1) - 1 + def validate_deptype(deptype): if isinstance(deptype, str): @@ -969,7 +972,7 @@ def dag_hash(self, length=None): return self._hash[:length] else: yaml_text = syaml.dump( - self.to_node_dict(), default_flow_style=True, width=sys.maxint) + self.to_node_dict(), default_flow_style=True, width=maxint) sha = hashlib.sha1(yaml_text) b32_hash = base64.b32encode(sha.digest()).lower()[:length] if self.concrete: diff --git a/lib/spack/spack/util/spack_yaml.py b/lib/spack/spack/util/spack_yaml.py index 7bcdf2d61f..506f56633a 100644 --- a/lib/spack/spack/util/spack_yaml.py +++ b/lib/spack/spack/util/spack_yaml.py @@ -32,6 +32,10 @@ """ import yaml +try: + from yaml import CLoader as Loader, CDumper as Dumper +except ImportError as e: + from yaml import Loader, Dumper from yaml.nodes import * from yaml.constructor import ConstructorError from ordereddict_backport import OrderedDict @@ -64,7 +68,7 @@ def mark(obj, node): obj._end_mark = node.end_mark -class OrderedLineLoader(yaml.Loader): +class OrderedLineLoader(Loader): """YAML loader that preserves order and line numbers. Mappings read in by this loader behave like an ordered dict. @@ -156,7 +160,7 @@ def construct_mapping(self, node, deep=False): u'tag:yaml.org,2002:str', OrderedLineLoader.construct_yaml_str) -class OrderedLineDumper(yaml.Dumper): +class OrderedLineDumper(Dumper): """Dumper that preserves ordering and formats ``syaml_*`` objects. This dumper preserves insertion ordering ``syaml_dict`` objects