autotools: extend patching of the libtool script (#30768)
* filter_file: introduce argument 'start_at' * autotools: extend patching of the libtool script * autotools: refactor _patch_usr_bin_file * autotools: improve readability of the filtering * autotools: keep the modification time of the configure scripts * autotools: do not try to patch directories * autotools: explain libtool patching for posterity Co-authored-by: Massimiliano Culpo <massimiliano.culpo@gmail.com> Co-authored-by: Harmen Stoppels <harmenstoppels@gmail.com>
This commit is contained in:
Sergey Kosukhin
GitHub
parent
8a5790514d
commit
4bc8f66388
4 changed files with 206 additions and 32 deletions
|
|
@ -233,9 +233,14 @@ def filter_file(regex, repl, *filenames, **kwargs):
|
|||
|
||||
Keyword Arguments:
|
||||
string (bool): Treat regex as a plain string. Default it False
|
||||
backup (bool): Make backup file(s) suffixed with ``~``. Default is True
|
||||
backup (bool): Make backup file(s) suffixed with ``~``. Default is False
|
||||
ignore_absent (bool): Ignore any files that don't exist.
|
||||
Default is False
|
||||
start_at (str): Marker used to start applying the replacements. If a
|
||||
text line matches this marker filtering is started at the next line.
|
||||
All contents before the marker and the marker itself are copied
|
||||
verbatim. Default is to start filtering from the first line of the
|
||||
file.
|
||||
stop_at (str): Marker used to stop scanning the file further. If a text
|
||||
line matches this marker filtering is stopped and the rest of the
|
||||
file is copied verbatim. Default is to filter until the end of the
|
||||
|
|
@ -244,6 +249,7 @@ def filter_file(regex, repl, *filenames, **kwargs):
|
|||
string = kwargs.get("string", False)
|
||||
backup = kwargs.get("backup", False)
|
||||
ignore_absent = kwargs.get("ignore_absent", False)
|
||||
start_at = kwargs.get("start_at", None)
|
||||
stop_at = kwargs.get("stop_at", None)
|
||||
|
||||
# Allow strings to use \1, \2, etc. for replacement, like sed
|
||||
|
|
@ -292,6 +298,7 @@ def groupid_to_group(x):
|
|||
# reached or we found a marker in the line if it was specified
|
||||
with open(tmp_filename, mode="r", **extra_kwargs) as input_file:
|
||||
with open(filename, mode="w", **extra_kwargs) as output_file:
|
||||
do_filtering = start_at is None
|
||||
# Using iter and readline is a workaround needed not to
|
||||
# disable input_file.tell(), which will happen if we call
|
||||
# input_file.next() implicitly via the for loop
|
||||
|
|
@ -301,8 +308,12 @@ def groupid_to_group(x):
|
|||
if stop_at == line.strip():
|
||||
output_file.write(line)
|
||||
break
|
||||
filtered_line = re.sub(regex, repl, line)
|
||||
output_file.write(filtered_line)
|
||||
if do_filtering:
|
||||
filtered_line = re.sub(regex, repl, line)
|
||||
output_file.write(filtered_line)
|
||||
else:
|
||||
do_filtering = start_at == line.strip()
|
||||
output_file.write(line)
|
||||
else:
|
||||
current_position = None
|
||||
|
||||
|
|
|
|||
|
|
@ -244,8 +244,11 @@ def _patch_usr_bin_file(self):
|
|||
scripts to use file from path."""
|
||||
|
||||
if self.spec.os.startswith("nixos"):
|
||||
for configure_file in fs.find(".", files=["configure"], recursive=True):
|
||||
fs.filter_file("/usr/bin/file", "file", configure_file, string=True)
|
||||
x = fs.FileFilter(
|
||||
*filter(fs.is_exe, fs.find(self.build_directory, "configure", recursive=True))
|
||||
)
|
||||
with fs.keep_modification_time(*x.filenames):
|
||||
x.filter(regex="/usr/bin/file", repl="file", string=True)
|
||||
|
||||
@run_before("configure")
|
||||
def _set_autotools_environment_variables(self):
|
||||
|
|
@ -262,35 +265,97 @@ def _set_autotools_environment_variables(self):
|
|||
"""
|
||||
os.environ["FORCE_UNSAFE_CONFIGURE"] = "1"
|
||||
|
||||
@run_before("configure")
|
||||
def _do_patch_libtool_configure(self):
|
||||
"""Patch bugs that propagate from libtool macros into "configure" and
|
||||
further into "libtool". Note that patches that can be fixed by patching
|
||||
"libtool" directly should be implemented in the _do_patch_libtool method
|
||||
below."""
|
||||
|
||||
# Exit early if we are required not to patch libtool-related problems:
|
||||
if not self.patch_libtool:
|
||||
return
|
||||
|
||||
x = fs.FileFilter(
|
||||
*filter(fs.is_exe, fs.find(self.build_directory, "configure", recursive=True))
|
||||
)
|
||||
|
||||
# There are distributed automatically generated files that depend on the configure script
|
||||
# and require additional tools for rebuilding.
|
||||
# See https://github.com/spack/spack/pull/30768#issuecomment-1219329860
|
||||
with fs.keep_modification_time(*x.filenames):
|
||||
# Fix parsing of compiler output when collecting predeps and postdeps
|
||||
# https://lists.gnu.org/archive/html/bug-libtool/2016-03/msg00003.html
|
||||
x.filter(regex=r'^(\s*if test x-L = )("\$p" \|\|\s*)$', repl=r"\1x\2")
|
||||
x.filter(
|
||||
regex=r'^(\s*test x-R = )("\$p")(; then\s*)$', repl=r'\1x\2 || test x-l = x"$p"\3'
|
||||
)
|
||||
# Support Libtool 2.4.2 and older:
|
||||
x.filter(regex=r'^(\s*test \$p = "-R")(; then\s*)$', repl=r'\1 || test x-l = x"$p"\2')
|
||||
|
||||
@run_after("configure")
|
||||
def _do_patch_libtool(self):
|
||||
"""If configure generates a "libtool" script that does not correctly
|
||||
detect the compiler (and patch_libtool is set), patch in the correct
|
||||
flags for the Arm, Clang/Flang, Fujitsu and NVHPC compilers. Also
|
||||
filter out spurious predep_objects for Intel dpcpp builds."""
|
||||
values for libtool variables.
|
||||
|
||||
# Exit early if we are required not to patch libtool
|
||||
The generated libtool script supports mixed compilers through tags:
|
||||
``libtool --tag=CC/CXX/FC/...```. For each tag there is a block with variables,
|
||||
which defines what flags to pass to the compiler. The default variables (which
|
||||
are used by the default tag CC) are set in a block enclosed by
|
||||
``# ### {BEGIN,END} LIBTOOL CONFIG``. For non-default tags, there are
|
||||
corresponding blocks ``# ### {BEGIN,END} LIBTOOL TAG CONFIG: {CXX,FC,F77}`` at
|
||||
the end of the file (after the exit command). libtool evals these blocks.
|
||||
Whenever we need to update variables that the configure script got wrong
|
||||
(for example cause it did not recognize the compiler), we should properly scope
|
||||
those changes to these tags/blocks so they only apply to the compiler we care
|
||||
about. Below, ``start_at`` and ``stop_at`` are used for that."""
|
||||
|
||||
# Exit early if we are required not to patch libtool:
|
||||
if not self.patch_libtool:
|
||||
return
|
||||
|
||||
for libtool_path in fs.find(self.build_directory, "libtool", recursive=True):
|
||||
self._patch_libtool(libtool_path)
|
||||
x = fs.FileFilter(
|
||||
*filter(fs.is_exe, fs.find(self.build_directory, "libtool", recursive=True))
|
||||
)
|
||||
|
||||
def _patch_libtool(self, libtool_path):
|
||||
if (
|
||||
self.spec.satisfies("%arm")
|
||||
or self.spec.satisfies("%clang")
|
||||
or self.spec.satisfies("%fj")
|
||||
or self.spec.satisfies("%nvhpc")
|
||||
):
|
||||
fs.filter_file('wl=""\n', 'wl="-Wl,"\n', libtool_path)
|
||||
fs.filter_file(
|
||||
'pic_flag=""\n', 'pic_flag="{0}"\n'.format(self.compiler.cc_pic_flag), libtool_path
|
||||
# Exit early if there is nothing to patch:
|
||||
if not x.filenames:
|
||||
return
|
||||
|
||||
markers = {"cc": "LIBTOOL CONFIG"}
|
||||
for tag in ["cxx", "fc", "f77"]:
|
||||
markers[tag] = "LIBTOOL TAG CONFIG: {0}".format(tag.upper())
|
||||
|
||||
# Replace empty linker flag prefixes:
|
||||
if self.compiler.name == "nag":
|
||||
# Nag is mixed with gcc and g++, which are recognized correctly.
|
||||
# Therefore, we change only Fortran values:
|
||||
for tag in ["fc", "f77"]:
|
||||
marker = markers[tag]
|
||||
x.filter(
|
||||
regex='^wl=""$',
|
||||
repl='wl="{0}"'.format(self.compiler.linker_arg),
|
||||
start_at="# ### BEGIN {0}".format(marker),
|
||||
stop_at="# ### END {0}".format(marker),
|
||||
)
|
||||
else:
|
||||
x.filter(regex='^wl=""$', repl='wl="{0}"'.format(self.compiler.linker_arg))
|
||||
|
||||
# Replace empty PIC flag values:
|
||||
for cc, marker in markers.items():
|
||||
x.filter(
|
||||
regex='^pic_flag=""$',
|
||||
repl='pic_flag="{0}"'.format(getattr(self.compiler, "{0}_pic_flag".format(cc))),
|
||||
start_at="# ### BEGIN {0}".format(marker),
|
||||
stop_at="# ### END {0}".format(marker),
|
||||
)
|
||||
if self.spec.satisfies("%fj"):
|
||||
fs.filter_file("-nostdlib", "", libtool_path)
|
||||
|
||||
# Other compiler-specific patches:
|
||||
if self.compiler.name == "fj":
|
||||
x.filter(regex="-nostdlib", repl="", string=True)
|
||||
rehead = r"/\S*/"
|
||||
objfile = [
|
||||
for o in [
|
||||
"fjhpctag.o",
|
||||
"fjcrt0.o",
|
||||
"fjlang08.o",
|
||||
|
|
@ -298,16 +363,86 @@ def _patch_libtool(self, libtool_path):
|
|||
"crti.o",
|
||||
"crtbeginS.o",
|
||||
"crtendS.o",
|
||||
]
|
||||
for o in objfile:
|
||||
fs.filter_file(rehead + o, "", libtool_path)
|
||||
# Hack to filter out spurious predp_objects when building with
|
||||
# Intel dpcpp; see issue #32863
|
||||
if self.spec.satisfies("%dpcpp"):
|
||||
fs.filter_file(
|
||||
r"^(predep_objects=.*)/tmp/conftest-[0-9A-Fa-f]+\.o", r"\1", libtool_path
|
||||
]:
|
||||
x.filter(regex=(rehead + o), repl="", string=True)
|
||||
elif self.compiler.name == "dpcpp":
|
||||
# Hack to filter out spurious predep_objects when building with Intel dpcpp
|
||||
# (see https://github.com/spack/spack/issues/32863):
|
||||
x.filter(regex=r"^(predep_objects=.*)/tmp/conftest-[0-9A-Fa-f]+\.o", repl=r"\1")
|
||||
x.filter(regex=r"^(predep_objects=.*)/tmp/a-[0-9A-Fa-f]+\.o", repl=r"\1")
|
||||
elif self.compiler.name == "nag":
|
||||
for tag in ["fc", "f77"]:
|
||||
marker = markers[tag]
|
||||
start_at = "# ### BEGIN {0}".format(marker)
|
||||
stop_at = "# ### END {0}".format(marker)
|
||||
# Libtool 2.4.2 does not know the shared flag:
|
||||
x.filter(
|
||||
regex=r"\$CC -shared",
|
||||
repl=r"\$CC -Wl,-shared",
|
||||
string=True,
|
||||
start_at=start_at,
|
||||
stop_at=stop_at,
|
||||
)
|
||||
# Libtool does not know how to inject whole archives
|
||||
# (e.g. https://github.com/pmodels/mpich/issues/4358):
|
||||
x.filter(
|
||||
regex=r'^whole_archive_flag_spec="\\\$({?wl}?)--whole-archive'
|
||||
r'\\\$convenience \\\$\1--no-whole-archive"$',
|
||||
repl=r'whole_archive_flag_spec="\$\1--whole-archive'
|
||||
r"\`for conv in \$convenience\\\\\"\\\\\"; do test -n \\\\\"\$conv\\\\\" && "
|
||||
r"new_convenience=\\\\\"\$new_convenience,\$conv\\\\\"; done; "
|
||||
r'func_echo_all \\\\\"\$new_convenience\\\\\"\` \$\1--no-whole-archive"',
|
||||
start_at=start_at,
|
||||
stop_at=stop_at,
|
||||
)
|
||||
# The compiler requires special treatment in certain cases:
|
||||
x.filter(
|
||||
regex=r"^(with_gcc=.*)$",
|
||||
repl="\\1\n\n# Is the compiler the NAG compiler?\nwith_nag=yes",
|
||||
start_at=start_at,
|
||||
stop_at=stop_at,
|
||||
)
|
||||
|
||||
# Disable the special treatment for gcc and g++:
|
||||
for tag in ["cc", "cxx"]:
|
||||
marker = markers[tag]
|
||||
x.filter(
|
||||
regex=r"^(with_gcc=.*)$",
|
||||
repl="\\1\n\n# Is the compiler the NAG compiler?\nwith_nag=no",
|
||||
start_at="# ### BEGIN {0}".format(marker),
|
||||
stop_at="# ### END {0}".format(marker),
|
||||
)
|
||||
|
||||
# The compiler does not support -pthread flag, which might come
|
||||
# from the inherited linker flags. We prepend the flag with -Wl,
|
||||
# before using it:
|
||||
x.filter(
|
||||
regex=r"^(\s*)(for tmp_inherited_linker_flag in \$tmp_inherited_linker_flags; "
|
||||
r"do\s*)$",
|
||||
repl='\\1if test "x$with_nag" = xyes; then\n'
|
||||
"\\1 revert_nag_pthread=$tmp_inherited_linker_flags\n"
|
||||
"\\1 tmp_inherited_linker_flags="
|
||||
"`$ECHO \"$tmp_inherited_linker_flags\" | $SED 's% -pthread% -Wl,-pthread%g'`\n"
|
||||
'\\1 test x"$revert_nag_pthread" = x"$tmp_inherited_linker_flags" && '
|
||||
"revert_nag_pthread=no || revert_nag_pthread=yes\n"
|
||||
"\\1fi\n\\1\\2",
|
||||
start_at='if test -n "$inherited_linker_flags"; then',
|
||||
stop_at='case " $new_inherited_linker_flags " in',
|
||||
)
|
||||
# And revert the modification to produce '*.la' files that can be
|
||||
# used with gcc (normally, we do not install the files but they can
|
||||
# still be used during the building):
|
||||
start_at = '# Time to change all our "foo.ltframework" stuff back to "-framework foo"'
|
||||
stop_at = "# installed libraries to the beginning of the library search list"
|
||||
x.filter(
|
||||
regex=r"(\s*)(# move library search paths that coincide with paths to not "
|
||||
r"yet\s*)$",
|
||||
repl='\\1test x"$with_nag$revert_nag_pthread" = xyesyes &&\n'
|
||||
'\\1 new_inherited_linker_flags=`$ECHO " $new_inherited_linker_flags" | '
|
||||
"$SED 's% -Wl,-pthread% -pthread%g'`\n\\1\\2",
|
||||
start_at=start_at,
|
||||
stop_at=stop_at,
|
||||
)
|
||||
fs.filter_file(r"^(predep_objects=.*)/tmp/a-[0-9A-Fa-f]+\.o", r"\1", libtool_path)
|
||||
|
||||
@property
|
||||
def configure_directory(self):
|
||||
|
|
|
|||
4
lib/spack/spack/test/data/filter_file/start_stop.txt
Normal file
4
lib/spack/spack/test/data/filter_file/start_stop.txt
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
A
|
||||
B
|
||||
C
|
||||
D
|
||||
|
|
@ -4,6 +4,7 @@
|
|||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
"""Tests for ``llnl/util/filesystem.py``"""
|
||||
import filecmp
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
|
|
@ -527,6 +528,29 @@ def test_filter_files_multiple(tmpdir):
|
|||
assert "<stdio.h>" not in f.read()
|
||||
|
||||
|
||||
def test_filter_files_start_stop(tmpdir):
|
||||
original_file = os.path.join(spack.paths.test_path, "data", "filter_file", "start_stop.txt")
|
||||
target_file = os.path.join(str(tmpdir), "start_stop.txt")
|
||||
shutil.copy(original_file, target_file)
|
||||
# None of the following should happen:
|
||||
# - filtering starts after A is found in the file:
|
||||
fs.filter_file("A", "X", target_file, string=True, start_at="B")
|
||||
# - filtering starts exactly when B is found:
|
||||
fs.filter_file("B", "X", target_file, string=True, start_at="B")
|
||||
# - filtering stops before D is found:
|
||||
fs.filter_file("D", "X", target_file, string=True, stop_at="C")
|
||||
|
||||
assert filecmp.cmp(original_file, target_file)
|
||||
|
||||
# All of the following should happen:
|
||||
fs.filter_file("A", "X", target_file, string=True)
|
||||
fs.filter_file("B", "X", target_file, string=True, start_at="X", stop_at="C")
|
||||
fs.filter_file(r"C|D", "X", target_file, start_at="X", stop_at="E")
|
||||
|
||||
with open(target_file, mode="r") as f:
|
||||
assert all("X" == line.strip() for line in f.readlines())
|
||||
|
||||
|
||||
# Each test input is a tuple of entries which prescribe
|
||||
# - the 'subdirs' to be created from tmpdir
|
||||
# - the 'files' in that directory
|
||||
|
|
|
|||
Loading…
Reference in a new issue