buildcache: do one less tar file extraction
The buildcache is now extracted in a temporary folder within the current store, moved to its final place and relocated. "spack clean -s" has been extended to also clean the temporary extraction directory. Add hardlinks with absolute paths for libraries in the corge, garply and quux packages to detect incorrect handling of hardlinks in tests.
This commit is contained in:
Patrick Gartung
GitHub
parent
e6b76578d2
commit
047c95aa8d
5 changed files with 40 additions and 38 deletions
|
|
@ -4,11 +4,9 @@
|
|||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
import codecs
|
||||
import glob
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import tarfile
|
||||
|
|
@ -1428,42 +1426,30 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False,
|
|||
buildinfo = spec_dict.get('buildinfo', {})
|
||||
old_relative_prefix = buildinfo.get('relative_prefix', new_relative_prefix)
|
||||
rel = buildinfo.get('relative_rpaths')
|
||||
# if the original relative prefix and new relative prefix differ the
|
||||
# directory layout has changed and the buildcache cannot be installed
|
||||
# if it was created with relative rpaths
|
||||
info = 'old relative prefix %s\nnew relative prefix %s\nrelative rpaths %s'
|
||||
tty.debug(info %
|
||||
(old_relative_prefix, new_relative_prefix, rel))
|
||||
# if (old_relative_prefix != new_relative_prefix and (rel)):
|
||||
# shutil.rmtree(tmpdir)
|
||||
# msg = "Package tarball was created from an install "
|
||||
# msg += "prefix with a different directory layout. "
|
||||
# msg += "It cannot be relocated because it "
|
||||
# msg += "uses relative rpaths."
|
||||
# raise NewLayoutException(msg)
|
||||
|
||||
# extract the tarball in a temp directory
|
||||
# Extract the tarball into the store root, presumably on the same filesystem.
|
||||
# The directory created is the base directory name of the old prefix.
|
||||
# Moving the old prefix name to the new prefix location should preserve
|
||||
# hard links and symbolic links.
|
||||
extract_tmp = os.path.join(spack.store.layout.root, '.tmp')
|
||||
mkdirp(extract_tmp)
|
||||
extracted_dir = os.path.join(extract_tmp,
|
||||
old_relative_prefix.split(os.path.sep)[-1])
|
||||
|
||||
with closing(tarfile.open(tarfile_path, 'r')) as tar:
|
||||
tar.extractall(path=tmpdir)
|
||||
# get the parent directory of the file .spack/binary_distribution
|
||||
# this should the directory unpacked from the tarball whose
|
||||
# name is unknown because the prefix naming is unknown
|
||||
bindist_file = glob.glob('%s/*/.spack/binary_distribution' % tmpdir)[0]
|
||||
workdir = re.sub('/.spack/binary_distribution$', '', bindist_file)
|
||||
tty.debug('workdir %s' % workdir)
|
||||
# install_tree copies hardlinks
|
||||
# create a temporary tarfile from prefix and exract it to workdir
|
||||
# tarfile preserves hardlinks
|
||||
temp_tarfile_name = tarball_name(spec, '.tar')
|
||||
temp_tarfile_path = os.path.join(tmpdir, temp_tarfile_name)
|
||||
with closing(tarfile.open(temp_tarfile_path, 'w')) as tar:
|
||||
tar.add(name='%s' % workdir,
|
||||
arcname='.')
|
||||
with closing(tarfile.open(temp_tarfile_path, 'r')) as tar:
|
||||
tar.extractall(spec.prefix)
|
||||
os.remove(temp_tarfile_path)
|
||||
|
||||
# cleanup
|
||||
try:
|
||||
tar.extractall(path=extract_tmp)
|
||||
except Exception as e:
|
||||
shutil.rmtree(extracted_dir)
|
||||
raise e
|
||||
try:
|
||||
shutil.move(extracted_dir, spec.prefix)
|
||||
except Exception as e:
|
||||
shutil.rmtree(extracted_dir)
|
||||
raise e
|
||||
os.remove(tarfile_path)
|
||||
os.remove(specfile_path)
|
||||
|
||||
|
|
|
|||
|
|
@ -76,7 +76,11 @@ def clean(parser, args):
|
|||
if args.stage:
|
||||
tty.msg('Removing all temporary build stages')
|
||||
spack.stage.purge()
|
||||
|
||||
# Temp directory where buildcaches are extracted
|
||||
extract_tmp = os.path.join(spack.store.layout.root, '.tmp')
|
||||
if os.path.exists(extract_tmp):
|
||||
tty.debug('Removing {0}'.format(extract_tmp))
|
||||
shutil.rmtree(extract_tmp)
|
||||
if args.downloads:
|
||||
tty.msg('Removing cached downloads')
|
||||
spack.caches.fetch_cache.destroy()
|
||||
|
|
|
|||
|
|
@ -142,10 +142,12 @@ class Corge
|
|||
'-Wl,-rpath,%s' % spec['quux'].prefix.lib64,
|
||||
'-Wl,-rpath,%s' % spec['garply'].prefix.lib64,
|
||||
'libcorge.dylib',
|
||||
'%s/libquux.dylib' % spec['quux'].prefix.lib64,
|
||||
'%s/libgarply.dylib' % spec['garply'].prefix.lib64)
|
||||
'%s/libquux.dylib.3.0' % spec['quux'].prefix.lib64,
|
||||
'%s/libgarply.dylib.3.0' % spec['garply'].prefix.lib64)
|
||||
mkdirp(prefix.lib64)
|
||||
copy('libcorge.dylib', '%s/libcorge.dylib' % prefix.lib64)
|
||||
os.link('%s/libcorge.dylib' % prefix.lib64,
|
||||
'%s/libcorge.dylib.3.0' % prefix.lib64)
|
||||
else:
|
||||
gpp('-fPIC', '-O2', '-g', '-DNDEBUG', '-shared',
|
||||
'-Wl,-soname,libcorge.so', '-o', 'libcorge.so', 'corge.cc.o',
|
||||
|
|
@ -159,10 +161,12 @@ class Corge
|
|||
'-Wl,-rpath,%s' % spec['quux'].prefix.lib64,
|
||||
'-Wl,-rpath,%s' % spec['garply'].prefix.lib64,
|
||||
'libcorge.so',
|
||||
'%s/libquux.so' % spec['quux'].prefix.lib64,
|
||||
'%s/libgarply.so' % spec['garply'].prefix.lib64)
|
||||
'%s/libquux.so.3.0' % spec['quux'].prefix.lib64,
|
||||
'%s/libgarply.so.3.0' % spec['garply'].prefix.lib64)
|
||||
mkdirp(prefix.lib64)
|
||||
copy('libcorge.so', '%s/libcorge.so' % prefix.lib64)
|
||||
os.link('%s/libcorge.so' % prefix.lib64,
|
||||
'%s/libcorge.so.3.0' % prefix.lib64)
|
||||
copy('corgegator', '%s/corgegator' % prefix.lib64)
|
||||
copy('%s/corge/corge.h' % self.stage.source_path,
|
||||
'%s/corge/corge.h' % prefix.include)
|
||||
|
|
|
|||
|
|
@ -109,6 +109,8 @@ class Garply
|
|||
'libgarply.dylib')
|
||||
mkdirp(prefix.lib64)
|
||||
copy('libgarply.dylib', '%s/libgarply.dylib' % prefix.lib64)
|
||||
os.link('%s/libgarply.dylib' % prefix.lib64,
|
||||
'%s/libgarply.dylib.3.0' % prefix.lib64)
|
||||
else:
|
||||
gpp('-fPIC', '-O2', '-g', '-DNDEBUG', '-shared',
|
||||
'-Wl,-soname,libgarply.so',
|
||||
|
|
@ -119,6 +121,8 @@ class Garply
|
|||
'libgarply.so')
|
||||
mkdirp(prefix.lib64)
|
||||
copy('libgarply.so', '%s/libgarply.so' % prefix.lib64)
|
||||
os.link('%s/libgarply.so' % prefix.lib64,
|
||||
'%s/libgarply.so.3.0' % prefix.lib64)
|
||||
copy('garplinator', '%s/garplinator' % prefix.lib64)
|
||||
copy('%s/garply/garply.h' % self.stage.source_path,
|
||||
'%s/garply/garply.h' % prefix.include)
|
||||
|
|
|
|||
|
|
@ -128,6 +128,8 @@ class Quux
|
|||
'%s/libgarply.dylib' % spec['garply'].prefix.lib64)
|
||||
mkdirp(prefix.lib64)
|
||||
copy('libquux.dylib', '%s/libquux.dylib' % prefix.lib64)
|
||||
os.link('%s/libquux.dylib' % prefix.lib64,
|
||||
'%s/libquux.dylib.3.0' % prefix.lib64)
|
||||
else:
|
||||
gpp('-fPIC', '-O2', '-g', '-DNDEBUG', '-shared',
|
||||
'-Wl,-soname,libquux.so', '-o', 'libquux.so', 'quux.cc.o',
|
||||
|
|
@ -142,6 +144,8 @@ class Quux
|
|||
'%s/libgarply.so' % spec['garply'].prefix.lib64)
|
||||
mkdirp(prefix.lib64)
|
||||
copy('libquux.so', '%s/libquux.so' % prefix.lib64)
|
||||
os.link('%s/libquux.so' % prefix.lib64,
|
||||
'%s/libquux.so.3.0' % prefix.lib64)
|
||||
copy('quuxifier', '%s/quuxifier' % prefix.lib64)
|
||||
copy('%s/quux/quux.h' % self.stage.source_path,
|
||||
'%s/quux/quux.h' % prefix.include)
|
||||
|
|
|
|||
Loading…
Reference in a new issue