hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

rebuild-index: fix race condition by avoiding parallelism (#39594)

Browse source
This commit is contained in:
Harmen Stoppels 2023-08-24 08:26:56 +02:00 committed by GitHub
parent fdea5e7624
commit 8914d26867
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 10 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
lib/spack/spack/binary_distribution.py
View file

@ -9,7 +9,6 @@
import io import io
import itertools import itertools
import json import json
import multiprocessing.pool
import os import os
import re import re
import shutil import shutil
@ -876,32 +875,18 @@ def _read_specs_and_push_index(file_list, read_method, cache_prefix, db, temp_di
db: A spack database used for adding specs and then writing the index. db: A spack database used for adding specs and then writing the index.
temp_dir (str): Location to write index.json and hash for pushing temp_dir (str): Location to write index.json and hash for pushing
concurrency (int): Number of parallel processes to use when fetching concurrency (int): Number of parallel processes to use when fetching
Return:
None
""" """
for file in file_list:
contents = read_method(file)
# Need full spec.json name or this gets confused with index.json.
if file.endswith(".json.sig"):
specfile_json = Spec.extract_json_from_clearsig(contents)
fetched_spec = Spec.from_dict(specfile_json)
elif file.endswith(".json"):
fetched_spec = Spec.from_json(contents)
else:
continue
def _fetch_spec_from_mirror(spec_url):
spec_file_contents = read_method(spec_url)
if spec_file_contents:
# Need full spec.json name or this gets confused with index.json.
if spec_url.endswith(".json.sig"):
specfile_json = Spec.extract_json_from_clearsig(spec_file_contents)
return Spec.from_dict(specfile_json)
if spec_url.endswith(".json"):
return Spec.from_json(spec_file_contents)
tp = multiprocessing.pool.ThreadPool(processes=concurrency)
try:
fetched_specs = tp.map(
llnl.util.lang.star(_fetch_spec_from_mirror), [(f,) for f in file_list]
)
finally:
tp.terminate()
tp.join()
for fetched_spec in fetched_specs:
db.add(fetched_spec, None) db.add(fetched_spec, None)
db.mark(fetched_spec, "in_buildcache", True) db.mark(fetched_spec, "in_buildcache", True)