From 397334a4befea7d76c5320824a0742259c225475 Mon Sep 17 00:00:00 2001
From: "John W. Parent" <45471568+johnwparent@users.noreply.github.com>
Date: Fri, 22 Mar 2024 17:06:29 -0400
Subject: [PATCH] Spack CI: Refactor process_command for Cross Platform support
 (#39739)

Generate CI scripts as powershell on Windows. This is intended to
output exactly the same bash scripts as before on Linux.

Co-authored-by: Ryan Krattiger <ryan.krattiger@kitware.com>
---
 lib/spack/spack/ci.py | 354 +++++++++++++++++++++++-------------------
 1 file changed, 191 insertions(+), 163 deletions(-)

diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py
index 7c2f3f2eb6..96dd58a4f4 100644
--- a/lib/spack/spack/ci.py
+++ b/lib/spack/spack/ci.py
@@ -70,7 +70,7 @@
 JOB_NAME_FORMAT = (
     "{name}{@version} {/hash:7} {%compiler.name}{@compiler.version}{arch=architecture}"
 )
-
+IS_WINDOWS = sys.platform == "win32"
 spack_gpg = spack.main.SpackCommand("gpg")
 spack_compiler = spack.main.SpackCommand("compiler")
 
@@ -103,7 +103,7 @@ def get_job_name(spec: spack.spec.Spec, build_group: str = ""):
     job_name = spec.format(JOB_NAME_FORMAT)
 
     if build_group:
-        job_name = "{0} {1}".format(job_name, build_group)
+        job_name = f"{job_name} {build_group}"
 
     return job_name[:255]
 
@@ -114,7 +114,7 @@ def _remove_reserved_tags(tags):
 
 
 def _spec_deps_key(s):
-    return "{0}/{1}".format(s.name, s.dag_hash(7))
+    return f"{s.name}/{s.dag_hash(7)}"
 
 
 def _add_dependency(spec_label, dep_label, deps):
@@ -213,7 +213,7 @@ def _print_staging_summary(spec_labels, stages, mirrors_to_check, rebuild_decisi
     mirrors = spack.mirror.MirrorCollection(mirrors=mirrors_to_check, binary=True)
     tty.msg("Checked the following mirrors for binaries:")
     for m in mirrors.values():
-        tty.msg("  {0}".format(m.fetch_url))
+        tty.msg(f"  {m.fetch_url}")
 
     tty.msg("Staging summary ([x] means a job needs rebuilding):")
     for stage_index, stage in enumerate(stages):
@@ -296,7 +296,7 @@ def append_dep(s, d):
     for spec in spec_list:
         for s in spec.traverse(deptype="all"):
             if s.external:
-                tty.msg("Will not stage external pkg: {0}".format(s))
+                tty.msg(f"Will not stage external pkg: {s}")
                 continue
 
             skey = _spec_deps_key(s)
@@ -305,7 +305,7 @@ def append_dep(s, d):
             for d in s.dependencies(deptype="all"):
                 dkey = _spec_deps_key(d)
                 if d.external:
-                    tty.msg("Will not stage external dep: {0}".format(d))
+                    tty.msg(f"Will not stage external dep: {d}")
                     continue
 
                 append_dep(skey, dkey)
@@ -374,8 +374,8 @@ def get_stack_changed(env_path, rev1="HEAD^", rev2="HEAD"):
 
             for path in lines:
                 if ".gitlab-ci.yml" in path or path in env_path:
-                    tty.debug("env represented by {0} changed".format(env_path))
-                    tty.debug("touched file: {0}".format(path))
+                    tty.debug(f"env represented by {env_path} changed")
+                    tty.debug(f"touched file: {path}")
                     return True
     return False
 
@@ -419,7 +419,7 @@ def get_spec_filter_list(env, affected_pkgs, dependent_traverse_depth=None):
     all_concrete_specs = env.all_specs()
     tty.debug("All concrete environment specs:")
     for s in all_concrete_specs:
-        tty.debug("  {0}/{1}".format(s.name, s.dag_hash()[:7]))
+        tty.debug(f"  {s.name}/{s.dag_hash()[:7]}")
     affected_pkgs = frozenset(affected_pkgs)
     env_matches = [s for s in all_concrete_specs if s.name in affected_pkgs]
     visited = set()
@@ -510,7 +510,7 @@ def __is_named(self, section):
         and if so return the name otherwise return none.
         """
         for _name in self.named_jobs:
-            keys = ["{0}-job".format(_name), "{0}-job-remove".format(_name)]
+            keys = [f"{_name}-job", f"{_name}-job-remove"]
             if any([key for key in keys if key in section]):
                 return _name
 
@@ -525,9 +525,9 @@ def __job_name(name, suffix=""):
 
         jname = name
         if suffix:
-            jname = "{0}-job{1}".format(name, suffix)
+            jname = f"{name}-job{suffix}"
         else:
-            jname = "{0}-job".format(name)
+            jname = f"{name}-job"
 
         return jname
 
@@ -739,7 +739,7 @@ def generate_gitlab_ci_yaml(
         # Requested to prune untouched packages, but assume we won't do that
         # unless we're actually in a git repo.
         rev1, rev2 = get_change_revisions()
-        tty.debug("Got following revisions: rev1={0}, rev2={1}".format(rev1, rev2))
+        tty.debug(f"Got following revisions: rev1={rev1}, rev2={rev2}")
         if rev1 and rev2:
             # If the stack file itself did not change, proceed with pruning
             if not get_stack_changed(env.manifest_path, rev1, rev2):
@@ -747,13 +747,13 @@ def generate_gitlab_ci_yaml(
                 affected_pkgs = compute_affected_packages(rev1, rev2)
                 tty.debug("affected pkgs:")
                 for p in affected_pkgs:
-                    tty.debug("  {0}".format(p))
+                    tty.debug(f"  {p}")
                 affected_specs = get_spec_filter_list(
                     env, affected_pkgs, dependent_traverse_depth=dependent_depth
                 )
                 tty.debug("all affected specs:")
                 for s in affected_specs:
-                    tty.debug("  {0}/{1}".format(s.name, s.dag_hash()[:7]))
+                    tty.debug(f"  {s.name}/{s.dag_hash()[:7]}")
 
     # Allow overriding --prune-dag cli opt with environment variable
     prune_dag_override = os.environ.get("SPACK_PRUNE_UP_TO_DATE", None)
@@ -978,7 +978,7 @@ def generate_gitlab_ci_yaml(
     rebuild_decisions = {}
 
     for stage_jobs in stages:
-        stage_name = "stage-{0}".format(stage_id)
+        stage_name = f"stage-{stage_id}"
         stage_names.append(stage_name)
         stage_id += 1
 
@@ -1009,7 +1009,7 @@ def generate_gitlab_ci_yaml(
             job_object = spack_ci_ir["jobs"][release_spec_dag_hash]["attributes"]
 
             if not job_object:
-                tty.warn("No match found for {0}, skipping it".format(release_spec))
+                tty.warn(f"No match found for {release_spec}, skipping it")
                 continue
 
             if spack_pipeline_type is not None:
@@ -1119,7 +1119,7 @@ def main_script_replacements(cmd):
 
             if artifacts_root:
                 job_object["needs"].append(
-                    {"job": generate_job_name, "pipeline": "{0}".format(parent_pipeline_id)}
+                    {"job": generate_job_name, "pipeline": f"{parent_pipeline_id}"}
                 )
 
             # Let downstream jobs know whether the spec needed rebuilding, regardless
@@ -1185,19 +1185,17 @@ def main_script_replacements(cmd):
         if spack_pipeline_type == "spack_pull_request":
             spack.mirror.remove("ci_shared_pr_mirror", cfg.default_modify_scope())
 
-    tty.debug("{0} build jobs generated in {1} stages".format(job_id, stage_id))
+    tty.debug(f"{job_id} build jobs generated in {stage_id} stages")
 
     if job_id > 0:
-        tty.debug(
-            "The max_needs_job is {0}, with {1} needs".format(max_needs_job, max_length_needs)
-        )
+        tty.debug(f"The max_needs_job is {max_needs_job}, with {max_length_needs} needs")
 
     # Use "all_job_names" to populate the build group for this set
     if cdash_handler and cdash_handler.auth_token:
         try:
             cdash_handler.populate_buildgroup(all_job_names)
         except (SpackError, HTTPError, URLError) as err:
-            tty.warn("Problem populating buildgroup: {0}".format(err))
+            tty.warn(f"Problem populating buildgroup: {err}")
     else:
         tty.warn("Unable to populate buildgroup without CDash credentials")
 
@@ -1211,9 +1209,7 @@ def main_script_replacements(cmd):
         sync_job = copy.deepcopy(spack_ci_ir["jobs"]["copy"]["attributes"])
         sync_job["stage"] = "copy"
         if artifacts_root:
-            sync_job["needs"] = [
-                {"job": generate_job_name, "pipeline": "{0}".format(parent_pipeline_id)}
-            ]
+            sync_job["needs"] = [{"job": generate_job_name, "pipeline": f"{parent_pipeline_id}"}]
 
         if "variables" not in sync_job:
             sync_job["variables"] = {}
@@ -1230,6 +1226,7 @@ def main_script_replacements(cmd):
             # TODO: Remove this condition in Spack 0.23
             buildcache_source = os.environ.get("SPACK_SOURCE_MIRROR", None)
         sync_job["variables"]["SPACK_BUILDCACHE_SOURCE"] = buildcache_source
+        sync_job["dependencies"] = []
 
         output_object["copy"] = sync_job
         job_id += 1
@@ -1348,7 +1345,7 @@ def main_script_replacements(cmd):
 
             copy_specs_file = os.path.join(
                 copy_specs_dir,
-                "copy_{}_specs.json".format(spack_stack_name if spack_stack_name else "rebuilt"),
+                f"copy_{spack_stack_name if spack_stack_name else 'rebuilt'}_specs.json",
             )
 
             with open(copy_specs_file, "w") as fd:
@@ -1440,7 +1437,7 @@ def import_signing_key(base64_signing_key):
             fd.write(decoded_key)
 
         key_import_output = spack_gpg("trust", sign_key_path, output=str)
-        tty.debug("spack gpg trust {0}".format(sign_key_path))
+        tty.debug(f"spack gpg trust {sign_key_path}")
         tty.debug(key_import_output)
 
     # Now print the keys we have for verifying and signing
@@ -1469,7 +1466,7 @@ def can_verify_binaries():
 def _push_mirror_contents(input_spec, sign_binaries, mirror_url):
     """Unchecked version of the public API, for easier mocking"""
     unsigned = not sign_binaries
-    tty.debug("Creating buildcache ({0})".format("unsigned" if unsigned else "signed"))
+    tty.debug(f"Creating buildcache ({'unsigned' if unsigned else 'signed'})")
     push_url = spack.mirror.Mirror.from_url(mirror_url).push_url
     return bindist.push(input_spec, push_url, bindist.PushOptions(force=True, unsigned=unsigned))
 
@@ -1498,9 +1495,9 @@ def push_mirror_contents(input_spec: spack.spec.Spec, mirror_url, sign_binaries)
         #     Exception
         #     BaseException
         #     object
-        err_msg = "Error msg: {0}".format(inst)
+        err_msg = f"Error msg: {inst}"
         if any(x in err_msg for x in ["Access Denied", "InvalidAccessKeyId"]):
-            tty.msg("Permission problem writing to {0}".format(mirror_url))
+            tty.msg(f"Permission problem writing to {mirror_url}")
             tty.msg(err_msg)
             return False
         else:
@@ -1531,8 +1528,9 @@ def copy_files_to_artifacts(src, artifacts_dir):
     try:
         fs.copy(src, artifacts_dir)
     except Exception as err:
-        msg = ("Unable to copy files ({0}) to artifacts {1} due to " "exception: {2}").format(
-            src, artifacts_dir, str(err)
+        msg = (
+            f"Unable to copy files ({src}) to artifacts {artifacts_dir} due to "
+            f"exception: {str(err)}"
         )
         tty.warn(msg)
 
@@ -1548,23 +1546,23 @@ def copy_stage_logs_to_artifacts(job_spec: spack.spec.Spec, job_log_dir: str) ->
         job_spec: spec associated with spack install log
         job_log_dir: path into which build log should be copied
     """
-    tty.debug("job spec: {0}".format(job_spec))
+    tty.debug(f"job spec: {job_spec}")
     if not job_spec:
-        msg = "Cannot copy stage logs: job spec ({0}) is required"
-        tty.error(msg.format(job_spec))
+        msg = f"Cannot copy stage logs: job spec ({job_spec}) is required"
+        tty.error(msg)
         return
 
     try:
         pkg_cls = spack.repo.PATH.get_pkg_class(job_spec.name)
         job_pkg = pkg_cls(job_spec)
-        tty.debug("job package: {0}".format(job_pkg))
+        tty.debug(f"job package: {job_pkg}")
     except AssertionError:
-        msg = "Cannot copy stage logs: job spec ({0}) must be concrete"
-        tty.error(msg.format(job_spec))
+        msg = f"Cannot copy stage logs: job spec ({job_spec}) must be concrete"
+        tty.error(msg)
         return
 
     stage_dir = job_pkg.stage.path
-    tty.debug("stage dir: {0}".format(stage_dir))
+    tty.debug(f"stage dir: {stage_dir}")
     for file in [job_pkg.log_path, job_pkg.env_mods_path, *job_pkg.builder.archive_files]:
         copy_files_to_artifacts(file, job_log_dir)
 
@@ -1577,10 +1575,10 @@ def copy_test_logs_to_artifacts(test_stage, job_test_dir):
         test_stage (str): test stage path
         job_test_dir (str): the destination artifacts test directory
     """
-    tty.debug("test stage: {0}".format(test_stage))
+    tty.debug(f"test stage: {test_stage}")
     if not os.path.exists(test_stage):
-        msg = "Cannot copy test logs: job test stage ({0}) does not exist"
-        tty.error(msg.format(test_stage))
+        msg = f"Cannot copy test logs: job test stage ({test_stage}) does not exist"
+        tty.error(msg)
         return
 
     copy_files_to_artifacts(os.path.join(test_stage, "*", "*.txt"), job_test_dir)
@@ -1595,7 +1593,7 @@ def download_and_extract_artifacts(url, work_dir):
         url (str): Complete url to artifacts.zip file
         work_dir (str): Path to destination where artifacts should be extracted
     """
-    tty.msg("Fetching artifacts from: {0}\n".format(url))
+    tty.msg(f"Fetching artifacts from: {url}\n")
 
     headers = {"Content-Type": "application/zip"}
 
@@ -1612,7 +1610,7 @@ def download_and_extract_artifacts(url, work_dir):
     response_code = response.getcode()
 
     if response_code != 200:
-        msg = "Error response code ({0}) in reproduce_ci_job".format(response_code)
+        msg = f"Error response code ({response_code}) in reproduce_ci_job"
         raise SpackError(msg)
 
     artifacts_zip_path = os.path.join(work_dir, "artifacts.zip")
@@ -1642,7 +1640,7 @@ def get_spack_info():
 
             return git_log
 
-    return "no git repo, use spack {0}".format(spack.spack_version)
+    return f"no git repo, use spack {spack.spack_version}"
 
 
 def setup_spack_repro_version(repro_dir, checkout_commit, merge_commit=None):
@@ -1665,8 +1663,8 @@ def setup_spack_repro_version(repro_dir, checkout_commit, merge_commit=None):
     """
     # figure out the path to the spack git version being used for the
     # reproduction
-    print("checkout_commit: {0}".format(checkout_commit))
-    print("merge_commit: {0}".format(merge_commit))
+    print(f"checkout_commit: {checkout_commit}")
+    print(f"merge_commit: {merge_commit}")
 
     dot_git_path = os.path.join(spack.paths.prefix, ".git")
     if not os.path.exists(dot_git_path):
@@ -1685,14 +1683,14 @@ def setup_spack_repro_version(repro_dir, checkout_commit, merge_commit=None):
         git("log", "-1", checkout_commit, output=str, error=os.devnull, fail_on_error=False)
 
         if git.returncode != 0:
-            tty.error("Missing commit: {0}".format(checkout_commit))
+            tty.error(f"Missing commit: {checkout_commit}")
             return False
 
         if merge_commit:
             git("log", "-1", merge_commit, output=str, error=os.devnull, fail_on_error=False)
 
             if git.returncode != 0:
-                tty.error("Missing commit: {0}".format(merge_commit))
+                tty.error(f"Missing commit: {merge_commit}")
                 return False
 
     # Next attempt to clone your local spack repo into the repro dir
@@ -1715,7 +1713,7 @@ def setup_spack_repro_version(repro_dir, checkout_commit, merge_commit=None):
         )
 
         if git.returncode != 0:
-            tty.error("Unable to checkout {0}".format(checkout_commit))
+            tty.error(f"Unable to checkout {checkout_commit}")
             tty.msg(co_out)
             return False
 
@@ -1734,7 +1732,7 @@ def setup_spack_repro_version(repro_dir, checkout_commit, merge_commit=None):
             )
 
             if git.returncode != 0:
-                tty.error("Unable to merge {0}".format(merge_commit))
+                tty.error(f"Unable to merge {merge_commit}")
                 tty.msg(merge_out)
                 return False
 
@@ -1755,6 +1753,7 @@ def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime):
     commands to run to reproduce the build once inside the container.
     """
     work_dir = os.path.realpath(work_dir)
+    platform_script_ext = "ps1" if IS_WINDOWS else "sh"
     download_and_extract_artifacts(url, work_dir)
 
     gpg_path = None
@@ -1765,13 +1764,13 @@ def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime):
     lock_file = fs.find(work_dir, "spack.lock")[0]
     repro_lock_dir = os.path.dirname(lock_file)
 
-    tty.debug("Found lock file in: {0}".format(repro_lock_dir))
+    tty.debug(f"Found lock file in: {repro_lock_dir}")
 
     yaml_files = fs.find(work_dir, ["*.yaml", "*.yml"])
 
     tty.debug("yaml files:")
     for yaml_file in yaml_files:
-        tty.debug("  {0}".format(yaml_file))
+        tty.debug(f"  {yaml_file}")
 
     pipeline_yaml = None
 
@@ -1786,10 +1785,10 @@ def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime):
                 pipeline_yaml = yaml_obj
 
     if pipeline_yaml:
-        tty.debug("\n{0} is likely your pipeline file".format(yf))
+        tty.debug(f"\n{yf} is likely your pipeline file")
 
     relative_concrete_env_dir = pipeline_yaml["variables"]["SPACK_CONCRETE_ENV_DIR"]
-    tty.debug("Relative environment path used by cloud job: {0}".format(relative_concrete_env_dir))
+    tty.debug(f"Relative environment path used by cloud job: {relative_concrete_env_dir}")
 
     # Using the relative concrete environment path found in the generated
     # pipeline variable above, copy the spack environment files so they'll
@@ -1803,10 +1802,11 @@ def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime):
     shutil.copyfile(orig_yaml_path, copy_yaml_path)
 
     # Find the install script in the unzipped artifacts and make it executable
-    install_script = fs.find(work_dir, "install.sh")[0]
-    st = os.stat(install_script)
-    os.chmod(install_script, st.st_mode | stat.S_IEXEC)
-
+    install_script = fs.find(work_dir, f"install.{platform_script_ext}")[0]
+    if not IS_WINDOWS:
+        # pointless on Windows
+        st = os.stat(install_script)
+        os.chmod(install_script, st.st_mode | stat.S_IEXEC)
     # Find the repro details file.  This just includes some values we wrote
     # during `spack ci rebuild` to make reproduction easier.  E.g. the job
     # name is written here so we can easily find the configuration of the
@@ -1844,7 +1844,7 @@ def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime):
             job_image = job_image_elt["name"]
         else:
             job_image = job_image_elt
-        tty.msg("Job ran with the following image: {0}".format(job_image))
+        tty.msg(f"Job ran with the following image: {job_image}")
 
         # Because we found this job was run with a docker image, so we will try
         # to print a "docker run" command that bind-mounts the directory where
@@ -1919,65 +1919,75 @@ def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime):
     job_tags = None
     if "tags" in job_yaml:
         job_tags = job_yaml["tags"]
-        tty.msg("Job ran with the following tags: {0}".format(job_tags))
+        tty.msg(f"Job ran with the following tags: {job_tags}")
 
     entrypoint_script = [
         ["git", "config", "--global", "--add", "safe.directory", mount_as_dir],
-        [".", os.path.join(mount_as_dir if job_image else work_dir, "share/spack/setup-env.sh")],
+        [
+            ".",
+            os.path.join(
+                mount_as_dir if job_image else work_dir,
+                f"share/spack/setup-env.{platform_script_ext}",
+            ),
+        ],
         ["spack", "gpg", "trust", mounted_gpg_path if job_image else gpg_path] if gpg_path else [],
         ["spack", "env", "activate", mounted_env_dir if job_image else repro_dir],
-        [os.path.join(mounted_repro_dir, "install.sh") if job_image else install_script],
+        [
+            (
+                os.path.join(mounted_repro_dir, f"install.{platform_script_ext}")
+                if job_image
+                else install_script
+            )
+        ],
     ]
-
+    entry_script = os.path.join(mounted_workdir, f"entrypoint.{platform_script_ext}")
     inst_list = []
     # Finally, print out some instructions to reproduce the build
     if job_image:
         # Allow interactive
-        entrypoint_script.extend(
-            [
-                [
-                    "echo",
-                    "Re-run install script using:\n\t{0}".format(
-                        os.path.join(mounted_repro_dir, "install.sh")
-                        if job_image
-                        else install_script
-                    ),
-                ],
-                # Allow interactive
-                ["exec", "$@"],
-            ]
+        install_mechanism = (
+            os.path.join(mounted_repro_dir, f"install.{platform_script_ext}")
+            if job_image
+            else install_script
         )
+        entrypoint_script.append(["echo", f"Re-run install script using:\n\t{install_mechanism}"])
+        # Allow interactive
+        if IS_WINDOWS:
+            entrypoint_script.extend(["&", "($args -Join ' ')", "-NoExit"])
+        else:
+            entrypoint_script.extend(["exec", "$@"])
+
         process_command(
             "entrypoint", entrypoint_script, work_dir, run=False, exit_on_failure=False
         )
 
         docker_command = [
-            [
-                runtime,
-                "run",
-                "-i",
-                "-t",
-                "--rm",
-                "--name",
-                "spack_reproducer",
-                "-v",
-                ":".join([work_dir, mounted_workdir, "Z"]),
-                "-v",
-                ":".join(
-                    [
-                        os.path.join(work_dir, "jobs_scratch_dir"),
-                        os.path.join(mount_as_dir, "jobs_scratch_dir"),
-                        "Z",
-                    ]
-                ),
-                "-v",
-                ":".join([os.path.join(work_dir, "spack"), mount_as_dir, "Z"]),
-                "--entrypoint",
-                os.path.join(mounted_workdir, "entrypoint.sh"),
-                job_image,
-                "bash",
-            ]
+            runtime,
+            "run",
+            "-i",
+            "-t",
+            "--rm",
+            "--name",
+            "spack_reproducer",
+            "-v",
+            ":".join([work_dir, mounted_workdir, "Z"]),
+            "-v",
+            ":".join(
+                [
+                    os.path.join(work_dir, "jobs_scratch_dir"),
+                    os.path.join(mount_as_dir, "jobs_scratch_dir"),
+                    "Z",
+                ]
+            ),
+            "-v",
+            ":".join([os.path.join(work_dir, "spack"), mount_as_dir, "Z"]),
+            "--entrypoint",
         ]
+        if IS_WINDOWS:
+            docker_command.extend(["powershell.exe", job_image, entry_script, "powershell.exe"])
+        else:
+            docker_command.extend([entry_script, job_image, "bash"])
+        docker_command = [docker_command]
         autostart = autostart and setup_result
         process_command("start", docker_command, work_dir, run=autostart)
 
@@ -1986,22 +1996,20 @@ def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime):
             inst_list.extend(
                 [
                     "    - Start the docker container install",
-                    "       $ {0}/start.sh".format(work_dir),
+                    f"       $ {work_dir}/start.{platform_script_ext}",
                 ]
             )
     else:
         process_command("reproducer", entrypoint_script, work_dir, run=False)
 
         inst_list.append("\nOnce on the tagged runner:\n\n")
-        inst_list.extent(
-            ["    - Run the reproducer script", "       $ {0}/reproducer.sh".format(work_dir)]
-        )
+        inst_list.extent(["    - Run the reproducer script", f"       $ {work_dir}/reproducer.py"])
 
     if not setup_result:
         inst_list.append("\n    - Clone spack and acquire tested commit")
-        inst_list.append("\n        {0}\n".format(spack_info))
+        inst_list.append(f"\n        {spack_info}\n")
         inst_list.append("\n")
-        inst_list.append("\n        Path to clone spack: {0}/spack\n\n".format(work_dir))
+        inst_list.append(f"\n        Path to clone spack: {work_dir}/spack\n\n")
 
     tty.msg("".join(inst_list))
 
@@ -2020,50 +2028,78 @@ def process_command(name, commands, repro_dir, run=True, exit_on_failure=True):
 
     Returns: the exit code from processing the command
     """
-    tty.debug("spack {0} arguments: {1}".format(name, commands))
 
+    tty.debug(f"spack {name} arguments: {commands}")
     if len(commands) == 0 or isinstance(commands[0], str):
         commands = [commands]
 
-    # Create a string [command 1] && [command 2] && ... && [command n] with commands
-    # quoted using double quotes.
-    args_to_string = lambda args: " ".join('"{}"'.format(arg) for arg in args)
-    full_command = " \n ".join(map(args_to_string, commands))
+    def compose_command_err_handling(args):
+        if not IS_WINDOWS:
+            args = [f'"{arg}"' for arg in args]
+        arg_str = " ".join(args)
+        result = arg_str + "\n"
+        # ErrorActionPreference will handle PWSH commandlets (Spack calls),
+        # but we need to handle EXEs (git, etc) ourselves
+        catch_exe_failure = (
+            """
+if ($LASTEXITCODE -ne 0){
+    throw "Command {} has failed"
+}
+"""
+            if IS_WINDOWS
+            else ""
+        )
+        if exit_on_failure and catch_exe_failure:
+            result += catch_exe_failure.format(arg_str)
+        return result
 
-    # Write the command to a shell script
-    script = "{0}.sh".format(name)
-    with open(script, "w") as fd:
-        fd.write("#!/bin/sh\n\n")
-        fd.write("\n# spack {0} command\n".format(name))
+    # Create a string [command 1] \n [command 2] \n ... \n [command n] with
+    # commands composed into a platform dependent shell script, pwsh on Windows,
+    full_command = "\n".join(map(compose_command_err_handling, commands))
+    # Write the command to a python script
+    if IS_WINDOWS:
+        script = f"{name}.ps1"
+        script_content = [f"\n# spack {name} command\n"]
         if exit_on_failure:
-            fd.write("set -e\n")
+            script_content.append('$ErrorActionPreference = "Stop"\n')
         if os.environ.get("SPACK_VERBOSE_SCRIPT"):
-            fd.write("set -x\n")
-        fd.write(full_command)
-        fd.write("\n")
+            script_content.append("Set-PSDebug -Trace 2\n")
+    else:
+        script = f"{name}.sh"
+        script_content = ["#!/bin/sh\n\n", f"\n# spack {name} command\n"]
+        if exit_on_failure:
+            script_content.append("set -e\n")
+        if os.environ.get("SPACK_VERBOSE_SCRIPT"):
+            script_content.append("set -x\n")
+    script_content.append(full_command)
+    script_content.append("\n")
 
-    st = os.stat(script)
-    os.chmod(script, st.st_mode | stat.S_IEXEC)
+    with open(script, "w") as fd:
+        for line in script_content:
+            fd.write(line)
 
     copy_path = os.path.join(repro_dir, script)
     shutil.copyfile(script, copy_path)
-    st = os.stat(copy_path)
-    os.chmod(copy_path, st.st_mode | stat.S_IEXEC)
+    if not IS_WINDOWS:
+        st = os.stat(copy_path)
+        os.chmod(copy_path, st.st_mode | stat.S_IEXEC)
 
-    # Run the generated install.sh shell script as if it were being run in
+    # Run the generated shell script as if it were being run in
     # a login shell.
     exit_code = None
     if run:
         try:
-            cmd_process = subprocess.Popen(["/bin/sh", "./{0}".format(script)])
+            # We use sh as executor on Linux like platforms, pwsh on Windows
+            interpreter = "powershell.exe" if IS_WINDOWS else "/bin/sh"
+            cmd_process = subprocess.Popen([interpreter, f"./{script}"])
             cmd_process.wait()
             exit_code = cmd_process.returncode
         except (ValueError, subprocess.CalledProcessError, OSError) as err:
-            tty.error("Encountered error running {0} script".format(name))
+            tty.error(f"Encountered error running {name} script")
             tty.error(err)
             exit_code = 1
 
-        tty.debug("spack {0} exited {1}".format(name, exit_code))
+        tty.debug(f"spack {name} exited {exit_code}")
     else:
         # Delete the script, it is copied to the destination dir
         os.remove(script)
@@ -2122,7 +2158,7 @@ def write_broken_spec(url, pkg_name, stack_name, job_url, pipeline_url, spec_dic
         # If there is an S3 error (e.g., access denied or connection
         # error), the first non boto-specific class in the exception
         # hierarchy is Exception.  Just print a warning and return
-        msg = "Error writing to broken specs list {0}: {1}".format(url, err)
+        msg = f"Error writing to broken specs list {url}: {err}"
         tty.warn(msg)
     finally:
         shutil.rmtree(tmpdir)
@@ -2135,7 +2171,7 @@ def read_broken_spec(broken_spec_url):
     try:
         _, _, fs = web_util.read_from_url(broken_spec_url)
     except (URLError, web_util.SpackWebError, HTTPError):
-        tty.warn("Unable to read broken spec from {0}".format(broken_spec_url))
+        tty.warn(f"Unable to read broken spec from {broken_spec_url}")
         return None
 
     broken_spec_contents = codecs.getreader("utf-8")(fs).read()
@@ -2150,14 +2186,14 @@ def display_broken_spec_messages(base_url, hashes):
     for spec_hash, broken_spec in [tup for tup in broken_specs if tup[1]]:
         details = broken_spec["broken-spec"]
         if "job-name" in details:
-            item_name = "{0}/{1}".format(details["job-name"], spec_hash[:7])
+            item_name = f"{details['job-name']}/{spec_hash[:7]}"
         else:
             item_name = spec_hash
 
         if "job-stack" in details:
-            item_name = "{0} (in stack {1})".format(item_name, details["job-stack"])
+            item_name = f"{item_name} (in stack {details['job-stack']})"
 
-        msg = "  {0} was reported broken here: {1}".format(item_name, details["job-url"])
+        msg = f"  {item_name} was reported broken here: {details['job-url']}"
         tty.msg(msg)
 
 
@@ -2180,7 +2216,7 @@ def run_standalone_tests(**kwargs):
     log_file = kwargs.get("log_file")
 
     if cdash and log_file:
-        tty.msg("The test log file {0} option is ignored with CDash reporting".format(log_file))
+        tty.msg(f"The test log file {log_file} option is ignored with CDash reporting")
         log_file = None
 
     # Error out but do NOT terminate if there are missing required arguments.
@@ -2206,10 +2242,10 @@ def run_standalone_tests(**kwargs):
             test_args.extend(["--log-file", log_file])
     test_args.append(job_spec.name)
 
-    tty.debug("Running {0} stand-alone tests".format(job_spec.name))
+    tty.debug(f"Running {job_spec.name} stand-alone tests")
     exit_code = process_command("test", test_args, repro_dir)
 
-    tty.debug("spack test exited {0}".format(exit_code))
+    tty.debug(f"spack test exited {exit_code}")
 
 
 class CDashHandler:
@@ -2232,7 +2268,7 @@ def __init__(self, ci_cdash):
         # append runner description to the site if available
         runner = os.environ.get("CI_RUNNER_DESCRIPTION")
         if runner:
-            self.site += " ({0})".format(runner)
+            self.site += f" ({runner})"
 
         # track current spec, if any
         self.current_spec = None
@@ -2260,21 +2296,13 @@ def build_name(self):
         Returns: (str) current spec's CDash build name."""
         spec = self.current_spec
         if spec:
-            build_name = "{0}@{1}%{2} hash={3} arch={4} ({5})".format(
-                spec.name,
-                spec.version,
-                spec.compiler,
-                spec.dag_hash(),
-                spec.architecture,
-                self.build_group,
-            )
-            tty.debug(
-                "Generated CDash build name ({0}) from the {1}".format(build_name, spec.name)
-            )
+            build_name = f"{spec.name}@{spec.version}%{spec.compiler} \
+hash={spec.dag_hash()} arch={spec.architecture} ({self.build_group})"
+            tty.debug(f"Generated CDash build name ({build_name}) from the {spec.name}")
             return build_name
 
         build_name = os.environ.get("SPACK_CDASH_BUILD_NAME")
-        tty.debug("Using CDash build name ({0}) from the environment".format(build_name))
+        tty.debug(f"Using CDash build name ({build_name}) from the environment")
         return build_name
 
     @property  # type: ignore
@@ -2288,25 +2316,25 @@ def build_stamp(self):
         Returns: (str) current CDash build stamp"""
         build_stamp = os.environ.get("SPACK_CDASH_BUILD_STAMP")
         if build_stamp:
-            tty.debug("Using build stamp ({0}) from the environment".format(build_stamp))
+            tty.debug(f"Using build stamp ({build_stamp}) from the environment")
             return build_stamp
 
         build_stamp = cdash_build_stamp(self.build_group, time.time())
-        tty.debug("Generated new build stamp ({0})".format(build_stamp))
+        tty.debug(f"Generated new build stamp ({build_stamp})")
         return build_stamp
 
     @property  # type: ignore
     @memoized
     def project_enc(self):
-        tty.debug("Encoding project ({0}): {1})".format(type(self.project), self.project))
+        tty.debug(f"Encoding project ({type(self.project)}): {self.project})")
         encode = urlencode({"project": self.project})
         index = encode.find("=") + 1
         return encode[index:]
 
     @property
     def upload_url(self):
-        url_format = "{0}/submit.php?project={1}"
-        return url_format.format(self.url, self.project_enc)
+        url_format = f"{self.url}/submit.php?project={self.project_enc}"
+        return url_format
 
     def copy_test_results(self, source, dest):
         """Copy test results to artifacts directory."""
@@ -2324,7 +2352,7 @@ def create_buildgroup(self, opener, headers, url, group_name, group_type):
         response_code = response.getcode()
 
         if response_code not in [200, 201]:
-            msg = "Creating buildgroup failed (response code = {0})".format(response_code)
+            msg = f"Creating buildgroup failed (response code = {response_code})"
             tty.warn(msg)
             return None
 
@@ -2335,10 +2363,10 @@ def create_buildgroup(self, opener, headers, url, group_name, group_type):
         return build_group_id
 
     def populate_buildgroup(self, job_names):
-        url = "{0}/api/v1/buildgroup.php".format(self.url)
+        url = f"{self.url}/api/v1/buildgroup.php"
 
         headers = {
-            "Authorization": "Bearer {0}".format(self.auth_token),
+            "Authorization": f"Bearer {self.auth_token}",
             "Content-Type": "application/json",
         }
 
@@ -2346,11 +2374,11 @@ def populate_buildgroup(self, job_names):
 
         parent_group_id = self.create_buildgroup(opener, headers, url, self.build_group, "Daily")
         group_id = self.create_buildgroup(
-            opener, headers, url, "Latest {0}".format(self.build_group), "Latest"
+            opener, headers, url, f"Latest {self.build_group}", "Latest"
         )
 
         if not parent_group_id or not group_id:
-            msg = "Failed to create or retrieve buildgroups for {0}".format(self.build_group)
+            msg = f"Failed to create or retrieve buildgroups for {self.build_group}"
             tty.warn(msg)
             return
 
@@ -2370,7 +2398,7 @@ def populate_buildgroup(self, job_names):
         response_code = response.getcode()
 
         if response_code != 200:
-            msg = "Error response code ({0}) in populate_buildgroup".format(response_code)
+            msg = f"Error response code ({response_code}) in populate_buildgroup"
             tty.warn(msg)
 
     def report_skipped(self, spec: spack.spec.Spec, report_dir: str, reason: Optional[str]):