# Copyright Spack Project Developers. See COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

import base64
import json
import os
import pathlib
import re
import shutil
import stat
import subprocess
import tempfile
import zipfile
from collections import namedtuple
from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple
from urllib.request import Request

import spack
import spack.binary_distribution
import spack.builder
import spack.config as cfg
import spack.environment as ev
import spack.llnl.path
import spack.llnl.util.filesystem as fs
import spack.llnl.util.tty as tty
import spack.main
import spack.mirrors.mirror
import spack.paths
import spack.repo
import spack.spec
import spack.stage
import spack.store
import spack.util.git
import spack.util.gpg as gpg_util
import spack.util.spack_yaml as syaml
import spack.util.url as url_util
import spack.util.web as web_util
from spack import traverse
from spack.error import SpackError
from spack.llnl.util.tty.color import cescape, colorize
from spack.reporters.cdash import SPACK_CDASH_TIMEOUT

from .common import (
    IS_WINDOWS,
    CDashHandler,
    PipelineDag,
    PipelineOptions,
    PipelineType,
    SpackCIConfig,
    SpackCIError,
    copy_files_to_artifacts,
)
from .generator_registry import UnknownGeneratorException, get_generator

# Import any modules with generator functions from here, so they get
# registered without introducing any import cycles.
from .gitlab import generate_gitlab_yaml  # noqa: F401

spack_gpg = spack.main.SpackCommand("gpg")
spack_compiler = spack.main.SpackCommand("compiler")

PushResult = namedtuple("PushResult", "success url")

urlopen = web_util.urlopen  # alias for mocking in tests


def get_git_root(path: str) -> Optional[str]:
    git = spack.util.git.git(required=True)
    try:
        with fs.working_dir(path):
            # Raises SpackError on command failure
            git_dir = git("rev-parse", "--show-toplevel", fail_on_error=True, output=str).strip()
            tty.debug(f"{path} git toplevel at {git_dir}")
            return git_dir
    except SpackError:
        return None


def get_change_revisions(path: str) -> Tuple[Optional[str], Optional[str]]:
    """If this is a git repo get the revisions to use when checking
    for changed packages and spack core modules."""

    if get_git_root(path):
        # TODO: This will only find changed packages from the last
        # TODO: commit.  While this may work for single merge commits
        # TODO: when merging the topic branch into the base, it will
        # TODO: require more thought outside of that narrow case.
        return "HEAD^", "HEAD"
    else:
        return None, None


def filter_added_checksums(
    checksums: Iterable[str], path: str, from_ref: str = "HEAD~1", to_ref: str = "HEAD"
) -> List[str]:
    """Get a list of the version checksums added between ``from_ref`` and ``to_ref``.

    Args:
       checksums: an iterable of checksums to look for in the diff
       path: path to the package.py
       from_ref: oldest git ref, defaults to ``HEAD~1``
       to_ref: newer git ref, defaults to ``HEAD``
    Returns: list of version checksums added between refs
    """
    git_exe = spack.util.git.git(required=True)

    # Gather git diff
    diff_lines = git_exe("diff", from_ref, to_ref, "--", path, output=str).split("\n")

    # Store added and removed versions
    # Removed versions are tracked here to determine when versions are moved in a file
    # and show up as both added and removed in a git diff.
    added_checksums: Set[str] = set()
    removed_checksums: Set[str] = set()

    # Scrape diff for modified versions and prune added versions if they show up
    # as also removed (which means they've actually just moved in the file and
    # we shouldn't need to rechecksum them)
    for checksum in checksums:
        for line in diff_lines:
            if checksum in line:
                if line.startswith("+"):
                    added_checksums.add(checksum)
                if line.startswith("-"):
                    removed_checksums.add(checksum)

    return list(added_checksums - removed_checksums)


def stack_changed(env_path: str) -> bool:
    """Given an environment manifest path, return whether or not the stack was changed.
    Returns True iff the environment manifest changed between the provided revisions (or
    additionally if the ``.gitlab-ci.yml`` file itself changed)."""
    # git returns posix paths always, normalize input to be compatible with that
    env_path = spack.llnl.path.convert_to_posix_path(os.path.dirname(env_path))

    git = spack.util.git.git(required=True)
    git_dir = get_git_root(env_path)

    if git_dir is None:
        return False

    with fs.working_dir(git_dir):
        diff = git(
            "diff",
            "--name-only",
            "HEAD^",
            "HEAD",
            output=str,
            error=os.devnull,
            fail_on_error=False,
        ).strip()

        if not diff:
            return False

        for path in diff.split():
            if ".gitlab-ci.yml" in path or path in env_path:
                tty.debug(f"env represented by {env_path} changed")
                tty.debug(f"touched file: {path}")
                return True
    return False


def compute_affected_packages(
    repo: spack.repo.Repo, rev1: str = "HEAD^", rev2: str = "HEAD"
) -> Set[str]:
    """Determine which packages were added, removed or changed
    between rev1 and rev2, and return the names as a set"""
    return spack.repo.get_all_package_diffs("ARC", repo, rev1=rev1, rev2=rev2)


def get_spec_filter_list(
    env: ev.Environment, affected_pkgs: Set[str], dependent_traverse_depth: Optional[int] = None
) -> Set[spack.spec.Spec]:
    """Given a list of package names and an active/concretized environment, return the set of all
    concrete specs from the environment that could have been affected by changing the list of
    packages.

    If a ``dependent_traverse_depth`` is given, it is used to limit upward (in the parent
    direction) traversal of specs of touched packages. E.g. if 1 is provided, then only direct
    dependents of touched package specs are traversed to produce specs that could have been
    affected by changing the package, while if 0 is provided, only the changed specs themselves
    are traversed. If ``None`` is given, upward traversal of touched package specs is done all the
    way to the environment roots. Providing a negative number results in no traversals at all,
    yielding an empty set.

    Arguments:
        env: Active concrete environment
        affected_pkgs: Affected package names
        dependent_traverse_depth: Integer to limit dependent traversal, None means no limit

    Returns:
        A set of concrete specs from the active environment including those associated with
        affected packages, their dependencies and dependents, as well as their dependents
        dependencies.
    """
    affected_specs: Set[spack.spec.Spec] = set()
    all_concrete_specs = env.all_specs()
    env_matches = [s for s in all_concrete_specs if s.name in affected_pkgs]
    visited: Set[str] = set()
    for depth, parent in traverse.traverse_nodes(
        env_matches, direction="parents", key=traverse.by_dag_hash, depth=True, order="breadth"
    ):
        if dependent_traverse_depth is not None and depth > dependent_traverse_depth:
            break
        affected_specs.update(
            parent.traverse(direction="children", visited=visited, key=traverse.by_dag_hash)
        )
    return affected_specs


# Pruning functions should take a spack.spec.Spec object and
# return a RebuildDecision containing the pruners opinion on
# whether or not to keep (rebuild) the spec and a message
# containing the reason for the decision.


class RebuildDecision:
    def __init__(self, rebuild: bool = True, reason: str = ""):
        self.rebuild = rebuild
        self.reason = reason


PrunerCallback = Callable[[spack.spec.Spec], RebuildDecision]


def create_unaffected_pruner(affected_specs: Set[spack.spec.Spec]) -> PrunerCallback:
    """Given a set of "affected" specs, return a filter that prunes specs
    not in the set."""

    def rebuild_filter(s: spack.spec.Spec) -> RebuildDecision:
        if s in affected_specs:
            return RebuildDecision(True, "affected by change")
        return RebuildDecision(False, "unaffected by change")

    return rebuild_filter


def create_already_built_pruner(check_index_only: bool = True) -> PrunerCallback:
    """Return a filter that prunes specs already present on any configured
    mirrors"""
    try:
        spack.binary_distribution.BINARY_INDEX.update()
    except spack.binary_distribution.FetchCacheError as e:
        tty.warn(e)

    def rebuild_filter(s: spack.spec.Spec) -> RebuildDecision:
        spec_locations = spack.binary_distribution.get_mirrors_for_spec(
            spec=s, index_only=check_index_only
        )

        if not spec_locations:
            return RebuildDecision(True, "not found anywhere")

        urls = ",".join(f"{loc:_url@v_version? (view: _view)}" for loc in spec_locations)
        message = f"up-to-date [{urls}]"
        return RebuildDecision(False, message)

    return rebuild_filter


def create_external_pruner() -> PrunerCallback:
    """Return a filter that prunes external specs"""

    def rebuild_filter(s: spack.spec.Spec) -> RebuildDecision:
        if not s.external:
            return RebuildDecision(True, "not external")
        return RebuildDecision(False, "external spec")

    return rebuild_filter


def _format_pruning_message(spec: spack.spec.Spec, prune: bool, reasons: List[str]) -> str:
    reason_msg = ", ".join(reasons)
    spec_fmt = "{name}{@version}{/hash:7}{compilers}"

    if not prune:
        status = colorize("@*g{[x]}  ")
        return f"  {status}{spec.cformat(spec_fmt)} ({reason_msg})"

    msg = f"{spec.format(spec_fmt)} ({reason_msg})"
    return colorize(f"  @K -   {cescape(msg)}@.")


def prune_pipeline(
    pipeline: PipelineDag, pruning_filters: List[PrunerCallback], print_summary: bool = False
) -> None:
    """Given a PipelineDag and a list of pruning filters, return a modified
    PipelineDag containing only the nodes that survive pruning by all of the
    filters."""
    keys_to_prune = set()
    keys_to_rebuild = set()
    specs: Dict[str, spack.spec.Spec] = {}
    reasons: Dict[str, List[str]] = {}

    for _, node in pipeline.traverse_nodes(direction="children"):
        filter_results = [keepSpec(node.spec) for keepSpec in pruning_filters]

        reasons[node.key] = [r.reason for r in filter_results]
        specs[node.key] = node.spec

        if not all(r.rebuild for r in filter_results):
            keys_to_prune.add(node.key)
        else:
            keys_to_rebuild.add(node.key)

    for key in keys_to_prune:
        pipeline.prune(key)

    if print_summary:
        sort_key = lambda k: f"{specs[k].name}/{specs[k].dag_hash(7)}"
        tty.msg("Pipeline pruning summary:")
        if keys_to_rebuild:
            tty.msg("  Rebuild list:")
            for key in sorted(keys_to_rebuild, key=sort_key):
                tty.msg(_format_pruning_message(specs[key], False, reasons[key]))
        if keys_to_prune:
            tty.msg("  Prune list:")
            for key in sorted(keys_to_prune, key=sort_key):
                tty.msg(_format_pruning_message(specs[key], True, reasons[key]))


def check_for_broken_specs(pipeline_specs: List[spack.spec.Spec], broken_specs_url: str) -> bool:
    """Check the pipeline specs against the list of known broken specs and return
    True if there were any matches, False otherwise."""
    if broken_specs_url.startswith("http"):
        # To make checking each spec against the list faster, we require
        # a url protocol that allows us to iterate the url in advance.
        tty.msg("Cannot use an http(s) url for broken specs, ignoring")
        return False

    broken_spec_urls = web_util.list_url(broken_specs_url)

    if broken_spec_urls is None:
        return False

    known_broken_specs_encountered = []
    for release_spec in pipeline_specs:
        release_spec_dag_hash = release_spec.dag_hash()
        if release_spec_dag_hash in broken_spec_urls:
            known_broken_specs_encountered.append(release_spec_dag_hash)

    if known_broken_specs_encountered:
        tty.error("This pipeline generated hashes known to be broken on develop:")
        display_broken_spec_messages(broken_specs_url, known_broken_specs_encountered)
        return True

    return False


def collect_pipeline_options(env: ev.Environment, args) -> PipelineOptions:
    """Gather pipeline options from cli args, spack environment, and
    os environment variables"""
    pipeline_mirrors = spack.mirrors.mirror.MirrorCollection(binary=True)
    if "buildcache-destination" not in pipeline_mirrors:
        raise SpackCIError("spack ci generate requires a mirror named 'buildcache-destination'")

    buildcache_destination = pipeline_mirrors["buildcache-destination"]
    options = PipelineOptions(env, buildcache_destination)

    options.env = env
    options.artifacts_root = args.artifacts_root
    options.output_file = args.output_file
    options.prune_up_to_date = args.prune_dag
    options.prune_unaffected = args.prune_unaffected
    options.prune_external = args.prune_externals
    options.check_index_only = args.index_only
    options.forward_variables = args.forward_variable or []

    ci_config = cfg.get("ci")

    cdash_config = cfg.get("cdash")
    if "build-group" in cdash_config:
        options.cdash_handler = CDashHandler(cdash_config)

    dependent_depth = os.environ.get("SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH", None)
    if dependent_depth is not None:
        try:
            options.untouched_pruning_dependent_depth = int(dependent_depth)
        except (TypeError, ValueError):
            tty.warn(
                f"Unrecognized value ({dependent_depth}) "
                "provided for SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH, "
                "ignoring it."
            )

    spack_prune_untouched = str(os.environ.get("SPACK_PRUNE_UNTOUCHED", options.prune_unaffected))
    options.prune_untouched = (
        spack_prune_untouched is not None and spack_prune_untouched.lower() == "true"
    )

    # Allow overriding --prune-dag cli opt with environment variable
    prune_dag_override = os.environ.get("SPACK_PRUNE_UP_TO_DATE", None)
    if prune_dag_override is not None:
        options.prune_up_to_date = True if prune_dag_override.lower() == "true" else False

    options.stack_name = os.environ.get("SPACK_CI_STACK_NAME", None)
    require_signing = os.environ.get("SPACK_REQUIRE_SIGNING", None)
    options.require_signing = (
        True if require_signing and require_signing.lower() == "true" else False
    )

    # Get the type of pipeline, which is optional
    spack_pipeline_type = os.environ.get("SPACK_PIPELINE_TYPE", None)
    if spack_pipeline_type:
        try:
            options.pipeline_type = PipelineType[spack_pipeline_type]
        except KeyError:
            options.pipeline_type = None

    if "broken-specs-url" in ci_config:
        options.broken_specs_url = ci_config["broken-specs-url"]

    if "rebuild-index" in ci_config and ci_config["rebuild-index"] is False:
        options.rebuild_index = False

    return options


def get_unaffected_pruners(
    env: ev.Environment, untouched_pruning_dependent_depth: Optional[int]
) -> Optional[PrunerCallback]:

    # If the stack env has changed, do not apply unaffected pruning
    if stack_changed(env.manifest_path):
        tty.info("Skipping unaffected pruning: stack environment changed")
        return None

    # TODO: This should be configurable to only check for changed packages
    # in specific configured repos that are being tested with CI. For now
    # it assumes all configured repos are merge commits that contain relevant
    # changes to run CI on.
    affected_pkgs: Set[str] = set()
    for repo in spack.repo.PATH.repos:
        rev1, rev2 = get_change_revisions(repo.root)
        if not (rev1 and rev2):
            continue

        tty.debug(f"repo {repo.namespace}: revisions rev1={rev1}, rev2={rev2}")

        repo_affected_pkgs = compute_affected_packages(repo, rev1=rev1, rev2=rev2)
        tty.debug(f"repo {repo.namespace}: affected pkgs")
        for p in repo_affected_pkgs:
            tty.debug(f"  {p}")

        affected_pkgs.update(repo_affected_pkgs)

    if not affected_pkgs:
        tty.info("Skipping unaffected pruning: no package changes were detected")
        return None

    affected_specs = get_spec_filter_list(
        env, affected_pkgs, dependent_traverse_depth=untouched_pruning_dependent_depth
    )
    tty.debug(f"dependent_traverse_depth={untouched_pruning_dependent_depth}, affected specs:")
    for s in affected_specs:
        tty.debug(f"  {PipelineDag.key(s)}")

    return create_unaffected_pruner(affected_specs)


def generate_pipeline(env: ev.Environment, args) -> None:
    """Given an environment and the command-line args, generate a pipeline.

    Arguments:
        env (spack.environment.Environment): Activated environment object
            which must contain a ci section describing attributes for
            all jobs and a target which should specify an existing
            pipeline generator.
        args: (spack.main.SpackArgumentParser): Parsed arguments from the command
            line.
    """
    with env.write_transaction():
        env.concretize()
        env.write()

    options = collect_pipeline_options(env, args)

    # Get the joined "ci" config with all of the current scopes resolved
    ci_config = cfg.get("ci")
    if not ci_config:
        raise SpackCIError("Environment does not have a `ci` configuration")

    # Get the target platform we should generate a pipeline for
    ci_target = ci_config.get("target", "gitlab")
    try:
        generate_method = get_generator(ci_target)
    except UnknownGeneratorException:
        raise SpackCIError(f"Spack CI module cannot generate a pipeline for format {ci_target}")

    # If we are not doing any kind of pruning, we are rebuilding everything
    rebuild_everything = not options.prune_up_to_date and not options.prune_untouched

    # Build a pipeline from the specs in the concrete environment
    pipeline = PipelineDag([env.specs_by_hash[x.hash] for x in env.concretized_roots])

    # Optionally add various pruning filters
    pruning_filters = []

    # Possibly prune specs that were unaffected by the change
    if options.prune_untouched:
        # If we don't have two revisions to compare, or if either the spack.yaml
        # associated with the active env or the .gitlab-ci.yml files changed
        # between the provided revisions, then don't do any "untouched spec"
        # pruning.  Otherwise, list the names of all packages touched between
        # rev1 and rev2, and prune from the pipeline any node whose spec has a
        # packagen name not in that list.
        unaffected_pruner = get_unaffected_pruners(env, options.untouched_pruning_dependent_depth)
        if unaffected_pruner:
            tty.info("Enabling Unaffected Pruner")
            pruning_filters.append(unaffected_pruner)

    # Possibly prune specs that are already built on some configured mirror
    if options.prune_up_to_date:
        tty.info("Enabling Up-to-date Pruner")
        pruning_filters.append(
            create_already_built_pruner(check_index_only=options.check_index_only)
        )

    # Possibly prune specs that are external
    if options.prune_external:
        tty.info("Enabling Externals Pruner")
        pruning_filters.append(create_external_pruner())

    # Do all the pruning
    prune_pipeline(pipeline, pruning_filters, options.print_summary)

    # List all specs remaining after any pruning
    pipeline_specs = [n.spec for _, n in pipeline.traverse_nodes(direction="children")]

    # If this is configured, spack will fail "spack ci generate" if it
    # generates any hash which exists under the broken specs url.
    if options.broken_specs_url and not options.pipeline_type == PipelineType.COPY_ONLY:
        broken = check_for_broken_specs(pipeline_specs, options.broken_specs_url)
        if broken and not rebuild_everything:
            raise SpackCIError("spack ci generate failed broken specs check")

    spack_ci_config = SpackCIConfig(ci_config)
    spack_ci_config.init_pipeline_jobs(pipeline)

    # Format the pipeline using the formatter specified in the configs
    generate_method(pipeline, spack_ci_config, options)

    # Use all unpruned specs to populate the build group for this set
    cdash_config = cfg.get("cdash")
    if options.cdash_handler and options.cdash_handler.auth_token:
        options.cdash_handler.create_buildgroup()
    elif cdash_config:
        # warn only if there was actually a CDash configuration.
        tty.warn("Unable to populate buildgroup without CDash credentials")


def import_signing_key(base64_signing_key: str) -> None:
    """Given Base64-encoded gpg key, decode and import it to use for signing packages.

    Arguments:
        base64_signing_key:
            A gpg key including the secret key, armor-exported and base64 encoded, so it can be
            stored in a gitlab CI variable. For an example of how to generate such a key, see
            https://github.com/spack/spack-infrastructure/blob/main/gitlab-docker/files/gen-key.
    """
    if not base64_signing_key:
        tty.warn("No key found for signing/verifying packages")
        return

    tty.debug("ci.import_signing_key() will attempt to import a key")

    # This command has the side-effect of creating the directory referred
    # to as GNUPGHOME in setup_environment()
    list_output = spack_gpg("list")

    tty.debug("spack gpg list:")
    tty.debug(list_output)

    decoded_key = base64.b64decode(base64_signing_key).decode("utf-8")

    with tempfile.TemporaryDirectory() as tmpdir:
        sign_key_path = os.path.join(tmpdir, "signing_key")
        with open(sign_key_path, "w", encoding="utf-8") as fd:
            fd.write(decoded_key)

        key_import_output = spack_gpg("trust", "-y", sign_key_path)
        tty.debug(f"spack gpg trust {sign_key_path}")
        tty.debug(key_import_output)

    # Now print the keys we have for verifying and signing
    trusted_keys_output = spack_gpg("list", "--trusted")
    signing_keys_output = spack_gpg("list", "--signing")

    tty.debug("spack gpg list --trusted")
    tty.debug(trusted_keys_output)
    tty.debug("spack gpg list --signing")
    tty.debug(signing_keys_output)


def can_sign_binaries():
    """Utility method to determine if this spack instance is capable of
    signing binary packages.  This is currently only possible if the
    spack gpg keystore contains exactly one secret key."""
    return len(gpg_util.signing_keys()) == 1


def can_verify_binaries():
    """Utility method to determine if this spack instance is capable (at
    least in theory) of verifying signed binaries."""
    return len(gpg_util.public_keys()) >= 1


def push_to_build_cache(spec: spack.spec.Spec, mirror_url: str, sign_binaries: bool) -> bool:
    """Push one or more binary packages to the mirror.

    Arguments:

        spec: Installed spec to push
        mirror_url: URL of target mirror
        sign_binaries: If True, spack will attempt to sign binary package before pushing.
    """
    tty.debug(f"Pushing to build cache ({'signed' if sign_binaries else 'unsigned'})")
    signing_key = spack.binary_distribution.select_signing_key() if sign_binaries else None
    mirror = spack.mirrors.mirror.Mirror.from_url(mirror_url)
    try:
        with spack.binary_distribution.make_uploader(mirror, signing_key=signing_key) as uploader:
            uploader.push_or_raise([spec])
        return True
    except spack.binary_distribution.PushToBuildCacheError as e:
        tty.error(f"Problem writing to {mirror_url}: {e}")
        return False


def copy_stage_logs_to_artifacts(job_spec: spack.spec.Spec, job_log_dir: str) -> None:
    """Copy selected build stage file(s) to the given artifacts directory

    Looks for build logs in the stage directory of the given
    job_spec, and attempts to copy the files into the directory given
    by job_log_dir.

    Parameters:
        job_spec: spec associated with spack install log
        job_log_dir: path into which build log should be copied
    """
    tty.debug(f"job spec: {job_spec}")
    if not job_spec.concrete:
        tty.warn("Cannot copy artifacts for non-concrete specs")
        return

    package_metadata_root = pathlib.Path(spack.store.STORE.layout.metadata_path(job_spec))
    if not os.path.isdir(package_metadata_root):
        # Fallback to using the stage directory
        job_pkg = job_spec.package

        package_metadata_root = pathlib.Path(job_pkg.stage.path)
        archive_files = spack.builder.create(job_pkg).archive_files
        tty.warn("Package not installed, falling back to use stage dir")
        tty.debug(f"stage dir: {package_metadata_root}")
    else:
        # Get the package's archived files
        archive_files = []
        archive_root = package_metadata_root / "archived-files"
        if os.path.isdir(archive_root):
            archive_files = [str(f) for f in archive_root.rglob("*") if os.path.isfile(f)]
        else:
            tty.debug(f"No archived files detected at {archive_root}")

    # Try zipped and unzipped versions of the build log
    build_log_zipped = package_metadata_root / "spack-build-out.txt.gz"
    build_log = package_metadata_root / "spack-build-out.txt"
    build_env_mods = package_metadata_root / "spack-build-env.txt"

    for f in [build_log_zipped, build_log, build_env_mods, *archive_files]:
        copy_files_to_artifacts(str(f), job_log_dir, compress_artifacts=True)


def copy_test_logs_to_artifacts(test_stage, job_test_dir):
    """
    Copy test log file(s) to the given artifacts directory

    Parameters:
        test_stage (str): test stage path
        job_test_dir (str): the destination artifacts test directory
    """
    tty.debug(f"test stage: {test_stage}")
    if not os.path.exists(test_stage):
        tty.error(f"Cannot copy test logs: job test stage ({test_stage}) does not exist")
        return

    copy_files_to_artifacts(
        os.path.join(test_stage, "*", "*.txt"), job_test_dir, compress_artifacts=True
    )


def download_and_extract_artifacts(url: str, work_dir: str) -> str:
    """Look for gitlab artifacts.zip at the given url, and attempt to download
    and extract the contents into the given work_dir

    Arguments:
        url: Complete url to artifacts.zip file
        work_dir: Path to destination where artifacts should be extracted

    Returns:
        Artifacts root path relative to the archive root
    """
    tty.msg(f"Fetching artifacts from: {url}")

    headers = {"Content-Type": "application/zip"}

    token = os.environ.get("GITLAB_PRIVATE_TOKEN", None)
    if token:
        headers["PRIVATE-TOKEN"] = token

    request = Request(url, headers=headers, method="GET")
    artifacts_zip_path = os.path.join(work_dir, "artifacts.zip")
    os.makedirs(work_dir, exist_ok=True)

    try:
        with urlopen(request, timeout=SPACK_CDASH_TIMEOUT) as response:
            with open(artifacts_zip_path, "wb") as out_file:
                shutil.copyfileobj(response, out_file)

        with zipfile.ZipFile(artifacts_zip_path) as zip_file:
            zip_file.extractall(work_dir)
            # Get the artifact root
            artifact_root = ""
            for f in zip_file.filelist:
                if "spack.lock" in f.filename:
                    artifact_root = os.path.dirname(os.path.dirname(f.filename))
                    break
    except OSError as e:
        raise SpackError(f"Error fetching artifacts: {e}")
    finally:
        try:
            os.remove(artifacts_zip_path)
        except FileNotFoundError:
            # If the file doesn't exist we are already raising
            pass

    return artifact_root


def get_spack_info():
    """If spack is running from a git repo, return the most recent git log
    entry, otherwise, return a string containing the spack version."""
    git_path = os.path.join(spack.paths.prefix, ".git")
    if os.path.exists(git_path):
        git = spack.util.git.git()
        if git:
            with fs.working_dir(spack.paths.prefix):
                git_log = git("log", "-1", output=str, error=os.devnull, fail_on_error=False)

            return git_log

    return f"no git repo, use spack {spack.spack_version}"


def setup_spack_repro_version(
    repro_dir: str, checkout_commit: str, merge_commit: Optional[str] = None
) -> bool:
    """Look in the local spack clone to find the checkout_commit, and if provided, the
    merge_commit given as arguments. If those commits can be found locally, then clone spack and
    attempt to recreate a merge commit with the same parent commits as tested in gitlab. This looks
    something like

    1. ``git clone repo && cd repo``
    2. ``git checkout <checkout_commit>``
    3. ``git merge <merge_commit>``

    If there is no merge_commit provided, then skip step (3).

    Arguments:

        repro_dir: Location where spack should be cloned
        checkout_commit: SHA of PR branch commit
        merge_commit: SHA of target branch parent

    Returns: True iff the git repo state was successfully recreated
    """
    # figure out the path to the spack git version being used for the
    # reproduction
    tty.info(f"checkout_commit: {checkout_commit}")
    tty.info(f"merge_commit: {merge_commit}")

    dot_git_path = os.path.join(spack.paths.prefix, ".git")
    if not os.path.exists(dot_git_path):
        tty.error("Unable to find the path to your local spack clone")
        return False

    spack_git_path = spack.paths.prefix

    git = spack.util.git.git()
    if not git:
        tty.error("reproduction of pipeline job requires git")
        return False

    # Check if we can find the tested commits in your local spack repo
    with fs.working_dir(spack_git_path):
        git("log", "-1", checkout_commit, output=str, error=os.devnull, fail_on_error=False)

        if git.returncode != 0:
            tty.error(f"Missing commit: {checkout_commit}")
            return False

        if merge_commit:
            git("log", "-1", merge_commit, output=str, error=os.devnull, fail_on_error=False)

            if git.returncode != 0:
                tty.error(f"Missing commit: {merge_commit}")
                return False

    # Next attempt to clone your local spack repo into the repro dir
    with fs.working_dir(repro_dir):
        clone_out = git(
            "clone", spack_git_path, "spack", output=str, error=os.devnull, fail_on_error=False
        )

        if git.returncode != 0:
            tty.error("Unable to clone your local spack repo:")
            tty.msg(clone_out)
            return False

    # Finally, attempt to put the cloned repo into the same state used during
    # the pipeline build job
    repro_spack_path = os.path.join(repro_dir, "spack")
    with fs.working_dir(repro_spack_path):
        co_out = git(
            "checkout", checkout_commit, output=str, error=os.devnull, fail_on_error=False
        )

        if git.returncode != 0:
            tty.error(f"Unable to checkout {checkout_commit}")
            tty.msg(co_out)
            return False

        if merge_commit:
            merge_out = git(
                "-c",
                "user.name=cirepro",
                "-c",
                "user.email=user@email.org",
                "merge",
                "--no-edit",
                merge_commit,
                output=str,
                error=os.devnull,
                fail_on_error=False,
            )

            if git.returncode != 0:
                tty.error(f"Unable to merge {merge_commit}")
                tty.msg(merge_out)
                return False

    return True


def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime, use_local_head):
    """Given a url to gitlab artifacts.zip from a failed ``spack ci rebuild`` job,
    attempt to setup an environment in which the failure can be reproduced
    locally.  This entails the following:

    First download and extract artifacts.  Then look through those artifacts
    to glean some information needed for the reproduer (e.g. one of the
    artifacts contains information about the version of spack tested by
    gitlab, another is the generated pipeline yaml containing details
    of the job like the docker image used to run it).  The output of this
    function is a set of printed instructions for running docker and then
    commands to run to reproduce the build once inside the container.
    """
    work_dir = os.path.realpath(work_dir)
    if os.path.exists(work_dir) and os.listdir(work_dir):
        raise SpackError(f"Cannot run reproducer in non-empty working dir:\n  {work_dir}")

    platform_script_ext = "ps1" if IS_WINDOWS else "sh"
    artifact_root = download_and_extract_artifacts(url, work_dir)

    gpg_path = None
    if gpg_url:
        gpg_path = web_util.fetch_url_text(gpg_url, dest_dir=os.path.join(work_dir, "_pgp"))
        rel_gpg_path = gpg_path.replace(work_dir, "").lstrip(os.path.sep)

    lock_file = fs.find(work_dir, "spack.lock")[0]
    repro_lock_dir = os.path.dirname(lock_file)

    tty.debug(f"Found lock file in: {repro_lock_dir}")

    yaml_files = fs.find(work_dir, ["*.yaml", "*.yml"])

    tty.debug("yaml files:")
    for yaml_file in yaml_files:
        tty.debug(f"  {yaml_file}")

    pipeline_yaml = None

    # Try to find the dynamically generated pipeline yaml file in the
    # reproducer.  If the user did not put it in the artifacts root,
    # but rather somewhere else and exported it as an artifact from
    # that location, we won't be able to find it.
    for yf in yaml_files:
        with open(yf, encoding="utf-8") as y_fd:
            yaml_obj = syaml.load(y_fd)
            if "variables" in yaml_obj and "stages" in yaml_obj:
                pipeline_yaml = yaml_obj

    if pipeline_yaml:
        tty.debug(f"\n{yf} is likely your pipeline file")

    relative_concrete_env_dir = pipeline_yaml["variables"]["SPACK_CONCRETE_ENV_DIR"]
    tty.debug(f"Relative environment path used by cloud job: {relative_concrete_env_dir}")

    # Using the relative concrete environment path found in the generated
    # pipeline variable above, copy the spack environment files so they'll
    # be found in the same location as when the job ran in the cloud.
    concrete_env_dir = os.path.join(work_dir, relative_concrete_env_dir)
    os.makedirs(concrete_env_dir, exist_ok=True)
    copy_lock_path = os.path.join(concrete_env_dir, "spack.lock")
    orig_yaml_path = os.path.join(repro_lock_dir, "spack.yaml")
    copy_yaml_path = os.path.join(concrete_env_dir, "spack.yaml")
    shutil.copyfile(lock_file, copy_lock_path)
    shutil.copyfile(orig_yaml_path, copy_yaml_path)

    # Find the install script in the unzipped artifacts and make it executable
    install_script = fs.find(work_dir, f"install.{platform_script_ext}")[0]
    if not IS_WINDOWS:
        # pointless on Windows
        st = os.stat(install_script)
        os.chmod(install_script, st.st_mode | stat.S_IEXEC)
    # Find the repro details file.  This just includes some values we wrote
    # during `spack ci rebuild` to make reproduction easier.  E.g. the job
    # name is written here so we can easily find the configuration of the
    # job from the generated pipeline file.
    repro_file = fs.find(work_dir, "repro.json")[0]
    repro_details = None
    with open(repro_file, encoding="utf-8") as fd:
        repro_details = json.load(fd)

    spec_file = fs.find(work_dir, repro_details["job_spec_json"])[0]
    reproducer_spec = spack.spec.Spec.from_specfile(spec_file)

    repro_dir = os.path.dirname(repro_file)
    rel_repro_dir = repro_dir.replace(work_dir, "").lstrip(os.path.sep)

    # Find the spack info text file that should contain the git log
    # of the HEAD commit used during the CI build
    spack_info_file = fs.find(work_dir, "spack_info.txt")[0]
    with open(spack_info_file, encoding="utf-8") as fd:
        spack_info = fd.read()

    # Access the specific job configuration
    job_name = repro_details["job_name"]
    job_yaml = None

    if job_name in pipeline_yaml:
        job_yaml = pipeline_yaml[job_name]

    if job_yaml:
        tty.debug("Found job:")
        tty.debug(job_yaml)

    job_image = None
    setup_result = False
    if "image" in job_yaml:
        job_image_elt = job_yaml["image"]
        if "name" in job_image_elt:
            job_image = job_image_elt["name"]
        else:
            job_image = job_image_elt
        tty.msg(f"Job ran with the following image: {job_image}")

        # Because we found this job was run with a docker image, so we will try
        # to print a "docker run" command that bind-mounts the directory where
        # we extracted the artifacts.

        # Destination of bind-mounted reproduction directory.  It makes for a
        # more faithful reproducer if everything appears to run in the same
        # absolute path used during the CI build.
        mount_as_dir = "/work"
        mounted_workdir = "/reproducer"
        if repro_details:
            mount_as_dir = repro_details["ci_project_dir"]
            mounted_repro_dir = os.path.join(mount_as_dir, rel_repro_dir)
            mounted_env_dir = os.path.join(mount_as_dir, relative_concrete_env_dir)
            if gpg_path:
                mounted_gpg_path = os.path.join(mounted_workdir, rel_gpg_path)

    # We will also try to clone spack from your local checkout and
    # reproduce the state present during the CI build, and put that into
    # the bind-mounted reproducer directory.

    # Regular expressions for parsing that HEAD commit.  If the pipeline
    # was on the gitlab spack mirror, it will have been a merge commit made by
    # github and pushed by the sync script.  If the pipeline was run on some
    # environment repo, then the tested spack commit will likely have been
    # a regular commit.
    commit_1 = None
    commit_2 = None
    commit_regex = re.compile(r"commit\s+([^\s]+)")
    merge_commit_regex = re.compile(r"Merge\s+([^\s]+)\s+into\s+([^\s]+)")

    if use_local_head:
        commit_1 = "HEAD"
    else:
        # Try the more specific merge commit regex first
        m = merge_commit_regex.search(spack_info)
        if m:
            # This was a merge commit and we captured the parents
            commit_1 = m.group(1)
            commit_2 = m.group(2)
        else:
            # Not a merge commit, just get the commit sha
            m = commit_regex.search(spack_info)
            if m:
                commit_1 = m.group(1)

    setup_result = False
    if commit_1:
        if commit_2:
            setup_result = setup_spack_repro_version(work_dir, commit_2, merge_commit=commit_1)
        else:
            setup_result = setup_spack_repro_version(work_dir, commit_1)

    if not setup_result:
        setup_msg = """
    This can happen if the spack you are using to run this command is not a git
    repo, or if it is a git repo, but it does not have the commits needed to
    recreate the tested merge commit.  If you are trying to reproduce a spack
    PR pipeline job failure, try fetching the latest develop commits from
    mainline spack and make sure you have the most recent commit of the PR
    branch in your local spack repo.  Then run this command again.
    Alternatively, you can also manually clone spack if you know the version
    you want to test.
        """
        tty.error(
            "Failed to automatically setup the tested version of spack "
            "in your local reproduction directory."
        )
        tty.info(setup_msg)

    # In cases where CI build was run on a shell runner, it might be useful
    # to see what tags were applied to the job so the user knows what shell
    # runner was used.  But in that case in general, we cannot do nearly as
    # much to set up the reproducer.
    job_tags = None
    if "tags" in job_yaml:
        job_tags = job_yaml["tags"]
        tty.msg(f"Job ran with the following tags: {job_tags}")

    entrypoint_script = [
        ["git", "config", "--global", "--add", "safe.directory", mount_as_dir],
        [
            ".",
            os.path.join(
                mount_as_dir if job_image else work_dir,
                f"share/spack/setup-env.{platform_script_ext}",
            ),
        ],
        (
            ["spack", "gpg", "trust", "-y", mounted_gpg_path if job_image else gpg_path]
            if gpg_path
            else []
        ),
        ["spack", "env", "activate", mounted_env_dir if job_image else repro_dir],
        [
            (
                os.path.join(mounted_repro_dir, f"install.{platform_script_ext}")
                if job_image
                else install_script
            )
        ],
    ]
    entry_script = os.path.join(mounted_workdir, f"entrypoint.{platform_script_ext}")
    inst_list = []
    # Finally, print out some instructions to reproduce the build
    if job_image:
        # Allow interactive
        install_mechanism = (
            os.path.join(mounted_repro_dir, f"install.{platform_script_ext}")
            if job_image
            else install_script
        )
        entrypoint_script.append(["echo", f"Re-run install script using:\n\t{install_mechanism}"])
        # Allow interactive
        if IS_WINDOWS:
            entrypoint_script.append(["&", "($args -Join ' ')", "-NoExit"])
        else:
            entrypoint_script.append(["exec", "$@"])

        process_command(
            "entrypoint", entrypoint_script, work_dir, run=False, exit_on_failure=False
        )

        # Attempt to create a unique name for the reproducer container
        container_suffix = "_" + reproducer_spec.dag_hash() if reproducer_spec else ""
        docker_command = [
            runtime,
            "run",
            "-i",
            "-t",
            "--rm",
            "--name",
            f"spack_reproducer{container_suffix}",
            "-v",
            ":".join([work_dir, mounted_workdir, "Z"]),
            "-v",
            ":".join(
                [
                    os.path.join(work_dir, artifact_root),
                    os.path.join(mount_as_dir, artifact_root),
                    "Z",
                ]
            ),
            "-v",
            ":".join([os.path.join(work_dir, "spack"), mount_as_dir, "Z"]),
            "--entrypoint",
        ]
        if IS_WINDOWS:
            docker_command.extend(["powershell.exe", job_image, entry_script, "powershell.exe"])
        else:
            docker_command.extend([entry_script, job_image, "bash"])
        docker_command = [docker_command]
        autostart = autostart and setup_result
        process_command("start", docker_command, work_dir, run=autostart)

        if not autostart:
            inst_list.append("\nTo run the docker reproducer:\n\n")
            inst_list.extend(
                [
                    "    - Start the docker container install",
                    f"       $ {work_dir}/start.{platform_script_ext}",
                ]
            )
    else:
        autostart = autostart and setup_result
        process_command("reproducer", entrypoint_script, work_dir, run=autostart)

        inst_list.append("\nOnce on the tagged runner:\n\n")
        inst_list.extend(
            [
                "    - Run the reproducer script",
                f"       $ {work_dir}/reproducer.{platform_script_ext}",
            ]
        )

    if not setup_result:
        inst_list.append("\n    - Clone spack and acquire tested commit")
        inst_list.append(f"\n        {spack_info}\n")
        inst_list.append("\n")
        inst_list.append(f"\n        Path to clone spack: {work_dir}/spack\n\n")

    tty.msg("".join(inst_list))


def process_command(name, commands, repro_dir, run=True, exit_on_failure=True):
    """
    Create a script for and run the command. Copy the script to the
    reproducibility directory.

    Arguments:
        name (str): name of the command being processed
        commands (list): list of arguments for single command or list of lists of
            arguments for multiple commands. No shell escape is performed.
        repro_dir (str): Job reproducibility directory
        run (bool): Run the script and return the exit code if True

    Returns: the exit code from processing the command
    """

    tty.debug(f"spack {name} arguments: {commands}")
    if len(commands) == 0 or isinstance(commands[0], str):
        commands = [commands]

    def compose_command_err_handling(args):
        if not IS_WINDOWS:
            args = [f'"{arg}"' for arg in args]
        arg_str = " ".join(args)
        result = arg_str + "\n"
        # ErrorActionPreference will handle PWSH commandlets (Spack calls),
        # but we need to handle EXEs (git, etc) ourselves
        catch_exe_failure = (
            """
if ($LASTEXITCODE -ne 0){{
    throw 'Command {} has failed'
}}
"""
            if IS_WINDOWS
            else ""
        )
        if exit_on_failure and catch_exe_failure:
            result += catch_exe_failure.format(arg_str)
        return result

    # Create a string [command 1] \n [command 2] \n ... \n [command n] with
    # commands composed into a platform dependent shell script, pwsh on Windows,
    full_command = "\n".join(map(compose_command_err_handling, commands))
    # Write the command to a python script
    if IS_WINDOWS:
        script = f"{name}.ps1"
        script_content = [f"\n# spack {name} command\n"]
        if exit_on_failure:
            script_content.append('$ErrorActionPreference = "Stop"\n')
        if os.environ.get("SPACK_VERBOSE_SCRIPT"):
            script_content.append("Set-PSDebug -Trace 2\n")
    else:
        script = f"{name}.sh"
        script_content = ["#!/bin/sh\n\n", f"\n# spack {name} command\n"]
        if exit_on_failure:
            script_content.append("set -e\n")
        if os.environ.get("SPACK_VERBOSE_SCRIPT"):
            script_content.append("set -x\n")
    script_content.append(full_command)
    script_content.append("\n")

    with open(script, "w", encoding="utf-8") as fd:
        for line in script_content:
            fd.write(line)

    copy_path = os.path.join(repro_dir, script)
    shutil.copyfile(script, copy_path)
    if not IS_WINDOWS:
        st = os.stat(copy_path)
        os.chmod(copy_path, st.st_mode | stat.S_IEXEC)

    # Run the generated shell script as if it were being run in
    # a login shell.
    exit_code = None
    if run:
        try:
            # We use sh as executor on Linux like platforms, pwsh on Windows
            interpreter = "powershell.exe" if IS_WINDOWS else "/bin/sh"
            cmd_process = subprocess.Popen([interpreter, f"./{script}"])
            cmd_process.wait()
            exit_code = cmd_process.returncode
        except (ValueError, subprocess.CalledProcessError, OSError) as err:
            tty.error(f"Encountered error running {name} script")
            tty.error(err)
            exit_code = 1

        tty.debug(f"spack {name} exited {exit_code}")
    else:
        # Delete the script, it is copied to the destination dir
        os.remove(script)

    return exit_code


def create_buildcache(
    input_spec: spack.spec.Spec, *, destination_mirror_urls: List[str], sign_binaries: bool = False
) -> List[PushResult]:
    """Create the buildcache at the provided mirror(s).

    Arguments:
        input_spec: Installed spec to package and push
        destination_mirror_urls: List of urls to push to
        sign_binaries: Whether or not to sign buildcache entry

    Returns: A list of PushResults, indicating success or failure.
    """
    results = []

    for mirror_url in destination_mirror_urls:
        results.append(
            PushResult(
                success=push_to_build_cache(input_spec, mirror_url, sign_binaries), url=mirror_url
            )
        )

    return results


def write_broken_spec(url, pkg_name, stack_name, job_url, pipeline_url, spec_dict):
    """Given a url to write to and the details of the failed job, write an entry
    in the broken specs list.
    """
    with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir:
        file_path = os.path.join(tmpdir, "broken.txt")

        broken_spec_details = {
            "broken-spec": {
                "job-name": pkg_name,
                "job-stack": stack_name,
                "job-url": job_url,
                "pipeline-url": pipeline_url,
                "concrete-spec-dict": spec_dict,
            }
        }

        try:
            with open(file_path, "w", encoding="utf-8") as fd:
                syaml.dump(broken_spec_details, fd)
            web_util.push_to_url(
                file_path, url, keep_original=False, extra_args={"ContentType": "text/plain"}
            )
        except Exception as err:
            # If there is an S3 error (e.g., access denied or connection
            # error), the first non boto-specific class in the exception
            # hierarchy is Exception.  Just print a warning and return
            msg = f"Error writing to broken specs list {url}: {err}"
            tty.warn(msg)


def read_broken_spec(broken_spec_url):
    """Read data from broken specs file located at the url, return as a yaml
    object.
    """
    try:
        broken_spec_contents = web_util.read_text(broken_spec_url)
    except web_util.SpackWebError:
        tty.warn(f"Unable to read broken spec from {broken_spec_url}")
        return None

    return syaml.load(broken_spec_contents)


def display_broken_spec_messages(base_url, hashes):
    """Fetch the broken spec file for each of the hashes under the base_url and
    print a message with some details about each one.
    """
    broken_specs = [(h, read_broken_spec(url_util.join(base_url, h))) for h in hashes]
    for spec_hash, broken_spec in [tup for tup in broken_specs if tup[1]]:
        details = broken_spec["broken-spec"]
        if "job-name" in details:
            item_name = f"{details['job-name']}/{spec_hash[:7]}"
        else:
            item_name = spec_hash

        if "job-stack" in details:
            item_name = f"{item_name} (in stack {details['job-stack']})"

        msg = f"  {item_name} was reported broken here: {details['job-url']}"
        tty.msg(msg)


def run_standalone_tests(
    *,
    cdash: Optional[CDashHandler] = None,
    fail_fast: bool = False,
    log_file: Optional[str] = None,
    job_spec: Optional[spack.spec.Spec] = None,
    repro_dir: Optional[str] = None,
    timeout: Optional[int] = None,
):
    """Run stand-alone tests on the current spec.

    Args:
        cdash: cdash handler instance
        fail_fast: terminate tests after the first failure
        log_file: test log file name if NOT CDash reporting
        job_spec: spec that was built
        repro_dir: reproduction directory
        timeout: maximum time (in seconds) that tests are allowed to run
    """
    if cdash and log_file:
        tty.msg(f"The test log file {log_file} option is ignored with CDash reporting")
        log_file = None

    # Error out but do NOT terminate if there are missing required arguments.
    if not job_spec:
        tty.error("Job spec is required to run stand-alone tests")
        return

    if not repro_dir:
        tty.error("Reproduction directory is required for stand-alone tests")
        return

    test_args = ["spack", "--color=always", "--backtrace", "--verbose", "test", "run"]
    if fail_fast:
        test_args.append("--fail-fast")

    if timeout is not None:
        test_args.extend(["--timeout", str(timeout)])

    if cdash:
        test_args.extend(cdash.args())
    else:
        test_args.extend(["--log-format", "junit"])
        if log_file:
            test_args.extend(["--log-file", log_file])
    test_args.append(job_spec.name)

    tty.debug(f"Running {job_spec.name} stand-alone tests")
    exit_code = process_command("test", test_args, repro_dir)

    tty.debug(f"spack test exited {exit_code}")
