bootstrap

#!/usr/bin/env python

"""
 This source file is part of the Swift.org open source project

 Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
 Licensed under Apache License v2.0 with Runtime Library Exception

 See http://swift.org/LICENSE.txt for license information
 See http://swift.org/CONTRIBUTORS.txt for Swift project authors

 -------------------------------------------------------------------------
 This script is used to bootstrap the Swift package manager build.

 It does so by writing out a build task file which can be used to build a
 stage1 package manager. That package manager is then expected to be able
 to build itself to produce the production tools.

 Note that currently this script is also responsible for building the package
 manager in such a way that it can be installed along with the Swift package.
 In particular, it knows how to build the runtime PackageDescription library
 correctly and install it.  It can also build libSwiftPM, allowing clients to
 access package manager functionality.

"""

from __future__ import print_function

try:
    from cStringIO import StringIO
except ImportError:
    from io import StringIO
import argparse
import codecs
import copy
import errno
import json
import os
import pipes
import platform
import re
import shlex
import shutil
import subprocess
import sys
import tempfile

def note(message):
    print("--- %s: note: %s" % (os.path.basename(sys.argv[0]), message))
    sys.stdout.flush()


def error(message):
    print("--- %s: error: %s" % (os.path.basename(sys.argv[0]), message))
    sys.stdout.flush()
    raise SystemExit(1)

def symlink_force(target, link_name):
    if os.path.isdir(link_name):
        link_name = os.path.join(link_name, os.path.basename(target))
    try:
        os.symlink(target, link_name)
    except OSError as e:
        if e.errno == errno.EEXIST:
            os.remove(link_name)
            os.symlink(target, link_name)
        else:
            raise e

def mkdir_p(path):
    """
    mkdir_p(path)

    Create the given directory, if it does not exist.
    """

    try:
        os.makedirs(path)
    except OSError as e:
        # Ignore EEXIST, which may occur during a race condition.
        if e.errno != errno.EEXIST:
            raise


# FIXME: Consider eliminating this once the build task format supports node
# hashing.
def write_file_if_changed(path, data):
    """
    write_file_if_changed(path, data)

    Write the given data to the path, only updating the file if the contents are
    different than the current ones.
    """

    try:
        with open(path) as f:
            old_data = f.read()
    except:
        old_data = None
    if old_data == data:
        return

    # Create directory if needed.
    mkdir_p(os.path.dirname(path))

    # Write the contents.
    with open(path, "w") as f:
        f.write(data)

###

def find_xcode_version():
    return subprocess.check_output(
        ["xcodebuild", "-version"],
        universal_newlines=True).strip().splitlines()[0]

g_num_cpus = os.sysconf("SC_NPROCESSORS_ONLN")
g_default_sysroot = None
if platform.system() == 'Darwin':
    g_platform_path = subprocess.check_output(
        ["xcrun", "--sdk", "macosx", "--show-sdk-platform-path"],
        universal_newlines=True).strip()
    g_default_sysroot = subprocess.check_output(
        ["xcrun", "--sdk", "macosx", "--show-sdk-path"],
        universal_newlines=True).strip()

if platform.system() == 'Linux':
    g_shared_lib_ext = ".so"
else:
    g_shared_lib_ext = ".dylib"


# Represents a target definition extracted from the SwiftPM package manifest.
class Target(object):
    @property
    def virtual_node(self):
        return "<target-%s>" % (self.name)

    @property
    def linked_virtual_node(self):
        return "<link-%s>" % (self.name)

    def __init__(self, name, dependencies=[], swiftflags=[], extra_libs=[],
                 subpath=None):
        self.name = name
        self.release = False
        self.dependencies = list(dependencies)
        self.swiftflags = list(swiftflags)
        self.extra_libs = list(extra_libs)

        # Discover the source files, and whether or not this is a library.
        self.is_library = True
        self.is_swift = False
        # FIXME: Currently only C libraries are supported in bootstrap script.
        self.is_c = False

        self.sources = []
        self.module_root_dir = os.path.join(g_source_root, subpath or self.name)

        for (dirpath, dirnames, filenames) in os.walk(self.module_root_dir):
            for name in filenames:
                path = os.path.join(dirpath, name)
                _, ext = os.path.splitext(name)
                if ext == '.swift':
                    self.is_swift = True
                    if name == 'main.swift':
                        self.is_library = False
                    self.sources.append(path)
                if ext == '.c':
                    self.is_c = True
                    self.sources.append(path)

        if self.is_swift and self.is_c:
            error("Target %s contains mixed C and Swift sources which is unsupported." % (self.name))

        self.sources.sort()

    def module_name(self):
        return self.name.replace("-", "_")

    def write_compile_commands(self, args, target_build_dir,
                                     module_dir, include_dir, output, objects,
                                     link_input_nodes, predecessor_node, target_map):
        if self.is_swift:
            self.write_swift_compile_commands(args, target_build_dir, module_dir,
                                              include_dir, output, objects, link_input_nodes,
                                              predecessor_node, target_map)
        elif self.is_c:
            self.write_c_compile_commands(args, target_build_dir, module_dir,
                                              include_dir, output, objects, link_input_nodes,
                                              predecessor_node)


    def write_swift_compile_commands(self, args, target_build_dir,
                                     module_dir, include_dir, output, objects,
                                     link_input_nodes, predecessor_node, target_map):
        # Compute the derived paths.
        module_path = os.path.join(module_dir, "%s.swiftmodule" % (self.name,))

        # Create the per-file entries.
        swift_objects = []
        for path in self.sources:
            filename = os.path.basename(path)
            base_path = os.path.join(
                target_build_dir, os.path.splitext(filename)[0])
            object_path = base_path + ".o"
            swift_objects.append(object_path)
            objects.append(object_path)

        # Form the command to build all the swift files.
        #
        # FIXME: The -j doesn't belong here, and should move into the
        # 'swift' tool.
        other_args = []
        if self.release:
            other_args.append("-O")
        else:
            other_args.extend(["-Onone", "-g"])
        other_args.extend(['-j%d' % g_num_cpus] + self.swiftflags)
        if platform.system() == 'Darwin':
            other_args.extend(["-target", "x86_64-apple-macosx10.10"])
        if args.sysroot:
            other_args.extend(["-sdk", args.sysroot])
        compile_swift_node = '<compile-swift-%s>' % (self.name,)
        link_input_nodes.append(compile_swift_node)

        if args.libdispatch_source_dir:
            other_args.extend(["-Xcc", "-fblocks"])

        other_args.extend(["-swift-version", "4"])
        
        print("  %s:" % json.dumps(compile_swift_node), file=output)
        print("    tool: swift-compiler", file=output)
        print("    executable: %s" % json.dumps(args.swiftc_path), file=output)
        # FIXME: We shouldn't even need to specify the sources here once we have
        # discovered dependencies support.
        print("    inputs: %s" % json.dumps(
            [predecessor_node] + self.sources), file=output)
        print("    outputs: %s" % json.dumps(
            [compile_swift_node, module_path] + swift_objects), file=output)
        print("    module-name: %s" % json.dumps(self.module_name()), file=output)
        print("    module-output-path: %s" % json.dumps(module_path),
              file=output)
        print("    sources: %s" % json.dumps(self.sources), file=output)
        print("    objects: %s" % json.dumps(swift_objects), file=output)

        import_paths = [module_dir, include_dir]

        # To import C target, add import path to where modulemap is located.
        for dependency in self.dependencies:
            dep_target = target_map[dependency]
            if dep_target.is_c:
                import_paths.append(os.path.join(dep_target.module_root_dir, "include"))

        if args.foundation_path:
            import_paths.append(args.foundation_path)
            import_paths.append(os.path.join(args.foundation_path,
                                             "usr/lib/swift/CoreFoundation"))
            import_paths.append(os.path.join(args.foundation_path,
                                             "usr/lib/swift"))
        if args.libdispatch_build_dir:
            import_paths.append(os.path.join(args.libdispatch_build_dir,
                                             "src"))
            import_paths.append(os.path.join(args.libdispatch_build_dir,
                                             "src", "swift"))
        if args.libdispatch_source_dir:
            import_paths.append(args.libdispatch_source_dir)
        if args.xctest_path:
            import_paths.append(args.xctest_path)
        print("    import-paths: %s" % json.dumps(import_paths), file=output)
        print("    other-args: %s" % json.dumps(other_args),
              file=output)
        print("    temps-path: %s" % json.dumps(target_build_dir), file=output)
        print("    is-library: %s" % json.dumps(
            str(bool(self.is_library)).lower()), file=output)
        print(file=output)

    def write_c_compile_commands(self, args, target_build_dir,
                                     module_dir, include_dir, output, objects,
                                     link_input_nodes, predecessor_node):

        common_args = ["-fobjc-arc", "-fmodule-name=%s" % self.module_name()]
        if platform.system() == 'Darwin':
            common_args.extend(["-arch", "x86_64", "-mmacosx-version-min=10.10"])

        if args.sysroot:
            common_args.extend(["-isysroot", args.sysroot])

        common_args.extend(["-g", "-O0", "-MD", "-MT", "dependencies", "-MF"])

        for source in self.sources:
            filename = os.path.basename(source)
            base_path = os.path.join(
                target_build_dir, os.path.splitext(filename)[0])
            object_path = base_path + ".o"
            deps_path = base_path + ".d"
            objects.append(object_path)

            link_input_nodes.append(object_path)

            args = ["clang"]
            args.extend(common_args)
            args.extend([deps_path, "-c", source,"-o", object_path])

            print("  %s:" % json.dumps(object_path), file=output)
            print("    tool: clang", file=output)
            print("    description: Compile %s" % filename, file=output)
            print("    inputs: %s" % json.dumps([source]), file=output)
            print("    outputs: %s" % json.dumps([object_path]), file=output)
            print("    args: %s" % json.dumps(args), file=output)
            print("    deps: %s" % json.dumps(deps_path), file=output)
            print(file=output)


# Represents the type of a product extracted from the manifest.
ProductType = type('Enum', (), dict(
    EXECUTABLE = 'executable',
    STATIC_LIBRARY = 'static_library',
    DYNAMIC_LIBRARY = 'dynamic_library'
))


# Represents a product definition extracted from the SwiftPM package manifest.
class Product(object):
    @property
    def product_name(self):
        return {
            ProductType.EXECUTABLE: self.name,
            ProductType.DYNAMIC_LIBRARY: "lib" + self.name + g_shared_lib_ext,
            ProductType.STATIC_LIBRARY: "lib" + self.name + ".a"
        }[self.type]

    def __init__(self, name, type, targets=[]):
        self.name = name
        self.type = type
        self.targets = list(targets)


# Loads and parses the Package.swift manifest, returning the lists of Targets
# and Product objects discovered by doing so.
def parse_manifest():
    # We have a *very* strict format for our manifest to make parsing more
    # robust.  We use this to determine the target and product definitions.
    target_pattern = re.compile(
        r'\.target\(.*?name: "(.*?)",\n *dependencies: (\[.*?\])\)',
        re.DOTALL | re.MULTILINE)
    product_pattern = re.compile(
        r'.library\([\n ]*name: \"(.*?)\",[\n ]*type: (.*?),[\n ]*targets: (\[.*?\])[\n ]*\)',
        re.DOTALL | re.MULTILINE)

    # Load the manifest as a string (we always assume UTF-8 encoding).
    manifest_data = codecs.open(os.path.join(g_project_root,
                                "Package.swift"), encoding='utf-8',
                                errors='strict').read()

    # Extract the target definitions.
    def make_target(match):
        name = match.group(1)
        deps = eval(match.group(2))
        return Target(name, deps)
    targets = list(map(make_target, target_pattern.finditer(manifest_data)))

    # Extract the product definitions.
    def make_product(match):
        name = match.group(1)
        try:
            type = {
                '.executable': ProductType.EXECUTABLE,
                '.dynamic': ProductType.DYNAMIC_LIBRARY,
                '.static': ProductType.STATIC_LIBRARY
            }[match.group(2)]
        except:
            error("unknown type '%s for product '%s' in manifest" % (match.group(2), name))
        targets = eval(match.group(3))
        return Product(name, type, targets)
    products = list(map(make_product, product_pattern.finditer(manifest_data)))

    # Filter out the targets that should be ignored in stage 1.
    targets = [target for target in targets
               if target.name not in g_ignored_targets and
                  not target.name.endswith("Tests")]

    # Convert each target's array of dependencies from strings to corresponding
    # Target objects (taking into account that some might be implicit).
    for target in targets:
        def convert(targetName):
            try:
                return next(a for a in targets if a.name == targetName)
            except StopIteration:
                # this target is not explicit in the manifest: it is an
                # implicit target
                b = Target(targetName)
                targets.append(b)
                return b
        target.dependencies = list(map(convert, target.dependencies))

    # Construct the dependency graph, and convert each Target's dependencies
    # back to an array of strings.
    def convert(target):
        myset = set()

        def recurse(root):
            deps = []
            for dep in root.dependencies:
                if dep.name not in myset:
                    myset.add(dep.name)
                    deps += recurse(dep) + [dep.name]
            return deps
        # `reversed` because Linux link order must be reverse-topological
        return Target(target.name, reversed(recurse(target)))
    targets = list(map(convert, targets))
    
    # Finally, return the lists of targets and products.
    return (targets, products)

# Hard-coded target definition.
g_project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
g_source_root = os.path.join(g_project_root, "Sources")
g_ignored_targets = ["swiftpm-xctest-helper", "TestSupport"]
(targets, products) = parse_manifest()
product_map = dict((p.name, p) for p in products)

# Create a quoted C string literal from an arbitrary string.
def make_c_string_literal(str):
    return "\"" + str.replace("\\", "\\\\").replace("\"", "\\\"") + "\""

# Create a quoted XML string literal from an arbitrary string.
def make_xml_string_literal(str):
    return "\"" + str.replace("&", "&amp;").replace("\"", "&quot;") + "\""


# Write out a header containing version information, and a module map for it.
def write_bootstrap_version_info(include_path, vendor_name, build_identifier):
    # Construct the header and write it out if it's changed.
    output = StringIO()
    vendor_name_literal = make_c_string_literal(vendor_name)
    build_identifier_literal = make_c_string_literal(build_identifier)
    contents = """
static inline const char* VendorNameString() {
    return %(vendor_name_literal)s;
}
static inline const char* BuildIdentifierString() {
    return %(build_identifier_literal)s;
}
""" % locals()
    write_file_if_changed(os.path.join(include_path, "VersionInfo.h"), contents)
    
    # Also construct the module map and write it out if it's changed.
    contents = """\
module VersionInfo [extern_c] {
    header \"VersionInfo.h\"
}
"""
    write_file_if_changed(os.path.join(include_path, "module.map"), contents)

class llbuild(object):

    def __init__(self, sandbox_path, targets, args):

        # Compute output directories paths.
        self.output_file = os.path.join(sandbox_path, "build.yaml")
        self.inc_dir = os.path.join(sandbox_path, "inc")
        self.lib_dir = os.path.join(sandbox_path, "lib")
        self.bin_dir = os.path.join(sandbox_path, "bin")
        self.module_dir = os.path.join(sandbox_path, "modules")
        self.targets = targets
        self.target_map = dict((t.name, t) for t in targets)
        self.sandbox_path = sandbox_path
        self.args = args

    def run(self):
        cmd = [self.args.sbt_path, "-f", self.output_file]
        if self.args.verbose:
            cmd.append("-v")
        result = subprocess.call(cmd)
        if result != 0:
            error("build failed with exit status %d" % (result,))

    def create_manifest(self):

        # Write out the build file.
        output = StringIO()

        # Write out the task file header.
        print("client:", file=output)
        print("  name: swift-build", file=output)
        print(file=output)

        # Write out the tools section.
        #
        # FIXME: Not yet defined.
        print("tools: {}", file=output)
        print(file=output)

        # Write out the targets list.
        #
        # We support 'all' (build all targets) and an individual name for each
        # target.
        #
        # FIXME: Need support for default target.
        print("targets:", file=output)
        print("  \"\": %s" % (json.dumps(
            [target.virtual_node for target in self.targets]),), file=output)
        print("  all: %s" % (json.dumps(
            [target.virtual_node for target in self.targets]),), file=output)
        for target in self.targets:
            print("  %s: %s" % (
                target.name, json.dumps([target.virtual_node])), file=output)
        print(file=output)

        print("commands:", file=output)
        for target in self.targets:
            print("  # Target Commands: %r" % (target.name,), file=output)
            target_build_dir = os.path.join(self.sandbox_path, target.name + ".build")
            mkdir_p(target_build_dir)

            predecessor_node = "<entry-%s>" % (target.name,)
            objects = []
            link_input_nodes = [predecessor_node]

            # Write out the predecessor node used to order w.r.t. other targets.
            print("  %s:" % json.dumps(predecessor_node), file=output)
            print("    tool: phony", file=output)
            print("    inputs: %s" % json.dumps(
                [self.target_map[name].virtual_node for name in target.dependencies]),
                file=output)
            print("    outputs: %s" % json.dumps([predecessor_node]), file=output)
            print(file=output)

            # Write out the target build commands (we just name the command and node
            # the same).

            # Write the compile commands, if used.
            if target.sources:
                target.write_compile_commands(
                    self.args, target_build_dir, self.module_dir, self.inc_dir, output, objects,
                    link_input_nodes, predecessor_node, self.target_map)


            # llbuild tool to use for this command.
            tool = "shell"

            # Form the command line to link.
            linked_libraries = []
            if target.is_swift and target.is_library:
                tool = "archive"
                link_output_path = os.path.join(self.lib_dir, "%s.a" % (target.name,))
                link_command = [] # Archive tool auto infers objects from inputs.
            elif target.is_c and target.is_library:
                tool = "archive"
                link_output_path = os.path.join(self.lib_dir, "%s.a" % (target.name,))
                # Archive tool auto infers objects from inputs.
                objects = []
                link_command = []
            elif target.is_c and not target.is_library:
                error("Executable C target not supported by bootstrap yet")
            elif target.is_swift and not target.is_library:
                link_output_path = os.path.join(self.bin_dir, target.name)
    
                link_command = [self.args.swiftc_path,
                                '-o', link_output_path]
                if self.args.sysroot:
                    link_command.extend(["-sdk", self.args.sysroot])
                if platform.system() == 'Darwin':
                    link_command.extend(["-target", "x86_64-apple-macosx10.10"])
                link_command.extend(objects)
                for dependency in target.dependencies:
                    dep_target = self.target_map[dependency]
                    dependency_lib_path = os.path.join(self.lib_dir, "%s.a" % dependency)
                    link_command.append(dependency_lib_path)
                    linked_libraries.append(dependency_lib_path)
                if platform.system() == 'Darwin':
                    link_command.extend(["-Xlinker", "-all_load"])
                for lib in target.extra_libs:
                    link_command.extend(["-Xlinker", "-l%s" % (lib,)])
                if platform.system() == 'Linux':
                    link_command.extend(
                        ["-Xlinker", "-rpath=$ORIGIN/../lib/swift/linux"])
                if self.args.foundation_path:
                    link_command.extend(["-L", self.args.foundation_path])
                if self.args.libdispatch_build_dir:
                    link_command.extend(["-L", os.path.join(self.args.libdispatch_build_dir, "src", ".libs")])

            # Write out the link command.
            if target.is_library:
                description = "Link %s" % target.name
            else:
                description = "Link %s" % link_output_path
            self.writeTool(
                tool,
                target.linked_virtual_node,
                description,
                link_input_nodes + objects + linked_libraries,
                [target.linked_virtual_node, link_output_path],
                link_command,
                output)

            # Write the top-level target group command.
            print("  %s:" % json.dumps(target.virtual_node), file=output)
            print("    tool: phony", file=output)
            print("    inputs: %s" % json.dumps(
                [target.linked_virtual_node]), file=output)
            print("    outputs: %s" % json.dumps([target.virtual_node]),
                  file=output)
            print(file=output)

        # Write the output file.
        write_file_if_changed(
                self.output_file, output.getvalue())

    # Write out the tool into output buffer.
    # Supported tools: shell and archive. args is ignored for archive tool.
    def writeTool(self, tool, node, description, inputs, outputs, args, output):
        print("  %s:" % json.dumps(node), file=output)
        print("    tool: %s" % tool, file=output)
        print("    description: %s" % description, file=output)
        print("    inputs: %s" % json.dumps(inputs), file=output)
        print("    outputs: %s" % json.dumps(outputs), file=output)
        if tool == "shell":
            print("    args: %s" % json.dumps(args), file=output)
        print(file=output)

def build_runtimes(targets, sandbox_path, args):
    target_map = dict((t.name, t) for t in targets)
    runtimes = {}
    runtimes['3'] = target_map['PackageDescription']
    runtimes['4'] = target_map['PackageDescription4']
    runtimes['4_2'] = target_map['PackageDescription4']

    built_runtimes = {}
    for version, target in runtimes.items():
        note("building runtime v%s target: %s: " % (version, target.name))
        target_copy = copy.copy(target)
        target_copy.name = 'PackageDescription'
        target_copy.release = args.release
        target_copy.swiftflags = ["-DPACKAGE_DESCRIPTION_%s" % (version)]
        build = llbuild(
                os.path.join(sandbox_path, "runtimes", version), [target_copy], args)
        build.create_manifest()
        build.run()
        built_runtimes[version] = build
    return built_runtimes

def process_runtime_libraries(build, args, lib_path):
    module_input_path = os.path.join(
        build.module_dir, "PackageDescription.swiftmodule")
    swiftdoc_input_path = os.path.join(
        build.module_dir, "PackageDescription.swiftdoc")
    input_lib_path = os.path.join(
        build.lib_dir, "PackageDescription.a")

    mkdir_p(lib_path)
    runtime_module_path = os.path.join(
        lib_path, "PackageDescription.swiftmodule")
    runtime_swiftdoc_path = os.path.join(
        lib_path, "PackageDescription.swiftdoc")

    cmd = ["rsync", "-a", module_input_path, runtime_module_path]
    subprocess.check_call(cmd)
    cmd = ["rsync", "-a", swiftdoc_input_path, runtime_swiftdoc_path]
    subprocess.check_call(cmd)

    if platform.system() == 'Darwin':
        runtime_lib_path = os.path.join(lib_path, "libPackageDescription.dylib")
        cmd = [args.swiftc_path, "-Xlinker", "-dylib", "-o",
               runtime_lib_path,
               "-Xlinker", "-all_load",
               input_lib_path,
               "-target", "x86_64-apple-macosx10.10"]
    else:
        # Derive the command line to use by querying to swiftc
        # driver. Unfortunately we cannot use it directly due to the inability
        # to use an -X... style arg to pass arguments to the Clang driver, which
        # it calls before calling the linker.

        # This is the command we would like to use.
        #
        # We include an RPATH entry so that the Swift libraries can be found
        # relative to where it will be installed (in 'lib/swift/pm/<version>/...').
        runtime_lib_path = os.path.join(lib_path, "libPackageDescription.so")
        cmd = [args.swiftc_path, "-emit-library", "-o", runtime_lib_path,
               "-Xlinker", "--whole-archive",
               "-Xlinker", input_lib_path,
               "-Xlinker", "--no-whole-archive", "-lswiftGlibc",
               "-Xlinker", "-rpath=$ORIGIN/../../linux"]

        # We need to pass one swift file here to bypass the "no input files"
        # error.
        tf = tempfile.NamedTemporaryFile(suffix=".swift")
        cmds = subprocess.check_output(
            cmd + [tf.name, "-###"],
            universal_newlines=True).strip().split("\n")

        # Get the link command 'swiftc' used.
        link_cmd = shlex.split(cmds[-1])

        # Unqoute any quoted characters.
        link_cmd = [arg.replace("\\", "") for arg in link_cmd]

        # Drop any .o files
        cmd = [arg for arg in link_cmd
               if arg.endswith(("swift_begin.o", "swift_end.o")) or
               (not arg.endswith((".o", ".autolink")))]
    subprocess.check_call(cmd)
    return (runtime_module_path, runtime_swiftdoc_path, runtime_lib_path)


def get_swift_build_tool_path():
    # Search for a 'swift-build-tool' to use.

    # First, look in $(BUILT_PRODUCTS_DIR) in case we are being built from Xcode
    # along with the llbuild project.
    built_products_dir = os.environ.get("BUILT_PRODUCTS_DIR")
    if built_products_dir:
        sbt_path = os.path.join(built_products_dir, "swift-build-tool")
        if os.path.exists(sbt_path):
            return sbt_path

    SWIFT_EXEC = os.getenv("SWIFT_EXEC")
    if SWIFT_EXEC:
        maybe_path = os.path.join(SWIFT_EXEC, "../swift-build-tool")
        if os.path.exists(maybe_path):
            return maybe_path

    # If that failed, on Darwin use xcrun to search for the tool.
    if platform.system() == 'Darwin':
        try:
            sbt_path = subprocess.check_output(
                ["xcrun", "--find", "swift-build-tool"],
                stderr=subprocess.PIPE, universal_newlines=True).strip()
            if os.path.exists(sbt_path):
                return sbt_path
        except subprocess.CalledProcessError:
            pass
    else:
        # Otherwise, search for it in PATH.
        try:
            return subprocess.check_output(["which", "swift-build-tool"],
                                           universal_newlines=True).strip()
        except:
            pass

    # If all else failed, report an error.
    error("unable to find 'swift-build-tool' tool for bootstrap build")


def get_swiftc_path():
    try:
        if os.getenv("SWIFT_EXEC"):
            swiftc_path=os.path.realpath(os.getenv("SWIFT_EXEC"))
            if os.path.basename(swiftc_path) == 'swift':
                swiftc_path = swiftc_path + 'c'
            return swiftc_path
        elif platform.system() == 'Darwin':
            return subprocess.check_output(["xcrun", "--find", "swiftc"],
                stderr=subprocess.PIPE, universal_newlines=True).strip()
        else:
            return subprocess.check_output(["which", "swiftc"],
            universal_newlines=True).strip()
    except:
        error("unable to find 'swiftc' tool for bootstrap build")

# Install a binary file at the install path.
#
# This method removes the hardcoded stdlib path from the binary.
def installBinary(binary_path, install_path, swiftc_path, add_rpath=None):
    mkdir_p(install_path)
    cmd = ["rsync", "-a", binary_path, install_path]
    note("installing %s: %s" % (
        os.path.basename(binary_path), ' '.join(cmd)))
    result = subprocess.call(cmd)
    if result != 0:
        error("install failed with exit status %d" % (result,))

    # Remove the hard-coded stdlib RPATHs that `swiftc` bakes in on
    # Darwin.
    #
    # FIXME: We need a way to control this, instead of having to rip it
    # out after the fact. https://bugs.swift.org/browse/SR-1967
    if platform.system() == 'Darwin':
        installed_path = os.path.join(
            install_path, os.path.basename(binary_path))
        stdlib_path = os.path.realpath(
            os.path.join(os.path.dirname(swiftc_path), "..",
                         "lib", "swift", "macosx"))
        cmd = ["install_name_tool", "-delete_rpath",
               stdlib_path, installed_path]
        note("removing stray RPATH from %s: %s" % (
                installed_path, ' '.join(cmd)))
        result = subprocess.call(cmd)
        if result != 0:
            error("command  failed with exit status %d" % (result,))

        # Add an additional RPATH, if requested.
        if add_rpath:
            cmd = ["install_name_tool", "-add_rpath",
                   add_rpath, installed_path]
            note("adding RPATH to %s: %s" % (
                    installed_path, ' '.join(cmd)))
            result = subprocess.call(cmd)
            if result != 0:
                error("command  failed with exit status %d" % (result,))
    else:
        if add_rpath:
            error("unable to add RPATHs on this platform")

def main():
    parser = argparse.ArgumentParser(
        usage="%(prog)s [options] [clean|all|test|install]",
        description="This script will build a bootstrapped copy of the Swift "
                    "Package Manager, and optionally perform extra actions "
                    "like installing the result (with 'install') to a "
                    "location ('--prefix').")
    parser.add_argument("build_actions",
                        help="Extra actions to perform. Can be any number of "
                             "the following: [clean, all, test, install]",
                        nargs="*", default=["all"])
    parser.add_argument("--show-bin-path", action='store_true',
                        help="output the path to which the swiftpm binaries "
                             "are built")
    parser.add_argument("--swiftc", dest="swiftc_path",
                        help="path to the swift compiler [%(default)s]",
                        metavar="PATH")
    parser.add_argument("--sbt", dest="sbt_path",
                        help="path to the 'swift-build-tool' tool "
                             "[%(default)s]",
                        metavar="PATH")
    parser.add_argument("--sysroot",
                        help="compiler sysroot to pass to Swift [%(default)s]",
                        default=g_default_sysroot, metavar="PATH")
    parser.add_argument("--build", dest="build_path",
                        help="create build products at PATH [%(default)s]",
                        default=".build", metavar="PATH")
    parser.add_argument("--prefix", dest="install_prefixes", nargs='*',
                        help="use PATHS as the prefixes for installing "
                             "[%(default)s]",
                        default=["/usr/local"], metavar="PATHS")
    parser.add_argument("-v", "--verbose", action="store_true",
                        help="use verbose output")
    parser.add_argument("--foundation", dest="foundation_path",
                        help="Path to Foundation build directory")
    parser.add_argument("--xctest", dest="xctest_path",
                        help="Path to XCTest build directory")
    parser.add_argument("--libdispatch-build-dir", dest="libdispatch_build_dir",
                        help="Path to the libdispatch build directory")
    parser.add_argument("--libdispatch-source-dir", dest="libdispatch_source_dir",
                        help="Path to the libdispatch source directory")
    parser.add_argument("--release", action="store_true",
                        help="Build stage 2 for release")
    parser.add_argument("--generate-xcodeproj", action="store_true",
                        help="Also generate an Xcode project for SwiftPM")
    parser.add_argument("--test-parallel", action="store_true",
                        help="Run tests in parallel")
    parser.add_argument("--test-filter", action="append",
                        help="Filter on a given test", default=[])
    parser.add_argument("--enable-perf-tests", action="store_true",
                        help="Enables performance tests in the generated Xcode project")
    parser.add_argument("--vendor-name", dest="vendor_name",
                        help="vendor name for use in --version")
    parser.add_argument("--build-identifier", dest="build_identifier",
                        help="build identifier for use in --version")
    parser.add_argument("--libswiftpm-library-dir", dest="libswiftpm_library_dir",
                        help="Directory in which to put libSwiftPM library")
    parser.add_argument("--libswiftpm-modules-dir", dest="libswiftpm_modules_dir",
                        help="Directory in which to put libSwiftPM modules")
    parser.add_argument("--libswiftpm-skip-c-headers", action="store_true",
                        help="Skip installing C headers in libSwiftPM install directory")
    args = parser.parse_args()

    if not args.swiftc_path:
        args.swiftc_path = get_swiftc_path()

    # Absolutize input paths.
    if args.build_path:
        args.build_path = os.path.abspath(args.build_path)
    if args.swiftc_path:
        args.swiftc_path = os.path.abspath(args.swiftc_path)
    if args.sbt_path:
        args.sbt_path = os.path.abspath(args.sbt_path)
    if args.foundation_path:
        args.foundation_path = os.path.abspath(args.foundation_path)
    if args.xctest_path:
        args.xctest_path = os.path.abspath(args.xctest_path)
        
    build_actions = set(args.build_actions)

    # Save the build configuration.
    if args.release:
        conf = "release"
    else:
        conf = "debug"
            
    # Validate the build actions.
    for action in build_actions:
        if action not in ("clean", "all", "test", "install"):
            raise SystemExit("unknown build action: %r" % (action,))

    # Compute the build paths.
    if platform.system() == 'Darwin':
        build_target = "x86_64-apple-macosx10.10"
    elif platform.system() == 'Linux':              
        if platform.machine() == 'x86_64':
            build_target = "x86_64-unknown-linux"
        elif platform.machine() == 's390x':
            build_target = "s390x-unknown-linux"
        elif platform.machine() == 'ppc64le':
            build_target = 'powerpc64le-unknown-linux'
        elif platform.machine().startswith("armv7"):
            build_target = 'armv7-unknown-linux-gnueabihf'
        elif platform.machine() == 'aarch64':
            build_target = 'aarch64-unknown-linux'
        else:
            raise SystemExit("ERROR: unsupported machine:",platform.machine())
    else:
        raise SystemExit("ERROR: unsupported system:",platform.system())

    build_path = os.path.join(g_project_root, args.build_path)
    sandbox_path = os.path.join(build_path, ".bootstrap")
    target_path = os.path.join(build_path, build_target)

    libdir = os.path.join(target_path, "lib")
    bindir = os.path.join(target_path, conf)

    if args.show_bin_path:
        print(bindir)
        sys.exit(0)

    # Confirm rsync is available.
    try:
        subprocess.check_output(['rsync', '--version'], stderr=subprocess.STDOUT)
    except Exception as e:
        error("rsync is not available with error: %s" % e)  
    
    # If the action is "clean", just remove the bootstrap and build directories.
    if "clean" in build_actions:
        cmd = ["rm", "-rf", sandbox_path]
        note("cleaning stage1: %s" % (' '.join(cmd),))
        result = subprocess.call(cmd)
        if result != 0:
            error("build failed with exit status %d" % (result,))

        cmd = ["rm", "-rf", build_path]
        note("cleaning self-hosted: %s" % (' '.join(cmd),))
        result = subprocess.call(cmd)
        if result != 0:
            error("build failed with exit status %d" % (result,))
        raise SystemExit(0)

    # All other actions build.

    # Create the sandbox.
    mkdir_p(sandbox_path)

    # Determine the swift-build-tool to use.
    args.sbt_path = os.path.abspath(
        args.sbt_path or get_swift_build_tool_path())

    # Run the stage1 build.
    note("building stage1")
    stage1 = llbuild(sandbox_path, targets, args)
    stage1.create_manifest()
    stage1.run()

    # Build runtime libraries.
    built_runtimes = build_runtimes(targets, sandbox_path, args)

    processed_runtimes = {}
    for version, build in built_runtimes.items():
        # Path where runtime library will be copied from sandbox.
        lib_path = os.path.join(sandbox_path, "lib", "swift", "pm", version)

        # Stage the stage1 runtime library.
        processed_runtimes[version] = process_runtime_libraries(
                build, args, lib_path)

    mkdir_p(bindir)
    bootstrapped_product = os.path.join(bindir, "swift-build-stage1")

    # Construct a fake toolchain so swift-build can build itself
    # without requiring it to have hacky-edge-case logic
    def make_fake_toolchain():
        symlink_force(args.swiftc_path, os.path.join(bindir, "swift"))
        symlink_force(args.swiftc_path, os.path.join(bindir, "swiftc"))
        symlink_force(args.sbt_path, os.path.join(bindir, "swift-build-tool"))
        symlink_force(os.path.join(sandbox_path, "bin", "swift-build"),
                      bootstrapped_product)
        if os.path.isdir(libdir) and not os.path.islink(libdir):
            # TODO remove, here to prevent revlock incremental CI build failures
            shutil.rmtree(libdir)
        symlink_force(os.path.join(sandbox_path, "lib"), target_path)

        # To simulate usr/lib directory.
        usrdir = os.path.join(target_path, "usr")
        mkdir_p(usrdir)
        symlink_force(os.path.join(sandbox_path, "lib"), usrdir)

        if args.foundation_path:
            libswiftdir = os.path.join(sandbox_path, "lib", "swift", "linux")
            mkdir_p(libswiftdir)
            symlink_force(os.path.join(args.foundation_path, 'libFoundation.so'),
                libswiftdir)
            if args.libdispatch_build_dir:
                symlink_force(os.path.join(args.libdispatch_build_dir, "src", ".libs", "libdispatch.so"),
                    libswiftdir)

    make_fake_toolchain()

    # Build the package manager with itself.

    # Compute the build flags, needed for swift-build and swift-test.
    build_flags = []

    # Disable sandboxing when bootstraping.
    build_flags.append("--disable-sandbox")
    
    # We need to embed an RPATH so swift-{build,test} can find the core
    # libraries.
    if platform.system() == 'Linux':
        embed_rpath = "$ORIGIN/../lib/swift/linux"
    else:
        embed_rpath = "@executable_path/../lib/swift/macosx"
    build_flags.extend(["-Xlinker", "-rpath", "-Xlinker", embed_rpath])
    if args.verbose:
        build_flags.append("-v")

    # If appropriate, write out the version info header and module map file,
    # and add flags for a custom version string:
    if args.vendor_name or args.build_identifier:
        if not args.vendor_name or not args.build_identifier:
            error("--build-identifier is required with --vendor-name")

        incdir = os.path.join(sandbox_path, "inc")
        write_bootstrap_version_info(
                incdir, args.vendor_name, args.build_identifier)

        build_flags.extend(["-Xswiftc", "-I{}".format(incdir)])
        build_flags.extend(["-Xswiftc", "-DHasCustomVersionString"])
    
    if args.foundation_path:
        core_foundation_path = os.path.join(
            args.foundation_path, "usr", "lib", "swift")
        # Tell the linker where to look for XCTest, but autolinking
        # knows to pass -lXCTest.
        build_flags.extend(["-Xlinker", "-L", "-Xlinker", args.foundation_path])
        # Add an RPATH, so that the tests can be run directly.
        build_flags.extend(["-Xlinker", "-rpath", "-Xlinker",
                            args.foundation_path])
        build_flags.extend(["-Xswiftc", "-I{}".format(args.foundation_path)])
        build_flags.extend(["-Xswiftc", "-I{}".format(core_foundation_path)])

    if args.xctest_path:
        # Tell the linker where to look for XCTest, but autolinking
        # knows to pass -lXCTest.
        build_flags.extend(["-Xlinker", "-L", "-Xlinker", args.xctest_path])
        # Add an RPATH, so that the tests can be run directly.
        build_flags.extend(["-Xlinker", "-rpath", "-Xlinker", args.xctest_path])
        build_flags.extend(["-Xswiftc", "-I{}".format(args.xctest_path)])

    if args.libdispatch_build_dir:
        build_flags.extend(["-Xlinker", "-L{}".format(
            os.path.join(args.libdispatch_build_dir, "src", ".libs"))])
        build_flags.extend(["-Xswiftc", "-I{}".format(
            os.path.join(args.libdispatch_build_dir, "src"))])
        build_flags.extend(["-Xswiftc", "-I{}".format(
            os.path.join(args.libdispatch_build_dir, "src", "swift"))])
        build_flags.extend(["-Xswiftc", "-I{}".format(
            args.libdispatch_source_dir)])
        build_flags.extend(["-Xcc", "-fblocks"])
    
    # Enable testing in release mode because SwiftPM's tests uses @testable imports.
    #
    # Note: Testing is already enabled in debug mode by SwiftPM itself.
    if args.release:
        build_flags.extend(["-Xswiftc", "-enable-testing"])
        build_flags.extend(["--configuration", "release"])

    env_cmd = ["env", "SWIFT_EXEC=" + os.path.join(bindir, "swiftc"),
                      "SWIFTPM_BUILD_DIR=" + build_path]
    if args.sysroot:
        env_cmd.append("SYSROOT=" + args.sysroot)
    cmd = env_cmd + [bootstrapped_product] + build_flags

    # Always build tests in stage2.
    cmd.extend(["--build-tests"])

    note("building self-hosted 'swift-build': %s" % (
        ' '.join(cmd),))
    result = subprocess.call(cmd, cwd=g_project_root)
    if result != 0:
        error("build failed with exit status %d" % (result,))

    swift_package_path = os.path.join(target_path, conf, "swift-package")
    swiftpm_build_dir = os.path.join(target_path, conf, "swift-build")
    swift_test_path = os.path.join(target_path, conf, "swift-test")
    swift_run_path = os.path.join(target_path, conf, "swift-run")
    swiftpm_xctest_helper_path = os.path.join(
        target_path, conf, "swiftpm-xctest-helper")

    # If testing, run each of the test bundles.
    if "test" in build_actions:
        # Construct the test environment.
        cmd = env_cmd + [swift_test_path] + build_flags
        if args.test_parallel:
            cmd.append("--parallel")
        for arg in args.test_filter:
            cmd.extend(["--filter", arg])
        note("testing with command: %s" % (
            ' '.join(cmd),))
        result = subprocess.call(cmd, cwd=g_project_root)
        if result != 0:
            error("tests failed with exit status %d" % (result,))

    # If installing, put the build products in the appropriate locations.
    if "install" in build_actions:
        # Install in each prefix.
        for install_prefix in args.install_prefixes:
            bin_install_path = os.path.join(g_project_root, install_prefix,
                                            "bin")
            lib_install_path = os.path.join(g_project_root, install_prefix,
                                            "lib", "swift", "pm")
            libexec_install_path = os.path.join(g_project_root, install_prefix,
                                            "libexec", "swift", "pm")
            mkdir_p(bin_install_path)
            mkdir_p(lib_install_path)
            mkdir_p(libexec_install_path)

            # Install XCTest helper inside libexec.
            if platform.system() == 'Darwin':
                # Append the required RPATH for finding the standard libraries from
                # the libexec install path.
                #
                # FIXME: We need to find a way to express this from within the
                # package manager. https://bugs.swift.org/browse/SR-1968
                installBinary(swiftpm_xctest_helper_path, libexec_install_path,
                              args.swiftc_path,
                              add_rpath=("@executable_path/../../../"
                                         "lib/swift/macosx"))

            # Install the main executables.
            for product_path in [swift_package_path, swiftpm_build_dir,
                                 swift_test_path, swift_run_path]:
                installBinary(product_path, bin_install_path, args.swiftc_path)

            # Install the runtimes.
            for version, runtime in processed_runtimes.items():
                runtime_module_path, runtime_swiftdoc_path, runtime_lib_path = runtime
                install_path = os.path.join(lib_install_path, version)

                # Install library.
                installBinary(runtime_lib_path, install_path, args.swiftc_path)

                # Install module.
                cmd = ["install", "-m", "0644",
                       runtime_module_path, install_path]
                note("installing %s: %s" % (
                    os.path.basename(runtime_module_path), ' '.join(cmd)))
                result = subprocess.call(cmd)
                if result != 0:
                    error("module install failed with exit status %d" % (result,))

                # Install swiftdoc.
                cmd = ["install", "-m", "0644",
                       runtime_swiftdoc_path, install_path]
                note("installing %s: %s" % (
                    os.path.basename(runtime_swiftdoc_path), ' '.join(cmd)))
                result = subprocess.call(cmd)
                if result != 0:
                    error("swiftdoc install failed with exit status %d" % (result,))

    
    # Copy the libSwiftPM library to a directory, if we've been asked to do so.
    if args.libswiftpm_library_dir:
        # Make the directory absolute, if needed.
        libswiftpm_library_dir = os.path.abspath(args.libswiftpm_library_dir)
        
        # Create the output directory, if needed.
        mkdir_p(libswiftpm_library_dir)
        
        # Find the libSwiftPM product.  It's an error if we don't.
        try:
            libswiftpm_product = product_map["SwiftPM"]
        except:
            error("unable to find 'SwiftPM' product in package manifest; cannot copy library to '%s'" % (libswiftpm_library_dir, ))
        
        # Copy the libSwiftPM library.
        # FIXME: This shouldn't require so much knowledge of the manifest.
        # FIXME: We should handle what happens if it's a static library.
        src_path = os.path.join(target_path, conf, libswiftpm_product.product_name)
        installBinary(src_path, libswiftpm_library_dir, args.swiftc_path)
        
        # If it's a dynamic library, and if we're on Darwin, fix the dylib id.
        if platform.system() == 'Darwin':
            if libswiftpm_product.type == ProductType.DYNAMIC_LIBRARY:
                dst_path = os.path.join(libswiftpm_library_dir, libswiftpm_product.product_name)
                dylib_id = os.path.join("@rpath", libswiftpm_product.product_name)
                cmd = ["install_name_tool", "-id", dylib_id, dst_path]
                note("resetting dylib id: %s" % ' '.join(cmd))
                result = subprocess.call(cmd)
                if result != 0:
                    error("command  failed with exit status %d" % (result,))
    
    # Copy the libSwiftPM modules to a directory, if we've been asked to do so.
    # FIXME: There is too much duplication between this part and the one above.
    if args.libswiftpm_modules_dir:
        # Make the directory absolute, if needed.
        libswiftpm_modules_dir = os.path.abspath(args.libswiftpm_modules_dir)
        
        # Create the output directory, if needed.
        mkdir_p(libswiftpm_modules_dir)
        
        # Find the libSwiftPM product.  It's an error if we don't find it.
        try:
            libswiftpm_product = product_map["SwiftPM"]
        except:
            error("unable to find 'SwiftPM' product in package manifest; cannot copy modules to '%s'", libswiftpm_library_dir)
        
        # Copy the libSwiftPM modules.
        # FIXME: This shouldn't require so much knowledge of the manifest.
        # FIXME: We should handle what happens if it's a static library.
        target_map = dict((t.name, t) for t in targets)
        for target_name in libswiftpm_product.targets:
            target = target_map[target_name]
            if target.is_c:
                # Skip if requested.
                if args.libswiftpm_skip_c_headers:
                    continue
                # Copy headers of the C target.
                include_dir = os.path.join(target.module_root_dir, "include/")
                if not os.path.exists(include_dir):
                    continue
                dst_path = os.path.join(libswiftpm_modules_dir, target.name)
                cmd = ["rsync", "-a", include_dir, dst_path]
                note("copying %s: %s" % (os.path.basename(src_path), ' '.join(cmd)))
                result = subprocess.call(cmd)
                if result != 0:
                    error("copying failed with exit status %d" % result)
            elif target.is_swift:
                # Copy swiftmodule and swiftdoc i.e. headers of the Swift target.
                for suffix in [".swiftmodule", ".swiftdoc"]:
                    src_path = os.path.join(target_path, conf, target.name + suffix)
                    if not os.path.exists(src_path):
                        continue
                    dst_path = os.path.join(libswiftpm_modules_dir, target.name + suffix)
                    cmd = ["rsync", "-a", src_path, dst_path]
                    note("copying %s: %s" % (os.path.basename(src_path), ' '.join(cmd)))
                    result = subprocess.call(cmd)
                    if result != 0:
                        error("copying failed with exit status %d" % result)
            else:
                error("Unknown target type " + target)
        
        # Also emit the version info, if appropriate.  This appears as a module
        # consisting of a header file and a module map.
        if args.vendor_name or args.build_identifier:
            if not args.vendor_name or not args.build_identifier:
                error("--build-identifier is required with --vendor-name")
            note("writing version info ('%s', '%s') to %s" % (args.vendor_name, args.build_identifier, libswiftpm_modules_dir))
            write_bootstrap_version_info(libswiftpm_modules_dir, args.vendor_name, args.build_identifier)
    
    # If generating an Xcode project, also use the build product to do that.
    if args.generate_xcodeproj:
        # We will use the `swift-package` binary we just built to generate the
        # Xcode project.
        cmd = [swift_package_path]
        if args.enable_perf_tests:
            cmd.extend(["-Xswiftc", "-DENABLE_PERF_TESTS"])
        cmd.extend(["generate-xcodeproj"])
        
        # If there is a `llvm-profdata` binary next to `swiftc`, we enable code
        # coverage in the generated project.
        llvm_profdata_path = os.path.join(os.path.dirname(args.swiftc_path),
            "llvm-profdata")
        if os.path.exists(llvm_profdata_path):
            cmd.append("--enable-code-coverage")

        # Generate the .xcconfig file.
        xcconfig_path = os.path.join("SwiftPM.xcodeproj/overrides.xcconfig")
        if not os.path.exists(os.path.dirname(xcconfig_path)):
            mkdir_p(os.path.dirname(xcconfig_path))
        with open(xcconfig_path, "w") as f:
            print("\n", file=f)
        cmd += ["--xcconfig-overrides", xcconfig_path]
            
        # Call `swift-package` to generate the Xcode project.
        note("generating Xcode project: %s" % (' '.join(cmd),))
        result = subprocess.call(cmd)
        if result != 0:
            error("xcodeproj generation failed with exit status %d" % (result,))

        # Copy the perf test scheme if perf tests should be enabled.
        #
        # This is hacky but we need a way to run only performance tests and there
        # is no way to do this with SwiftPM right now.
        if args.enable_perf_tests:
            shutil.copyfile(os.path.join(os.path.dirname(os.path.abspath(__file__)), "PerformanceTests.xcscheme"), "SwiftPM.xcodeproj/xcshareddata/xcschemes/PerformanceTests.xcscheme")
                
if __name__ == '__main__':
    main()