# Copyright (c) 2017 CEF Python, see the Authors file.
# All rights reserved. Licensed under BSD 3-clause license.
# Project website: https://github.com/cztomczak/cefpython

"""
Build distribution packages for all architectures and all supported
python versions.

Usage:
    build_distrib.py VERSION [--no-run-examples] [--no-rebuild]

Options:
    VERSION            Version number eg. 50.0
    --no-run-examples  Do not run examples while building cefpython modules.
                       Examples require interaction, closing window before
                       proceeding. Only unit tests will be run in such case.
    --no-rebuild       Do not rebuild cefpython modules. For internal use
                       so that changes to packaging can be quickly tested.


This script does the following:
1. Expects that all supported python versions are installed
   a) On Windows search for Pythons in the multiple default install
      locations
   b) On Linux use only Pythons from ~/.pyenv/versions/
      directory
   c) On Mac use Pythons from ~/.pyenv/versions/ and /usr/local/bin/python
      For example will use Python 2.7.13 from /usr/local/bin/ only
      when 2.7 was not found in ~/.pyenv/versions/.
2. Expects that all python compilers for supported python versions
   are installed. See docs/Build-instructions.md > Requirements.
3. Expects cef_binary*/ directories from Spotify Automated Builds
   to be in the build/ directory. It does not rebuild cefclient
   nor libcef_dll_wrapper libraries in these directories. If you
   would like to rebuild everything from scratch then delete subdirs
   manually (build_cefclient/, build_wrapper*/).
   When building CEF from sources copy build/chromium/src/cef/binary_distrib
   /cef_binary_*/ to the build/ directory.
4. Install and/or upgrade tools/requirements.txt and uninstall
   cefpython3 packages for all python versions
5. Run automate.py --prebuilt-cef using both Python 32-bit and Python 64-bit
6. Pack the prebuilt biaries using zip on Win/Mac and .tar.gz on Linux
   and move to build/distrib/
7. Reduce packages size (Issue #321). After packing prebuilt binaries,
   reduce its size so that packages will use the reduced prebuilt binaries.
8. Build cefpython modules for all supported Python versions on both
   32-bit and 64-bit. Backup and restore subprocess executable on Windows
   built with Python 2.7 (Issue #342).
9. Make setup installers and pack them to zip (Win/Mac) or .tar.gz (Linux)
10. Make wheel packages
11. Move setup and wheel packages to the build/distrib/ directory
12. Test wheel packages installation and run unit tests using the
    installed wheel package.
13. Show summary
"""

from common import *
import glob
import os
import pprint
import re
import shutil
import subprocess
# import tarfile  # Currently using zip on all platforms
import zipfile

# Command line args
VERSION = ""
NO_RUN_EXAMPLES = False
NO_REBUILD = False

# Python versions
SUPPORTED_PYTHON_VERSIONS = [(2, 7), (3, 4), (3, 5), (3, 6)]

# Python search paths. It will use first Python found for specific version.
# Supports replacement of one environment variable in path eg.: %ENV_KEY%.
PYTHON_SEARCH_PATHS = dict(
    WINDOWS=[
        "C:\\Python*\\",
        "%LOCALAPPDATA%\\Programs\\Python\\Python*\\",
        "C:\\Program Files\\Python*\\",
        "C:\\Program Files (x86)\\Python*\\",
    ],
    LINUX=[
        "%PYENV_ROOT%/versions/*/bin",
    ],
    MAC=[
        "%PYENV_ROOT%/versions/*/bin",
        "/usr/local/bin",
    ],
)


def main():
    command_line_args()
    supported = list()
    for version in SUPPORTED_PYTHON_VERSIONS:
        supported.append("{major}.{minor}".format(major=version[0],
                                                  minor=version[1]))
    print("[build_distrib.py] Supported python versions: {supported}"
          .format(supported=" / ".join(supported)))
    clean_build_directories()
    if WINDOWS:
        pythons_32bit = search_for_pythons("32bit")
        pythons_64bit = search_for_pythons("64bit")
    elif LINUX:
        pythons_32bit = search_for_pythons("32bit") if ARCH32 else list()
        pythons_64bit = search_for_pythons("64bit") if ARCH64 else list()
    elif MAC:
        pythons_32bit = list()
        pythons_64bit = search_for_pythons("64bit")
    else:
        print("ERROR: Unsupported OS")
        sys.exit(1)
    check_pythons(pythons_32bit, pythons_64bit)
    install_upgrade_requirements(pythons_32bit + pythons_64bit)
    uninstall_cefpython3_packages(pythons_32bit + pythons_64bit)
    if not os.path.exists(DISTRIB_DIR):
        os.makedirs(DISTRIB_DIR)
    if pythons_32bit:
        run_automate_prebuilt_cef(pythons_32bit[0])
        pack_prebuilt_cef("32bit")
        if LINUX:
            reduce_package_size_issue_262("32bit")
        remove_unnecessary_package_files("32bit")
    if pythons_64bit:
        run_automate_prebuilt_cef(pythons_64bit[0])
        pack_prebuilt_cef("64bit")
        if LINUX:
            reduce_package_size_issue_262("64bit")
        remove_unnecessary_package_files("64bit")
    if not NO_REBUILD:
        build_cefpython_modules(pythons_32bit, "32bit")
        build_cefpython_modules(pythons_64bit, "64bit")
    if pythons_32bit:
        make_packages(pythons_32bit[0], "32bit")
    if pythons_64bit:
        make_packages(pythons_64bit[0], "64bit")
    test_wheel_packages(pythons_32bit + pythons_64bit)
    show_summary(pythons_32bit, pythons_64bit)


def command_line_args():
    global VERSION, NO_RUN_EXAMPLES, NO_REBUILD
    version = get_version_from_command_line_args(__file__)
    if not version or "--help" in sys.argv:
        print(__doc__)
        sys.exit(1)
    VERSION = version
    if "--no-run-examples" in sys.argv:
        NO_RUN_EXAMPLES = True
        sys.argv.remove("--no-run-examples")
    if "--no-rebuild" in sys.argv:
        NO_REBUILD = True
        sys.argv.remove("--no-rebuild")
    args = sys.argv[1:]
    for arg in args:
        if arg == version:
            continue
        print("[build_distrib.py] Invalid argument: {arg}".format(arg=arg))
        sys.exit(1)


def clean_build_directories():
    print("[build_distrib.py] Clean build directories")

    # Delete distrib dir
    if os.path.exists(DISTRIB_DIR):
        print("[build_distrib.py] Delete directory: {distrib_dir}/"
              .format(distrib_dir=os.path.basename(DISTRIB_DIR)))
        shutil.rmtree(DISTRIB_DIR)

    if not NO_REBUILD:
        # Delete build_cefpython/ dir
        if os.path.exists(BUILD_CEFPYTHON):
            print("[build_distirb.py] Delete directory: {dir}/"
                  .format(dir=os.path.basename(BUILD_CEFPYTHON)))
            shutil.rmtree(BUILD_CEFPYTHON)
        # Delete cefpython_binary_*/ dirs
        delete_cefpython_binary_dir("32bit")
        delete_cefpython_binary_dir("64bit")

    # Delete cef binaries and libraries dirs
    delete_cef_binaries_libraries_dir("32bit")
    delete_cef_binaries_libraries_dir("64bit")


def delete_cefpython_binary_dir(arch):
    cefpython_binary = get_cefpython_binary_basename(
            postfix2=get_os_postfix2_for_arch(arch))
    assert cefpython_binary, cefpython_binary
    cefpython_binary = os.path.join(BUILD_DIR, cefpython_binary)
    if os.path.exists(cefpython_binary):
        print("[build_distrib.py] Delete directory: {dir}/"
              .format(dir=os.path.basename(cefpython_binary)))
        shutil.rmtree(cefpython_binary)


def delete_cef_binaries_libraries_dir(arch):
    cef_binlib = get_cef_binaries_libraries_basename(
            postfix2=get_os_postfix2_for_arch(arch))
    assert cef_binlib, cef_binlib
    cef_binlib = os.path.join(BUILD_DIR, cef_binlib)
    if os.path.exists(cef_binlib):
        print("[build_distrib.py] Delete directory: {dir}/"
              .format(dir=os.path.basename(cef_binlib)))
        shutil.rmtree(cef_binlib)


def search_for_pythons(search_arch):
    """Returns pythons ordered by version from lowest to highest."""
    pythons_found = list()
    for pattern in PYTHON_SEARCH_PATHS[SYSTEM]:
        # Replace env variable in path
        match = re.search(r"%(\w+)%", pattern)
        if match:
            env_key = match.group(1)
            if env_key in os.environ:
                pattern = pattern.replace(match.group(0), os.environ[env_key])
            else:
                print("ERROR: Env variable not found: {env_key}"
                      .format(env_key=env_key))
                sys.exit(1)
        results = glob.glob(pattern)
        for path in results:
            if os.path.isdir(path):
                python = os.path.join(path,
                                      "python{ext}".format(ext=EXECUTABLE_EXT))
                version_code = ("import sys;"
                                "print(str(sys.version_info[:3]));")
                if not os.path.isfile(python):
                    print("ERROR: Python executable not found: {executable}"
                          .format(executable=python))
                    sys.exit(1)
                version_str = subprocess.check_output([python, "-c",
                                                       version_code])
                version_str = version_str.strip()
                match = re.search("^\((\d+), (\d+), (\d+)\)$", version_str)
                assert match, version_str
                major = match.group(1)
                minor = match.group(2)
                micro = match.group(3)
                version_tuple2 = (int(major), int(minor))
                version_tuple3 = (int(major), int(minor), int(micro))
                arch_code = ("import platform;"
                             "print(str(platform.architecture()[0]));")
                arch = subprocess.check_output([python, "-c", arch_code])
                arch = arch.strip()
                if version_tuple2 in SUPPORTED_PYTHON_VERSIONS \
                        and arch == search_arch:
                    name = ("Python {major}.{minor}.{micro} {arch}"
                            .format(major=major, minor=minor, micro=micro,
                                    arch=arch))
                    pythons_found.append(dict(
                        version2=version_tuple2,
                        version3=version_tuple3,
                        arch=arch,
                        executable=python,
                        name=name))
    ret_pythons = list()
    for version_tuple in SUPPORTED_PYTHON_VERSIONS:
        supported_python = None
        for python in pythons_found:
            if python["version2"] == version_tuple:
                supported_python = python
                break
        if supported_python:
            ret_pythons.append(supported_python)
    return ret_pythons


def check_pythons(pythons_32bit, pythons_64bit):
    check_32bit = True
    check_64bit = True
    if MAC:
        check_32bit = False
    elif LINUX:
        if pythons_64bit:
            check_32bit = False
        elif pythons_32bit:
            check_64bit = False

    pp = pprint.PrettyPrinter(indent=4)
    if pythons_32bit:
        print("[build_distrib.py] Pythons 32-bit found:")
        pp.pprint(pythons_32bit)
    if check_32bit and len(pythons_32bit) != len(SUPPORTED_PYTHON_VERSIONS):
        print("[build_distrib.py] ERROR: Couldn't find all supported"
              " python 32-bit installations. Found: {found}."
              .format(found=len(pythons_32bit)))
        sys.exit(1)
    if pythons_64bit:
        print("[build_distrib.py] Pythons 64-bit found:")
        pp.pprint(pythons_64bit)
    if check_64bit and len(pythons_64bit) != len(SUPPORTED_PYTHON_VERSIONS):
        print("[build_distrib.py] ERROR: Couldn't find all supported"
              " python 64-bit installations. Found: {found}."
              .format(found=len(pythons_64bit)))
        sys.exit(1)


def install_upgrade_requirements(pythons):
    for python in pythons:
        print("[build_distrib.py] pip install/upgrade requirements.txt"
              " for: {name}".format(name=python["name"]))

        # Upgrade pip
        command = ("\"{python}\" -m pip install --upgrade pip"
                   .format(python=python["executable"]))
        command = sudo_command(command, python=python["executable"])
        pcode = subprocess.call(command, shell=True)
        if pcode != 0:
            print("[build_distrib.py] ERROR while upgrading pip")
            sys.exit(1)

        # Install/upgrade requirements.txt
        requirements = os.path.join(TOOLS_DIR, "requirements.txt")
        command = ("\"{python}\" -m pip install --upgrade -r {requirements}"
                   .format(python=python["executable"],
                           requirements=requirements))
        command = sudo_command(command, python=python["executable"])
        pcode = subprocess.call(command, shell=True)
        if pcode != 0:
            print("[build_distrib.py] ERROR while running pip install/upgrade")
            sys.exit(1)


def uninstall_cefpython3_packages(pythons):
    for python in pythons:
        print("[build_distrib.py] Uninstall cefpython3 package"
              " for: {name}".format(name=python["name"]))

        # Check if package is installed
        command = ("\"{python}\" -m pip show cefpython3"
                   .format(python=python["executable"]))
        try:
            output = subprocess.check_output(command, shell=True)
        except subprocess.CalledProcessError, exc:
            # pip show returns error code when package is not installed
            output = exc.output
        if not len(output.strip()):
            # Package is not installed - info is an empty string
            print("[build_distrib.py] Not installed")
            continue

        # Uninstall package. Only uninstall if package is installed,
        # otherwise error code is returned.
        command = ("\"{python}\" -m pip uninstall -y cefpython3"
                   .format(python=python["executable"]))
        command = sudo_command(command, python=python["executable"])
        pcode = subprocess.call(command, shell=True)
        if pcode != 0:
            print("[build_distrib.py] ERROR while uninstall cefpython3"
                  " package using pip")
            sys.exit(1)


def run_automate_prebuilt_cef(python):
    print("[build_distrib.py] Run automate.py --prebuilt-cef for {arch}"
          .format(arch=python["arch"]))
    automate = os.path.join(TOOLS_DIR, "automate.py")
    command = ("\"{python}\" {automate} --prebuilt-cef"
               .format(python=python["executable"], automate=automate))
    code = subprocess.call(command, shell=True)
    if code != 0:
        print("[build_distrib.py] ERROR while running automate.py")
        sys.exit(1)


def pack_prebuilt_cef(arch):
    prebuilt_basename = get_cef_binaries_libraries_basename(
                get_os_postfix2_for_arch(arch))
    print("[build_distrib.py] Pack directory: {dir}/ ..."
          .format(dir=prebuilt_basename))
    prebuilt_dir = os.path.join(BUILD_DIR, prebuilt_basename)
    assert os.path.exists(prebuilt_dir), prebuilt_dir
    archive = pack_directory(prebuilt_dir, base_path=BUILD_DIR)
    shutil.move(archive, DISTRIB_DIR)
    print("[build_distrib.py] Created archive in distrib dir: {archive}"
          .format(archive=os.path.basename(archive)))


def pack_directory(path, base_path):
    if path.endswith(os.path.sep):
        path = path[:-1]
    # ext = ".zip" if WINDOWS or MAC else ".tar.gz"
    ext = ".zip"
    archive = path + ext
    if os.path.exists(archive):
        os.remove(archive)
    if WINDOWS or MAC:
        zip_directory(path, base_path=base_path, archive=archive)
    else:
        zip_directory(path, base_path=base_path, archive=archive)
        # with tarfile.open(archive, "w:gz") as tar:
        #     tar.add(path, arcname=os.path.basename(path))
    assert os.path.isfile(archive), archive
    return archive


def zip_directory(path, base_path, archive):
    original_dir = os.getcwd()
    os.chdir(base_path)
    path = path.replace(base_path, "")
    if path[0] == os.path.sep:
        path = path[1:]
    zipf = zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED)
    for root, dirs, files in os.walk(path):
        for file_ in files:
            zipf.write(os.path.join(root, file_))
    zipf.close()
    os.chdir(original_dir)


def reduce_package_size_issue_262(arch):
    """Linux only: libcef.so is huge (500 MB) in Chrome v54+. Issue #262."""
    print("[build_distrib.py] Reduce package size for {arch} (Issue #262)"
          .format(arch=arch))
    prebuilt_basename = get_cef_binaries_libraries_basename(
                get_os_postfix2_for_arch(arch))
    bin_dir = os.path.join(prebuilt_basename, "bin")

    # Run `strip` command on `libcef.so`
    libcef_so = os.path.join(bin_dir, "libcef.so")
    print("[build_distrib.py] Strip {libcef_so}"
          .format(libcef_so=os.path.basename(libcef_so)))
    command = "strip {libcef_so}".format(libcef_so=libcef_so)
    pcode = subprocess.call(command, shell=True)
    assert pcode == 0, "strip command failed"


def remove_unnecessary_package_files(arch):
    """Do not ship sample applications (cefclient etc) with the package.
    They increase size and also are an additional unnecessary factor
    when dealing with false-positives in Anti-Virus software."""
    print("[build_distrib.py] Reduce package size for {arch} (Issue #321)"
          .format(arch=arch))
    prebuilt_basename = get_cef_binaries_libraries_basename(
                get_os_postfix2_for_arch(arch))
    bin_dir = os.path.join(prebuilt_basename, "bin")
    delete_cef_sample_apps(caller_script=__file__, bin_dir=bin_dir)


def build_cefpython_modules(pythons, arch):
    for python in pythons:
        print("[build_distrib.py] Build cefpython module for {python_name}"
              .format(python_name=python["name"]))
        flags = ""
        if NO_RUN_EXAMPLES:
            flags += " --no-run-examples"
        # On Linux/Mac Makefiles are used and must pass --clean flag
        command = ("\"{python}\" {build_py} {version} --clean {flags}"
                   .format(python=python["executable"],
                           build_py=os.path.join(TOOLS_DIR, "build.py"),
                           version=VERSION,
                           flags=flags))
        # build.py may require sudo if system python, so shell=True
        pcode = subprocess.call(command, shell=True)
        if pcode != 0:
            print("[build_distrib.py] ERROR: failed to build cefpython"
                  " module for {python_name}"
                  .format(python_name=python["name"]))
            sys.exit(1)
        print("[build_distrib.py] Built successfully cefpython module for"
              " {python_name}".format(python_name=python["name"]))
        # Issue #342
        backup_subprocess_executable_issue342(python)

    # Issue #342
    restore_subprocess_executable_issue342(arch)

    print("[build_distrib.py] Successfully built cefpython modules for {arch}"
          .format(arch=arch))


def backup_subprocess_executable_issue342(python):
    """Use subprocess executable build by Python 2.7 to avoid
    false-positives by AVs when building subprocess with Python 3.
    Windows-only issue."""
    if not WINDOWS:
        return
    if python["version2"] == (2, 7):
        print("[build_distrib.py] Backup subprocess executable built"
              " with Python 2.7 (Issue #342)")
        cefpython_binary_basename = get_cefpython_binary_basename(
                get_os_postfix2_for_arch(python["arch"]))
        cefpython_binary = os.path.join(BUILD_DIR, cefpython_binary_basename)
        assert os.path.isdir(cefpython_binary)
        src = os.path.join(cefpython_binary, "subprocess.exe")
        dst = os.path.join(BUILD_CEFPYTHON,
                           "subprocess_py27_{arch}_issue342.exe"
                           .format(arch=python["arch"]))
        shutil.copy(src, dst)


def restore_subprocess_executable_issue342(arch):
    """Use subprocess executable build by Python 2.7 to avoid
    false-positives by AVs when building subprocess with Python 3.
    Windows-only issue."""
    if not WINDOWS:
        return
    print("[build_distrib.py] Restore subprocess executable built"
          " with Python 2.7 (Issue #342)")
    cefpython_binary_basename = get_cefpython_binary_basename(
            get_os_postfix2_for_arch(arch))
    cefpython_binary = os.path.join(BUILD_DIR, cefpython_binary_basename)
    assert os.path.isdir(cefpython_binary)
    src = os.path.join(BUILD_CEFPYTHON,
                       "subprocess_py27_{arch}_issue342.exe"
                       .format(arch=arch))
    assert os.path.isfile(src)
    dst = os.path.join(cefpython_binary, "subprocess.exe")
    shutil.copy(src, dst)


def make_packages(python, arch):
    """Make setup and wheel packages."""
    print("[build_distrib.py] Make setup package for {arch}..."
          .format(arch=arch))

    # Call make_installer.py
    make_installer_py = os.path.join(TOOLS_DIR, "make_installer.py")
    installer_command = ("\"{python}\" {make_installer_py} {version}"
                         .format(python=python["executable"],
                                 make_installer_py=make_installer_py,
                                 version=VERSION))
    pcode = subprocess.call(installer_command, cwd=BUILD_DIR, shell=True)
    if pcode != 0:
        print("[build_distrib.py] ERROR: failed to make setup package for"
              " {arch}".format(arch=arch))
        sys.exit(1)

    # Pack setup package and move to distrib dir
    print("[build_distrib.py] Pack setup package for {arch}..."
          .format(arch=arch))
    setup_basename = get_setup_installer_basename(
            VERSION, get_os_postfix2_for_arch(arch))
    setup_dir = os.path.join(BUILD_DIR, setup_basename)
    archive = pack_directory(setup_dir, BUILD_DIR)
    shutil.move(archive, DISTRIB_DIR)

    # Make wheel package
    print("[build_distrib.py] Make wheel package for {arch}..."
          .format(arch=arch))
    wheel_args = "bdist_wheel --universal"
    wheel_command = ("\"{python}\" setup.py {wheel_args}"
                     .format(python=python["executable"],
                             wheel_args=wheel_args))
    pcode = subprocess.call(wheel_command, cwd=setup_dir, shell=True)
    if pcode != 0:
        print("[build_distrib.py] ERROR: failed to make wheel package for"
              " {arch}".format(arch=arch))
        sys.exit(1)

    # Move wheel package
    files = glob.glob(os.path.join(setup_dir, "dist", "*.whl"))
    assert len(files) == 1, ".whl file not found"
    shutil.move(files[0], DISTRIB_DIR)

    # Delete setup directory
    print("[build_distrib.py] Delete setup directory: {setup_dir}/"
          .format(setup_dir=os.path.basename(setup_dir)))
    shutil.rmtree(setup_dir)


def test_wheel_packages(pythons):
    """Test wheel packages installation and run unit tests."""
    uninstall_cefpython3_packages(pythons)
    for python in pythons:
        print("[build_distrib.py] Test wheel package (install, unittests) for"
              " {python_name}".format(python_name=python["name"]))
        platform_tag = get_pypi_postfix2_for_arch(python["arch"])
        whl_pattern = (r"*{platform_tag}.whl"
                       .format(platform_tag=platform_tag))
        wheels = glob.glob(os.path.join(DISTRIB_DIR, whl_pattern))
        assert len(wheels) == 1, ("No wheels found in distrib dir for %s"
                                  % python["arch"])

        # Install wheel
        command = ("\"{python}\" -m pip install {wheel}"
                   .format(python=python["executable"],
                           wheel=os.path.basename(wheels[0])))
        command = sudo_command(command, python=python["executable"])
        pcode = subprocess.call(command, cwd=DISTRIB_DIR, shell=True)
        if pcode != 0:
            print("[build_distrib.py] Wheel package installation failed for"
                  " {python_name}".format(python_name=python["name"]))
            sys.exit(1)

        # Run unittests using the installed wheel package
        command = ("\"{python}\" {unittests}"
                   .format(python=python["executable"],
                           unittests=os.path.join(UNITTESTS_DIR,
                                                  "main_test.py")))
        pcode = subprocess.call(command, cwd=DISTRIB_DIR, shell=True)
        if pcode != 0:
            print("[build_distrib.py] ERROR: Unit tests failed for"
                  " {python_name}".format(python_name=python["name"]))
            sys.exit(1)


def show_summary(pythons_32bit, pythons_64bit):
    print("[build_distrib.py] SUMMARY:")
    print("  Pythons 32bit ({count})".format(count=len(pythons_32bit)))
    for python in pythons_32bit:
        print("    {python_name}".format(python_name=python["name"]))
    print("  Pythons 64bit ({count})".format(count=len(pythons_64bit)))
    for python in pythons_64bit:
        print("    {python_name}".format(python_name=python["name"]))
    files = glob.glob(os.path.join(DISTRIB_DIR, "*"))
    print("  Files in the build/{distrib_basename}/ directory ({count})"
          .format(distrib_basename=os.path.basename(DISTRIB_DIR),
                  count=len(files)))
    for file_ in files:
        print("    {filename}".format(filename=os.path.basename(file_)))
    print("[build_distrib.py] Everything OK. Distribution packages created.")


if __name__ == "__main__":
    main()
