build.py

#!/usr/bin/env python3

from tempfile import mkdtemp
import argparse
import errno
import hashlib
import multiprocessing
import os
import os.path as p
import platform
import re
import shlex
import shutil
import subprocess
import sys
import sysconfig
import tarfile
from zipfile import ZipFile
import tempfile
import urllib.request


class InstallationFailed( Exception ):
  def __init__( self, message = None, exit_code = 1 ):
    self.message = message
    self.exit_code = exit_code

  def Print( self ):
    if self.message:
      print( '', file = sys.stderr )
      print( self.message, file = sys.stderr )

  def Exit( self ):
    sys.exit( self.exit_code )


IS_MSYS = 'MSYS' == os.environ.get( 'MSYSTEM' )

IS_64BIT = sys.maxsize > 2**32
PY_MAJOR, PY_MINOR = sys.version_info[ 0 : 2 ]
PY_VERSION = sys.version_info[ 0 : 3 ]
if PY_VERSION < ( 3, 8, 0 ):
  sys.exit( 'ycmd requires Python >= 3.8.0; '
            'your version of Python is ' + sys.version +
            '\nHint: Try running python3 ' + ' '.join( sys.argv ) )

DIR_OF_THIS_SCRIPT = p.dirname( p.abspath( __file__ ) )
DIR_OF_THIRD_PARTY = p.join( DIR_OF_THIS_SCRIPT, 'third_party' )
LIBCLANG_DIR = p.join( DIR_OF_THIRD_PARTY, 'clang', 'lib' )

for folder in os.listdir( DIR_OF_THIRD_PARTY ):
  abs_folder_path = p.join( DIR_OF_THIRD_PARTY, folder )
  if p.isdir( abs_folder_path ) and not os.listdir( abs_folder_path ):
    sys.exit(
      f'ERROR: folder { folder } in { DIR_OF_THIRD_PARTY } is empty; '
      'you probably forgot to run:\n'
      '\tgit submodule update --init --recursive\n'
    )


NO_DYNAMIC_PYTHON_ERROR = (
  'ERROR: found static Python library ({library}) but a dynamic one is '
  'required. You must use a Python compiled with the {flag} flag. '
  'If using pyenv, you need to run the command:\n'
  '  export PYTHON_CONFIGURE_OPTS="{flag}"\n'
  'before installing a Python version.' )
NO_PYTHON_LIBRARY_ERROR = 'ERROR: unable to find an appropriate Python library.'
NO_PYTHON_HEADERS_ERROR = 'ERROR: Python headers are missing in {include_dir}.'

# Regular expressions used to find static and dynamic Python libraries.
# Notes:
#  - Python 3 library name may have an 'm' suffix on Unix platforms, for
#    instance libpython3.6m.so;
#  - the linker name (the soname without the version) does not always
#    exist so we look for the versioned names too;
#  - on Windows, the .lib extension is used instead of the .dll one. See
#    https://en.wikipedia.org/wiki/Dynamic-link_library#Import_libraries
STATIC_PYTHON_LIBRARY_REGEX = '^libpython{major}\\.{minor}m?\\.a$'
DYNAMIC_PYTHON_LIBRARY_REGEX = """
  ^(?:
  # Linux, BSD
  libpython{major}\\.{minor}m?\\.so(\\.\\d+)*|
  # OS X
  libpython{major}\\.{minor}m?\\.dylib|
  # Windows
  python{major}{minor}\\.lib|
  # Cygwin
  libpython{major}\\.{minor}\\.dll\\.a
  )$
"""

JDTLS_MILESTONE = '1.26.0'
JDTLS_BUILD_STAMP = '202307271613'
JDTLS_SHA256 = (
  'ba5fe5ee3b2a8395287e24aef20ce6e17834cf8e877117e6caacac6a688a6c53'
)

DEFAULT_RUST_TOOLCHAIN = 'nightly-2023-08-18'
RUST_ANALYZER_DIR = p.join( DIR_OF_THIRD_PARTY, 'rust-analyzer' )

BUILD_ERROR_MESSAGE = (
  'ERROR: the build failed.\n\n'
  'NOTE: it is *highly* unlikely that this is a bug but rather '
  'that this is a problem with the configuration of your system '
  'or a missing dependency. Please carefully read CONTRIBUTING.md '
  'and if you\'re sure that it is a bug, please raise an issue on the '
  'issue tracker, including the entire output of this script (with --verbose) '
  'and the invocation line used to run it.' )

CLANGD_VERSION = '17.0.1'
CLANGD_BINARIES_ERROR_MESSAGE = (
  'No prebuilt Clang {version} binaries for {platform}. '
  'You\'ll have to compile Clangd {version} from source '
  'or use your system Clangd. '
  'See the YCM docs for details on how to use a custom Clangd.' )


def FindLatestMSVC( quiet ):
  ACCEPTABLE_VERSIONS = [ 17, 16, 15 ]

  VSWHERE_EXE = os.path.join( os.environ[ 'ProgramFiles(x86)' ],
                             'Microsoft Visual Studio',
                             'Installer', 'vswhere.exe' )

  if os.path.exists( VSWHERE_EXE ):
    if not quiet:
      print( "Calling vswhere -latest -installationVersion" )
    latest_full_v = subprocess.check_output(
      [ VSWHERE_EXE, '-latest', '-property', 'installationVersion' ]
    ).strip().decode()
    if '.' in latest_full_v:
      try:
        latest_v = int( latest_full_v.split( '.' )[ 0 ] )
      except ValueError:
        raise ValueError( f"{latest_full_v} is not a version number." )

      if not quiet:
        print( f'vswhere -latest returned version {latest_full_v}' )

      if latest_v not in ACCEPTABLE_VERSIONS:
        if latest_v > 17:
          if not quiet:
            print( f'MSVC Version {latest_full_v} is newer than expected.' )
        else:
          raise ValueError(
            f'vswhere returned {latest_full_v} which is unexpected.'
            'Pass --msvc <version> argument.' )
      return latest_v
    else:
      if not quiet:
        print( f'vswhere returned nothing usable, {latest_full_v}' )

  # Fall back to registry parsing, which works at least until MSVC 2019 (16)
  # but is likely failing on MSVC 2022 (17)
  if not quiet:
    print( "vswhere method failed, falling back to searching the registry" )

  import winreg
  handle = winreg.ConnectRegistry( None, winreg.HKEY_LOCAL_MACHINE )
  msvc = None
  for i in ACCEPTABLE_VERSIONS:
    if not quiet:
      print( 'Trying to find '
             rf'HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\{i}.0' )
    try:
      winreg.OpenKey( handle, rf'SOFTWARE\Microsoft\VisualStudio\{i}.0' )
      if not quiet:
        print( f"Found MSVC version {i}" )
      msvc = i
      break
    except FileNotFoundError:
      pass
  return msvc


def RemoveDirectory( directory ):
  try_number = 0
  max_tries = 10
  while try_number < max_tries:
    try:
      shutil.rmtree( directory )
      return
    except OSError:
      try_number += 1
  raise RuntimeError(
    f'Cannot remove directory { directory } after { max_tries } tries.' )



def RemoveDirectoryIfExists( directory_path ):
  if p.exists( directory_path ):
    RemoveDirectory( directory_path )


def MakeCleanDirectory( directory_path ):
  RemoveDirectoryIfExists( directory_path )
  os.makedirs( directory_path )


def CheckFileIntegrity( file_path, check_sum ):
  with open( file_path, 'rb' ) as existing_file:
    existing_sha256 = hashlib.sha256( existing_file.read() ).hexdigest()
  return existing_sha256 == check_sum


def DownloadFileTo( download_url, file_path ):
  with urllib.request.urlopen( download_url ) as response:
    with open( file_path, 'wb' ) as package_file:
      package_file.write( response.read() )


def OnMac():
  return platform.system() == 'Darwin'


def OnWindows():
  return platform.system() == 'Windows'


def OnAArch64():
  return platform.machine().lower().startswith( 'aarch64' )


def OnArm():
  return platform.machine().lower().startswith( 'arm' )


def OnX86_64():
  return platform.machine().lower().startswith( 'x86_64' )


def FindExecutableOrDie( executable, message ):
  path = FindExecutable( executable )

  if not path:
    raise InstallationFailed(
      f"ERROR: Unable to find executable '{ executable }'. { message }" )

  return path


# On Windows, distutils.spawn.find_executable only works for .exe files
# but .bat and .cmd files are also executables, so we use our own
# implementation.
def FindExecutable( executable ):
  # Executable extensions used on Windows
  WIN_EXECUTABLE_EXTS = [ '.exe', '.bat', '.cmd' ]

  paths = os.environ[ 'PATH' ].split( os.pathsep )
  base, extension = os.path.splitext( executable )

  if OnWindows() and extension.lower() not in WIN_EXECUTABLE_EXTS:
    extensions = WIN_EXECUTABLE_EXTS
  else:
    extensions = [ '' ]

  for extension in extensions:
    executable_name = executable + extension
    if not os.path.isfile( executable_name ):
      for path in paths:
        executable_path = p.join( path, executable_name )
        if os.path.isfile( executable_path ):
          return executable_path
    else:
      return executable_name
  return None


def PathToFirstExistingExecutable( executable_name_list ):
  for executable_name in executable_name_list:
    path = FindExecutable( executable_name )
    if path:
      return path
  return None


def NumCores():
  ycm_cores = os.environ.get( 'YCM_CORES' )
  if ycm_cores:
    return int( ycm_cores )
  try:
    return multiprocessing.cpu_count()
  except NotImplementedError:
    return 1


def CheckCall( args, **kwargs ):
  quiet = kwargs.pop( 'quiet', False )
  status_message = kwargs.pop( 'status_message', None )

  if quiet:
    _CheckCallQuiet( args, status_message, **kwargs )
  else:
    _CheckCall( args, **kwargs )


def _CheckCallQuiet( args, status_message, **kwargs ):
  if status_message:
    print( status_message + '...', flush = True, end = '' )

  with tempfile.NamedTemporaryFile() as temp_file:
    _CheckCall( args, stdout=temp_file, stderr=subprocess.STDOUT, **kwargs )

  if status_message:
    print( "OK" )


def _CheckCall( args, **kwargs ):
  exit_message = kwargs.pop( 'exit_message', None )
  stdout = kwargs.get( 'stdout', None )
  on_failure = kwargs.pop( 'on_failure', None )

  try:
    subprocess.check_call( args, **kwargs )
  except subprocess.CalledProcessError as error:
    if stdout is not None:
      stdout.seek( 0 )
      print( stdout.read().decode( 'utf-8' ) )
      print( "FAILED" )

    if on_failure:
      on_failure( exit_message, error.returncode )
    elif exit_message:
      raise InstallationFailed( exit_message )
    else:
      raise InstallationFailed( exit_code = error.returncode )


def GetGlobalPythonPrefix():
  # In a virtualenv, sys.real_prefix points to the parent Python prefix.
  if hasattr( sys, 'real_prefix' ):
    return sys.real_prefix
  # In a pyvenv (only available on Python 3), sys.base_prefix points to the
  # parent Python prefix. Outside a pyvenv, it is equal to sys.prefix.
  return sys.base_prefix


def GetPossiblePythonLibraryDirectories():
  prefix = GetGlobalPythonPrefix()

  if OnWindows() and not IS_MSYS:
    return [ p.join( prefix, 'libs' ) ]
  # On pyenv and some distributions, there is no Python dynamic library in the
  # directory returned by the LIBPL variable. Such library can be found in the
  # "lib" or "lib64" folder of the base Python installation.
  return [
    sysconfig.get_config_var( 'LIBPL' ),
    p.join( prefix, 'lib64' ),
    p.join( prefix, 'lib/64' ),
    p.join( prefix, 'lib' )
  ]


def FindPythonLibraries():
  include_dir = sysconfig.get_config_var( 'INCLUDEPY' )
  if not p.isfile( p.join( include_dir, 'Python.h' ) ):
    raise InstallationFailed(
      NO_PYTHON_HEADERS_ERROR.format( include_dir = include_dir ) )

  library_dirs = GetPossiblePythonLibraryDirectories()

  # Since ycmd is compiled as a dynamic library, we can't link it to a Python
  # static library. If we try, the following error will occur on Mac:
  #
  #   Fatal Python error: PyThreadState_Get: no current thread
  #
  # while the error happens during linking on Linux and looks something like:
  #
  #   relocation R_X86_64_32 against `a local symbol' can not be used when
  #   making a shared object; recompile with -fPIC
  #
  # On Windows, the Python library is always a dynamic one (an import library to
  # be exact). To obtain a dynamic library on other platforms, Python must be
  # compiled with the --enable-shared flag on Linux or the --enable-framework
  # flag on Mac.
  #
  # So we proceed like this:
  #  - look for a dynamic library and return its path;
  #  - if a static library is found instead, raise an error with instructions
  #    on how to build Python as a dynamic library.
  #  - if no libraries are found, raise a generic error.
  dynamic_name = re.compile( DYNAMIC_PYTHON_LIBRARY_REGEX.format(
    major = PY_MAJOR, minor = PY_MINOR ), re.X )
  static_name = re.compile( STATIC_PYTHON_LIBRARY_REGEX.format(
    major = PY_MAJOR, minor = PY_MINOR ), re.X )
  static_libraries = []

  for library_dir in library_dirs:
    if not p.exists( library_dir ):
      continue

    # Files are sorted so that we found the non-versioned Python library before
    # the versioned one.
    for filename in sorted( os.listdir( library_dir ) ):
      if dynamic_name.match( filename ):
        return p.join( library_dir, filename ), include_dir

      if static_name.match( filename ):
        static_libraries.append( p.join( library_dir, filename ) )

  if static_libraries and not OnWindows():
    dynamic_flag = ( '--enable-framework' if OnMac() else
                     '--enable-shared' )
    raise InstallationFailed(
      NO_DYNAMIC_PYTHON_ERROR.format( library = static_libraries[ 0 ],
                                      flag = dynamic_flag ) )

  raise InstallationFailed( NO_PYTHON_LIBRARY_ERROR )


def CustomPythonCmakeArgs( args ):
  # The CMake 'FindPythonLibs' Module does not work properly.
  # So we are forced to do its job for it.
  if not args.quiet:
    print( f'Searching Python { PY_MAJOR }.{ PY_MINOR } libraries...' )

  python_library, python_include = FindPythonLibraries()

  if not args.quiet:
    print( f'Found Python library: { python_library }' )
    print( f'Found Python headers folder: { python_include }' )

  return [
    f'-DPython3_LIBRARY={ python_library }',
    f'-DPython3_EXECUTABLE={ sys.executable }',
    f'-DPython3_INCLUDE_DIR={ python_include }'
  ]


def GetGenerator( args ):
  if args.ninja:
    return 'Ninja'
  if OnWindows() and not IS_MSYS:
    # The architecture must be specified through the -A option for the Visual
    # Studio 16 generator.
    if args.msvc == 16:
      return 'Visual Studio 16'
    if args.msvc == 17:
      return 'Visual Studio 17 2022'
    return f"Visual Studio { args.msvc }{ ' Win64' if IS_64BIT else '' }"
  return 'Unix Makefiles'


def ParseArguments():
  parser = argparse.ArgumentParser()
  parser.add_argument( '--clang-completer', action = 'store_true',
                       help = 'Enable C-family semantic completion engine '
                              'through libclang.' )
  parser.add_argument( '--clangd-completer', action = 'store_true',
                       help = 'Enable C-family semantic completion engine '
                              'through clangd lsp server.(EXPERIMENTAL)' )
  parser.add_argument( '--cs-completer', action = 'store_true',
                       help = 'Enable C# semantic completion engine.' )
  parser.add_argument( '--go-completer', action = 'store_true',
                       help = 'Enable Go semantic completion engine.' )
  parser.add_argument( '--rust-completer', action = 'store_true',
                       help = 'Enable Rust semantic completion engine.' )
  parser.add_argument( '--rust-toolchain-version',
                       action = 'store',
                       default = DEFAULT_RUST_TOOLCHAIN,
                       help = 'For advanced users ***NO SUPPORT***, specify '
                              'the rust toolchain version to install from '
                              'rustup. Only the default vaule "' +
                               DEFAULT_RUST_TOOLCHAIN + '" is tested/'
                              'supported by the maintainers of YCM/ycmd' )
  parser.add_argument( '--java-completer', action = 'store_true',
                       help = 'Enable Java semantic completion engine.' ),
  parser.add_argument( '--ts-completer', action = 'store_true',
                       help = 'Enable JavaScript and TypeScript semantic '
                              'completion engine.' ),
  parser.add_argument( '--system-libclang', action = 'store_true',
                       help = 'Use system libclang instead of downloading one '
                       'from llvm.org. NOT RECOMMENDED OR SUPPORTED!' )
  if OnWindows():
    parser.add_argument( '--msvc', type = int, choices = [ 15, 16, 17 ],
                          default=None,
                          help= 'Choose the Microsoft Visual Studio version '
                                '(default: %(default)s).' )
  parser.add_argument( '--ninja', action = 'store_true',
                       help = 'Use Ninja build system.' )
  parser.add_argument( '--all',
                       action = 'store_true',
                       help   = 'Enable all supported completers',
                       dest   = 'all_completers' )
  parser.add_argument( '--enable-coverage',
                       action = 'store_true',
                       help   = 'For developers: Enable gcov coverage for the '
                                'c++ module' )
  parser.add_argument( '--enable-debug',
                       action = 'store_true',
                       help   = 'For developers: build ycm_core library with '
                                'debug symbols' )
  parser.add_argument( '--build-dir',
                       help   = 'For developers: perform the build in the '
                                'specified directory, and do not delete the '
                                'build output. This is useful for incremental '
                                'builds, and required for coverage data' )

  # Historically, "verbose" mode was the default and --quiet was added. Now,
  # quiet is the default (but the argument is still allowed, to avoid breaking
  # scripts), and --verbose is added to get the full output.
  parser.add_argument( '--quiet',
                       action = 'store_true',
                       default = True, # This argument is deprecated
                       help = 'Quiet installation mode. Just print overall '
                              'progress and errors. This is the default, so '
                              'this flag is actually ignored. Ues --verbose '
                              'to see more output.' )
  parser.add_argument( '--verbose',
                       action = 'store_false',
                       dest = 'quiet',
                       help = 'Verbose installation mode; prints output from '
                              'build operations. Useful for debugging '
                              'build failures.' )

  parser.add_argument( '--skip-build',
                       action = 'store_true',
                       help = "Don't build ycm_core lib, just install deps" )
  parser.add_argument( '--valgrind',
                       action = 'store_true',
                       help = 'For developers: '
                              'Run core tests inside valgrind.' )
  parser.add_argument( '--clang-tidy',
                       action = 'store_true',
                       help = 'For developers: Run clang-tidy static analysis '
                              'on the ycm_core code itself.' )
  parser.add_argument( '--core-tests', nargs = '?', const = '*',
                       help = 'Run core tests and optionally filter them.' )
  parser.add_argument( '--cmake-path',
                       help = 'For developers: specify the cmake executable. '
                              'Useful for testing with specific versions, or '
                              'if the system is unable to find cmake.' )
  parser.add_argument( '--force-sudo',
                       action = 'store_true',
                       help = 'Compiling with sudo causes problems. If you'
                              ' know what you are doing, proceed.' )

  # These options are deprecated.
  parser.add_argument( '--omnisharp-completer', action = 'store_true',
                       help = argparse.SUPPRESS )
  parser.add_argument( '--gocode-completer', action = 'store_true',
                       help = argparse.SUPPRESS )
  parser.add_argument( '--racer-completer', action = 'store_true',
                       help = argparse.SUPPRESS )
  parser.add_argument( '--tern-completer', action = 'store_true',
                       help = argparse.SUPPRESS )
  parser.add_argument( '--js-completer', action = 'store_true',
                       help = argparse.SUPPRESS )

  args = parser.parse_args()

  # coverage is not supported for c++ on MSVC
  if not OnWindows() and args.enable_coverage:
    # We always want a debug build when running with coverage enabled
    args.enable_debug = True

  if OnWindows() and args.msvc is None:
    args.msvc = FindLatestMSVC( args.quiet )
    if args.msvc is None:
      raise FileNotFoundError( "Could not find a valid MSVC version." )

  if args.core_tests:
    os.environ[ 'YCM_TESTRUN' ] = '1'
  elif os.environ.get( 'YCM_TESTRUN' ):
    args.core_tests = '*'

  if not args.clang_tidy and os.environ.get( 'YCM_CLANG_TIDY' ):
    args.clang_tidy = True

  if ( args.system_libclang and
       not args.clang_completer and
       not args.all_completers ):
    raise InstallationFailed(
      'ERROR: you can\'t pass --system-libclang without also passing '
      '--clang-completer or --all as well.' )
  return args


def FindCmake( args ):
  cmake_exe = [ 'cmake3', 'cmake' ]

  if args.cmake_path:
    cmake_exe.insert( 0, args.cmake_path )

  cmake = PathToFirstExistingExecutable( cmake_exe )
  if cmake is None:
    raise InstallationFailed(
      "ERROR: Unable to find cmake executable in any of"
      f" { cmake_exe }. CMake is required to build ycmd" )
  return cmake


def GetCmakeCommonArgs( args ):
  cmake_args = [ '-G', GetGenerator( args ) ]

  # Set the architecture for the Visual Studio 16/17 generator.
  if OnWindows() and args.msvc >= 16 and not args.ninja and not IS_MSYS:
    arch = 'x64' if IS_64BIT else 'Win32'
    cmake_args.extend( [ '-A', arch ] )

  cmake_args.extend( CustomPythonCmakeArgs( args ) )

  return cmake_args


def GetCmakeArgs( parsed_args ):
  cmake_args = []
  if parsed_args.clang_completer or parsed_args.all_completers:
    cmake_args.append( '-DUSE_CLANG_COMPLETER=ON' )

  if parsed_args.clang_tidy:
    cmake_args.append( '-DUSE_CLANG_TIDY=ON' )

  if parsed_args.system_libclang:
    cmake_args.append( '-DUSE_SYSTEM_LIBCLANG=ON' )

  if parsed_args.enable_debug:
    cmake_args.append( '-DCMAKE_BUILD_TYPE=Debug' )
    cmake_args.append( '-DUSE_DEV_FLAGS=ON' )
  else:
    cmake_args.append( '-DCMAKE_BUILD_TYPE=Release' )

  # coverage is not supported for c++ on MSVC
  if not OnWindows() and parsed_args.enable_coverage:
    cmake_args.append( '-DCMAKE_CXX_FLAGS=-coverage' )

  extra_cmake_args = os.environ.get( 'EXTRA_CMAKE_ARGS', '' )
  # We use shlex split to properly parse quoted CMake arguments.
  cmake_args.extend( shlex.split( extra_cmake_args ) )
  return cmake_args


def RunYcmdTests( args, build_dir ):
  tests_dir = p.join( build_dir, 'ycm', 'tests' )
  new_env = os.environ.copy()

  if OnWindows():
    # We prepend the ycm_core and Clang third-party directories to the PATH
    # instead of overwriting it so that the executable is able to find the
    # Python library.
    new_env[ 'PATH' ] = ( DIR_OF_THIS_SCRIPT + ';' +
                          LIBCLANG_DIR + ';' +
                          new_env[ 'PATH' ] )
  else:
    new_env[ 'LD_LIBRARY_PATH' ] = LIBCLANG_DIR

  tests_cmd = [ p.join( tests_dir, 'ycm_core_tests' ), '--gtest_brief' ]
  if args.core_tests != '*':
    tests_cmd.append( f'--gtest_filter={ args.core_tests }' )
  if args.valgrind:
    new_env[ 'PYTHONMALLOC' ] = 'malloc'
    tests_cmd = [ 'valgrind',
            '--gen-suppressions=all',
            '--error-exitcode=1',
            '--leak-check=full',
            '--show-leak-kinds=definite,indirect',
            '--errors-for-leak-kinds=definite,indirect',
            '--suppressions=' + p.join( DIR_OF_THIS_SCRIPT,
                                        'valgrind.suppressions' ) ] + tests_cmd
  CheckCall( tests_cmd,
      env = new_env,
      quiet = args.quiet,
      status_message = 'Running ycmd tests' )


def RunYcmdBenchmarks( args, build_dir ):
  benchmarks_dir = p.join( build_dir, 'ycm', 'benchmarks' )
  new_env = os.environ.copy()

  if OnWindows():
    # We prepend the ycm_core and Clang third-party directories to the PATH
    # instead of overwriting it so that the executable is able to find the
    # Python library.
    new_env[ 'PATH' ] = ( DIR_OF_THIS_SCRIPT + ';' +
                          LIBCLANG_DIR + ';' +
                          new_env[ 'PATH' ] )
  else:
    new_env[ 'LD_LIBRARY_PATH' ] = LIBCLANG_DIR

  # Note we don't pass the quiet flag here because the output of the benchmark
  # is the only useful info.
  CheckCall( p.join( benchmarks_dir, 'ycm_core_benchmarks' ), env = new_env )


# On Windows, if the ycmd library is in use while building it, a LNK1104
# fatal error will occur during linking. Exit the script early with an
# error message if this is the case.
def ExitIfYcmdLibInUseOnWindows():
  if not OnWindows():
    return

  ycmd_library = p.join( DIR_OF_THIS_SCRIPT, 'ycm_core.pyd' )

  if not p.exists( ycmd_library ):
    return

  try:
    open( p.join( ycmd_library ), 'a' ).close()
  except IOError as error:
    if error.errno == errno.EACCES:
      raise InstallationFailed( 'ERROR: ycmd library is currently in use. '
                                'Stop all ycmd instances before compilation.' )


def GetCMakeBuildConfiguration( args ):
  if OnWindows():
    if args.enable_debug:
      return [ '--config', 'Debug' ]
    return [ '--config', 'Release' ]
  return [ '--', '-j', str( NumCores() ) ]


def BuildYcmdLib( cmake, cmake_common_args, script_args ):
  if script_args.build_dir:
    build_dir = os.path.abspath( script_args.build_dir )
    if not os.path.exists( build_dir ):
      os.makedirs( build_dir )
  else:
    build_dir = mkdtemp( prefix = 'ycm_build_' )

  try:
    os.chdir( build_dir )

    configure_command = ( [ cmake ] + cmake_common_args +
                          GetCmakeArgs( script_args ) )
    configure_command.append( p.join( DIR_OF_THIS_SCRIPT, 'cpp' ) )

    CheckCall( configure_command,
               exit_message = BUILD_ERROR_MESSAGE,
               quiet = script_args.quiet,
               status_message = 'Generating ycmd build configuration' )

    build_targets = [ 'ycm_core' ]
    if script_args.core_tests:
      build_targets.append( 'ycm_core_tests' )
    if 'YCM_BENCHMARK' in os.environ:
      build_targets.append( 'ycm_core_benchmarks' )

    build_config = GetCMakeBuildConfiguration( script_args )

    for target in build_targets:
      build_command = ( [ cmake, '--build', '.', '--target', target ] +
                        build_config )
      CheckCall( build_command,
                 exit_message = BUILD_ERROR_MESSAGE,
                 quiet = script_args.quiet,
                 status_message = f'Compiling ycmd target: { target }' )

    if script_args.core_tests:
      RunYcmdTests( script_args, build_dir )
    if 'YCM_BENCHMARK' in os.environ:
      RunYcmdBenchmarks( script_args, build_dir )
  finally:
    os.chdir( DIR_OF_THIS_SCRIPT )

    if script_args.build_dir:
      print( 'The build files are in: ' + build_dir )
    else:
      RemoveDirectory( build_dir )


def BuildRegexModule( script_args ):
  build_dir = p.join( DIR_OF_THIRD_PARTY, 'regex-build', '3' )
  lib_dir = p.join( DIR_OF_THIRD_PARTY, 'regex-build' )

  try:
    os.chdir( p.join( DIR_OF_THIRD_PARTY, 'mrab-regex' ) )

    RemoveDirectoryIfExists( build_dir )
    RemoveDirectoryIfExists( lib_dir )

    try:
      import setuptools # noqa
      CheckCall( [ sys.executable,
                   'setup.py',
                   'build',
                   '--build-base=' + build_dir,
                   '--build-lib=' + lib_dir ],
                 exit_message = 'Failed to build regex module.',
                 quiet = script_args.quiet,
                 status_message = 'Building regex module' )
    except ImportError:
      pass # Swallow the error - ycmd will fall back to the standard `re`.

  finally:
    RemoveDirectoryIfExists( build_dir )
    os.chdir( DIR_OF_THIS_SCRIPT )


def EnableCsCompleter( args ):
  def WriteStdout( text ):
    if not args.quiet:
      sys.stdout.write( text )
      sys.stdout.flush()

  if args.quiet:
    sys.stdout.write( 'Installing Omnisharp for C# support...' )
    sys.stdout.flush()

  build_dir = p.join( DIR_OF_THIRD_PARTY, "omnisharp-roslyn" )
  try:
    MkDirIfMissing( build_dir )
    os.chdir( build_dir )

    download_data = GetCsCompleterDataForPlatform()
    version = download_data[ 'version' ]

    WriteStdout( f"Installing Omnisharp { version }\n" )

    CleanCsCompleter( build_dir, version )
    package_path = DownloadCsCompleter( WriteStdout, download_data )
    ExtractCsCompleter( WriteStdout, build_dir, package_path )

    WriteStdout( "Done installing Omnisharp\n" )

    if args.quiet:
      print( 'OK' )
  finally:
    os.chdir( DIR_OF_THIS_SCRIPT )


def MkDirIfMissing( path ):
  try:
    os.mkdir( path )
  except OSError:
    pass


def CleanCsCompleter( build_dir, version ):
  for file_name in os.listdir( build_dir ):
    file_path = p.join( build_dir, file_name )
    if file_name == version:
      continue
    if os.path.isfile( file_path ):
      os.remove( file_path )
    elif os.path.isdir( file_path ):
      import shutil
      shutil.rmtree( file_path )


def DownloadCsCompleter( writeStdout, download_data ):
  file_name = download_data[ 'file_name' ]
  download_url = download_data[ 'download_url' ]
  check_sum = download_data[ 'check_sum' ]
  version = download_data[ 'version' ]

  MkDirIfMissing( version )

  package_path = p.join( version, file_name )
  if ( p.exists( package_path )
       and not CheckFileIntegrity( package_path, check_sum ) ):
    writeStdout( 'Cached Omnisharp file does not match checksum.\n' )
    writeStdout( 'Removing...' )
    os.remove( package_path )
    writeStdout( 'DONE\n' )

  if p.exists( package_path ):
    writeStdout( f'Using cached Omnisharp: { file_name }\n' )
  else:
    writeStdout( f'Downloading Omnisharp from { download_url }...' )
    DownloadFileTo( download_url, package_path )
    writeStdout( 'DONE\n' )

  return package_path


def ExtractCsCompleter( writeStdout, build_dir, package_path ):
  writeStdout( f'Extracting Omnisharp to { build_dir }...' )
  if OnWindows():
    with ZipFile( package_path, 'r' ) as package_zip:
      package_zip.extractall()
  else:
    with tarfile.open( package_path ) as package_tar:
      package_tar.extractall()
  writeStdout( 'DONE\n' )


def GetCsCompleterDataForPlatform():
  ####################################
  # GENERATED BY update_omnisharp.py #
  # DON'T MANUALLY EDIT              #
  ####################################
  DATA = {
    'win32': {
      'version': 'v1.37.11',
      'download_url': ( 'https://github.com/OmniSharp/omnisharp-roslyn/release'
                        's/download/v1.37.11/omnisharp.http-win-x86.zip' ),
      'file_name': 'omnisharp.http-win-x86.zip',
      'check_sum': ( '461544056b144c97e8413de8c1aa1ddd9e2902f5a9f2223af8046d65'
                     '4d95f2a0' ),
    },
    'win64': {
      'version': 'v1.37.11',
      'download_url': ( 'https://github.com/OmniSharp/omnisharp-roslyn/release'
                        's/download/v1.37.11/omnisharp.http-win-x64.zip' ),
      'file_name': 'omnisharp.http-win-x64.zip',
      'check_sum': ( '7f6f0abfac00d028b90aaf1f56813e4fbb73d84bdf2c4704862aa976'
                     '1b61a59c' ),
    },
    'macos': {
      'version': 'v1.37.11',
      'download_url': ( 'https://github.com/OmniSharp/omnisharp-roslyn/release'
                        's/download/v1.37.11/omnisharp.http-osx.tar.gz' ),
      'file_name': 'omnisharp.http-osx.tar.gz',
      'check_sum': ( '84b84a8a3cb8fd3986ea795d9230457c43bf130b482fcb77fef57c67'
                     'e151828a' ),
    },
    'linux32': {
      'version': 'v1.37.11',
      'download_url': ( 'https://github.com/OmniSharp/omnisharp-roslyn/release'
                        's/download/v1.37.11/omnisharp.http-linux-x86.tar.gz' ),
      'file_name': 'omnisharp.http-linux-x86.tar.gz',
      'check_sum': ( 'a5ab39380a5d230c75f08bf552980cdc5bd8c31a43348acbfa66f1f4'
                     '6f12851f' ),
    },
    'linux64': {
      'version': 'v1.37.11',
      'download_url': ( 'https://github.com/OmniSharp/omnisharp-roslyn/release'
                        's/download/v1.37.11/omnisharp.http-linux-x64.tar.gz' ),
      'file_name': 'omnisharp.http-linux-x64.tar.gz',
      'check_sum': ( '9a6e9a246babd777229eebb57f0bee86e7ef5da271c67275eae5ed9d'
                     '7b0ad563' ),
    },
  }
  if OnWindows():
    return DATA[ 'win64' if IS_64BIT else 'win32' ]
  else:
    if OnMac():
      return DATA[ 'macos' ]
    return DATA[ 'linux64' if IS_64BIT else 'linux32' ]


def EnableGoCompleter( args ):
  go = FindExecutableOrDie( 'go', 'go is required to build gopls.' )

  new_env = os.environ.copy()
  new_env[ 'GO111MODULE' ] = 'on'
  new_env[ 'GOPATH' ] = p.join( DIR_OF_THIS_SCRIPT, 'third_party', 'go' )
  new_env.pop( 'GOROOT', None )
  new_env[ 'GOBIN' ] = p.join( new_env[ 'GOPATH' ], 'bin' )

  gopls = 'golang.org/x/tools/gopls@v0.13.2'
  CheckCall( [ go, 'install', gopls ],
             env = new_env,
             quiet = args.quiet,
             status_message = 'Building gopls for go completion',
             on_failure = lambda msg, code: CheckCall(
               [ go, 'get', gopls ],
               env = new_env,
               quiet = args.quiet,
               status_message = 'Trying legacy get get' ) )


def WriteToolchainVersion( version ):
  path = p.join( RUST_ANALYZER_DIR, 'TOOLCHAIN_VERSION' )
  with open( path, 'w' ) as f:
    f.write( version )


def ReadToolchainVersion():
  try:
    filepath = p.join( RUST_ANALYZER_DIR, 'TOOLCHAIN_VERSION' )
    with open( filepath ) as f:
      return f.read().strip()
  except OSError:
    return None


def RustToolchainNeedsRefresh( cur, req ):
  if cur == 'stable' and req == 'stable':
    return True

  return cur != req


def EnableRustCompleter( switches ):
  cur_toolchain_version = ReadToolchainVersion()
  req_toolchain_version = switches.rust_toolchain_version

  if switches.quiet:
    sys.stdout.write( f'Installing rust-analyzer "{req_toolchain_version}" '
                        'for Rust support...' )
    sys.stdout.flush()

  if RustToolchainNeedsRefresh( cur_toolchain_version, req_toolchain_version ):
    install_dir = mkdtemp( prefix = 'rust_install_' )

    new_env = os.environ.copy()
    new_env[ 'RUSTUP_HOME' ] = install_dir

    rustup_init = p.join( install_dir, 'rustup-init' )

    if OnWindows():
      rustup_cmd = [ rustup_init ]
      rustup_url = f"https://win.rustup.rs/{ 'x86_64' if IS_64BIT else 'i686' }"
    else:
      rustup_cmd = [ 'sh', rustup_init ]
      rustup_url = 'https://sh.rustup.rs'

    DownloadFileTo( rustup_url, rustup_init )

    new_env[ 'CARGO_HOME' ] = install_dir

    CheckCall( rustup_cmd + [ '-y',
                              '--default-toolchain', 'none',
                              '--no-modify-path' ],
               env = new_env,
               quiet = switches.quiet )

    rustup = p.join( install_dir, 'bin', 'rustup' )

    try:
      CheckCall( [ rustup, 'toolchain', 'install', req_toolchain_version ],
                 env = new_env,
                 quiet = switches.quiet )

      for component in [ 'rust-src',
                         'rust-analyzer-preview',
                         'rustfmt',
                         'clippy' ]:
        CheckCall( [ rustup, 'component', 'add', component,
                     '--toolchain', req_toolchain_version ],
                   env = new_env,
                   quiet = switches.quiet )

      toolchain_dir = subprocess.check_output(
        [ rustup, 'run', req_toolchain_version, 'rustc', '--print', 'sysroot' ],
        env = new_env
      ).rstrip().decode( 'utf8' )

      if p.exists( RUST_ANALYZER_DIR ):
        RemoveDirectory( RUST_ANALYZER_DIR )
      os.makedirs( RUST_ANALYZER_DIR )

      for folder in os.listdir( toolchain_dir ):
        shutil.move( p.join( toolchain_dir, folder ), RUST_ANALYZER_DIR )

      WriteToolchainVersion( req_toolchain_version )
    finally:
      RemoveDirectory( install_dir )

  if switches.quiet:
    print( 'OK' )


def EnableJavaScriptCompleter( args ):
  npm = FindExecutableOrDie( 'npm', 'npm is required to set up Tern.' )
  os.chdir( p.join( DIR_OF_THIS_SCRIPT, 'third_party', 'tern_runtime' ) )
  CheckCall( [ npm, 'install', '--production' ],
             quiet = args.quiet,
             status_message = 'Setting up Tern for JavaScript completion' )


def CheckJavaVersion( required_version ):
  java = FindExecutableOrDie(
    'java',
    f'java { required_version } is required to install JDT.LS' )
  java_version = None
  try:
    new_env = os.environ.copy()
    new_env.pop( 'JAVA_TOOL_OPTIONS', None )
    new_env.pop( '_JAVA_OPTIONS', None )
    java_version = int(
      subprocess.check_output(
        [ java, p.join( DIR_OF_THIS_SCRIPT, 'CheckJavaVersion.java' ) ],
        stderr=subprocess.STDOUT,
        env = new_env )
      .decode( 'utf-8' )
      .strip() )
  except subprocess.CalledProcessError:
    pass

  if java_version is None or java_version < required_version:
    print( f'\n\n*** WARNING ***: jdt.ls requires Java { required_version }.'
           ' You must set the option java_binary_path to point to a working '
           f'java { required_version }.\n\n' )


def EnableJavaCompleter( switches ):
  def Print( *args, **kwargs ):
    if not switches.quiet:
      print( *args, **kwargs )

  if switches.quiet:
    sys.stdout.write( 'Installing jdt.ls for Java support...' )
    sys.stdout.flush()

  CheckJavaVersion( 17 )

  TARGET = p.join( DIR_OF_THIRD_PARTY, 'eclipse.jdt.ls', 'target', )
  REPOSITORY = p.join( TARGET, 'repository' )
  CACHE = p.join( TARGET, 'cache' )

  JDTLS_SERVER_URL_FORMAT = ( 'https://download.eclipse.org/jdtls/milestones/'
                              '{jdtls_milestone}/'
                              '{jdtls_package_name}' )
  JDTLS_PACKAGE_NAME_FORMAT = ( 'jdt-language-server-{jdtls_milestone}-'
                                '{jdtls_build_stamp}.tar.gz' )

  package_name = JDTLS_PACKAGE_NAME_FORMAT.format(
      jdtls_milestone = JDTLS_MILESTONE,
      jdtls_build_stamp = JDTLS_BUILD_STAMP )
  url = JDTLS_SERVER_URL_FORMAT.format(
      jdtls_milestone = JDTLS_MILESTONE,
      jdtls_package_name = package_name )
  file_name = p.join( CACHE, package_name )

  MakeCleanDirectory( REPOSITORY )

  if not p.exists( CACHE ):
    os.makedirs( CACHE )
  elif p.exists( file_name ) and not CheckFileIntegrity( file_name,
                                                         JDTLS_SHA256 ):
    Print( 'Cached tar file does not match checksum. Removing...' )
    os.remove( file_name )


  if p.exists( file_name ):
    Print( f'Using cached jdt.ls: { file_name }' )
  else:
    Print( f"Downloading jdt.ls from { url }..." )
    DownloadFileTo( url, file_name )

  Print( f"Extracting jdt.ls to { REPOSITORY }..." )
  with tarfile.open( file_name ) as package_tar:
    package_tar.extractall( REPOSITORY )

  Print( "Done installing jdt.ls" )

  if switches.quiet:
    print( 'OK' )


def EnableTypeScriptCompleter( args ):
  RemoveDirectoryIfExists( p.join( DIR_OF_THIRD_PARTY, 'tsserver', 'bin' ) )
  RemoveDirectoryIfExists( p.join( DIR_OF_THIRD_PARTY, 'tsserver', 'lib' ) )
  npm = FindExecutableOrDie( 'npm', 'npm is required to install TSServer.' )
  os.chdir( p.join( DIR_OF_THIRD_PARTY, 'tsserver' ) )
  CheckCall( [ npm, 'install', '--production' ],
             quiet = args.quiet,
             status_message = 'Setting up TSserver for TypeScript completion' )


def GetClangdTarget():
  if OnWindows():
    return [
      ( 'clangd-{version}-win64',
        '66a1e4d527b451d1e9f21183416fd53ef7f395266bbf7fd74b470ec326d19c98' ),
      ( 'clangd-{version}-win32',
        'c4c351da9f528a2cfacbc669cfb656ef34791ed637aeed051274adf611f3ba5a' ) ]
  if OnMac():
    if OnArm():
      return [
        ( 'clangd-{version}-arm64-apple-darwin',
          '38b0335306193cfe7978af9b2bb9dffc48406739b23f19158e7f000f910df5b0' ) ]
    return [
      ( 'clangd-{version}-x86_64-apple-darwin',
        'e3dcbefda4a10d7e1e2f8ce8db820219d78ac48ade247048fc0c6a821105ca26' ) ]
  if OnAArch64():
    return [
      ( 'clangd-{version}-aarch64-linux-gnu',
        'a3074a5d3c955b3326881617d36438e2cf36140d8de4b5f7d98e73eda92797a8' ) ]
  if OnArm():
    return [
      None, # First list index is for 64bit archives. ARMv7 is 32bit only.
      ( 'clangd-{version}-armv7a-linux-gnueabihf',
        'f167c13d3741ad7869a6ee57621af2cb9c2477bb300ab2fac91ea64c19f8df43' ) ]
  if OnX86_64():
    return [
      ( 'clangd-{version}-x86_64-unknown-linux-gnu',
        '70a9cf4c9e288941f0193dbfe0ab164e1805b622c2df522ea7319dabdeae3b4c' ) ]
  raise InstallationFailed(
    CLANGD_BINARIES_ERROR_MESSAGE.format( version = CLANGD_VERSION,
                                          platform = 'this system' ) )


def DownloadClangd( printer ):
  CLANGD_DIR = p.join( DIR_OF_THIRD_PARTY, 'clangd', )
  CLANGD_CACHE_DIR = p.join( CLANGD_DIR, 'cache' )
  CLANGD_OUTPUT_DIR = p.join( CLANGD_DIR, 'output' )

  target = GetClangdTarget()
  target_name, check_sum = target[ not IS_64BIT ]
  target_name = target_name.format( version = CLANGD_VERSION )
  file_name = f'{ target_name }.tar.bz2'
  download_url = ( 'https://github.com/ycm-core/llvm/releases/download/'
                   f'{ CLANGD_VERSION }/{ file_name }' )

  file_name = p.join( CLANGD_CACHE_DIR, file_name )

  MakeCleanDirectory( CLANGD_OUTPUT_DIR )

  if not p.exists( CLANGD_CACHE_DIR ):
    os.makedirs( CLANGD_CACHE_DIR )
  elif p.exists( file_name ) and not CheckFileIntegrity( file_name, check_sum ):
    printer( 'Cached Clangd archive does not match checksum. Removing...' )
    os.remove( file_name )

  if p.exists( file_name ):
    printer( f'Using cached Clangd: { file_name }' )
  else:
    printer( f"Downloading Clangd from { download_url }..." )
    DownloadFileTo( download_url, file_name )
    if not CheckFileIntegrity( file_name, check_sum ):
      raise InstallationFailed(
        'ERROR: downloaded Clangd archive does not match checksum.' )

  printer( f"Extracting Clangd to { CLANGD_OUTPUT_DIR }..." )
  with tarfile.open( file_name ) as package_tar:
    package_tar.extractall( CLANGD_OUTPUT_DIR )

  printer( "Done installing Clangd" )


def EnableClangdCompleter( Args ):
  if Args.quiet:
    sys.stdout.write( 'Setting up Clangd completer...' )
    sys.stdout.flush()

  def Print( msg ):
    if not Args.quiet:
      print( msg )

  DownloadClangd( Print )

  if Args.quiet:
    print( 'OK' )

  if not Args.quiet:
    print( 'Clangd completer enabled. If you are using .ycm_extra_conf.py '
           'files, make sure they use Settings() instead of the old and '
           'deprecated FlagsForFile().' )


def WritePythonUsedDuringBuild():
  path = p.join( DIR_OF_THIS_SCRIPT, 'PYTHON_USED_DURING_BUILDING' )
  with open( path, 'w' ) as f:
    f.write( sys.executable )


def BuildWatchdogModule( script_args ):
  DIR_OF_WATCHDOG_DEPS = p.join( DIR_OF_THIRD_PARTY, 'watchdog_deps' )
  build_dir = p.join( DIR_OF_WATCHDOG_DEPS, 'watchdog', 'build', '3' )
  lib_dir = p.join( DIR_OF_WATCHDOG_DEPS, 'watchdog', 'build', 'lib3' )
  try:
    os.chdir( p.join( DIR_OF_WATCHDOG_DEPS, 'watchdog' ) )

    RemoveDirectoryIfExists( build_dir )
    RemoveDirectoryIfExists( lib_dir )

    try:
      import setuptools # noqa
      CheckCall( [ sys.executable,
                   'setup.py',
                   'build',
                   '--build-base=' + build_dir,
                   '--build-lib=' + lib_dir ],
                 exit_message = 'Failed to build watchdog module.',
                 quiet = script_args.quiet,
                 status_message = 'Building watchdog module' )
    except ImportError:
      if OnMac():
        print( 'WARNING: setuptools unavailable. Watchdog will fall back to '
               'the slower kqueue filesystem event API.\n'
               'To use the faster fsevents, install setuptools and '
               'rerun this script.' )
      os.makedirs( lib_dir )
      shutil.copytree( p.join( 'src', 'watchdog' ),
                       p.join( lib_dir, 'watchdog' ) )
  finally:
    RemoveDirectoryIfExists( build_dir )
    os.chdir( DIR_OF_THIS_SCRIPT )


def DoCmakeBuilds( args ):
  cmake = FindCmake( args )
  cmake_common_args = GetCmakeCommonArgs( args )

  ExitIfYcmdLibInUseOnWindows()
  BuildYcmdLib( cmake, cmake_common_args, args )
  WritePythonUsedDuringBuild()

  BuildRegexModule( args )
  BuildWatchdogModule( args )


def PrintReRunMessage():
  print( '',
         'The installation failed; please see above for the actual error. '
         'In order to get more information, please re-run the command, '
         'adding the --verbose flag. If you think this is a bug and you '
         'raise an issue, you MUST include the *full verbose* output.',
         '',
         'For example, run:' + ' '.join( shlex.quote( arg )
                                         for arg in [ sys.executable ] +
                                                    sys.argv +
                                                    [ '--verbose' ] ),
         '',
         file = sys.stderr,
         sep = '\n' )


def Main(): # noqa: C901
  args = ParseArguments()

  if not OnWindows() and os.geteuid() == 0:
    if args.force_sudo:
      print( 'Forcing build with root privileges. '
             'If it breaks, keep the pieces.' )
    else:
      sys.exit( 'This script should not be run with root privileges.' )

  try:
    if not args.skip_build:
      DoCmakeBuilds( args )
    if args.cs_completer or args.omnisharp_completer or args.all_completers:
      EnableCsCompleter( args )
    if args.go_completer or args.gocode_completer or args.all_completers:
      EnableGoCompleter( args )
    if args.js_completer or args.tern_completer or args.all_completers:
      EnableJavaScriptCompleter( args )
    if args.rust_completer or args.racer_completer or args.all_completers:
      EnableRustCompleter( args )
    if args.java_completer or args.all_completers:
      EnableJavaCompleter( args )
    if args.ts_completer or args.all_completers:
      EnableTypeScriptCompleter( args )
    if args.clangd_completer or args.all_completers:
      EnableClangdCompleter( args )
  except InstallationFailed as e:
    e.Print()
    if args.quiet:
      PrintReRunMessage()
    e.Exit()
  except Exception as e:
    if args.quiet:
      print( f"FAILED with exception { type( e ).__name__ }: { e }" )
      PrintReRunMessage()
    else:
      raise


if __name__ == '__main__':
  Main()