CMakeLists.txt

# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# The TOP cmake file, provides tensorflow build options, 
# as well we break the whole cmake into small pieces. 
# It's recommended to run cmake under VS2017 x64 prompt.
# ==============================================================================

# Minimum CMake required
cmake_minimum_required(VERSION 3.12)

# Project
project(tensorflow C CXX)

# Set C++14 as standard for the whole project
set(CMAKE_CXX_STANDARD 14)

# [CLEANUP] Not sure if this is needed(copied from Protobuf)
# CMake policies
cmake_policy(SET CMP0022 NEW)

# enable folder scope in ide
set_property(GLOBAL PROPERTY USE_FOLDERS ON)

# Options
option(tensorflow_VERBOSE "Enable for verbose output" OFF)

option(tensorflow_ENABLE_GRPC_SUPPORT "Enable gRPC support" ON)
option(tensorflow_ENABLE_HDFS_SUPPORT "Enable HDFS support" OFF)
option(tensorflow_ENABLE_JEMALLOC_SUPPORT "Enable jemalloc support" OFF)
option(tensorflow_BUILD_CC_EXAMPLE "Build the C++ tutorial example" ON)
option(tensorflow_BUILD_PYTHON_BINDINGS "Build the Python bindings" ON)
option(tensorflow_BUILD_ALL_KERNELS "Build all OpKernels" ON)
option(tensorflow_BUILD_CONTRIB_KERNELS "Build OpKernels from tensorflow/contrib/..." ON)
option(tensorflow_BUILD_CC_TESTS "Build cc unit tests " OFF)
option(tensorflow_BUILD_PYTHON_TESTS "Build python unit tests " OFF)
option(tensorflow_BUILD_MORE_PYTHON_TESTS "Build more python unit tests for contrib packages" OFF)
option(tensorflow_BUILD_SHARED_LIB "Build TensorFlow as a shared library" OFF)
option(tensorflow_OPTIMIZE_FOR_NATIVE_ARCH "Enable compiler optimizations for the native processor architecture(if available)" ON)
option(tensorflow_ENABLE_SNAPPY_SUPPORT "Enable SNAPPY compression support" ON)
option(tensorflow_DISABLE_EIGEN_FORCEINLINE "Disable forceinline, to speed up build on windows." OFF)

# SIMD, MKL and MKLDNN options
option(tensorflow_WIN_CPU_SIMD_OPTIONS "Enables CPU SIMD instructions" OFF)
option(tensorflow_ENABLE_MKL_SUPPORT "Enable Intel MKL support" OFF)
option(tensorflow_ENABLE_MKLDNN_SUPPORT "Enable Intel MKLDNN support, requires MKL enabled" OFF)

# GPU, CUDA and cuDNN options
option(tensorflow_ENABLE_GPU "Enable GPU support" OFF)
set(tensorflow_CUDA_VERSION "9.0" CACHE STRING "CUDA version to build against")
set(tensorflow_CUDNN_VERSION "7" CACHE STRING "cuDNN version to build against")

if(HAIKU)
	option(tensorflow_ENABLE_POSITION_INDEPENDENT_CODE "Enable PIE support" OFF)
else()
	option(tensorflow_ENABLE_POSITION_INDEPENDENT_CODE "Enable PIE support" ON)
endif()

if(WIN32)
  # BoringSSL is disabled for windows as it currently doesn't build with
  # MSBuild.(Ninja is required.)
  option(tensorflow_ENABLE_SSL_SUPPORT "Enable boringssl support" OFF)
else(WIN32)
  # [TODO]
  message(WARNING "Build on Linux or Mac hasn't been verified!")
  # BoringSSL is enabled for gRPC.
  option(tensorflow_ENABLE_SSL_SUPPORT "Enable boringssl support" ON)

  # Threads: defines CMAKE_THREAD_LIBS_INIT and adds -pthread compile option
  # for targets that link ${CMAKE_THREAD_LIBS_INIT}.
  find_package(Threads)

  # Options for linking CUDA/CUDNN libraries
  option(tensorflow_PATH_STATIC_LIB "Additional library search path for libcudnn_static.a, libnccl_static.a, libculibos.a" /usr/local/cuda/lib64/)
  option(tensorflow_CUDNN_INCLUDE "cudnn.h header install path" /usr/include/)
  if(NOT tensorflow_CUDNN_INCLUDE)
    # option's default value is OFF. Fill it with real default values
    set(tensorflow_CUDNN_INCLUDE /usr/include)
  endif(NOT tensorflow_CUDNN_INCLUDE)
  option(tensorflow_PATH_CUDNN_STATIC_LIB "Override PATH_STATIC_LIB for libcudnn_static.a" ${tensorflow_PATH_STATIC_LIB})
  if(NOT tensorflow_PATH_CUDNN_STATIC_LIB)
    # option's default value is OFF. Fill it with real default values
    set(tensorflow_PATH_CUDNN_STATIC_LIB ${tensorflow_PATH_STATIC_LIB})
  endif(NOT tensorflow_PATH_CUDNN_STATIC_LIB)
  option(tensorflow_PATH_NCCL_STATIC_LIB "Override PATH_STATIC_LIB for libnccl_static.a" ${tensorflow_PATH_STATIC_LIB})
  if(NOT tensorflow_PATH_NCCL_STATIC_LIB)
    # option's default value is OFF. Fill it with real default values
    set(tensorflow_PATH_NCCL_STATIC_LIB ${tensorflow_PATH_STATIC_LIB})
  endif(NOT tensorflow_PATH_NCCL_STATIC_LIB)
  option(tensorflow_CUDA_LIBRARY_PATH "Designate the default CUDA library paths" /usr/local/cuda/lib64)
  if(NOT tensorflow_CUDA_LIBRARY_PATH)
    # option's default value is OFF. Fill it with real default values
    set(tensorflow_CUDA_LIBRARY_PATH /usr/local/cuda/lib64)
  endif(NOT tensorflow_CUDA_LIBRARY_PATH)

  # Options for linking other libraries
  option(systemlib_ZLIB "Use the system installed library as shared objects instead of downloading ZLIB and statically linking to it: ZLIB" OFF)

  option(systemlib_ALL "Turn on every possible systemlib_* options" OFF)
  if(systemlib_ALL)
    set(systemlib_ZLIB ON)
  endif(systemlib_ALL)
endif(WIN32)

# External dependencies
list(APPEND CMAKE_MODULE_PATH
    ${tensorflow_SOURCE_DIR}/cmake
    ${tensorflow_SOURCE_DIR}/cmake/external)
# Location where external projects will be downloaded
set(DOWNLOAD_LOCATION "${CMAKE_BINARY_DIR}/downloads"
    CACHE PATH "Location where external projects will be downloaded.")
mark_as_advanced(DOWNLOAD_LOCATION)
# External dependencies
include(zlib)
include(gif)
include(png)
include(jpeg)
include(lmdb)
include(eigen)
include(gemmlowp)
include(jsoncpp)
include(farmhash)
include(fft2d)
include(highwayhash)
include(nsync)
include(protobuf)
include(re2)
include(cub)
include(sqlite)
include(double_conversion)
include(swig)
if(tensorflow_BUILD_CC_TESTS)
  include(googletest)
endif()


# Set -fPIC flag for non-Windows
if(WIN32)
  set(BOOL_WIN32 ON)
else()
  set(BOOL_WIN32 OFF)
  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC")
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
endif()

if(tensorflow_ENABLE_POSITION_INDEPENDENT_CODE)
	set(CMAKE_POSITION_INDEPENDENT_CODE ON)
else()
	set(CMAKE_POSITION_INDEPENDENT_CODE OFF)
endif()

# TODO(jart): We should make this only apply to snapfn.cc
add_definitions(-DSQLITE_OMIT_LOAD_EXTENSION)

if(tensorflow_DISABLE_EIGEN_FORCEINLINE)
  add_definitions(-DEIGEN_STRONG_INLINE=inline)
endif()
add_definitions(-DEIGEN_AVOID_STL_ARRAY)

if(WIN32)
  if(CMAKE_SIZEOF_VOID_P EQUAL 8)
      # 64 bits
      add_definitions(-DWIN64)
  elseif(CMAKE_SIZEOF_VOID_P EQUAL 4)
      # 32 bits
      # temporary fix for #18241
      add_definitions(-DEIGEN_DEFAULT_DENSE_INDEX_TYPE=std::int64_t)
  endif()
  add_definitions(-DNOMINMAX -D_WIN32_WINNT=0x0A00)
  add_definitions(-DWIN32_LEAN_AND_MEAN -DNOGDI -DPLATFORM_WINDOWS)
  add_definitions(-DTENSORFLOW_USE_EIGEN_THREADPOOL -DEIGEN_HAS_C99_MATH)
  add_definitions(-DTF_COMPILE_LIBRARY)
  add_compile_options(/bigobj /GF /MP /Gm-)
  # Suppress warnings to reduce build log size.
  add_compile_options(
    /wd4267 /wd4244 /wd4800 /wd4503 /wd4554 /wd4996 /wd4348 /wd4018
    /wd4099 /wd4146 /wd4267 /wd4305 /wd4307 /wd4715 /wd4722 /wd4723
    /wd4838 /wd4309 /wd4334 /wd4003 /wd4244 /wd4267 /wd4503 /wd4506
    /wd4800 /wd4996 /wd4190
  )
  # Suppress linker warnings. (no effect on VS2017)
  set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /ignore:4049 /ignore:4197 /ignore:4217 /ignore:4221")
  set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} /ignore:4049 /ignore:4197 /ignore:4217 /ignore:4221")
  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /ignore:4049 /ignore:4197 /ignore:4217 /ignore:4221")
  set(CMAKE_CXX_FLAGS_DEBUG "/D_DEBUG /MDd /Ob2")
  set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /D_ITERATOR_DEBUG_LEVEL=0")
  set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} /D_ITERATOR_DEBUG_LEVEL=0")
  set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} /D_ITERATOR_DEBUG_LEVEL=0")

  set(compiler_flags
      CMAKE_CXX_FLAGS
      CMAKE_CXX_FLAGS_DEBUG
      CMAKE_CXX_FLAGS_RELEASE
      CMAKE_C_FLAGS
      CMAKE_C_FLAGS_DEBUG
      CMAKE_C_FLAGS_RELEASE
  )
  # No exception
  foreach(flag ${compiler_flags})
    string(REPLACE "/EHsc" "/EHs-c-" ${flag} "${${flag}}")
  endforeach()
  add_definitions(/D_HAS_EXCEPTIONS=0)
  # Suppress 'noexcept used with no exception handling mode specified' warning
  add_compile_options(/wd4577)

  # Try to avoid flaky failures due to failed generation of generate.stamp files.
  set(CMAKE_SUPPRESS_REGENERATION ON)
endif()

if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions -std=c++11")
endif()

include(CheckCXXCompilerFlag)

if(tensorflow_OPTIMIZE_FOR_NATIVE_ARCH)
  CHECK_CXX_COMPILER_FLAG("-march=native" COMPILER_OPT_ARCH_NATIVE_SUPPORTED)
  if(COMPILER_OPT_ARCH_NATIVE_SUPPORTED)
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
  endif()
endif()

# OpenMP Support
CHECK_CXX_COMPILER_FLAG("-fopenmp" GCC_OPENMP_SUPPORT)
if(GCC_OPENMP_SUPPORT)
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp")
endif()
CHECK_CXX_COMPILER_FLAG("/openmp" MSVC_OPENMP_SUPPORT)
if(MSVC_OPENMP_SUPPORT)
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /openmp")
endif()

# MSVC SIMD instructions
if(tensorflow_WIN_CPU_SIMD_OPTIONS)
  if(WIN32)
    CHECK_CXX_COMPILER_FLAG(${tensorflow_WIN_CPU_SIMD_OPTIONS} COMPILER_OPT_WIN_CPU_SIMD_SUPPORTED)
    if(COMPILER_OPT_WIN_CPU_SIMD_SUPPORTED)
      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${tensorflow_WIN_CPU_SIMD_OPTIONS}")
    endif()
  endif()
endif()

if(tensorflow_ENABLE_JEMALLOC_SUPPORT)
  add_definitions(-DTENSORFLOW_USE_JEMALLOC -DJEMALLOC_EXPORT=)
endif()

add_definitions(${ADD_CFLAGS})
link_directories(${ADD_LINK_DIRECTORY})

set(tensorflow_EXTERNAL_LIBRARIES
    ${gif_STATIC_LIBRARIES}
    ${png_STATIC_LIBRARIES}
    ${jpeg_STATIC_LIBRARIES}
    ${lmdb_STATIC_LIBRARIES}
    ${jsoncpp_STATIC_LIBRARIES}
    ${farmhash_STATIC_LIBRARIES}
    ${fft2d_STATIC_LIBRARIES}
    ${highwayhash_STATIC_LIBRARIES}
    ${nsync_STATIC_LIBRARIES}
    ${protobuf_STATIC_LIBRARIES}
    ${re2_STATIC_LIBRARIES}
    ${sqlite_STATIC_LIBRARIES}
    ${double_conversion_STATIC_LIBRARIES}
)

if(systemlib_ZLIB)
  set(tensorflow_EXTERNAL_LIBRARIES ${tensorflow_EXTERNAL_LIBRARIES}
      ${ZLIB_LIBRARIES})
else(systemlib_ZLIB)
  set(tensorflow_EXTERNAL_LIBRARIES ${tensorflow_EXTERNAL_LIBRARIES}
      ${zlib_STATIC_LIBRARIES})
endif(systemlib_ZLIB)

set(tensorflow_EXTERNAL_DEPENDENCIES
    zlib_copy_headers_to_destination
    gif_copy_headers_to_destination
    png_copy_headers_to_destination
    jpeg_copy_headers_to_destination
    lmdb_copy_headers_to_destination
    jsoncpp
    farmhash_copy_headers_to_destination
    highwayhash_copy_headers_to_destination
    nsync_copy_headers_to_destination
    eigen
    gemmlowp
    cub
    fft2d
    re2
    sqlite_copy_headers_to_destination
    double_conversion
)

include_directories(
    # Source and generated code.
    ${tensorflow_SOURCE_DIR}
    ${CMAKE_BINARY_DIR}
    # External dependencies.
    ${zlib_INCLUDE_DIR}
    ${gif_INCLUDE_DIR}
    ${png_INCLUDE_DIR}
    ${jpeg_INCLUDE_DIR}
    ${lmdb_INCLUDE_DIR}
    ${eigen_INCLUDE_DIRS}
    ${gemmlowp_INCLUDE_DIR}
    ${jsoncpp_INCLUDE_DIR}
    ${farmhash_INCLUDE_DIR}
    ${highwayhash_INCLUDE_DIR}
    ${cub_INCLUDE_DIR}
    ${nsync_INCLUDE_DIR}
    ${PROTOBUF_INCLUDE_DIRS}
    ${re2_INCLUDE_DIR}
    ${sqlite_INCLUDE_DIR}
    ${double_conversion_INCLUDE_DIR}
)

if(tensorflow_ENABLE_GRPC_SUPPORT)
  if(tensorflow_ENABLE_SSL_SUPPORT)
    include(boringssl)
    include_directories(${boringssl_INCLUDE_DIR})
  endif()
  include(grpc)
  include_directories(${GRPC_INCLUDE_DIRS})
  # Place boringssl after grpc as grpc depends on boringssl.
  list(APPEND tensorflow_EXTERNAL_LIBRARIES ${grpc_STATIC_LIBRARIES})
  list(APPEND tensorflow_EXTERNAL_DEPENDENCIES grpc)
  if(tensorflow_ENABLE_SSL_SUPPORT)
    list(APPEND tensorflow_EXTERNAL_LIBRARIES ${boringssl_STATIC_LIBRARIES})
    list(APPEND tensorflow_EXTERNAL_DEPENDENCIES boringssl)
  endif()
endif()

if(tensorflow_ENABLE_JEMALLOC_SUPPORT)
  include(jemalloc)
  list(APPEND tensorflow_EXTERNAL_LIBRARIES ${jemalloc_STATIC_LIBRARIES})
  list(APPEND tensorflow_EXTERNAL_DEPENDENCIES jemalloc)
  include_directories(${jemalloc_INCLUDE_DIRS})
endif()

if(tensorflow_ENABLE_SNAPPY_SUPPORT)
  include(snappy)
  list(APPEND tensorflow_EXTERNAL_LIBRARIES ${snappy_STATIC_LIBRARIES})
  list(APPEND tensorflow_EXTERNAL_DEPENDENCIES snappy)
  include_directories(${snappy_INCLUDE_DIR})
endif()

if(WIN32)
  # windows socket libraries
  list(APPEND tensorflow_EXTERNAL_LIBRARIES wsock32 ws2_32 shlwapi)
endif()
if(UNIX)
  list(APPEND tensorflow_EXTERNAL_LIBRARIES ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS})
endif()
if(HAIKU)
  list(APPEND tensorflow_EXTERNAL_LIBRARIES network)
endif()

# MKL Support
if(tensorflow_ENABLE_MKL_SUPPORT)
  add_definitions(-DINTEL_MKL -DEIGEN_USE_VML)
  include(mkl)
  list(APPEND tensorflow_EXTERNAL_LIBRARIES ${mkl_STATIC_LIBRARIES})
  list(APPEND tensorflow_EXTERNAL_DEPENDENCIES mkl_copy_shared_to_destination)
  include_directories(${mkl_INCLUDE_DIRS})
  if(tensorflow_ENABLE_MKLDNN_SUPPORT)
    include(mkldnn)
    list(APPEND tensorflow_EXTERNAL_LIBRARIES ${mkldnn_STATIC_LIBRARIES})
    list(APPEND tensorflow_EXTERNAL_DEPENDENCIES mkldnn_copy_shared_to_destination)
    include_directories(${mkldnn_INCLUDE_DIRS})
  else(tensorflow_ENABLE_MKLDNN_SUPPORT)
    add_definitions(-DINTEL_MKL_ML)
  endif(tensorflow_ENABLE_MKLDNN_SUPPORT)
endif(tensorflow_ENABLE_MKL_SUPPORT)

if(tensorflow_ENABLE_GPU)
  if(NOT WIN32)
    # Default install paths for cuda libraries in Linux
    # In some Linux distros, find_package(CUDA) seems to require CMAKE_LIBRARY_PATH to include cuda-lib paths
    list(APPEND CMAKE_LIBRARY_PATH "${tensorflow_CUDA_LIBRARY_PATH}")
    list(APPEND CMAKE_LIBRARY_PATH "${tensorflow_CUDA_LIBRARY_PATH}/stubs")
  endif(NOT WIN32)

  # later command will make use of the value in tensorflow_CUDA_VERSION
  find_package(CUDA ${tensorflow_CUDA_VERSION} REQUIRED EXACT)

  # Test compatibility of compiler on CUDA
  try_compile(CUDA_TEST_COMPILE_C
      ${CMAKE_BINARY_DIR}/tests/cuda
      ${CMAKE_SOURCE_DIR}/cmake/tests/cuda/compatibility_test.c
      CMAKE_FLAGS -DINCLUDE_DIRECTORIES=${CUDA_INCLUDE_DIRS})
  try_compile(CUDA_TEST_COMPILE_CXX
      ${CMAKE_BINARY_DIR}/tests/cuda
      ${CMAKE_SOURCE_DIR}/cmake/tests/cuda/compatibility_test.cc
      CMAKE_FLAGS -DINCLUDE_DIRECTORIES=${CUDA_INCLUDE_DIRS})
  if(NOT(CUDA_TEST_COMPILE_C AND CUDA_TEST_COMPILE_CXX))
    message(FATAL_ERROR "Selected compiler(or version) is not supported for CUDA")
  endif()

  # by default we assume compute cabability 3.5 and 5.2. If you change this change it in
  # CUDA_NVCC_FLAGS and cuda_config.h below
  set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};-gencode arch=compute_30,code=\"sm_30,compute_30\";-gencode arch=compute_35,code=\"sm_35,compute_35\";-gencode arch=compute_52,code=\"sm_52,compute_52\")
  set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};--include-path ${PROJECT_BINARY_DIR}/$\{build_configuration\};--expt-relaxed-constexpr)
  set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};-ftz=true)  # Flush denormals to zero
  set(CUDA_INCLUDE ${CUDA_TOOLKIT_TARGET_DIR} ${CUDA_TOOLKIT_TARGET_DIR}/extras/CUPTI/include)
  include_directories(${CUDA_INCLUDE})
  if(WIN32)
    add_definitions(-DGOOGLE_CUDA=1 -DTF_EXTRA_CUDA_CAPABILITIES=3.0,3.5,5.2)
  else(WIN32)
    # Without these double quotes, cmake in Linux makes it "-DTF_EXTRA_CUDA_CAPABILITIES=3.0, -D3.5, -D5.2" for cc, which incurs build breaks
    add_definitions(-DGOOGLE_CUDA=1 -D"TF_EXTRA_CUDA_CAPABILITIES=3.0,3.5,5.2")
  endif(WIN32)

  if(WIN32)
    # add cudnn
    if(NOT CUDNN_HOME)
      set(CUDNN_HOME ${CUDA_TOOLKIT_TARGET_DIR})
    endif(NOT CUDNN_HOME)
    set(CUDNN_INCLUDE "${CUDNN_HOME}/include")

    set(CUDA_LIBRARIES ${CUDA_LIBRARIES} ${CUDA_CUDA_LIBRARY} ${CUDA_CUBLAS_LIBRARIES} ${CUDA_CUFFT_LIBRARIES}
        ${CUDA_curand_LIBRARY} ${CUDA_cupti_LIBRARY} ${CUDA_cusolver_LIBRARY} ${CUDNN_HOME}/lib/x64/cudnn.lib)
  else(WIN32)
    set(CUDNN_INCLUDE "${tensorflow_CUDNN_INCLUDE}")

    find_library(nccl_STATIC_LIBRARY NAMES libnccl_static.a PATHS ${tensorflow_PATH_NCCL_STATIC_LIB} ${CUDA_TOOLKIT_ROOT_DIR})
    if(NOT nccl_STATIC_LIBRARY)
      message(FATAL_ERROR "NCCL is required for GPU-build")
    else(NOT nccl_STATIC_LIBRARY)
      message("nccl-static: ${nccl_STATIC_LIBRARY}")
      # something like /usr/lib64/libnccl_static.a
    endif(NOT nccl_STATIC_LIBRARY)

    find_library(cudnn_STATIC_LIBRARY NAMES libcudnn_static.a PATHS ${tensorflow_PATH_CUDNN_STATIC_LIB} ${CUDA_TOOLKIT_ROOT_DIR})
    if(NOT cudnn_STATIC_LIBRARY)
      message(FATAL_ERROR "CUDNN is required for GPU-build")
    else(NOT cudnn_STATIC_LIBRARY)
      message("cudnn-static: ${cudnn_STATIC_LIBRARY}")
    endif(NOT cudnn_STATIC_LIBRARY)

    find_library(culibos_STATIC_LIBRARY NAMES libculibos.a PATHS ${tensorflow_PATH_STATIC_LIB} ${CUDA_TOOLKIT_ROOT_DIR})
    if(NOT culibos_STATIC_LIBRARY)
      message(FATAL_ERROR "CULIBOS is required for GPU-build")
    else(NOT culibos_STATIC_LIBRARY)
      message("culibos-static: ${culibos_STATIC_LIBRARY}")
    endif(NOT culibos_STATIC_LIBRARY)

    set(CUDA_LIBRARIES ${CUDA_LIBRARIES} ${CUDA_CUDA_LIBRARY} ${CUDA_CUBLAS_LIBRARIES} ${CUDA_CUFFT_LIBRARIES}
        ${CUDA_curand_LIBRARY} ${CUDA_cupti_LIBRARY} ${CUDA_cusolver_LIBRARY} ${cudnn_STATIC_LIBRARY} ${culibos_STATIC_LIBRARY} ${nccl_STATIC_LIBRARY})
  endif(WIN32)
  include_directories(${CUDNN_INCLUDE})

  # Remove "." from CUDA version variable.
  string(REPLACE "." "" short_CUDA_VER ${tensorflow_CUDA_VERSION})

  # create cuda_config.h
  FILE(WRITE ${tensorflow_SOURCE_DIR}/third_party/gpus/cuda/cuda_config.h
      "#ifndef CUDA_CUDA_CONFIG_H_\n"
      "#define CUDA_CUDA_CONFIG_H_\n"
      "#define TF_CUDA_CAPABILITIES CudaVersion(\"3.0\"),CudaVersion(\"3.5\"),CudaVersion(\"5.2\")\n"
      "#define TF_CUDA_VERSION \"64_${short_CUDA_VER}\"\n"
      "#define TF_CUDNN_VERSION \"64_${tensorflow_CUDNN_VERSION}\"\n"
      "#define TF_CUDA_TOOLKIT_PATH \"${CUDA_TOOLKIT_ROOT_DIR}\"\n"
      "#endif  // CUDA_CUDA_CONFIG_H_\n"
  )

  # tf assumes in various places header files to be in cuda/include. On windows the cuda sdk
  # installs them under cuda/version/include and to avoid that we need to change tf we copy a
  # few files to cuda/include
  FILE(COPY
      ${CUDA_TOOLKIT_TARGET_DIR}/include/cuda.h
      ${CUDA_TOOLKIT_TARGET_DIR}/include/cuComplex.h
      ${CUDA_TOOLKIT_TARGET_DIR}/include/cublas_v2.h
      ${CUDA_TOOLKIT_TARGET_DIR}/include/cusolverDn.h
      ${CUDA_TOOLKIT_TARGET_DIR}/include/cuda_fp16.h
      ${CUDA_TOOLKIT_TARGET_DIR}/include/device_functions.h
      ${CUDA_TOOLKIT_TARGET_DIR}/include/cufft.h
      ${CUDA_TOOLKIT_TARGET_DIR}/include/curand.h
      ${CUDA_TOOLKIT_TARGET_DIR}/include/cuda_runtime_api.h
      ${CUDNN_INCLUDE}/cudnn.h
      DESTINATION ${tensorflow_SOURCE_DIR}/third_party/gpus/cuda/include
  )

  include_directories(${tensorflow_SOURCE_DIR}/third_party/gpus)
  # add cuda libraries to tensorflow_EXTERNAL_LIBRARIES
  list(APPEND tensorflow_EXTERNAL_LIBRARIES ${CUDA_LIBRARIES})
  if(NOT WIN32)
    # add gomp to tensorflow_EXTERNAL_LIBRARIES, needed by libcusolver.so
    list(APPEND tensorflow_EXTERNAL_LIBRARIES gomp)
  endif()

  # NOTE(mrry): Update these flags when the version of CUDA or cuDNN used
  # in the default build is upgraded.
  if(WIN32)
    set(tensorflow_BUILD_INFO_FLAGS --build_config cuda --key_value
        msvcp_dll_name=msvcp140.dll
        cudart_dll_name=cudart64_${short_CUDA_VER}.dll
        cuda_version_number=${tensorflow_CUDA_VERSION}
        nvcuda_dll_name=nvcuda.dll
        cudnn_dll_name=cudnn64_${tensorflow_CUDNN_VERSION}.dll
        cudnn_version_number=${tensorflow_CUDNN_VERSION})
  else(WIN32)
    set(tensorflow_BUILD_INFO_FLAGS --build_config cuda --key_value
        cuda_version_number=${tensorflow_CUDA_VERSION}
        cudnn_version_number=${tensorflow_CUDNN_VERSION})
  endif(WIN32)
else(tensorflow_ENABLE_GPU)
  set(tensorflow_BUILD_INFO_FLAGS --build_config cpu --key_value
      msvcp_dll_name=msvcp140.dll)
endif(tensorflow_ENABLE_GPU)

# Find python executable
include(FindPythonInterp)
if(NOT ${PYTHONINTERP_FOUND})
  message(FATAL_ERROR "CMake was unable to find a python interpreter.")
endif()

add_subdirectory(tensorflow)