Adding install script to be consumed by pybuda. Misc - env variable r…

…ename (#119)

* Fixing ttmlir project for integration with pybuda cmake

* Fixing lint errors

CMakeLists.txt

@@ -22,13 +22,20 @@ if(NOT CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
   message(FATAL_ERROR "In-source builds are not allowed. Please make a new directory (called a build directory) and run CMake from there.")
 endif()
 
+# If there are multiple python env we need to explicitly set the python3 executable
+set(Python3_EXECUTABLE $ENV{TTMLIR_VENV_DIR}/bin/python3)
+
 include(FindMLIR)
 include(TTMLIRVersion)
 
-set(TTMLIR_TOOLCHAIN "/opt/ttmlir-toolchain" CACHE STRING "Path to the toolchain")
+if (NOT DEFINED ENV{TTMLIR_TOOLCHAIN_DIR})
+    message(FATAL_ERROR "TTMLIR_TOOLCHAIN_DIR environment variable not set. Please run 'source env/activate'.")
+endif()
+
+set(TTMLIR_TOOLCHAIN_DIR $ENV{TTMLIR_TOOLCHAIN_DIR})
 set(TTMLIR_SOURCE_DIR ${PROJECT_SOURCE_DIR})
 set(TTMLIR_BINARY_DIR ${PROJECT_BINARY_DIR})
-set(LLVM_LIT_TOOLS_DIR "${TTMLIR_TOOLCHAIN}/src/llvm-project/llvm/utils/lit")
+set(LLVM_LIT_TOOLS_DIR "${TTMLIR_TOOLCHAIN_DIR}/src/llvm-project/llvm/utils/lit")
 include_directories(SYSTEM ${LLVM_INCLUDE_DIRS})
 include_directories(SYSTEM ${MLIR_INCLUDE_DIRS})
 include_directories(${TTMLIR_SOURCE_DIR}/include)

cmake/modules/LintTools.cmake

@@ -1,5 +1,5 @@
 # clang-tidy setup
-add_custom_target(run-clang-tidy-install COMMAND cp ${TTMLIR_TOOLCHAIN}/src/llvm-project/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py ${TTMLIR_TOOLCHAIN}/bin/)
+add_custom_target(run-clang-tidy-install COMMAND cp ${TTMLIR_TOOLCHAIN_DIR}/src/llvm-project/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py ${TTMLIR_TOOLCHAIN_DIR}/bin/)
 add_custom_target(clang-tidy-filter-out-external-srcs COMMAND python3 ${TTMLIR_SOURCE_DIR}/tools/scripts/filter-compile-commands.py ${TTMLIR_BINARY_DIR}/compile_commands.json ${TTMLIR_SOURCE_DIR})
 add_custom_target(clang-tidy COMMAND run-clang-tidy.py -p ${PROJECT_BINARY_DIR} -warnings-as-errors '*' DEPENDS clang-tidy-filter-out-external-srcs run-clang-tidy-install)
 add_custom_target(clang-format COMMAND git-clang-format)

cmake/modules/TTMLIRPythonSitePackages.cmake

@@ -2,4 +2,4 @@ execute_process(
   COMMAND python -c "from distutils import sysconfig; print(sysconfig.get_python_lib(prefix='', plat_specific=True))"
   OUTPUT_VARIABLE PYTHON_SITE_PACKAGES
   OUTPUT_STRIP_TRAILING_WHITESPACE)
-set(TTMLIR_PYTHON_SITE_PACKAGES "${TTMLIR_TOOLCHAIN}/venv/${PYTHON_SITE_PACKAGES}" CACHE STRING "Path to the Python site-packages directory")
+set(TTMLIR_PYTHON_SITE_PACKAGES "${TTMLIR_TOOLCHAIN_DIR}/venv/${PYTHON_SITE_PACKAGES}" CACHE STRING "Path to the Python site-packages directory")

env/CMakeLists.txt

@@ -2,23 +2,28 @@ cmake_minimum_required(VERSION 3.20.0)
 project(ttmlir-toolchain LANGUAGES CXX C)
 include(ExternalProject)
 
-set(TTMLIR_TOOLCHAIN "/opt/ttmlir-toolchain" CACHE STRING "Path to the toolchain")
+if (NOT DEFINED ENV{TTMLIR_TOOLCHAIN_DIR})
+    message(FATAL_ERROR "Please set TTMLIR_TOOLCHAIN_DIR to the directory where the toolchain will be installed")
+else()
+    set(TTMLIR_TOOLCHAIN_DIR $ENV{TTMLIR_TOOLCHAIN_DIR})
+endif()
 
-if (NOT EXISTS "${TTMLIR_TOOLCHAIN}")
-  message( FATAL_ERROR "The directory ${TTMLIR_TOOLCHAIN} does not exist. Please create it before running this script.\n  sudo mkdir -p ${TTMLIR_TOOLCHAIN}\n  sudo chown -R $ENV{USER} ${TTMLIR_TOOLCHAIN}")
+if (NOT EXISTS "${TTMLIR_TOOLCHAIN_DIR}")
+    message( FATAL_ERROR "The directory ${TTMLIR_TOOLCHAIN_DIR} does not exist. Please create it before running this script.\n  sudo mkdir -p ${TTMLIR_TOOLCHAIN_DIR}\n  sudo chown -R $ENV{USER} ${TTMLIR_TOOLCHAIN_DIR}")
 endif()
 
-add_custom_target(python-venv ALL COMMAND TTMLIR_TOOLCHAIN=${TTMLIR_TOOLCHAIN} CURRENT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} bash ${CMAKE_CURRENT_SOURCE_DIR}/build_venv.sh)
+
+add_custom_target(python-venv ALL COMMAND TTMLIR_TOOLCHAIN=${TTMLIR_TOOLCHAIN_DIR} CURRENT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} bash ${CMAKE_CURRENT_SOURCE_DIR}/build_venv.sh)
 
 ExternalProject_Add(
     flatbuffers
-    PREFIX ${TTMLIR_TOOLCHAIN}
+    PREFIX ${TTMLIR_TOOLCHAIN_DIR}
     CMAKE_GENERATOR Ninja
     CMAKE_ARGS
-      -DCMAKE_BUILD_TYPE=Release
-      -DCMAKE_INSTALL_PREFIX=${TTMLIR_TOOLCHAIN}
-      -DFLATBUFFERS_BUILD_TESTS=OFF
-      -DCMAKE_POSITION_INDEPENDENT_CODE=ON # Required for linking into shared libraries (python bindings)
+    -DCMAKE_BUILD_TYPE=Release
+    -DCMAKE_INSTALL_PREFIX=${TTMLIR_TOOLCHAIN_DIR}
+    -DFLATBUFFERS_BUILD_TESTS=OFF
+    -DCMAKE_POSITION_INDEPENDENT_CODE=ON # Required for linking into shared libraries (python bindings)
     GIT_REPOSITORY https://github.com/google/flatbuffers.git
     GIT_TAG v24.3.7
     DEPENDS python-venv
@@ -27,29 +32,29 @@ ExternalProject_Add(
 # -DMLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS=ON set if debug
 ExternalProject_Add(
     llvm-project
-    PREFIX ${TTMLIR_TOOLCHAIN}
+    PREFIX ${TTMLIR_TOOLCHAIN_DIR}
     # Super hacky way to install the python dependencies before the build
-    PATCH_COMMAND bash -c "source ${CMAKE_CURRENT_SOURCE_DIR}/activate && pip install -r ${TTMLIR_TOOLCHAIN}/src/llvm-project/mlir/python/requirements.txt"
+    PATCH_COMMAND bash -c "source ${CMAKE_CURRENT_SOURCE_DIR}/activate && pip install -r ${TTMLIR_TOOLCHAIN_DIR}/src/llvm-project/mlir/python/requirements.txt"
     CMAKE_GENERATOR Ninja
     CMAKE_ARGS
-      -DPython3_FIND_VIRTUALENV=ONLY
-      -DPython3_EXECUTABLE=${TTMLIR_TOOLCHAIN}/venv/bin/python
-      -DCMAKE_INSTALL_PREFIX=${TTMLIR_TOOLCHAIN}
-      -DLLVM_ENABLE_PROJECTS=mlir
-      -DLLVM_INSTALL_UTILS=ON
-      # Build shared libraries
-      # ======================
-      -DLLVM_ENABLE_PIC=ON
-      -DLLVM_BUILD_LLVM_DYLIB=ON
-      -DLLVM_LINK_LLVM_DYLIB=ON
-      -DMLIR_BUILD_MLIR_C_DYLIB=ON
-      -DMLIR_LINK_MLIR_DYLIB=ON
-      -DMLIR_BUILD_MLIR_C_DYLIB=ON
-      # ======================
-      -DCMAKE_BUILD_TYPE=MinSizeRel
-      -DLLVM_ENABLE_ASSERTIONS=ON
-      -DMLIR_ENABLE_BINDINGS_PYTHON=ON
-      -DCMAKE_C_FLAGS=-D_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION
+    -DPython3_FIND_VIRTUALENV=ONLY
+    -DPython3_EXECUTABLE=${TTMLIR_TOOLCHAIN_DIR}/venv/bin/python
+    -DCMAKE_INSTALL_PREFIX=${TTMLIR_TOOLCHAIN_DIR}
+    -DLLVM_ENABLE_PROJECTS=mlir
+    -DLLVM_INSTALL_UTILS=ON
+    # Build shared libraries
+    # ======================
+    -DLLVM_ENABLE_PIC=ON
+    -DLLVM_BUILD_LLVM_DYLIB=ON
+    -DLLVM_LINK_LLVM_DYLIB=ON
+    -DMLIR_BUILD_MLIR_C_DYLIB=ON
+    -DMLIR_LINK_MLIR_DYLIB=ON
+    -DMLIR_BUILD_MLIR_C_DYLIB=ON
+    # ======================
+    -DCMAKE_BUILD_TYPE=MinSizeRel
+    -DLLVM_ENABLE_ASSERTIONS=ON
+    -DMLIR_ENABLE_BINDINGS_PYTHON=ON
+    -DCMAKE_C_FLAGS=-D_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION
     SOURCE_SUBDIR llvm
     LIST_SEPARATOR ","
     GIT_REPOSITORY https://github.com/llvm/llvm-project.git
@@ -58,4 +63,4 @@ ExternalProject_Add(
     DEPENDS python-venv
 )
 
-add_custom_target(llvm-lit ALL COMMAND cp ${TTMLIR_TOOLCHAIN}/src/llvm-project-build/bin/llvm-lit ${TTMLIR_TOOLCHAIN}/bin/llvm-lit DEPENDS llvm-project)
+add_custom_target(llvm-lit ALL COMMAND cp ${TTMLIR_TOOLCHAIN_DIR}/src/llvm-project-build/bin/llvm-lit ${TTMLIR_TOOLCHAIN_DIR}/bin/llvm-lit DEPENDS llvm-project)

env/activate

@@ -1,8 +1,16 @@
-TTMLIR_VENV="${TTMLIR_VENV:-/opt/ttmlir-toolchain/venv}"
-TOOLCHAIN_ENV="${TOOLCHAIN_ENV:-/opt/ttmlir-toolchain}"
-[ -f $TTMLIR_VENV/bin/activate ] && source $TTMLIR_VENV/bin/activate
+# Step 1: Check if TTMLIR_TOOLCHAIN_DIR is already set, if not, set it to /opt/ttmlir-toolchain
+if [ -z "${TTMLIR_TOOLCHAIN_DIR}" ]; then
+  export TTMLIR_TOOLCHAIN_DIR="/opt/ttmlir-toolchain"
+fi
+
+# Step 2: Check if TTMLIR_VENV_DIR is already set, if not, set it to the venv directory inside TTMLIR_TOOLCHAIN_DIR
+if [ -z "${TTMLIR_VENV_DIR}" ]; then
+  export TTMLIR_VENV_DIR="${TTMLIR_TOOLCHAIN_DIR}/venv"
+fi
+
+[ -f $TTMLIR_VENV_DIR/bin/activate ] && source $TTMLIR_VENV_DIR/bin/activate
 export TTMLIR_ENV_ACTIVATED=1
-export PATH=$TOOLCHAIN_ENV/bin:$TOOLCHAIN_ENV/venv/bin:$PATH
+export PATH=$TTMLIR_TOOLCHAIN_DIR/bin:$TTMLIR_TOOLCHAIN_DIR/venv/bin:$PATH
 export TT_METAL_HOME="$(pwd)/third_party/tt-metal/src/tt-metal"
 export PYTHONPATH="$(pwd)/build/python_packages:$(pwd)/.local/toolchain/python_packages/mlir_core"
 export ARCH_NAME="${ARCH_NAME:-wormhole_b0}"

env/build_venv.sh

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 OS=$(uname)
-TTMLIR_VENV=$TTMLIR_TOOLCHAIN/venv
+TTMLIR_VENV=$TTMLIR_TOOLCHAIN_DIR/venv
 
 python3 -m venv $TTMLIR_VENV
 source $CURRENT_SOURCE_DIR/activate

runtime/tools/python/setup.py

@@ -13,7 +13,7 @@
     "SOURCE_ROOT",
     os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", ".."),
 )
-toolchain = os.environ.get("TOOLCHAIN_ENV", "/opt/ttmlir-toolchain")
+toolchain = os.environ.get("TTMLIR_TOOLCHAIN_DIR", "/opt/ttmlir-toolchain")
 metallibdir = f"{src_dir}/third_party/tt-metal/src/tt-metal-build/lib"
 
 os.environ["LDFLAGS"] = "-Wl,-rpath,'$ORIGIN'"