diff --git a/include/onnxruntime/core/common/make_string.h b/include/onnxruntime/core/common/make_string.h
index c47e6cc35e48..826898de852a 100644
--- a/include/onnxruntime/core/common/make_string.h
+++ b/include/onnxruntime/core/common/make_string.h
@@ -21,29 +21,27 @@
 #include <sstream>
 #include <type_traits>
 
-#include "core/util/force_inline.h"
-
 namespace onnxruntime {
 
 namespace detail {
 
-ORT_FORCEINLINE void MakeStringImpl(std::ostringstream& /*ss*/) noexcept {
+inline void MakeStringImpl(std::ostringstream& /*ss*/) noexcept {
 }
 
 template <typename T>
-ORT_FORCEINLINE void MakeStringImpl(std::ostringstream& ss, const T& t) noexcept {
+inline void MakeStringImpl(std::ostringstream& ss, const T& t) noexcept {
   ss << t;
 }
 
 template <typename T, typename... Args>
-ORT_FORCEINLINE void MakeStringImpl(std::ostringstream& ss, const T& t, const Args&... args) noexcept {
+inline void MakeStringImpl(std::ostringstream& ss, const T& t, const Args&... args) noexcept {
   MakeStringImpl(ss, t);
   MakeStringImpl(ss, args...);
 }
 
 // see MakeString comments for explanation of why this is necessary
 template <typename... Args>
-ORT_FORCEINLINE std::string MakeStringImpl(const Args&... args) noexcept {
+inline std::string MakeStringImpl(const Args&... args) noexcept {
   std::ostringstream ss;
   MakeStringImpl(ss, args...);
   return ss.str();
@@ -80,7 +78,7 @@ using if_char_array_make_ptr_t = typename if_char_array_make_ptr<T>::type;
  * This version uses the current locale.
  */
 template <typename... Args>
-ORT_FORCEINLINE std::string MakeString(const Args&... args) {
+std::string MakeString(const Args&... args) {
   // We need to update the types from the MakeString template instantiation to decay any char[n] to char*.
   //   e.g. MakeString("in", "out") goes from MakeString<char[2], char[3]> to MakeStringImpl<char*, char*>
   //        so that MakeString("out", "in") will also match MakeStringImpl<char*, char*> instead of requiring
@@ -100,7 +98,7 @@ ORT_FORCEINLINE std::string MakeString(const Args&... args) {
  * This version uses std::locale::classic().
  */
 template <typename... Args>
-ORT_FORCEINLINE std::string MakeStringWithClassicLocale(const Args&... args) {
+std::string MakeStringWithClassicLocale(const Args&... args) {
   std::ostringstream ss;
   ss.imbue(std::locale::classic());
   detail::MakeStringImpl(ss, args...);
@@ -109,19 +107,19 @@ ORT_FORCEINLINE std::string MakeStringWithClassicLocale(const Args&... args) {
 
 // MakeString versions for already-a-string types.
 
-ORT_FORCEINLINE std::string MakeString(const std::string& str) {
+inline std::string MakeString(const std::string& str) {
   return str;
 }
 
-ORT_FORCEINLINE std::string MakeString(const char* cstr) {
+inline std::string MakeString(const char* cstr) {
   return cstr;
 }
 
-ORT_FORCEINLINE std::string MakeStringWithClassicLocale(const std::string& str) {
+inline std::string MakeStringWithClassicLocale(const std::string& str) {
   return str;
 }
 
-ORT_FORCEINLINE std::string MakeStringWithClassicLocale(const char* cstr) {
+inline std::string MakeStringWithClassicLocale(const char* cstr) {
   return cstr;
 }
 
diff --git a/onnxruntime/contrib_ops/cpu/quantization/blockwise_quant_block_bnb4.h b/onnxruntime/contrib_ops/cpu/quantization/blockwise_quant_block_bnb4.h
index f1692a1ab743..cb8e97a592d8 100644
--- a/onnxruntime/contrib_ops/cpu/quantization/blockwise_quant_block_bnb4.h
+++ b/onnxruntime/contrib_ops/cpu/quantization/blockwise_quant_block_bnb4.h
@@ -7,17 +7,21 @@
 #include <algorithm>
 #include <cmath>
 
-#include "core/util/force_inline.h"
-
 namespace onnxruntime {
 namespace contrib {
 
+#if defined(_MSC_VER)
+#define FORCEINLINE __forceinline
+#else
+#define FORCEINLINE __attribute__((always_inline)) inline
+#endif
+
 typedef enum Bnb_DataType_t {
   FP4 = 0,
   NF4 = 1,
 } Bnb_DataType_t;
 
-ORT_FORCEINLINE uint8_t QuantizeOneFP4(float x) {
+FORCEINLINE uint8_t QuantizeOneFP4(float x) {
   // FP4 with bias of 3
   // first bit is a sign
   // subnormals
@@ -65,7 +69,7 @@ ORT_FORCEINLINE uint8_t QuantizeOneFP4(float x) {
   }
 }
 
-ORT_FORCEINLINE uint8_t QuantizeOneNF4(float x) {
+FORCEINLINE uint8_t QuantizeOneNF4(float x) {
   if (x > 0.03979014977812767f) {
     if (x > 0.3893125355243683f) {      // 1
       if (x > 0.6427869200706482f) {    // 11
@@ -116,7 +120,7 @@ ORT_FORCEINLINE uint8_t QuantizeOneNF4(float x) {
 }
 
 template <int32_t DATA_TYPE>
-ORT_FORCEINLINE uint8_t QuantizeOneBnb4(float x) {
+FORCEINLINE uint8_t QuantizeOneBnb4(float x) {
   if constexpr (DATA_TYPE == FP4)
     return QuantizeOneFP4(x);
   else
@@ -124,7 +128,7 @@ ORT_FORCEINLINE uint8_t QuantizeOneBnb4(float x) {
 }
 
 template <typename T, int32_t block_size, int32_t DATA_TYPE>
-ORT_FORCEINLINE void QuantizeBlockBnb4(const T* src, uint8_t* dst, T& absmax_block, int32_t block_idx, int32_t numel) {
+FORCEINLINE void QuantizeBlockBnb4(const T* src, uint8_t* dst, T& absmax_block, int32_t block_idx, int32_t numel) {
   float local_absmax = 0.0f;
 
   int32_t block_len = std::min(block_size, numel - block_idx * block_size);
@@ -173,7 +177,7 @@ static float nf4_qaunt_map[16] = {-1.0f,
                                   1.0f};
 
 template <typename T, int32_t DATA_TYPE>
-ORT_FORCEINLINE T DequantizeOneBnb4(uint8_t x) {
+FORCEINLINE T DequantizeOneBnb4(uint8_t x) {
   if constexpr (DATA_TYPE == FP4)
     return static_cast<T>(fp4_qaunt_map[x]);
   else
@@ -181,7 +185,7 @@ ORT_FORCEINLINE T DequantizeOneBnb4(uint8_t x) {
 }
 
 template <typename T, int32_t block_size, int32_t DATA_TYPE>
-ORT_FORCEINLINE void DequantizeBlockBnb4(const uint8_t* src, T* dst, T absmax_block, int32_t block_idx, int32_t numel) {
+FORCEINLINE void DequantizeBlockBnb4(const uint8_t* src, T* dst, T absmax_block, int32_t block_idx, int32_t numel) {
   int32_t block_len = std::min(block_size, numel - block_idx * block_size);
   int32_t src_offset = block_idx * block_size / 2;
   int32_t dst_offset = block_idx * block_size;
diff --git a/onnxruntime/core/framework/murmurhash3.cc b/onnxruntime/core/framework/murmurhash3.cc
index 802f0a4c58a6..e2dbba9b079b 100644
--- a/onnxruntime/core/framework/murmurhash3.cc
+++ b/onnxruntime/core/framework/murmurhash3.cc
@@ -17,8 +17,6 @@
 
 #include "core/framework/endian.h"
 
-#include "core/util/force_inline.h"
-
 //-----------------------------------------------------------------------------
 // Platform-specific functions and macros
 
@@ -26,6 +24,8 @@
 
 #if defined(_MSC_VER)
 
+#define FORCE_INLINE __forceinline
+
 #include <stdlib.h>
 
 #define ROTL32(x, y) _rotl(x, y)
@@ -37,6 +37,8 @@
 
 #else  // defined(_MSC_VER)
 
+#define FORCE_INLINE inline __attribute__((always_inline))
+
 inline uint32_t rotl32(uint32_t x, int8_t r) {
   return (x << r) | (x >> (32 - r));
 }
@@ -59,7 +61,7 @@ inline uint64_t rotl64(uint64_t x, int8_t r) {
 //
 // Changes to support big-endian from https://github.com/explosion/murmurhash/pull/27/
 // were manually applied to original murmurhash3 source code.
-ORT_FORCEINLINE uint32_t getblock32(const uint32_t* p, int i) {
+FORCE_INLINE uint32_t getblock32(const uint32_t* p, int i) {
   if constexpr (onnxruntime::endian::native == onnxruntime::endian::little) {
     return p[i];
   } else {
@@ -71,7 +73,7 @@ ORT_FORCEINLINE uint32_t getblock32(const uint32_t* p, int i) {
   }
 }
 
-ORT_FORCEINLINE uint64_t getblock64(const uint64_t* p, int i) {
+FORCE_INLINE uint64_t getblock64(const uint64_t* p, int i) {
   if constexpr (onnxruntime::endian::native == onnxruntime::endian::little) {
     return p[i];
   } else {
@@ -90,7 +92,7 @@ ORT_FORCEINLINE uint64_t getblock64(const uint64_t* p, int i) {
 //-----------------------------------------------------------------------------
 // Finalization mix - force all bits of a hash block to avalanche
 
-ORT_FORCEINLINE constexpr uint32_t fmix32(uint32_t h) {
+FORCE_INLINE constexpr uint32_t fmix32(uint32_t h) {
   h ^= h >> 16;
   h *= 0x85ebca6b;
   h ^= h >> 13;
@@ -102,7 +104,7 @@ ORT_FORCEINLINE constexpr uint32_t fmix32(uint32_t h) {
 
 //----------
 
-ORT_FORCEINLINE constexpr uint64_t fmix64(uint64_t k) {
+FORCE_INLINE constexpr uint64_t fmix64(uint64_t k) {
   k ^= k >> 33;
   k *= BIG_CONSTANT(0xff51afd7ed558ccd);
   k ^= k >> 33;
diff --git a/onnxruntime/core/providers/cpu/tensor/gather_elements.cc b/onnxruntime/core/providers/cpu/tensor/gather_elements.cc
index 495ff19d8613..a2239844eb1d 100644
--- a/onnxruntime/core/providers/cpu/tensor/gather_elements.cc
+++ b/onnxruntime/core/providers/cpu/tensor/gather_elements.cc
@@ -5,8 +5,6 @@
 #include "gather_elements.h"
 #include "onnxruntime_config.h"
 
-#include "core/util/force_inline.h"
-
 namespace onnxruntime {
 
 ONNX_CPU_OPERATOR_VERSIONED_KERNEL(
@@ -68,8 +66,14 @@ static inline size_t CalculateOffset(size_t inner_dim, const TensorPitches& inpu
   return base_offset;
 }
 
+#if defined(_MSC_VER)
+#define FORCEINLINE __forceinline
+#else
+#define FORCEINLINE __attribute__((always_inline)) inline
+#endif
+
 template <typename T>
-ORT_FORCEINLINE int64_t GetIndex(size_t i, const T* indices, int64_t axis_size) {
+FORCEINLINE int64_t GetIndex(size_t i, const T* indices, int64_t axis_size) {
   int64_t index = indices[i];
   if (index < 0)  // Handle negative indices
     index += axis_size;
diff --git a/onnxruntime/core/util/force_inline.h b/onnxruntime/core/util/force_inline.h
deleted file mode 100644
index cd1510700445..000000000000
--- a/onnxruntime/core/util/force_inline.h
+++ /dev/null
@@ -1,10 +0,0 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT License.
-
-#pragma once
-
-#if defined(_MSC_VER)
-#define ORT_FORCEINLINE __forceinline
-#else
-#define ORT_FORCEINLINE __attribute__((always_inline)) inline
-#endif
diff --git a/onnxruntime/core/util/matrix_layout.h b/onnxruntime/core/util/matrix_layout.h
index 105da1f5e285..dbf961ab5b03 100644
--- a/onnxruntime/core/util/matrix_layout.h
+++ b/onnxruntime/core/util/matrix_layout.h
@@ -17,7 +17,11 @@
 #include <cstdint>
 #include <gsl/gsl>
 
-#include "core/util/force_inline.h"
+#if defined(_MSC_VER)
+#define ORT_FORCEINLINE __forceinline
+#else
+#define ORT_FORCEINLINE __attribute__((always_inline)) inline
+#endif
 
 namespace onnxruntime {