diff --git a/R-package/src/Makevars.in b/R-package/src/Makevars.in
index f7f6dca97bb0..4d0e1d40a9e2 100644
--- a/R-package/src/Makevars.in
+++ b/R-package/src/Makevars.in
@@ -37,10 +37,6 @@ OBJECTS = \
     io/parser.o \
     io/train_share_states.o \
     io/tree.o \
-    io/dense_bin.o \
-    io/sparse_bin.o \
-    io/multi_val_dense_bin.o \
-    io/multi_val_sparse_bin.o \
     metric/dcg_calculator.o \
     metric/metric.o \
     objective/objective_function.o \
diff --git a/R-package/src/Makevars.win.in b/R-package/src/Makevars.win.in
index 354a3fda210f..10e3ce949396 100644
--- a/R-package/src/Makevars.win.in
+++ b/R-package/src/Makevars.win.in
@@ -38,10 +38,6 @@ OBJECTS = \
     io/parser.o \
     io/train_share_states.o \
     io/tree.o \
-    io/dense_bin.o \
-    io/sparse_bin.o \
-    io/multi_val_dense_bin.o \
-    io/multi_val_sparse_bin.o \
     metric/dcg_calculator.o \
     metric/metric.o \
     objective/objective_function.o \
diff --git a/src/io/bin.cpp b/src/io/bin.cpp
index c55bd7600e84..1ba3d7f103c5 100644
--- a/src/io/bin.cpp
+++ b/src/io/bin.cpp
@@ -719,4 +719,368 @@ namespace LightGBM {
     }
   }
 
+  template <>
+  const void* DenseBin<uint8_t, false>::GetColWiseData(
+    uint8_t* bit_type,
+    bool* is_sparse,
+    std::vector<BinIterator*>* bin_iterator,
+    const int /*num_threads*/) const {
+    *is_sparse = false;
+    *bit_type = 8;
+    bin_iterator->clear();
+    return reinterpret_cast<const void*>(data_.data());
+  }
+
+  template <>
+  const void* DenseBin<uint16_t, false>::GetColWiseData(
+    uint8_t* bit_type,
+    bool* is_sparse,
+    std::vector<BinIterator*>* bin_iterator,
+    const int /*num_threads*/) const {
+    *is_sparse = false;
+    *bit_type = 16;
+    bin_iterator->clear();
+    return reinterpret_cast<const void*>(data_.data());
+  }
+
+  template <>
+  const void* DenseBin<uint32_t, false>::GetColWiseData(
+    uint8_t* bit_type,
+    bool* is_sparse,
+    std::vector<BinIterator*>* bin_iterator,
+    const int /*num_threads*/) const {
+    *is_sparse = false;
+    *bit_type = 32;
+    bin_iterator->clear();
+    return reinterpret_cast<const void*>(data_.data());
+  }
+
+  template <>
+  const void* DenseBin<uint8_t, true>::GetColWiseData(
+    uint8_t* bit_type,
+    bool* is_sparse,
+    std::vector<BinIterator*>* bin_iterator,
+    const int /*num_threads*/) const {
+    *is_sparse = false;
+    *bit_type = 4;
+    bin_iterator->clear();
+    return reinterpret_cast<const void*>(data_.data());
+  }
+
+  template <>
+  const void* DenseBin<uint8_t, false>::GetColWiseData(
+    uint8_t* bit_type,
+    bool* is_sparse,
+    BinIterator** bin_iterator) const {
+    *is_sparse = false;
+    *bit_type = 8;
+    *bin_iterator = nullptr;
+    return reinterpret_cast<const void*>(data_.data());
+  }
+
+  template <>
+  const void* DenseBin<uint16_t, false>::GetColWiseData(
+    uint8_t* bit_type,
+    bool* is_sparse,
+    BinIterator** bin_iterator) const {
+    *is_sparse = false;
+    *bit_type = 16;
+    *bin_iterator = nullptr;
+    return reinterpret_cast<const void*>(data_.data());
+  }
+
+  template <>
+  const void* DenseBin<uint32_t, false>::GetColWiseData(
+    uint8_t* bit_type,
+    bool* is_sparse,
+    BinIterator** bin_iterator) const {
+    *is_sparse = false;
+    *bit_type = 32;
+    *bin_iterator = nullptr;
+    return reinterpret_cast<const void*>(data_.data());
+  }
+
+  template <>
+  const void* DenseBin<uint8_t, true>::GetColWiseData(
+    uint8_t* bit_type,
+    bool* is_sparse,
+    BinIterator** bin_iterator) const {
+    *is_sparse = false;
+    *bit_type = 4;
+    *bin_iterator = nullptr;
+    return reinterpret_cast<const void*>(data_.data());
+  }
+
+  template <>
+  const void* SparseBin<uint8_t>::GetColWiseData(
+    uint8_t* bit_type,
+    bool* is_sparse,
+    std::vector<BinIterator*>* bin_iterator,
+    const int num_threads) const {
+    *is_sparse = true;
+    *bit_type = 8;
+    for (int thread_index = 0; thread_index < num_threads; ++thread_index) {
+      bin_iterator->emplace_back(new SparseBinIterator<uint8_t>(this, 0));
+    }
+    return nullptr;
+  }
+
+  template <>
+  const void* SparseBin<uint16_t>::GetColWiseData(
+    uint8_t* bit_type,
+    bool* is_sparse,
+    std::vector<BinIterator*>* bin_iterator,
+    const int num_threads) const {
+    *is_sparse = true;
+    *bit_type = 16;
+    for (int thread_index = 0; thread_index < num_threads; ++thread_index) {
+      bin_iterator->emplace_back(new SparseBinIterator<uint16_t>(this, 0));
+    }
+    return nullptr;
+  }
+
+  template <>
+  const void* SparseBin<uint32_t>::GetColWiseData(
+    uint8_t* bit_type,
+    bool* is_sparse,
+    std::vector<BinIterator*>* bin_iterator,
+    const int num_threads) const {
+    *is_sparse = true;
+    *bit_type = 32;
+    for (int thread_index = 0; thread_index < num_threads; ++thread_index) {
+      bin_iterator->emplace_back(new SparseBinIterator<uint32_t>(this, 0));
+    }
+    return nullptr;
+  }
+
+  template <>
+  const void* SparseBin<uint8_t>::GetColWiseData(
+    uint8_t* bit_type,
+    bool* is_sparse,
+    BinIterator** bin_iterator) const {
+    *is_sparse = true;
+    *bit_type = 8;
+    *bin_iterator = new SparseBinIterator<uint8_t>(this, 0);
+    return nullptr;
+  }
+
+  template <>
+  const void* SparseBin<uint16_t>::GetColWiseData(
+    uint8_t* bit_type,
+    bool* is_sparse,
+    BinIterator** bin_iterator) const {
+    *is_sparse = true;
+    *bit_type = 16;
+    *bin_iterator = new SparseBinIterator<uint16_t>(this, 0);
+    return nullptr;
+  }
+
+  template <>
+  const void* SparseBin<uint32_t>::GetColWiseData(
+    uint8_t* bit_type,
+    bool* is_sparse,
+    BinIterator** bin_iterator) const {
+    *is_sparse = true;
+    *bit_type = 32;
+    *bin_iterator = new SparseBinIterator<uint32_t>(this, 0);
+    return nullptr;
+  }
+
+  #ifdef USE_CUDA_EXP
+  template <>
+  const void* MultiValDenseBin<uint8_t>::GetRowWiseData(uint8_t* bit_type,
+      size_t* total_size,
+      bool* is_sparse,
+      const void** out_data_ptr,
+      uint8_t* data_ptr_bit_type) const {
+    const uint8_t* to_return = data_.data();
+    *bit_type = 8;
+    *total_size = static_cast<size_t>(num_data_) * static_cast<size_t>(num_feature_);
+    CHECK_EQ(*total_size, data_.size());
+    *is_sparse = false;
+    *out_data_ptr = nullptr;
+    *data_ptr_bit_type = 0;
+    return to_return;
+  }
+
+  template <>
+  const void* MultiValDenseBin<uint16_t>::GetRowWiseData(uint8_t* bit_type,
+    size_t* total_size,
+    bool* is_sparse,
+    const void** out_data_ptr,
+    uint8_t* data_ptr_bit_type) const {
+    const uint16_t* data_ptr = data_.data();
+    const uint8_t* to_return = reinterpret_cast<const uint8_t*>(data_ptr);
+    *bit_type = 16;
+    *total_size = static_cast<size_t>(num_data_) * static_cast<size_t>(num_feature_);
+    CHECK_EQ(*total_size, data_.size());
+    *is_sparse = false;
+    *out_data_ptr = nullptr;
+    *data_ptr_bit_type = 0;
+    return to_return;
+  }
+
+  template <>
+  const void* MultiValDenseBin<uint32_t>::GetRowWiseData(uint8_t* bit_type,
+    size_t* total_size,
+    bool* is_sparse,
+    const void** out_data_ptr,
+    uint8_t* data_ptr_bit_type) const {
+    const uint32_t* data_ptr = data_.data();
+    const uint8_t* to_return = reinterpret_cast<const uint8_t*>(data_ptr);
+    *bit_type = 32;
+    *total_size = static_cast<size_t>(num_data_) * static_cast<size_t>(num_feature_);
+    CHECK_EQ(*total_size, data_.size());
+    *is_sparse = false;
+    *out_data_ptr = nullptr;
+    *data_ptr_bit_type = 0;
+    return to_return;
+  }
+
+  template <>
+  const void* MultiValSparseBin<uint16_t, uint8_t>::GetRowWiseData(
+    uint8_t* bit_type,
+    size_t* total_size,
+    bool* is_sparse,
+    const void** out_data_ptr,
+    uint8_t* data_ptr_bit_type) const {
+    const uint8_t* to_return = data_.data();
+    *bit_type = 8;
+    *total_size = data_.size();
+    *is_sparse = true;
+    *out_data_ptr = reinterpret_cast<const uint8_t*>(row_ptr_.data());
+    *data_ptr_bit_type = 16;
+    return to_return;
+  }
+
+  template <>
+  const void* MultiValSparseBin<uint16_t, uint16_t>::GetRowWiseData(
+    uint8_t* bit_type,
+    size_t* total_size,
+    bool* is_sparse,
+    const void** out_data_ptr,
+    uint8_t* data_ptr_bit_type) const {
+    const uint8_t* to_return = reinterpret_cast<const uint8_t*>(data_.data());
+    *bit_type = 16;
+    *total_size = data_.size();
+    *is_sparse = true;
+    *out_data_ptr = reinterpret_cast<const uint8_t*>(row_ptr_.data());
+    *data_ptr_bit_type = 16;
+    return to_return;
+  }
+
+  template <>
+  const void* MultiValSparseBin<uint16_t, uint32_t>::GetRowWiseData(
+    uint8_t* bit_type,
+    size_t* total_size,
+    bool* is_sparse,
+    const void** out_data_ptr,
+    uint8_t* data_ptr_bit_type) const {
+    const uint8_t* to_return = reinterpret_cast<const uint8_t*>(data_.data());
+    *bit_type = 32;
+    *total_size = data_.size();
+    *is_sparse = true;
+    *out_data_ptr = reinterpret_cast<const uint8_t*>(row_ptr_.data());
+    *data_ptr_bit_type = 16;
+    return to_return;
+  }
+
+  template <>
+  const void* MultiValSparseBin<uint32_t, uint8_t>::GetRowWiseData(
+    uint8_t* bit_type,
+    size_t* total_size,
+    bool* is_sparse,
+    const void** out_data_ptr,
+    uint8_t* data_ptr_bit_type) const {
+    const uint8_t* to_return = data_.data();
+    *bit_type = 8;
+    *total_size = data_.size();
+    *is_sparse = true;
+    *out_data_ptr = reinterpret_cast<const uint8_t*>(row_ptr_.data());
+    *data_ptr_bit_type = 32;
+    return to_return;
+  }
+
+  template <>
+  const void* MultiValSparseBin<uint32_t, uint16_t>::GetRowWiseData(
+    uint8_t* bit_type,
+    size_t* total_size,
+    bool* is_sparse,
+    const void** out_data_ptr,
+    uint8_t* data_ptr_bit_type) const {
+    const uint8_t* to_return = reinterpret_cast<const uint8_t*>(data_.data());
+    *bit_type = 16;
+    *total_size = data_.size();
+    *is_sparse = true;
+    *out_data_ptr = reinterpret_cast<const uint8_t*>(row_ptr_.data());
+    *data_ptr_bit_type = 32;
+    return to_return;
+  }
+
+  template <>
+  const void* MultiValSparseBin<uint32_t, uint32_t>::GetRowWiseData(
+    uint8_t* bit_type,
+    size_t* total_size,
+    bool* is_sparse,
+    const void** out_data_ptr,
+    uint8_t* data_ptr_bit_type) const {
+    const uint8_t* to_return = reinterpret_cast<const uint8_t*>(data_.data());
+    *bit_type = 32;
+    *total_size = data_.size();
+    *is_sparse = true;
+    *out_data_ptr = reinterpret_cast<const uint8_t*>(row_ptr_.data());
+    *data_ptr_bit_type = 32;
+    return to_return;
+  }
+
+  template <>
+  const void* MultiValSparseBin<uint64_t, uint8_t>::GetRowWiseData(
+    uint8_t* bit_type,
+    size_t* total_size,
+    bool* is_sparse,
+    const void** out_data_ptr,
+    uint8_t* data_ptr_bit_type) const {
+    const uint8_t* to_return = data_.data();
+    *bit_type = 8;
+    *total_size = data_.size();
+    *is_sparse = true;
+    *out_data_ptr = reinterpret_cast<const uint8_t*>(row_ptr_.data());
+    *data_ptr_bit_type = 64;
+    return to_return;
+  }
+
+  template <>
+  const void* MultiValSparseBin<uint64_t, uint16_t>::GetRowWiseData(
+    uint8_t* bit_type,
+    size_t* total_size,
+    bool* is_sparse,
+    const void** out_data_ptr,
+    uint8_t* data_ptr_bit_type) const {
+    const uint8_t* to_return = reinterpret_cast<const uint8_t*>(data_.data());
+    *bit_type = 16;
+    *total_size = data_.size();
+    *is_sparse = true;
+    *out_data_ptr = reinterpret_cast<const uint8_t*>(row_ptr_.data());
+    *data_ptr_bit_type = 64;
+    return to_return;
+  }
+
+  template <>
+  const void* MultiValSparseBin<uint64_t, uint32_t>::GetRowWiseData(
+    uint8_t* bit_type,
+    size_t* total_size,
+    bool* is_sparse,
+    const void** out_data_ptr,
+    uint8_t* data_ptr_bit_type) const {
+    const uint8_t* to_return = reinterpret_cast<const uint8_t*>(data_.data());
+    *bit_type = 32;
+    *total_size = data_.size();
+    *is_sparse = true;
+    *out_data_ptr = reinterpret_cast<const uint8_t*>(row_ptr_.data());
+    *data_ptr_bit_type = 64;
+    return to_return;
+  }
+
+  #endif  // USE_CUDA_EXP
+
 }  // namespace LightGBM
diff --git a/src/io/dense_bin.cpp b/src/io/dense_bin.cpp
deleted file mode 100644
index 89475f57110a..000000000000
--- a/src/io/dense_bin.cpp
+++ /dev/null
@@ -1,103 +0,0 @@
-/*!
- * Copyright (c) 2021 Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See LICENSE file in the project root for
- * license information.
- */
-
-#include "dense_bin.hpp"
-
-namespace LightGBM {
-
-template <>
-const void* DenseBin<uint8_t, false>::GetColWiseData(
-  uint8_t* bit_type,
-  bool* is_sparse,
-  std::vector<BinIterator*>* bin_iterator,
-  const int /*num_threads*/) const {
-  *is_sparse = false;
-  *bit_type = 8;
-  bin_iterator->clear();
-  return reinterpret_cast<const void*>(data_.data());
-}
-
-template <>
-const void* DenseBin<uint16_t, false>::GetColWiseData(
-  uint8_t* bit_type,
-  bool* is_sparse,
-  std::vector<BinIterator*>* bin_iterator,
-  const int /*num_threads*/) const {
-  *is_sparse = false;
-  *bit_type = 16;
-  bin_iterator->clear();
-  return reinterpret_cast<const void*>(data_.data());
-}
-
-template <>
-const void* DenseBin<uint32_t, false>::GetColWiseData(
-  uint8_t* bit_type,
-  bool* is_sparse,
-  std::vector<BinIterator*>* bin_iterator,
-  const int /*num_threads*/) const {
-  *is_sparse = false;
-  *bit_type = 32;
-  bin_iterator->clear();
-  return reinterpret_cast<const void*>(data_.data());
-}
-
-template <>
-const void* DenseBin<uint8_t, true>::GetColWiseData(
-  uint8_t* bit_type,
-  bool* is_sparse,
-  std::vector<BinIterator*>* bin_iterator,
-  const int /*num_threads*/) const {
-  *is_sparse = false;
-  *bit_type = 4;
-  bin_iterator->clear();
-  return reinterpret_cast<const void*>(data_.data());
-}
-
-template <>
-const void* DenseBin<uint8_t, false>::GetColWiseData(
-  uint8_t* bit_type,
-  bool* is_sparse,
-  BinIterator** bin_iterator) const {
-  *is_sparse = false;
-  *bit_type = 8;
-  *bin_iterator = nullptr;
-  return reinterpret_cast<const void*>(data_.data());
-}
-
-template <>
-const void* DenseBin<uint16_t, false>::GetColWiseData(
-  uint8_t* bit_type,
-  bool* is_sparse,
-  BinIterator** bin_iterator) const {
-  *is_sparse = false;
-  *bit_type = 16;
-  *bin_iterator = nullptr;
-  return reinterpret_cast<const void*>(data_.data());
-}
-
-template <>
-const void* DenseBin<uint32_t, false>::GetColWiseData(
-  uint8_t* bit_type,
-  bool* is_sparse,
-  BinIterator** bin_iterator) const {
-  *is_sparse = false;
-  *bit_type = 32;
-  *bin_iterator = nullptr;
-  return reinterpret_cast<const void*>(data_.data());
-}
-
-template <>
-const void* DenseBin<uint8_t, true>::GetColWiseData(
-  uint8_t* bit_type,
-  bool* is_sparse,
-  BinIterator** bin_iterator) const {
-  *is_sparse = false;
-  *bit_type = 4;
-  *bin_iterator = nullptr;
-  return reinterpret_cast<const void*>(data_.data());
-}
-
-}  // namespace LightGBM
diff --git a/src/io/multi_val_dense_bin.cpp b/src/io/multi_val_dense_bin.cpp
deleted file mode 100644
index f6cf41b9bb21..000000000000
--- a/src/io/multi_val_dense_bin.cpp
+++ /dev/null
@@ -1,64 +0,0 @@
-/*!
- * Copyright (c) 2021 Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See LICENSE file in the project root for license information.
- */
-
-#include "multi_val_dense_bin.hpp"
-
-namespace LightGBM {
-
-
-#ifdef USE_CUDA_EXP
-template <>
-const void* MultiValDenseBin<uint8_t>::GetRowWiseData(uint8_t* bit_type,
-    size_t* total_size,
-    bool* is_sparse,
-    const void** out_data_ptr,
-    uint8_t* data_ptr_bit_type) const {
-  const uint8_t* to_return = data_.data();
-  *bit_type = 8;
-  *total_size = static_cast<size_t>(num_data_) * static_cast<size_t>(num_feature_);
-  CHECK_EQ(*total_size, data_.size());
-  *is_sparse = false;
-  *out_data_ptr = nullptr;
-  *data_ptr_bit_type = 0;
-  return to_return;
-}
-
-template <>
-const void* MultiValDenseBin<uint16_t>::GetRowWiseData(uint8_t* bit_type,
-  size_t* total_size,
-  bool* is_sparse,
-  const void** out_data_ptr,
-  uint8_t* data_ptr_bit_type) const {
-  const uint16_t* data_ptr = data_.data();
-  const uint8_t* to_return = reinterpret_cast<const uint8_t*>(data_ptr);
-  *bit_type = 16;
-  *total_size = static_cast<size_t>(num_data_) * static_cast<size_t>(num_feature_);
-  CHECK_EQ(*total_size, data_.size());
-  *is_sparse = false;
-  *out_data_ptr = nullptr;
-  *data_ptr_bit_type = 0;
-  return to_return;
-}
-
-template <>
-const void* MultiValDenseBin<uint32_t>::GetRowWiseData(uint8_t* bit_type,
-  size_t* total_size,
-  bool* is_sparse,
-  const void** out_data_ptr,
-  uint8_t* data_ptr_bit_type) const {
-  const uint32_t* data_ptr = data_.data();
-  const uint8_t* to_return = reinterpret_cast<const uint8_t*>(data_ptr);
-  *bit_type = 32;
-  *total_size = static_cast<size_t>(num_data_) * static_cast<size_t>(num_feature_);
-  CHECK_EQ(*total_size, data_.size());
-  *is_sparse = false;
-  *out_data_ptr = nullptr;
-  *data_ptr_bit_type = 0;
-  return to_return;
-}
-
-#endif  // USE_CUDA_EXP
-
-}  // namespace LightGBM
diff --git a/src/io/multi_val_sparse_bin.cpp b/src/io/multi_val_sparse_bin.cpp
deleted file mode 100644
index 359bf31c1053..000000000000
--- a/src/io/multi_val_sparse_bin.cpp
+++ /dev/null
@@ -1,158 +0,0 @@
-/*!
- * Copyright (c) 2021 Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See LICENSE file in the project root for license information.
- */
-
-#include "multi_val_sparse_bin.hpp"
-
-namespace LightGBM {
-
-#ifdef USE_CUDA_EXP
-
-template <>
-const void* MultiValSparseBin<uint16_t, uint8_t>::GetRowWiseData(
-  uint8_t* bit_type,
-  size_t* total_size,
-  bool* is_sparse,
-  const void** out_data_ptr,
-  uint8_t* data_ptr_bit_type) const {
-  const uint8_t* to_return = data_.data();
-  *bit_type = 8;
-  *total_size = data_.size();
-  *is_sparse = true;
-  *out_data_ptr = reinterpret_cast<const uint8_t*>(row_ptr_.data());
-  *data_ptr_bit_type = 16;
-  return to_return;
-}
-
-template <>
-const void* MultiValSparseBin<uint16_t, uint16_t>::GetRowWiseData(
-  uint8_t* bit_type,
-  size_t* total_size,
-  bool* is_sparse,
-  const void** out_data_ptr,
-  uint8_t* data_ptr_bit_type) const {
-  const uint8_t* to_return = reinterpret_cast<const uint8_t*>(data_.data());
-  *bit_type = 16;
-  *total_size = data_.size();
-  *is_sparse = true;
-  *out_data_ptr = reinterpret_cast<const uint8_t*>(row_ptr_.data());
-  *data_ptr_bit_type = 16;
-  return to_return;
-}
-
-template <>
-const void* MultiValSparseBin<uint16_t, uint32_t>::GetRowWiseData(
-  uint8_t* bit_type,
-  size_t* total_size,
-  bool* is_sparse,
-  const void** out_data_ptr,
-  uint8_t* data_ptr_bit_type) const {
-  const uint8_t* to_return = reinterpret_cast<const uint8_t*>(data_.data());
-  *bit_type = 32;
-  *total_size = data_.size();
-  *is_sparse = true;
-  *out_data_ptr = reinterpret_cast<const uint8_t*>(row_ptr_.data());
-  *data_ptr_bit_type = 16;
-  return to_return;
-}
-
-template <>
-const void* MultiValSparseBin<uint32_t, uint8_t>::GetRowWiseData(
-  uint8_t* bit_type,
-  size_t* total_size,
-  bool* is_sparse,
-  const void** out_data_ptr,
-  uint8_t* data_ptr_bit_type) const {
-  const uint8_t* to_return = data_.data();
-  *bit_type = 8;
-  *total_size = data_.size();
-  *is_sparse = true;
-  *out_data_ptr = reinterpret_cast<const uint8_t*>(row_ptr_.data());
-  *data_ptr_bit_type = 32;
-  return to_return;
-}
-
-template <>
-const void* MultiValSparseBin<uint32_t, uint16_t>::GetRowWiseData(
-  uint8_t* bit_type,
-  size_t* total_size,
-  bool* is_sparse,
-  const void** out_data_ptr,
-  uint8_t* data_ptr_bit_type) const {
-  const uint8_t* to_return = reinterpret_cast<const uint8_t*>(data_.data());
-  *bit_type = 16;
-  *total_size = data_.size();
-  *is_sparse = true;
-  *out_data_ptr = reinterpret_cast<const uint8_t*>(row_ptr_.data());
-  *data_ptr_bit_type = 32;
-  return to_return;
-}
-
-template <>
-const void* MultiValSparseBin<uint32_t, uint32_t>::GetRowWiseData(
-  uint8_t* bit_type,
-  size_t* total_size,
-  bool* is_sparse,
-  const void** out_data_ptr,
-  uint8_t* data_ptr_bit_type) const {
-  const uint8_t* to_return = reinterpret_cast<const uint8_t*>(data_.data());
-  *bit_type = 32;
-  *total_size = data_.size();
-  *is_sparse = true;
-  *out_data_ptr = reinterpret_cast<const uint8_t*>(row_ptr_.data());
-  *data_ptr_bit_type = 32;
-  return to_return;
-}
-
-template <>
-const void* MultiValSparseBin<uint64_t, uint8_t>::GetRowWiseData(
-  uint8_t* bit_type,
-  size_t* total_size,
-  bool* is_sparse,
-  const void** out_data_ptr,
-  uint8_t* data_ptr_bit_type) const {
-  const uint8_t* to_return = data_.data();
-  *bit_type = 8;
-  *total_size = data_.size();
-  *is_sparse = true;
-  *out_data_ptr = reinterpret_cast<const uint8_t*>(row_ptr_.data());
-  *data_ptr_bit_type = 64;
-  return to_return;
-}
-
-template <>
-const void* MultiValSparseBin<uint64_t, uint16_t>::GetRowWiseData(
-  uint8_t* bit_type,
-  size_t* total_size,
-  bool* is_sparse,
-  const void** out_data_ptr,
-  uint8_t* data_ptr_bit_type) const {
-  const uint8_t* to_return = reinterpret_cast<const uint8_t*>(data_.data());
-  *bit_type = 16;
-  *total_size = data_.size();
-  *is_sparse = true;
-  *out_data_ptr = reinterpret_cast<const uint8_t*>(row_ptr_.data());
-  *data_ptr_bit_type = 64;
-  return to_return;
-}
-
-template <>
-const void* MultiValSparseBin<uint64_t, uint32_t>::GetRowWiseData(
-  uint8_t* bit_type,
-  size_t* total_size,
-  bool* is_sparse,
-  const void** out_data_ptr,
-  uint8_t* data_ptr_bit_type) const {
-  const uint8_t* to_return = reinterpret_cast<const uint8_t*>(data_.data());
-  *bit_type = 32;
-  *total_size = data_.size();
-  *is_sparse = true;
-  *out_data_ptr = reinterpret_cast<const uint8_t*>(row_ptr_.data());
-  *data_ptr_bit_type = 64;
-  return to_return;
-}
-
-#endif  // USE_CUDA_EXP
-
-}  // namespace LightGBM
diff --git a/src/io/sparse_bin.cpp b/src/io/sparse_bin.cpp
deleted file mode 100644
index 8c45fd512a04..000000000000
--- a/src/io/sparse_bin.cpp
+++ /dev/null
@@ -1,86 +0,0 @@
-/*!
- * Copyright (c) 2021 Microsoft Corporation. All rights reserved.
- * Licensed under the MIT License. See LICENSE file in the project root for
- * license information.
- */
-
-#include "sparse_bin.hpp"
-
-namespace LightGBM {
-
-template <>
-const void* SparseBin<uint8_t>::GetColWiseData(
-  uint8_t* bit_type,
-  bool* is_sparse,
-  std::vector<BinIterator*>* bin_iterator,
-  const int num_threads) const {
-  *is_sparse = true;
-  *bit_type = 8;
-  for (int thread_index = 0; thread_index < num_threads; ++thread_index) {
-    bin_iterator->emplace_back(new SparseBinIterator<uint8_t>(this, 0));
-  }
-  return nullptr;
-}
-
-template <>
-const void* SparseBin<uint16_t>::GetColWiseData(
-  uint8_t* bit_type,
-  bool* is_sparse,
-  std::vector<BinIterator*>* bin_iterator,
-  const int num_threads) const {
-  *is_sparse = true;
-  *bit_type = 16;
-  for (int thread_index = 0; thread_index < num_threads; ++thread_index) {
-    bin_iterator->emplace_back(new SparseBinIterator<uint16_t>(this, 0));
-  }
-  return nullptr;
-}
-
-template <>
-const void* SparseBin<uint32_t>::GetColWiseData(
-  uint8_t* bit_type,
-  bool* is_sparse,
-  std::vector<BinIterator*>* bin_iterator,
-  const int num_threads) const {
-  *is_sparse = true;
-  *bit_type = 32;
-  for (int thread_index = 0; thread_index < num_threads; ++thread_index) {
-    bin_iterator->emplace_back(new SparseBinIterator<uint32_t>(this, 0));
-  }
-  return nullptr;
-}
-
-template <>
-const void* SparseBin<uint8_t>::GetColWiseData(
-  uint8_t* bit_type,
-  bool* is_sparse,
-  BinIterator** bin_iterator) const {
-  *is_sparse = true;
-  *bit_type = 8;
-  *bin_iterator = new SparseBinIterator<uint8_t>(this, 0);
-  return nullptr;
-}
-
-template <>
-const void* SparseBin<uint16_t>::GetColWiseData(
-  uint8_t* bit_type,
-  bool* is_sparse,
-  BinIterator** bin_iterator) const {
-  *is_sparse = true;
-  *bit_type = 16;
-  *bin_iterator = new SparseBinIterator<uint16_t>(this, 0);
-  return nullptr;
-}
-
-template <>
-const void* SparseBin<uint32_t>::GetColWiseData(
-  uint8_t* bit_type,
-  bool* is_sparse,
-  BinIterator** bin_iterator) const {
-  *is_sparse = true;
-  *bit_type = 32;
-  *bin_iterator = new SparseBinIterator<uint32_t>(this, 0);
-  return nullptr;
-}
-
-}  // namespace LightGBM
diff --git a/windows/LightGBM.vcxproj b/windows/LightGBM.vcxproj
index b34ec20318fe..653b86c00ac5 100644
--- a/windows/LightGBM.vcxproj
+++ b/windows/LightGBM.vcxproj
@@ -317,14 +317,10 @@
     <ClCompile Include="..\src\io\config_auto.cpp" />
     <ClCompile Include="..\src\io\dataset.cpp" />
     <ClCompile Include="..\src\io\dataset_loader.cpp" />
-    <ClCompile Include="..\src\io\dense_bin.cpp" />
     <ClCompile Include="..\src\io\file_io.cpp" />
     <ClCompile Include="..\src\io\json11.cpp" />
     <ClCompile Include="..\src\io\metadata.cpp" />
-    <ClCompile Include="..\src\io\multi_val_dense_bin.cpp" />
-    <ClCompile Include="..\src\io\multi_val_sparse_bin.cpp" />
     <ClCompile Include="..\src\io\parser.cpp" />
-    <ClCompile Include="..\src\io\sparse_bin.cpp" />
     <ClCompile Include="..\src\io\train_share_states.cpp" />
     <ClCompile Include="..\src\io\tree.cpp" />
     <ClCompile Include="..\src\metric\dcg_calculator.cpp" />
diff --git a/windows/LightGBM.vcxproj.filters b/windows/LightGBM.vcxproj.filters
index b0e3d7744f3e..0f48c7564580 100644
--- a/windows/LightGBM.vcxproj.filters
+++ b/windows/LightGBM.vcxproj.filters
@@ -326,17 +326,5 @@
     <ClCompile Include="..\src\treelearner\linear_tree_learner.cpp">
       <Filter>src\treelearner</Filter>
     </ClCompile>
-    <ClCompile Include="..\src\io\multi_val_dense_bin.cpp">
-      <Filter>src\io</Filter>
-    </ClCompile>
-    <ClCompile Include="..\src\io\multi_val_sparse_bin.cpp">
-      <Filter>src\io</Filter>
-    </ClCompile>
-    <ClCompile Include="..\src\io\dense_bin.cpp">
-      <Filter>src\io</Filter>
-    </ClCompile>
-    <ClCompile Include="..\src\io\sparse_bin.cpp">
-      <Filter>src\io</Filter>
-    </ClCompile>
   </ItemGroup>
 </Project>
\ No newline at end of file