diff --git a/cpp/src/arrow/compute/kernels/vector_rank.cc b/cpp/src/arrow/compute/kernels/vector_rank.cc
index 4fdc83788c6f4..466bb8417c5cb 100644
--- a/cpp/src/arrow/compute/kernels/vector_rank.cc
+++ b/cpp/src/arrow/compute/kernels/vector_rank.cc
@@ -36,9 +36,10 @@ constexpr bool NeedsDuplicates(RankOptions::Tiebreaker tiebreaker) {
   return tiebreaker != RankOptions::First;
 }
 
-template <typename ValueSelector>
-void MarkDuplicates(const NullPartitionResult& sorted, ValueSelector&& value_selector) {
-  using T = decltype(value_selector(int64_t{}));
+template <typename IndexType, typename ValueSelector>
+void MarkDuplicates(const GenericNullPartitionResult<IndexType>& sorted,
+                    ValueSelector&& value_selector) {
+  using T = decltype(value_selector(*sorted.non_nulls_begin));
 
   // Process non-nulls
   if (sorted.non_nulls_end != sorted.non_nulls_begin) {