diff --git a/bucket.go b/bucket.go
index 2dbb307..aa3f046 100644
--- a/bucket.go
+++ b/bucket.go
@@ -118,3 +118,16 @@ func (b *bucket) split(cpl int, target ID) *bucket {
 	}
 	return newbuck
 }
+
+// maxCommonPrefix returns the maximum common prefix length between any peer in
+// the bucket with the target ID.
+func (b *bucket) maxCommonPrefix(target ID) uint {
+	maxCpl := uint(0)
+	for e := b.list.Front(); e != nil; e = e.Next() {
+		cpl := uint(CommonPrefixLen(e.Value.(*PeerInfo).dhtId, target))
+		if cpl > maxCpl {
+			maxCpl = cpl
+		}
+	}
+	return maxCpl
+}
diff --git a/table.go b/table.go
index b760aab..eaac2f5 100644
--- a/table.go
+++ b/table.go
@@ -365,3 +365,16 @@ func (rt *RoutingTable) bucketIdForPeer(p peer.ID) int {
 	}
 	return bucketID
 }
+
+// maxCommonPrefix returns the maximum common prefix length between any peer in
+// the table and the current peer.
+func (rt *RoutingTable) maxCommonPrefix() uint {
+	rt.tabLock.RLock()
+	defer rt.tabLock.RUnlock()
+
+	if len(rt.buckets) == 0 {
+		return 0
+	}
+
+	return rt.buckets[len(rt.buckets)-1].maxCommonPrefix(rt.local)
+}
diff --git a/table_refresh.go b/table_refresh.go
index 927f0e7..a7bfa09 100644
--- a/table_refresh.go
+++ b/table_refresh.go
@@ -15,25 +15,22 @@ import (
 // This limit exists because we can only generate 'maxCplForRefresh' bit prefixes for now.
 const maxCplForRefresh uint = 15
 
-// CplRefresh contains a CPL(common prefix length) with the host & the last time
-// we refreshed that cpl/searched for an ID which has that cpl with the host.
-type CplRefresh struct {
-	Cpl           uint
-	LastRefreshAt time.Time
-}
-
 // GetTrackedCplsForRefresh returns the Cpl's we are tracking for refresh.
 // Caller is free to modify the returned slice as it is a defensive copy.
-func (rt *RoutingTable) GetTrackedCplsForRefresh() []CplRefresh {
+func (rt *RoutingTable) GetTrackedCplsForRefresh() []time.Time {
+	maxCommonPrefix := rt.maxCommonPrefix()
+	if maxCommonPrefix > maxCplForRefresh {
+		maxCommonPrefix = maxCplForRefresh
+	}
+
 	rt.cplRefreshLk.RLock()
 	defer rt.cplRefreshLk.RUnlock()
 
-	cpls := make([]CplRefresh, 0, len(rt.cplRefreshedAt))
-
-	for c, t := range rt.cplRefreshedAt {
-		cpls = append(cpls, CplRefresh{c, t})
+	cpls := make([]time.Time, maxCommonPrefix)
+	for i := uint(0); i < maxCommonPrefix; i++ {
+		// defaults to the zero value if we haven't refreshed it yet.
+		cpls[i] = rt.cplRefreshedAt[i]
 	}
-
 	return cpls
 }
 
diff --git a/table_refresh_test.go b/table_refresh_test.go
index 4548c4c..f645ea5 100644
--- a/table_refresh_test.go
+++ b/table_refresh_test.go
@@ -35,28 +35,61 @@ func TestGenRandPeerID(t *testing.T) {
 
 func TestRefreshAndGetTrackedCpls(t *testing.T) {
 	t.Parallel()
+
+	const testCpl = 10
+
 	local := test.RandPeerIDFatal(t)
 	m := pstore.NewMetrics()
 	rt, err := NewRoutingTable(1, ConvertPeerID(local), time.Hour, m, NoOpThreshold)
 	require.NoError(t, err)
 
-	// push cpl's for tracking
-	for cpl := uint(0); cpl < maxCplForRefresh; cpl++ {
-		peerID, err := rt.GenRandPeerID(cpl)
-		require.NoError(t, err)
-		rt.ResetCplRefreshedAtForID(ConvertPeerID(peerID), time.Now())
-	}
-
 	// fetch cpl's
 	trackedCpls := rt.GetTrackedCplsForRefresh()
+	// should have nothing.
+	require.Len(t, trackedCpls, 0)
+
+	// add a peer ID
+	peerID, err := rt.GenRandPeerID(testCpl)
+	require.NoError(t, err)
+	added, err := rt.TryAddPeer(peerID, true)
+	require.NoError(t, err)
+	require.True(t, added)
+
+	// should be tracking 10
+	trackedCpls = rt.GetTrackedCplsForRefresh()
+	require.Len(t, trackedCpls, testCpl)
+	// they should all be zero
+	for _, refresh := range trackedCpls {
+		require.True(t, refresh.IsZero(), "tracked cpl's should be zero")
+	}
+
+	// add our peer ID to max out the table
+	added, err = rt.TryAddPeer(local, true)
+	require.NoError(t, err)
+	require.True(t, added)
+
+	// should be tracking the max
+	trackedCpls = rt.GetTrackedCplsForRefresh()
 	require.Len(t, trackedCpls, int(maxCplForRefresh))
-	actualCpls := make(map[uint]struct{})
-	for i := 0; i < len(trackedCpls); i++ {
-		actualCpls[trackedCpls[i].Cpl] = struct{}{}
+
+	// and not refreshed
+	for _, refresh := range trackedCpls {
+		require.True(t, refresh.IsZero(), "tracked cpl's should be zero")
 	}
 
-	for i := uint(0); i < maxCplForRefresh; i++ {
-		_, ok := actualCpls[i]
-		require.True(t, ok, "tracked cpl's should have cpl %d", i)
+	now := time.Now()
+	// reset the test peer ID.
+	rt.ResetCplRefreshedAtForID(ConvertPeerID(peerID), now)
+
+	// should still be tracking all buckets
+	trackedCpls = rt.GetTrackedCplsForRefresh()
+	require.Len(t, trackedCpls, int(maxCplForRefresh))
+
+	for i, refresh := range trackedCpls {
+		if i == testCpl {
+			require.True(t, now.Equal(refresh), "test cpl should have the correct refresh time")
+		} else {
+			require.True(t, refresh.IsZero(), "other cpl's should be 0")
+		}
 	}
 }