region_cache: reload region cache when flag needDelayedReloadReady is set (#1738)

internal/locate/region_cache.go

@@ -1322,7 +1322,7 @@ func (c *RegionCache) findRegionByKey(bo *retry.Backoffer, key []byte, isEndKey
 func (c *RegionCache) tryFindRegionByKey(key []byte, isEndKey bool) (r *Region) {
 	var expired bool
 	r, expired = c.searchCachedRegionByKey(key, isEndKey)
-	if r == nil || expired || r.checkSyncFlags(needReloadOnAccess) {
+	if r == nil || expired || r.checkSyncFlags(needReloadOnAccess|needDelayedReloadReady) {
 		return nil
 	}
 	return r
@@ -1898,20 +1898,25 @@ func (c *RegionCache) loadRegionByID(bo *retry.Backoffer, regionID uint64) (*Reg
 }
 
 // TODO(youjiali1995): for optimizing BatchLoadRegionsWithKeyRange, not used now.
-func (c *RegionCache) scanRegionsFromCache(bo *retry.Backoffer, startKey, endKey []byte, limit int) ([]*Region, error) {
+func (c *RegionCache) scanRegionsFromCache(startKey, endKey []byte, limit int) []*Region {
 	if limit == 0 {
-		return nil, nil
+		return nil
 	}
 
 	var regions []*Region
 	c.mu.RLock()
 	defer c.mu.RUnlock()
 	regions = c.mu.sorted.AscendGreaterOrEqual(startKey, endKey, limit)
-
-	if len(regions) == 0 {
-		return nil, errors.New("no regions in the cache")
+	// in-place filter out the regions which need reload.
+	i := 0
+	for _, region := range regions {
+		if !region.checkSyncFlags(needReloadOnAccess | needDelayedReloadReady) {
+			regions[i] = region
+			i++
+		}
 	}
-	return regions, nil
+	regions = regions[:i]
+	return regions
 }
 
 func (c *RegionCache) refreshRegionIndex(bo *retry.Backoffer) error {

internal/locate/region_cache_test.go

@@ -2233,27 +2233,50 @@ func (s *testRegionCacheSuite) testSplitThenLocateKey(markRegion func(r *Region)
 	markRegion(r1)
 
 	// locate key
+	s.Nil(s.cache.TryLocateKey(r1.StartKey()))
+	s.Nil(s.cache.TryLocateKey(k))
+	s.Len(s.cache.scanRegionsFromCache(r1.StartKey(), nil, 2), 0)
+	loc1, err := s.cache.LocateKey(s.bo, r1.StartKey())
+	s.NoError(err)
+	s.False(loc1.Contains(k))
 	loc, err := s.cache.LocateKey(s.bo, k)
 	s.NoError(err)
 	s.True(loc.Contains(k))
 }
 
 func (s *testRegionRequestToSingleStoreSuite) TestRefreshCache() {
 	_ = s.cache.refreshRegionIndex(s.bo)
-	r, _ := s.cache.scanRegionsFromCache(s.bo, []byte{}, nil, 10)
+	r := s.cache.scanRegionsFromCache([]byte{}, nil, 10)
 	s.Equal(len(r), 1)
 
 	region, _ := s.cache.LocateRegionByID(s.bo, s.region)
 	v2 := region.Region.confVer + 1
-	r2 := metapb.Region{Id: region.Region.id, RegionEpoch: &metapb.RegionEpoch{Version: region.Region.ver, ConfVer: v2}, StartKey: []byte{1}}
+	r2 := metapb.Region{Id: region.Region.id, RegionEpoch: &metapb.RegionEpoch{Version: region.Region.ver, ConfVer: v2}, StartKey: []byte{2}}
 	st := newUninitializedStore(s.store)
 	s.cache.insertRegionToCache(&Region{meta: &r2, store: unsafe.Pointer(st), ttl: nextTTLWithoutJitter(time.Now().Unix())}, true, true)
 
-	r, _ = s.cache.scanRegionsFromCache(s.bo, []byte{}, nil, 10)
+	// Since region cache doesn't remove the first intersected region(it scan intersected region by AscendGreaterOrEqual), the outdated region (-inf, inf) is still alive.
+	// The new inserted valid region [{2}, inf) is ignored because the first seen region (-inf, inf) contains all the required ranges.
+	r = s.cache.scanRegionsFromCache([]byte{}, nil, 10)
+	s.Equal(len(r), 1)
+	s.Equal(r[0].StartKey(), []byte(nil))
+
+	// regions: (-inf,2), [2, +inf).  Get all regions.
+	v3 := region.Region.confVer + 2
+	r3 := metapb.Region{Id: region.Region.id, RegionEpoch: &metapb.RegionEpoch{Version: region.Region.ver, ConfVer: v3}, StartKey: []byte{}, EndKey: []byte{2}}
+	s.cache.insertRegionToCache(&Region{meta: &r3, store: unsafe.Pointer(st), ttl: nextTTLWithoutJitter(time.Now().Unix())}, true, true)
+	r = s.cache.scanRegionsFromCache([]byte{}, nil, 10)
 	s.Equal(len(r), 2)
 
+	// regions: (-inf,1), [2, +inf).  Get region (-inf, 1).
+	v4 := region.Region.confVer + 3
+	r4 := metapb.Region{Id: region.Region.id, RegionEpoch: &metapb.RegionEpoch{Version: region.Region.ver, ConfVer: v4}, StartKey: []byte{}, EndKey: []byte{1}}
+	s.cache.insertRegionToCache(&Region{meta: &r4, store: unsafe.Pointer(st), ttl: nextTTLWithoutJitter(time.Now().Unix())}, true, true)
+	r = s.cache.scanRegionsFromCache([]byte{}, nil, 10)
+	s.Equal(len(r), 1)
+
 	_ = s.cache.refreshRegionIndex(s.bo)
-	r, _ = s.cache.scanRegionsFromCache(s.bo, []byte{}, nil, 10)
+	r = s.cache.scanRegionsFromCache([]byte{}, nil, 10)
 	s.Equal(len(r), 1)
 }
 
@@ -2354,7 +2377,7 @@ func (s *testRegionCacheWithDelaySuite) TestInsertStaleRegion() {
 	stale = !s.cache.insertRegionToCache(fakeRegion, true, true)
 	s.True(stale)
 
-	rs, err := s.cache.scanRegionsFromCache(s.bo, []byte(""), []byte(""), 100)
+	rs := s.cache.scanRegionsFromCache([]byte(""), []byte(""), 100)
 	s.NoError(err)
 	s.Greater(len(rs), 1)
 	s.NotEqual(rs[0].EndKey(), "")

internal/locate/sorted_btree.go

@@ -36,6 +36,7 @@ package locate
 
 import (
 	"bytes"
+	"time"
 
 	"github.com/google/btree"
 	"github.com/tikv/client-go/v2/internal/logutil"
@@ -79,12 +80,22 @@ func (s *SortedRegions) SearchByKey(key []byte, isEndKey bool) (r *Region) {
 }
 
 // AscendGreaterOrEqual returns all items that are greater than or equal to the key.
+// It is the caller's responsibility to make sure that startKey is a node in the B-tree, otherwise, the startKey will not be included in the return regions.
 func (s *SortedRegions) AscendGreaterOrEqual(startKey, endKey []byte, limit int) (regions []*Region) {
+	now := time.Now().Unix()
+	lastStartKey := startKey
 	s.b.AscendGreaterOrEqual(newBtreeSearchItem(startKey), func(item *btreeItem) bool {
 		region := item.cachedRegion
 		if len(endKey) > 0 && bytes.Compare(region.StartKey(), endKey) >= 0 {
 			return false
 		}
+		if !region.checkRegionCacheTTL(now) {
+			return false
+		}
+		if !region.Contains(lastStartKey) { // uncached hole
+			return false
+		}
+		lastStartKey = region.EndKey()
 		regions = append(regions, region)
 		return len(regions) < limit
 	})