Skip to content

Commit

Permalink
*: enhance the table regions HTTP API (pingcap#12357)
Browse files Browse the repository at this point in the history
  • Loading branch information
@tiancaiamao @sre-bot
tiancaiamao authored and sre-bot committed Sep 26, 2019
1 parent 510a851 commit db1eb1c
Show file tree
Hide file tree
Showing 8 changed files with 58 additions and 30 deletions.
3 changes: 1 addition & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ require (
github.com/grpc-ecosystem/grpc-gateway v1.5.1 // indirect
github.com/json-iterator/go v1.1.6 // indirect
github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808 // indirect
Expand All @@ -42,7 +41,7 @@ require (
github.com/pingcap/kvproto v0.0.0-20190910074005-0e61b6f435c1
github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd
github.com/pingcap/parser v0.0.0-20190923031704-33636bc5e5d6
github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b
github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0
github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible
github.com/pingcap/tipb v0.0.0-20190806070524-16909e03435e
github.com/prometheus/client_golang v0.9.0
Expand Down
10 changes: 5 additions & 5 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,24 +152,22 @@ github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FI
github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ=
github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9 h1:KH4f4Si9XK6/IW50HtoaiLIFHGkapOM6w83za47UYik=
github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM=
github.com/pingcap/errors v0.10.1/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pingcap/failpoint v0.0.0-20190512135322-30cc7431d99c h1:hvQd3aOLKLF7xvRV6DzvPkKY4QXzfVbjU1BhW0d9yL8=
github.com/pingcap/failpoint v0.0.0-20190512135322-30cc7431d99c/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI=
github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e h1:P73/4dPCL96rGrobssy1nVy2VaVpNCuLpCbr+FEaTA8=
github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw=
github.com/pingcap/kvproto v0.0.0-20190516013202-4cf58ad90b6c/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY=
github.com/pingcap/kvproto v0.0.0-20190822090350-11ea838aedf7/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY=
github.com/pingcap/kvproto v0.0.0-20190910074005-0e61b6f435c1 h1:DNvxkdcjA0TBIIIF+K2w9KMlTzMZzLZ5JVF26kTCPhg=
github.com/pingcap/kvproto v0.0.0-20190910074005-0e61b6f435c1/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY=
github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w=
github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd h1:hWDol43WY5PGhsh3+8794bFHY1bPrmu6bTalpssCrGg=
github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw=
github.com/pingcap/parser v0.0.0-20190923031704-33636bc5e5d6 h1:PyjsTUD8gJ6QGilbwiy/TTn89J84/69Pj9LixOd/fFE=
github.com/pingcap/parser v0.0.0-20190923031704-33636bc5e5d6/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA=
github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b h1:oS9PftxQqgcRouKhhdaB52tXhVLEP7Ng3Qqsd6Z18iY=
github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs=
github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0 h1:GIEq+wZfrl2bcJxpuSrEH4H7/nlf5YdmpS+dU9lNIt8=
github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0/go.mod h1:G/6rJpnYwM0LKMec2rI82/5Kg6GaZMvlfB+e6/tvYmI=
github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU=
github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
github.com/pingcap/tipb v0.0.0-20190806070524-16909e03435e h1:H7meq8QPmWGImOkHTQYAWw82zwIqndJaCDPVUknOHbM=
Expand Down Expand Up @@ -259,6 +257,7 @@ golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnf
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7 h1:0hQKqeLdqlt5iIwVOBErRisrHJAN57yOiPRQItI20fU=
golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
Expand Down Expand Up @@ -286,6 +285,7 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190909082730-f460065e899a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b h1:3S2h5FadpNr0zUUCVZjlKIEYF+KaX/OBplTGo89CYHI=
golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
Expand Down
34 changes: 26 additions & 8 deletions server/http_handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -994,13 +994,23 @@ func (h tableHandler) handleRegionRequest(schema infoschema.InfoSchema, tbl tabl
func (h tableHandler) getRegionsByID(tbl table.Table, id int64, name string) (*TableRegions, error) {
// for record
startKey, endKey := tablecodec.GetTableHandleKeyRange(id)
recordRegionIDs, err := h.RegionCache.ListRegionIDsInKeyRange(tikv.NewBackoffer(context.Background(), 500), startKey, endKey)
ctx := context.Background()
pdCli := h.RegionCache.PDClient()
regions, peers, err := pdCli.ScanRegions(ctx, startKey, endKey, -1)
if err != nil {
return nil, err
}
recordRegions, err := h.getRegionsMeta(recordRegionIDs)
if err != nil {
return nil, err

recordRegions := make([]RegionMeta, 0, len(peers))
for idx, leader := range peers {
region := regions[idx]
meta := RegionMeta{
ID: region.Id,
Leader: leader,
Peers: region.Peers,
RegionEpoch: region.RegionEpoch,
}
recordRegions = append(recordRegions, meta)
}

// for indices
Expand All @@ -1010,14 +1020,22 @@ func (h tableHandler) getRegionsByID(tbl table.Table, id int64, name string) (*T
indices[i].Name = index.Meta().Name.String()
indices[i].ID = indexID
startKey, endKey := tablecodec.GetTableIndexKeyRange(id, indexID)
rIDs, err := h.RegionCache.ListRegionIDsInKeyRange(tikv.NewBackoffer(context.Background(), 500), startKey, endKey)
regions, peers, err := pdCli.ScanRegions(ctx, startKey, endKey, -1)
if err != nil {
return nil, err
}
indices[i].Regions, err = h.getRegionsMeta(rIDs)
if err != nil {
return nil, err
indexRegions := make([]RegionMeta, 0, len(peers))
for idx, leader := range peers {
region := regions[idx]
meta := RegionMeta{
ID: region.Id,
Leader: leader,
Peers: region.Peers,
RegionEpoch: region.RegionEpoch,
}
indexRegions = append(indexRegions, meta)
}
indices[i].Regions = indexRegions
}

return &TableRegions{
Expand Down
21 changes: 14 additions & 7 deletions store/mockstore/mocktikv/cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ func (c *Cluster) GetRegionByID(regionID uint64) (*metapb.Region, *metapb.Peer)
}

// ScanRegions returns at most `limit` regions from given `key` and their leaders.
func (c *Cluster) ScanRegions(key []byte, limit int) ([]*metapb.Region, []*metapb.Peer) {
func (c *Cluster) ScanRegions(startKey, endKey []byte, limit int) ([]*metapb.Region, []*metapb.Peer) {
c.RLock()
defer c.RUnlock()

Expand All @@ -284,15 +284,22 @@ func (c *Cluster) ScanRegions(key []byte, limit int) ([]*metapb.Region, []*metap
return bytes.Compare(regions[i].Meta.GetStartKey(), regions[j].Meta.GetStartKey()) < 0
})

keyLocation := sort.Search(len(regions), func(i int) bool {
endKey := regions[i].Meta.GetEndKey()
if len(endKey) == 0 {
startPos := sort.Search(len(regions), func(i int) bool {
if len(regions[i].Meta.GetEndKey()) == 0 {
return true
}
return bytes.Compare(regions[i].Meta.GetEndKey(), key) > 0
return bytes.Compare(regions[i].Meta.GetEndKey(), startKey) > 0
})
regions = regions[keyLocation:]
if len(regions) > limit {
regions = regions[startPos:]
if len(endKey) > 0 {
endPos := sort.Search(len(regions), func(i int) bool {
return bytes.Compare(regions[i].Meta.GetStartKey(), endKey) >= 0
})
if endPos > 0 {
regions = regions[:endPos]
}
}
if limit > 0 && len(regions) > limit {
regions = regions[:limit]
}

Expand Down
4 changes: 2 additions & 2 deletions store/mockstore/mocktikv/pd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,8 @@ func (c *pdClient) GetRegionByID(ctx context.Context, regionID uint64) (*metapb.
return region, peer, nil
}

func (c *pdClient) ScanRegions(ctx context.Context, key []byte, limit int) ([]*metapb.Region, []*metapb.Peer, error) {
regions, peers := c.cluster.ScanRegions(key, limit)
func (c *pdClient) ScanRegions(ctx context.Context, startKey []byte, endKey []byte, limit int) ([]*metapb.Region, []*metapb.Peer, error) {
regions, peers := c.cluster.ScanRegions(startKey, endKey, limit)
return regions, peers, nil
}

Expand Down
10 changes: 7 additions & 3 deletions store/tikv/pd_codec.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,13 @@ func (c *codecPDClient) GetRegionByID(ctx context.Context, regionID uint64) (*me
return processRegionResult(region, peer, err)
}

func (c *codecPDClient) ScanRegions(ctx context.Context, startKey []byte, limit int) ([]*metapb.Region, []*metapb.Peer, error) {
encodedKey := codec.EncodeBytes([]byte(nil), startKey)
regions, peers, err := c.Client.ScanRegions(ctx, encodedKey, limit)
func (c *codecPDClient) ScanRegions(ctx context.Context, startKey []byte, endKey []byte, limit int) ([]*metapb.Region, []*metapb.Peer, error) {
startKey = codec.EncodeBytes([]byte(nil), startKey)
if len(endKey) > 0 {
endKey = codec.EncodeBytes([]byte(nil), endKey)
}

regions, peers, err := c.Client.ScanRegions(ctx, startKey, endKey, limit)
if err != nil {
return nil, nil, errors.Trace(err)
}
Expand Down
2 changes: 1 addition & 1 deletion store/tikv/region_cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -819,7 +819,7 @@ func (c *RegionCache) scanRegions(bo *Backoffer, startKey []byte, limit int) ([]
return nil, errors.Trace(err)
}
}
metas, leaders, err := c.pdClient.ScanRegions(bo.ctx, startKey, limit)
metas, leaders, err := c.pdClient.ScanRegions(bo.ctx, startKey, nil, limit)
if err != nil {
tikvRegionCacheCounterWithScanRegionsError.Inc()
backoffErr = errors.Errorf(
Expand Down
4 changes: 2 additions & 2 deletions store/tikv/store_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,14 +168,14 @@ func (c *mockPDClient) GetRegionByID(ctx context.Context, regionID uint64) (*met
return c.client.GetRegionByID(ctx, regionID)
}

func (c *mockPDClient) ScanRegions(ctx context.Context, startKey []byte, limit int) ([]*metapb.Region, []*metapb.Peer, error) {
func (c *mockPDClient) ScanRegions(ctx context.Context, startKey []byte, endKey []byte, limit int) ([]*metapb.Region, []*metapb.Peer, error) {
c.RLock()
defer c.RUnlock()

if c.stop {
return nil, nil, errors.Trace(errStopped)
}
return c.client.ScanRegions(ctx, startKey, limit)
return c.client.ScanRegions(ctx, startKey, endKey, limit)
}

func (c *mockPDClient) GetStore(ctx context.Context, storeID uint64) (*metapb.Store, error) {
Expand Down

0 comments on commit db1eb1c

Please sign in to comment.