Skip to content

Instantly share code, notes, and snippets.

@tabokie

tabokie/leader

Created December 7, 2021 09:49
Show Gist options
  • Save tabokie/cad2cf6b622e021cfffc6307cf828beb to your computer and use it in GitHub Desktop.
Save tabokie/cad2cf6b622e021cfffc6307cf828beb to your computer and use it in GitHub Desktop.
Download ZIP
oncall-4167
Raw
leader
[2021/12/03 10:58:15.059 +08:00] [Info] [raft.rs:1200] ["became leader at term 7"] [term=7] [raft_id=97957681] [region_id=97957472]
[2021/12/03 10:58:15.059 +08:00] [Info] [peer.rs:3771] ["require updating max ts"] [initial_status=30064791044] [region_id=97957472]
[2021/12/03 10:58:15.059 +08:00] [Info] [pd.rs:1271] ["succeed to update max timestamp"] [region_id=97957472]
[2021/12/03 10:58:15.059 +08:00] [Info] [pd.rs:1108] ["try to change peer"] [kind=EnterJoint] [changes="[peer { id: 97957752 store_id: 80511561 }, peer { id: 97957473 store_id: 80511562 role: Learner } change_type: AddLearnerNode]"] [region_id=97957472]
[2021/12/03 10:58:15.062 +08:00] [Info] [pd.rs:1108] ["try to change peer"] [kind=EnterJoint] [changes="[peer { id: 97957752 store_id: 80511561 }, peer { id: 97957473 store_id: 80511562 role: Learner } change_type: AddLearnerNode]"] [region_id=97957472]
[2021/12/03 10:58:15.062 +08:00] [Info] [pd.rs:1108] ["try to change peer"] [kind=EnterJoint] [changes="[peer { id: 97957752 store_id: 80511561 }, peer { id: 97957473 store_id: 80511562 role: Learner } change_type: AddLearnerNode]"] [region_id=97957472]
[2021/12/03 10:58:15.063 +08:00] [Info] [pd.rs:1108] ["try to change peer"] [kind=EnterJoint] [changes="[peer { id: 97957752 store_id: 80511561 }, peer { id: 97957473 store_id: 80511562 role: Learner } change_type: AddLearnerNode]"] [region_id=97957472]
[2021/12/03 10:58:15.063 +08:00] [Info] [peer.rs:3250] ["propose conf change peer"] [kind=EnterJoint] [changes="[peer { id: 97957752 store_id: 80511561 }, change_type: AddLearnerNode peer { id: 97957473 store_id: 80511562 role: Learner }]"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:58:15.063 +08:00] [Info] [endpoint.rs:309] ["register observe region"] [region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8DAB0FF3000000001376236FF3431306264FF2D65FF6663302D3464FF37FF372D386264622DFFFF6363326439353435FFFF37313634000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0132353033666365FF36FF2D623932332DFF3432FF61622D3965FF32612DFF31643339FF39666165FF373132FF6400000000FB0380FF00000042FE4F4900FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1674 version: 9986 } peers { id: 97957473 store_id: 80511562 } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 } peers { id: 97957752 store_id: 80511561 role: Learner }"]
[2021/12/03 10:58:15.063 +08:00] [Info] [peer.rs:3190] ["there is a pending conf change, try later"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:58:15.063 +08:00] [Info] [peer.rs:3190] ["there is a pending conf change, try later"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:58:15.064 +08:00] [Info] [endpoint.rs:233] ["Resolver initialized"] [pending_data_index=0] [snapshot_index=6344] [observe_id=ObserveID(9414)] [region=97957472]
[2021/12/03 10:58:15.066 +08:00] [Info] [apply.rs:1393] ["execute admin command"] [command="cmd_type: ChangePeerV2 change_peer_v2 { changes { peer { id: 97957752 store_id: 80511561 } } changes { change_type: AddLearnerNode peer { id: 97957473 store_id: 80511562 role: Learner } } }"] [index=6345] [term=7] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:58:15.066 +08:00] [Info] [apply.rs:1940] ["exec ConfChangeV2"] [epoch="conf_ver: 1674 version: 9986"] [kind=EnterJoint] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:58:15.066 +08:00] [Info] [apply.rs:2120] ["conf change successfully"] ["current region"="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8DAB0FF3000000001376236FF3431306264FF2D65FF6663302D3464FF37FF372D386264622DFFFF6363326439353435FFFF37313634000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0132353033666365FF36FF2D623932332DFF3432FF61622D3965FF32612DFF31643339FF39666165FF373132FF6400000000FB0380FF00000042FE4F4900FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1676 version: 9986 } peers { id: 97957473 store_id: 80511562 role: DemotingVoter } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 } peers { id: 97957752 store_id: 80511561 role: IncomingVoter }"] ["original region"="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8DAB0FF3000000001376236FF3431306264FF2D65FF6663302D3464FF37FF372D386264622DFFFF6363326439353435FFFF37313634000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0132353033666365FF36FF2D623932332DFF3432FF61622D3965FF32612DFF31643339FF39666165FF373132FF6400000000FB0380FF00000042FE4F4900FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1674 version: 9986 } peers { id: 97957473 store_id: 80511562 } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 } peers { id: 97957752 store_id: 80511561 role: Learner }"] [changes="[peer { id: 97957752 store_id: 80511561 }, change_type: AddLearnerNode peer { id: 97957473 store_id: 80511562 role: Learner }]"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:58:15.067 +08:00] [Info] [raft.rs:2609] ["switched to configuration"] [config="Configuration { voters: Configuration { incoming: Configuration { voters: {97957752, 97957681, 97957475} }, outgoing: Configuration { voters: {97957473, 97957681, 97957475} } }, learners: {}, learners_next: {97957473}, auto_leave: false }"] [raft_id=97957681] [region_id=97957472]
[2021/12/03 10:58:15.068 +08:00] [Info] [peer.rs:2307] ["notify pd with change peer region"] [region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8DAB0FF3000000001376236FF3431306264FF2D65FF6663302D3464FF37FF372D386264622DFFFF6363326439353435FFFF37313634000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0132353033666365FF36FF2D623932332DFF3432FF61622D3965FF32612DFF31643339FF39666165FF373132FF6400000000FB0380FF00000042FE4F4900FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1676 version: 9986 } peers { id: 97957473 store_id: 80511562 role: DemotingVoter } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 } peers { id: 97957752 store_id: 80511561 role: IncomingVoter }"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:58:15.068 +08:00] [Info] [pd.rs:1108] ["try to change peer"] [kind=LeaveJoint] [changes="[]"] [region_id=97957472]
[2021/12/03 10:58:15.068 +08:00] [Info] [peer.rs:3250] ["propose conf change peer"] [kind=LeaveJoint] [changes="[]"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:58:15.068 +08:00] [Info] [pd.rs:1108] ["try to change peer"] [kind=LeaveJoint] [changes="[]"] [region_id=97957472]
[2021/12/03 10:58:15.068 +08:00] [Info] [pd.rs:1108] ["try to change peer"] [kind=LeaveJoint] [changes="[]"] [region_id=97957472]
[2021/12/03 10:58:15.069 +08:00] [Info] [peer.rs:3190] ["there is a pending conf change, try later"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:58:15.069 +08:00] [Info] [peer.rs:3190] ["there is a pending conf change, try later"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:58:15.069 +08:00] [Info] [apply.rs:1393] ["execute admin command"] [command="cmd_type: ChangePeerV2 change_peer_v2 {}"] [index=6346] [term=7] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:58:15.069 +08:00] [Info] [apply.rs:1940] ["exec ConfChangeV2"] [epoch="conf_ver: 1676 version: 9986"] [kind=LeaveJoint] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:58:15.069 +08:00] [Info] [apply.rs:2150] ["leave joint state successfully"] [region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8DAB0FF3000000001376236FF3431306264FF2D65FF6663302D3464FF37FF372D386264622DFFFF6363326439353435FFFF37313634000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0132353033666365FF36FF2D623932332DFF3432FF61622D3965FF32612DFF31643339FF39666165FF373132FF6400000000FB0380FF00000042FE4F4900FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1678 version: 9986 } peers { id: 97957473 store_id: 80511562 role: Learner } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 } peers { id: 97957752 store_id: 80511561 }"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:58:15.070 +08:00] [Info] [raft.rs:2609] ["switched to configuration"] [config="Configuration { voters: Configuration { incoming: Configuration { voters: {97957752, 97957681, 97957475} }, outgoing: Configuration { voters: {} } }, learners: {97957473}, learners_next: {}, auto_leave: false }"] [raft_id=97957681] [region_id=97957472]
[2021/12/03 10:58:15.070 +08:00] [Info] [peer.rs:2307] ["notify pd with change peer region"] [region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8DAB0FF3000000001376236FF3431306264FF2D65FF6663302D3464FF37FF372D386264622DFFFF6363326439353435FFFF37313634000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0132353033666365FF36FF2D623932332DFF3432FF61622D3965FF32612DFF31643339FF39666165FF373132FF6400000000FB0380FF00000042FE4F4900FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1678 version: 9986 } peers { id: 97957473 store_id: 80511562 role: Learner } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 } peers { id: 97957752 store_id: 80511561 }"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:58:15.070 +08:00] [Info] [pd.rs:1089] ["try to change peer"] [peer="id: 97957473 store_id: 80511562 role: Learner"] [change_type=RemoveNode] [region_id=97957472]
[2021/12/03 10:58:15.070 +08:00] [Info] [peer.rs:3250] ["propose conf change peer"] [kind=Simple] [changes="[change_type: RemoveNode peer { id: 97957473 store_id: 80511562 role: Learner }]"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:58:15.070 +08:00] [Info] [apply.rs:1393] ["execute admin command"] [command="cmd_type: ChangePeer change_peer { change_type: RemoveNode peer { id: 97957473 store_id: 80511562 role: Learner } }"] [index=6347] [term=7] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:58:15.070 +08:00] [Info] [apply.rs:1759] ["exec ConfChange"] [epoch="conf_ver: 1678 version: 9986"] [type=RemoveNode] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:58:15.070 +08:00] [Info] [apply.rs:1867] ["remove peer successfully"] [region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8DAB0FF3000000001376236FF3431306264FF2D65FF6663302D3464FF37FF372D386264622DFFFF6363326439353435FFFF37313634000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0132353033666365FF36FF2D623932332DFF3432FF61622D3965FF32612DFF31643339FF39666165FF373132FF6400000000FB0380FF00000042FE4F4900FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1678 version: 9986 } peers { id: 97957473 store_id: 80511562 role: Learner } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 } peers { id: 97957752 store_id: 80511561 }"] [peer="id: 97957473 store_id: 80511562 role: Learner"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:58:15.071 +08:00] [Info] [raft.rs:2609] ["switched to configuration"] [config="Configuration { voters: Configuration { incoming: Configuration { voters: {97957752, 97957681, 97957475} }, outgoing: Configuration { voters: {} } }, learners: {}, learners_next: {}, auto_leave: false }"] [raft_id=97957681] [region_id=97957472]
[2021/12/03 10:58:15.071 +08:00] [Info] [peer.rs:2307] ["notify pd with change peer region"] [region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8DAB0FF3000000001376236FF3431306264FF2D65FF6663302D3464FF37FF372D386264622DFFFF6363326439353435FFFF37313634000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0132353033666365FF36FF2D623932332DFF3432FF61622D3965FF32612DFF31643339FF39666165FF373132FF6400000000FB0380FF00000042FE4F4900FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1679 version: 9986 } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 } peers { id: 97957752 store_id: 80511561 }"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:07.845 +08:00] [Info] [size.rs:168] ["approximate size over threshold, need to do split check"] [threshold=150994944] [size=213341546] [region_id=97957472]
[2021/12/03 10:59:08.047 +08:00] [Info] [split_check.rs:289] ["update approximate size and keys with accurate value"] [keys=830333] [size=183218633] [region_id=97957472]
[2021/12/03 10:59:08.122 +08:00] [Info] [peer.rs:3827] ["on split"] [source="split checker"] [split_keys="key 748000000000000FFF645F698000000000FF00000C0419A8E2ADFF7500000001646438FF3463383632FF2D36FF3866302D3432FF36FF342D386337332DFFFF3938643532363637FFFF31393031000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0135666239373633FF63FF2D303037382DFF3433FF36332D3866FF36632DFF66616634FF61396336FF386333FF3200000000FB0380FF000000462553FC00FEFA0A071424AFFFF9"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:08.122 +08:00] [Info] [pd.rs:761] ["try to batch split region"] [task=batch_split] [region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8DAB0FF3000000001376236FF3431306264FF2D65FF6663302D3464FF37FF372D386264622DFFFF6363326439353435FFFF37313634000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0132353033666365FF36FF2D623932332DFF3432FF61622D3965FF32612DFF31643339FF39666165FF373132FF6400000000FB0380FF00000042FE4F4900FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1679 version: 9986 } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 } peers { id: 97957752 store_id: 80511561 }"] [new_region_ids="[new_region_id: 97957798 new_peer_ids: 97957799 new_peer_ids: 97957800 new_peer_ids: 97957801]"] [region_id=97957472]
[2021/12/03 10:59:08.298 +08:00] [Info] [apply.rs:1393] ["execute admin command"] [command="cmd_type: BatchSplit splits { requests { split_key: 748000000000000FFF645F698000000000FF00000C0419A8E2ADFF7500000001646438FF3463383632FF2D36FF3866302D3432FF36FF342D386337332DFFFF3938643532363637FFFF31393031000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0135666239373633FF63FF2D303037382DFF3433FF36332D3866FF36632DFF66616634FF61396336FF386333FF3200000000FB0380FF000000462553FC00FE new_region_id: 97957798 new_peer_ids: 97957799 new_peer_ids: 97957800 new_peer_ids: 97957801 } right_derive: true }"] [index=9163] [term=7] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:08.298 +08:00] [Info] [apply.rs:2228] ["split region"] [keys="key 748000000000000FFF645F698000000000FF00000C0419A8E2ADFF7500000001646438FF3463383632FF2D36FF3866302D3432FF36FF342D386337332DFFFF3938643532363637FFFF31393031000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0135666239373633FF63FF2D303037382DFF3433FF36332D3866FF36632DFF66616634FF61396336FF386333FF3200000000FB0380FF000000462553FC00FE"] [region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8DAB0FF3000000001376236FF3431306264FF2D65FF6663302D3464FF37FF372D386264622DFFFF6363326439353435FFFF37313634000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0132353033666365FF36FF2D623932332DFF3432FF61622D3965FF32612DFF31643339FF39666165FF373132FF6400000000FB0380FF00000042FE4F4900FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1679 version: 9986 } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 } peers { id: 97957752 store_id: 80511561 }"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:08.309 +08:00] [Info] [peer.rs:2411] ["notify pd with split"] [split_count=2] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:08.329 +08:00] [Info] [endpoint.rs:382] ["deregister observe region"] [observe_id=ObserveID(9414)] [region_id=97957472] [store_id=Some(80512890)]
[2021/12/03 10:59:08.330 +08:00] [Info] [endpoint.rs:309] ["register observe region"] [region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8E2ADFF7500000001646438FF3463383632FF2D36FF3866302D3432FF36FF342D386337332DFFFF3938643532363637FFFF31393031000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0135666239373633FF63FF2D303037382DFF3433FF36332D3866FF36632DFF66616634FF61396336FF386333FF3200000000FB0380FF000000462553FC00FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1679 version: 9987 } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 } peers { id: 97957752 store_id: 80511561 }"]
[2021/12/03 10:59:08.497 +08:00] [Info] [endpoint.rs:233] ["Resolver initialized"] [pending_data_index=0] [snapshot_index=9163] [observe_id=ObserveID(9417)] [region=97957472]
[2021/12/03 10:59:47.491 +08:00] [Info] [pd.rs:1089] ["try to change peer"] [peer="id: 97957831 store_id: 80511562 role: Learner"] [change_type=AddLearnerNode] [region_id=97957472]
[2021/12/03 10:59:48.894 +08:00] [Info] [peer.rs:3250] ["propose conf change peer"] [kind=Simple] [changes="[change_type: AddLearnerNode peer { id: 97957831 store_id: 80511562 role: Learner }]"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:49.043 +08:00] [Info] [apply.rs:1393] ["execute admin command"] [command="cmd_type: ChangePeer change_peer { change_type: AddLearnerNode peer { id: 97957831 store_id: 80511562 role: Learner } }"] [index=9225] [term=7] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:49.043 +08:00] [Info] [apply.rs:1759] ["exec ConfChange"] [epoch="conf_ver: 1679 version: 9987"] [type=AddLearner] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:49.043 +08:00] [Info] [apply.rs:1899] ["add learner successfully"] [region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8E2ADFF7500000001646438FF3463383632FF2D36FF3866302D3432FF36FF342D386337332DFFFF3938643532363637FFFF31393031000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0135666239373633FF63FF2D303037382DFF3433FF36332D3866FF36632DFF66616634FF61396336FF386333FF3200000000FB0380FF000000462553FC00FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1679 version: 9987 } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 } peers { id: 97957752 store_id: 80511561 }"] [peer="id: 97957831 store_id: 80511562 role: Learner"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:49.127 +08:00] [Info] [raft.rs:2609] ["switched to configuration"] [config="Configuration { voters: Configuration { incoming: Configuration { voters: {97957752, 97957681, 97957475} }, outgoing: Configuration { voters: {} } }, learners: {97957831}, learners_next: {}, auto_leave: false }"] [raft_id=97957681] [region_id=97957472]
[2021/12/03 10:59:49.129 +08:00] [Info] [peer.rs:2307] ["notify pd with change peer region"] [region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8E2ADFF7500000001646438FF3463383632FF2D36FF3866302D3432FF36FF342D386337332DFFFF3938643532363637FFFF31393031000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0135666239373633FF63FF2D303037382DFF3433FF36332D3866FF36632DFF66616634FF61396336FF386333FF3200000000FB0380FF000000462553FC00FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1680 version: 9987 } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 } peers { id: 97957752 store_id: 80511561 } peers { id: 97957831 store_id: 80511562 role: Learner }"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:49.260 +08:00] [Info] [peer_storage.rs:1075] ["requesting snapshot"] [request_index=0] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:49.260 +08:00] [Info] [snap.rs:726] ["scan snapshot of one cf"] [size=0] [key_count=0] [cf=default] [snapshot=/tidb1/tidb/tikv-20161/tidb-data/snap/gen_97957472_7_9225_(default|lock|write).sst] [region_id=97957472]
[2021/12/03 10:59:49.260 +08:00] [Info] [snap.rs:726] ["scan snapshot of one cf"] [size=0] [key_count=0] [cf=lock] [snapshot=/tidb1/tidb/tikv-20161/tidb-data/snap/gen_97957472_7_9225_(default|lock|write).sst] [region_id=97957472]
[2021/12/03 10:59:50.123 +08:00] [Info] [snap.rs:726] ["scan snapshot of one cf"] [size=83759284] [key_count=380761] [cf=write] [snapshot=/tidb1/tidb/tikv-20161/tidb-data/snap/gen_97957472_7_9225_(default|lock|write).sst] [region_id=97957472]
[2021/12/03 10:59:50.127 +08:00] [Info] [snap.rs:801] ["scan snapshot"] [takes=867.067149ms] [size=7633688] [key_count=380761] [snapshot=/tidb1/tidb/tikv-20161/tidb-data/snap/gen_97957472_7_9225_(default|lock|write).sst] [region_id=97957472]
[2021/12/03 10:59:50.159 +08:00] [Info] [snap.rs:446] ["sent snapshot"] [duration=28.247215ms] [size=7633688] [snap_key=97957472_7_9225] [region_id=97957472]
[2021/12/03 10:59:50.159 +08:00] [Info] [peer.rs:1015] ["report snapshot status"] [status=Finish] [to="id: 97957831 store_id: 80511562 role: Learner"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:50.900 +08:00] [Info] [pd.rs:1108] ["try to change peer"] [kind=EnterJoint] [changes="[peer { id: 97957831 store_id: 80511562 }, peer { id: 97957752 store_id: 80511561 role: Learner } change_type: AddLearnerNode]"] [region_id=97957472]
[2021/12/03 10:59:50.901 +08:00] [Info] [peer.rs:3250] ["propose conf change peer"] [kind=EnterJoint] [changes="[peer { id: 97957831 store_id: 80511562 }, change_type: AddLearnerNode peer { id: 97957752 store_id: 80511561 role: Learner }]"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:50.905 +08:00] [Info] [apply.rs:1393] ["execute admin command"] [command="cmd_type: ChangePeerV2 change_peer_v2 { changes { peer { id: 97957831 store_id: 80511562 } } changes { change_type: AddLearnerNode peer { id: 97957752 store_id: 80511561 role: Learner } } }"] [index=9226] [term=7] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:50.905 +08:00] [Info] [apply.rs:1940] ["exec ConfChangeV2"] [epoch="conf_ver: 1680 version: 9987"] [kind=EnterJoint] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:50.905 +08:00] [Info] [apply.rs:2120] ["conf change successfully"] ["current region"="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8E2ADFF7500000001646438FF3463383632FF2D36FF3866302D3432FF36FF342D386337332DFFFF3938643532363637FFFF31393031000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0135666239373633FF63FF2D303037382DFF3433FF36332D3866FF36632DFF66616634FF61396336FF386333FF3200000000FB0380FF000000462553FC00FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1682 version: 9987 } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 } peers { id: 97957752 store_id: 80511561 role: DemotingVoter } peers { id: 97957831 store_id: 80511562 role: IncomingVoter }"] ["original region"="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8E2ADFF7500000001646438FF3463383632FF2D36FF3866302D3432FF36FF342D386337332DFFFF3938643532363637FFFF31393031000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0135666239373633FF63FF2D303037382DFF3433FF36332D3866FF36632DFF66616634FF61396336FF386333FF3200000000FB0380FF000000462553FC00FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1680 version: 9987 } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 } peers { id: 97957752 store_id: 80511561 } peers { id: 97957831 store_id: 80511562 role: Learner }"] [changes="[peer { id: 97957831 store_id: 80511562 }, change_type: AddLearnerNode peer { id: 97957752 store_id: 80511561 role: Learner }]"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:50.911 +08:00] [Info] [raft.rs:2609] ["switched to configuration"] [config="Configuration { voters: Configuration { incoming: Configuration { voters: {97957831, 97957681, 97957475} }, outgoing: Configuration { voters: {97957752, 97957681, 97957475} } }, learners: {}, learners_next: {97957752}, auto_leave: false }"] [raft_id=97957681] [region_id=97957472]
[2021/12/03 10:59:50.911 +08:00] [Info] [peer.rs:2307] ["notify pd with change peer region"] [region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8E2ADFF7500000001646438FF3463383632FF2D36FF3866302D3432FF36FF342D386337332DFFFF3938643532363637FFFF31393031000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0135666239373633FF63FF2D303037382DFF3433FF36332D3866FF36632DFF66616634FF61396336FF386333FF3200000000FB0380FF000000462553FC00FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1682 version: 9987 } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 } peers { id: 97957752 store_id: 80511561 role: DemotingVoter } peers { id: 97957831 store_id: 80511562 role: IncomingVoter }"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:50.912 +08:00] [Info] [pd.rs:1108] ["try to change peer"] [kind=LeaveJoint] [changes="[]"] [region_id=97957472]
[2021/12/03 10:59:50.912 +08:00] [Info] [peer.rs:3250] ["propose conf change peer"] [kind=LeaveJoint] [changes="[]"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:50.913 +08:00] [Info] [pd.rs:1108] ["try to change peer"] [kind=LeaveJoint] [changes="[]"] [region_id=97957472]
[2021/12/03 10:59:50.913 +08:00] [Info] [pd.rs:1108] ["try to change peer"] [kind=LeaveJoint] [changes="[]"] [region_id=97957472]
[2021/12/03 10:59:50.927 +08:00] [Info] [peer.rs:3190] ["there is a pending conf change, try later"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:50.927 +08:00] [Info] [peer.rs:3190] ["there is a pending conf change, try later"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:50.927 +08:00] [Info] [apply.rs:1393] ["execute admin command"] [command="cmd_type: ChangePeerV2 change_peer_v2 {}"] [index=9227] [term=7] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:50.927 +08:00] [Info] [apply.rs:1940] ["exec ConfChangeV2"] [epoch="conf_ver: 1682 version: 9987"] [kind=LeaveJoint] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:50.927 +08:00] [Info] [apply.rs:2150] ["leave joint state successfully"] [region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8E2ADFF7500000001646438FF3463383632FF2D36FF3866302D3432FF36FF342D386337332DFFFF3938643532363637FFFF31393031000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0135666239373633FF63FF2D303037382DFF3433FF36332D3866FF36632DFF66616634FF61396336FF386333FF3200000000FB0380FF000000462553FC00FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1684 version: 9987 } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 } peers { id: 97957752 store_id: 80511561 role: Learner } peers { id: 97957831 store_id: 80511562 }"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:50.928 +08:00] [Info] [raft.rs:2609] ["switched to configuration"] [config="Configuration { voters: Configuration { incoming: Configuration { voters: {97957831, 97957681, 97957475} }, outgoing: Configuration { voters: {} } }, learners: {97957752}, learners_next: {}, auto_leave: false }"] [raft_id=97957681] [region_id=97957472]
[2021/12/03 10:59:50.928 +08:00] [Info] [peer.rs:2307] ["notify pd with change peer region"] [region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8E2ADFF7500000001646438FF3463383632FF2D36FF3866302D3432FF36FF342D386337332DFFFF3938643532363637FFFF31393031000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0135666239373633FF63FF2D303037382DFF3433FF36332D3866FF36632DFF66616634FF61396336FF386333FF3200000000FB0380FF000000462553FC00FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1684 version: 9987 } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 } peers { id: 97957752 store_id: 80511561 role: Learner } peers { id: 97957831 store_id: 80511562 }"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:50.929 +08:00] [Info] [pd.rs:1089] ["try to change peer"] [peer="id: 97957752 store_id: 80511561 role: Learner"] [change_type=RemoveNode] [region_id=97957472]
[2021/12/03 10:59:50.929 +08:00] [Info] [peer.rs:3250] ["propose conf change peer"] [kind=Simple] [changes="[change_type: RemoveNode peer { id: 97957752 store_id: 80511561 role: Learner }]"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:50.930 +08:00] [Info] [apply.rs:1393] ["execute admin command"] [command="cmd_type: ChangePeer change_peer { change_type: RemoveNode peer { id: 97957752 store_id: 80511561 role: Learner } }"] [index=9228] [term=7] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:50.930 +08:00] [Info] [apply.rs:1759] ["exec ConfChange"] [epoch="conf_ver: 1684 version: 9987"] [type=RemoveNode] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:50.930 +08:00] [Info] [apply.rs:1867] ["remove peer successfully"] [region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8E2ADFF7500000001646438FF3463383632FF2D36FF3866302D3432FF36FF342D386337332DFFFF3938643532363637FFFF31393031000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0135666239373633FF63FF2D303037382DFF3433FF36332D3866FF36632DFF66616634FF61396336FF386333FF3200000000FB0380FF000000462553FC00FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1684 version: 9987 } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 } peers { id: 97957752 store_id: 80511561 role: Learner } peers { id: 97957831 store_id: 80511562 }"] [peer="id: 97957752 store_id: 80511561 role: Learner"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 10:59:50.931 +08:00] [Info] [raft.rs:2609] ["switched to configuration"] [config="Configuration { voters: Configuration { incoming: Configuration { voters: {97957831, 97957681, 97957475} }, outgoing: Configuration { voters: {} } }, learners: {}, learners_next: {}, auto_leave: false }"] [raft_id=97957681] [region_id=97957472]
[2021/12/03 10:59:50.931 +08:00] [Info] [peer.rs:2307] ["notify pd with change peer region"] [region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8E2ADFF7500000001646438FF3463383632FF2D36FF3866302D3432FF36FF342D386337332DFFFF3938643532363637FFFF31393031000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0135666239373633FF63FF2D303037382DFF3433FF36332D3866FF36632DFF66616634FF61396336FF386333FF3200000000FB0380FF000000462553FC00FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1685 version: 9987 } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 } peers { id: 97957831 store_id: 80511562 }"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 11:04:06.660 +08:00] [Info] [size.rs:168] ["approximate size over threshold, need to do split check"] [threshold=150994944] [size=171374385] [region_id=97957472]
[2021/12/03 11:04:06.853 +08:00] [Info] [split_check.rs:289] ["update approximate size and keys with accurate value"] [keys=729135] [size=160781058] [region_id=97957472]
[2021/12/03 11:04:06.853 +08:00] [Info] [peer.rs:3827] ["on split"] [source="split checker"] [split_keys="key 748000000000000FFF645F698000000000FF00000C0419A8EA06FF4100000001383431FF3265323866FF2D39FF3732312D3436FF36FF312D616539662DFFFF3830646239613237FFFF32363235000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0166336532613330FF35FF2D333238302DFF3431FF36372D6234FF63372DFF35363263FF39363534FF386261FF3600000000FB0380FF0000004A2345D800FEFA0A0703E3BFFFF2"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 11:04:06.853 +08:00] [Info] [pd.rs:761] ["try to batch split region"] [task=batch_split] [region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8E2ADFF7500000001646438FF3463383632FF2D36FF3866302D3432FF36FF342D386337332DFFFF3938643532363637FFFF31393031000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0135666239373633FF63FF2D303037382DFF3433FF36332D3866FF36632DFF66616634FF61396336FF386333FF3200000000FB0380FF000000462553FC00FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1685 version: 9987 } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 } peers { id: 97957831 store_id: 80511562 }"] [new_region_ids="[new_region_id: 97958173 new_peer_ids: 97958174 new_peer_ids: 97958175 new_peer_ids: 97958176]"] [region_id=97957472]
[2021/12/03 11:04:06.862 +08:00] [Info] [apply.rs:1393] ["execute admin command"] [command="cmd_type: BatchSplit splits { requests { split_key: 748000000000000FFF645F698000000000FF00000C0419A8EA06FF4100000001383431FF3265323866FF2D39FF3732312D3436FF36FF312D616539662DFFFF3830646239613237FFFF32363235000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0166336532613330FF35FF2D333238302DFF3431FF36372D6234FF63372DFF35363263FF39363534FF386261FF3600000000FB0380FF0000004A2345D800FE new_region_id: 97958173 new_peer_ids: 97958174 new_peer_ids: 97958175 new_peer_ids: 97958176 } right_derive: true }"] [index=13041] [term=7] [peer_id=97957681] [region_id=97957472]
[2021/12/03 11:04:06.862 +08:00] [Info] [apply.rs:2228] ["split region"] [keys="key 748000000000000FFF645F698000000000FF00000C0419A8EA06FF4100000001383431FF3265323866FF2D39FF3732312D3436FF36FF312D616539662DFFFF3830646239613237FFFF32363235000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0166336532613330FF35FF2D333238302DFF3431FF36372D6234FF63372DFF35363263FF39363534FF386261FF3600000000FB0380FF0000004A2345D800FE"] [region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8E2ADFF7500000001646438FF3463383632FF2D36FF3866302D3432FF36FF342D386337332DFFFF3938643532363637FFFF31393031000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0135666239373633FF63FF2D303037382DFF3433FF36332D3866FF36632DFF66616634FF61396336FF386333FF3200000000FB0380FF000000462553FC00FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1685 version: 9987 } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 } peers { id: 97957831 store_id: 80511562 }"] [peer_id=97957681] [region_id=97957472]
[2021/12/03 11:04:06.868 +08:00] [Info] [peer.rs:2411] ["notify pd with split"] [split_count=2] [peer_id=97957681] [region_id=97957472]
[2021/12/03 11:04:06.873 +08:00] [Info] [endpoint.rs:382] ["deregister observe region"] [observe_id=ObserveID(9417)] [region_id=97957472] [store_id=Some(80512890)]
[2021/12/03 11:04:06.873 +08:00] [Info] [endpoint.rs:309] ["register observe region"] [region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8EA06FF4100000001383431FF3265323866FF2D39FF3732312D3436FF36FF312D616539662DFFFF3830646239613237FFFF32363235000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0166336532613330FF35FF2D333238302DFF3431FF36372D6234FF63372DFF35363263FF39363534FF386261FF3600000000FB0380FF0000004A2345D800FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1685 version: 9988 } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 } peers { id: 97957831 store_id: 80511562 }"]
[2021/12/03 11:04:06.876 +08:00] [Info] [endpoint.rs:233] ["Resolver initialized"] [pending_data_index=0] [snapshot_index=13041] [observe_id=ObserveID(9428)] [region=97957472]
[2021/12/03 11:04:07.208 +08:00] [Info] [scheduler.rs:458] ["get snapshot failed"] [err="Error(Request(message: \"EpochNotMatch current epoch of region 97957472 is conf_ver: 1685 version: 9988, but you sent conf_ver: 1679 version: 9987\" epoch_not_match { current_regions { id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8EA06FF4100000001383431FF3265323866FF2D39FF3732312D3436FF36FF312D616539662DFFFF3830646239613237FFFF32363235000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0166336532613330FF35FF2D333238302DFF3431FF36372D6234FF63372DFF35363263FF39363534FF386261FF3600000000FB0380FF0000004A2345D800FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1685 version: 9988 } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 } peers { id: 97957831 store_id: 80511562 } } current_regions { id: 97958173 start_key: 748000000000000FFF645F698000000000FF00000C0419A8E2ADFF7500000001646438FF3463383632FF2D36FF3866302D3432FF36FF342D386337332DFFFF3938643532363637FFFF31393031000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0135666239373633FF63FF2D303037382DFF3433FF36332D3866FF36632DFF66616634FF61396336FF386333FF3200000000FB0380FF000000462553FC00FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EA06FF4100000001383431FF3265323866FF2D39FF3732312D3436FF36FF312D616539662DFFFF3830646239613237FFFF32363235000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0166336532613330FF35FF2D333238302DFF3431FF36372D6234FF63372DFF35363263FF39363534FF386261FF3600000000FB0380FF0000004A2345D800FE region_epoch { conf_ver: 1685 version: 9988 } peers { id: 97958174 store_id: 97823516 } peers { id: 97958175 store_id: 80512890 } peers { id: 97958176 store_id: 80511562 } } }))"] [cid=17075881]
Raw
peer
[2021/12/03 10:54:18.660 +08:00] [INFO] [scheduler.rs:458] ["get snapshot failed"] [err="Error(Request(message: \"region 97957472 is missing\" region_not_found { region_id: 97957472 }))"] [cid=15770212]
[2021/12/03 10:54:39.170 +08:00] [INFO] [peer.rs:259] ["replicate peer"] [peer_id=97957474] [region_id=97957472]
[2021/12/03 10:54:39.170 +08:00] [INFO] [raft.rs:2609] ["switched to configuration"] [config="Configuration { voters: Configuration { incoming: Configuration { voters: {} }, outgoing: Configuration { voters: {} } }, learners: {}, learners_next: {}, auto_leave: false }"] [raft_id=97957474] [region_id=97957472]
[2021/12/03 10:54:39.170 +08:00] [INFO] [raft.rs:1092] ["became follower at term 0"] [term=0] [raft_id=97957474] [region_id=97957472]
[2021/12/03 10:54:39.170 +08:00] [INFO] [raft.rs:384] [newRaft] [peers="Configuration { incoming: Configuration { voters: {} }, outgoing: Configuration { voters: {} } }"] ["last term"=0] ["last index"=0] [applied=0] [commit=0] [term=0] [raft_id=97957474] [region_id=97957472]
[2021/12/03 10:54:39.170 +08:00] [INFO] [raw_node.rs:315] ["RawNode created with id 97957474."] [id=97957474] [raft_id=97957474] [region_id=97957472]
[2021/12/03 10:54:39.872 +08:00] [INFO] [raft.rs:1336] ["received a message with higher term from 97957473"] ["msg type"=MsgHeartbeat] [message_term=6] [term=0] [from=97957473] [raft_id=97957474] [region_id=97957472]
[2021/12/03 10:54:39.872 +08:00] [INFO] [raft.rs:1092] ["became follower at term 6"] [term=6] [raft_id=97957474] [region_id=97957472]
[2021/12/03 10:54:41.915 +08:00] [INFO] [snap.rs:247] ["saving snapshot file"] [file=/tidb2/tidb/tikv-20162/tidb-data/snap/rev_97957472_6_331_(default|lock|write).sst] [snap_key=97957472_6_331]
[2021/12/03 10:54:43.503 +08:00] [INFO] [raft_log.rs:614] ["log [committed=0, persisted=0, applied=0, unstable.offset=1, unstable.entries.len()=0] starts to restore snapshot [index: 331, term: 6]"] [snapshot_term=6] [snapshot_index=331] [log="committed=0, persisted=0, applied=0, unstable.offset=1, unstable.entries.len()=0"] [raft_id=97957474] [region_id=97957472]
[2021/12/03 10:54:43.507 +08:00] [INFO] [raft.rs:2609] ["switched to configuration"] [config="Configuration { voters: Configuration { incoming: Configuration { voters: {97957473, 97957474, 97957475} }, outgoing: Configuration { voters: {} } }, learners: {}, learners_next: {}, auto_leave: false }"] [raft_id=97957474] [region_id=97957472]
[2021/12/03 10:54:43.507 +08:00] [INFO] [raft.rs:2593] ["restored snapshot"] [snapshot_term=6] [snapshot_index=331] [last_term=6] [last_index=331] [commit=331] [raft_id=97957474] [region_id=97957472]
[2021/12/03 10:54:43.507 +08:00] [INFO] [raft.rs:2474] ["[commit: 331, term: 6] restored snapshot [index: 331, term: 6]"] [snapshot_term=6] [snapshot_index=331] [commit=331] [term=6] [raft_id=97957474] [region_id=97957472]
[2021/12/03 10:54:43.507 +08:00] [INFO] [peer_storage.rs:1244] ["begin to apply snapshot"] [peer_id=97957474] [region_id=97957472]
[2021/12/03 10:54:43.507 +08:00] [INFO] [peer_storage.rs:1287] ["apply snapshot with state ok"] [state="applied_index: 331 truncated_state { index: 331 term: 6 }"] [region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8D482FF1F00000001396665FF6636363563FF2D37FF6634662D3464FF62FF302D623838322DFFFF3762666166373031FFFF35613865000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0164323832333733FF31FF2D346333372DFF3435FF62372D3935FF63612DFF35616135FF32633335FF363337FF6500000000FB0380FF00000036E44E0000FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1667 version: 9985 } peers { id: 97957473 store_id: 80511562 } peers { id: 97957474 store_id: 80512891 } peers { id: 97957475 store_id: 97823516 }"] [peer_id=97957474] [region_id=97957472]
[2021/12/03 10:54:44.352 +08:00] [INFO] [peer.rs:3171] ["snapshot is applied"] [region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8D482FF1F00000001396665FF6636363563FF2D37FF6634662D3464FF62FF302D623838322DFFFF3762666166373031FFFF35613865000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0164323832333733FF31FF2D346333372DFF3435FF62372D3935FF63612DFF35616135FF32633335FF363337FF6500000000FB0380FF00000036E44E0000FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1667 version: 9985 } peers { id: 97957473 store_id: 80511562 } peers { id: 97957474 store_id: 80512891 } peers { id: 97957475 store_id: 97823516 }"] [peer_id=97957474] [region_id=97957472]
[2021/12/03 10:54:47.610 +08:00] [INFO] [region.rs:328] ["begin apply snap data"] [region_id=97957472]
[2021/12/03 10:54:49.501 +08:00] [INFO] [region.rs:401] ["apply new data"] [time_takes=185.850339ms] [region_id=97957472]
[2021/12/03 10:55:09.637 +08:00] [INFO] [peer.rs:862] ["deleting applied snap file"] [snap_file=97957472_6_331] [peer_id=97957474] [region_id=97957472]
[2021/12/03 10:55:39.703 +08:00] [INFO] [apply.rs:1393] ["execute admin command"] [command="cmd_type: BatchSplit splits { requests { split_key: 748000000000000FFF645F698000000000FF00000C0419A8DAB0FF3000000001376236FF3431306264FF2D65FF6663302D3464FF37FF372D386264622DFFFF6363326439353435FFFF37313634000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0132353033666365FF36FF2D623932332DFF3432FF61622D3965FF32612DFF31643339FF39666165FF373132FF6400000000FB0380FF00000042FE4F4900FE new_region_id: 97957583 new_peer_ids: 97957584 new_peer_ids: 97957585 new_peer_ids: 97957586 } right_derive: true }"] [index=3791] [term=6] [peer_id=97957474] [region_id=97957472]
[2021/12/03 10:55:39.703 +08:00] [INFO] [apply.rs:2228] ["split region"] [keys="key 748000000000000FFF645F698000000000FF00000C0419A8DAB0FF3000000001376236FF3431306264FF2D65FF6663302D3464FF37FF372D386264622DFFFF6363326439353435FFFF37313634000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0132353033666365FF36FF2D623932332DFF3432FF61622D3965FF32612DFF31643339FF39666165FF373132FF6400000000FB0380FF00000042FE4F4900FE"] [region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8D482FF1F00000001396665FF6636363563FF2D37FF6634662D3464FF62FF302D623838322DFFFF3762666166373031FFFF35613865000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0164323832333733FF31FF2D346333372DFF3435FF62372D3935FF63612DFF35616135FF32633335FF363337FF6500000000FB0380FF00000036E44E0000FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1667 version: 9985 } peers { id: 97957473 store_id: 80511562 } peers { id: 97957474 store_id: 80512891 } peers { id: 97957475 store_id: 97823516 }"] [peer_id=97957474] [region_id=97957472]
[2021/12/03 10:57:30.733 +08:00] [INFO] [snap.rs:247] ["saving snapshot file"] [file=/tidb2/tidb/tikv-20162/tidb-data/snap/rev_97957472_6_6338_(default|lock|write).sst] [snap_key=97957472_6_6338]
[2021/12/03 10:57:31.519 +08:00] [INFO] [raft_log.rs:614] ["log [committed=5622, persisted=5622, applied=5622, unstable.offset=5623, unstable.entries.len()=0] starts to restore snapshot [index: 6338, term: 6]"] [snapshot_term=6] [snapshot_index=6338] [log="committed=5622, persisted=5622, applied=5622, unstable.offset=5623, unstable.entries.len()=0"] [raft_id=97957474] [region_id=97957472]
[2021/12/03 10:57:31.519 +08:00] [INFO] [raft.rs:2609] ["switched to configuration"] [config="Configuration { voters: Configuration { incoming: Configuration { voters: {97957473, 97957474, 97957475} }, outgoing: Configuration { voters: {} } }, learners: {97957681}, learners_next: {}, auto_leave: false }"] [raft_id=97957474] [region_id=97957472]
[2021/12/03 10:57:31.519 +08:00] [INFO] [raft.rs:2593] ["restored snapshot"] [snapshot_term=6] [snapshot_index=6338] [last_term=6] [last_index=6338] [commit=6338] [raft_id=97957474] [region_id=97957472]
[2021/12/03 10:57:31.519 +08:00] [INFO] [raft.rs:2474] ["[commit: 6338, term: 6] restored snapshot [index: 6338, term: 6]"] [snapshot_term=6] [snapshot_index=6338] [commit=6338] [term=6] [raft_id=97957474] [region_id=97957472]
[2021/12/03 10:57:31.519 +08:00] [INFO] [peer_storage.rs:1244] ["begin to apply snapshot"] [peer_id=97957474] [region_id=97957472]
[2021/12/03 10:57:31.519 +08:00] [INFO] [peer_storage.rs:1638] ["finish clear peer meta"] [takes=305.723µs] [raft_key=1] [apply_key=1] [meta_key=1] [region_id=97957472]
[2021/12/03 10:57:31.519 +08:00] [INFO] [peer_storage.rs:1287] ["apply snapshot with state ok"] [state="applied_index: 6338 commit_index: 5622 commit_term: 6 truncated_state { index: 6338 term: 6 }"] [region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8DAB0FF3000000001376236FF3431306264FF2D65FF6663302D3464FF37FF372D386264622DFFFF6363326439353435FFFF37313634000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0132353033666365FF36FF2D623932332DFF3432FF61622D3965FF32612DFF31643339FF39666165FF373132FF6400000000FB0380FF00000042FE4F4900FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1668 version: 9986 } peers { id: 97957473 store_id: 80511562 } peers { id: 97957474 store_id: 80512891 } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 role: Learner }"] [peer_id=97957474] [region_id=97957472]
[2021/12/03 10:57:33.703 +08:00] [INFO] [region.rs:328] ["begin apply snap data"] [region_id=97957472]
[2021/12/03 10:57:33.703 +08:00] [INFO] [peer.rs:3171] ["snapshot is applied"] [region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8DAB0FF3000000001376236FF3431306264FF2D65FF6663302D3464FF37FF372D386264622DFFFF6363326439353435FFFF37313634000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0132353033666365FF36FF2D623932332DFF3432FF61622D3965FF32612DFF31643339FF39666165FF373132FF6400000000FB0380FF00000042FE4F4900FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1668 version: 9986 } peers { id: 97957473 store_id: 80511562 } peers { id: 97957474 store_id: 80512891 } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 role: Learner }"] [peer_id=97957474] [region_id=97957472]
[2021/12/03 10:57:33.703 +08:00] [INFO] [apply.rs:3182] ["re-register to apply delegates"] [term=6] [peer_id=97957474] [region_id=97957472]
[2021/12/03 10:57:33.703 +08:00] [INFO] [peer.rs:3222] ["region changed after applying snapshot"] [region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8DAB0FF3000000001376236FF3431306264FF2D65FF6663302D3464FF37FF372D386264622DFFFF6363326439353435FFFF37313634000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0132353033666365FF36FF2D623932332DFF3432FF61622D3965FF32612DFF31643339FF39666165FF373132FF6400000000FB0380FF00000042FE4F4900FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1668 version: 9986 } peers { id: 97957473 store_id: 80511562 } peers { id: 97957474 store_id: 80512891 } peers { id: 97957475 store_id: 97823516 } peers { id: 97957681 store_id: 80512890 role: Learner }"] [prev_region="id: 97957472 start_key: 748000000000000FFF645F698000000000FF00000C0419A8DAB0FF3000000001376236FF3431306264FF2D65FF6663302D3464FF37FF372D386264622DFFFF6363326439353435FFFF37313634000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0132353033666365FF36FF2D623932332DFF3432FF61622D3965FF32612DFF31643339FF39666165FF373132FF6400000000FB0380FF00000042FE4F4900FE end_key: 748000000000000FFF645F698000000000FF00000C0419A8EC81FFD500000001386236FF6536343332FF2D63FF3630332D3432FF35FF382D396332312DFFFF3033396562333233FFFF61613536000000FF00FB013538666434FF643438FF2D636265FF352D3433FF30632DFF386363612DFF3862FF356233393436FF39FF62643500000000FBFF0139646262313739FF38FF2D313839392DFF3435FF39382D3866FF32652DFF31373637FF32653937FF383831FF6500000000FB0380FF0000003D6CAF5C00FE region_epoch { conf_ver: 1667 version: 9986 } peers { id: 97957473 store_id: 80511562 } peers { id: 97957474 store_id: 80512891 } peers { id: 97957475 store_id: 97823516 }"] [peer_id=97957474] [region_id=97957472]
[2021/12/03 10:57:33.703 +08:00] [INFO] [region_info_accessor.rs:238] ["trying to create region but it already exists, try to update it"] [region_id=97957472]
[2021/12/03 10:57:33.703 +08:00] [INFO] [peer.rs:1801] ["receives gc message, trying to remove"] [to_peer="id: 97957474 store_id: 80512891"] [peer_id=97957474] [region_id=97957472]
[2021/12/03 10:57:33.703 +08:00] [INFO] [peer.rs:1801] ["receives gc message, trying to remove"] [to_peer="id: 97957474 store_id: 80512891"] [peer_id=97957474] [region_id=97957472]
[2021/12/03 10:57:33.762 +08:00] [INFO] [peer.rs:784] ["stale peer is applying snapshot, will destroy next time"] [peer_id=97957474] [region_id=97957472]
[2021/12/03 10:57:33.762 +08:00] [INFO] [peer.rs:784] ["stale peer is applying snapshot, will destroy next time"] [peer_id=97957474] [region_id=97957472]
[2021/12/03 10:57:37.142 +08:00] [WARN] [region.rs:428] ["applying snapshot is aborted"] [region_id=97957472]
[2021/12/03 10:58:14.973 +08:00] [INFO] [peer.rs:862] ["deleting applied snap file"] [snap_file=97957472_6_6338] [peer_id=97957474] [region_id=97957472]
[2021/12/03 11:09:40.014 +08:00] [WARN] [peer.rs:4067] ["leader missing longer than abnormal_leader_missing_duration"] [expect=10m] [peer_id=97957474] [region_id=97957472]
[2021/12/03 11:09:48.476 +08:00] [FATAL] [lib.rs:465] ["unexpected error getting unapplied entries [5623, 6339): Store(Compacted), raft_id: 97957474, region_id: 97957472"] [backtrace="stack backtrace:\n 6: raft::raft::Raft<T>::hup::{{closure}}\n at /rust/git/checkouts/raft-rs-42b8049ef2e3af07/e6d28ef/src/raft.rs:1498\n 7: core::result::Result<T,E>::unwrap_or_else\n at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/core/src/result.rs:1065\n raft::raft::Raft<T>::hup\n at /rust/git/checkouts/raft-rs-42b8049ef2e3af07/e6d28ef/src/raft.rs:1494\n 8: raft::raft::Raft<T>::step\n at /rust/git/checkouts/raft-rs-42b8049ef2e3af07/e6d28ef/src/raft.rs:1421\n 9: raft::raft::Raft<T>::tick_election\n at /rust/git/checkouts/raft-rs-42b8049ef2e3af07/e6d28ef/src/raft.rs:1049\n raft::raft::Raft<T>::tick\n at /rust/git/checkouts/raft-rs-42b8049ef2e3af07/e6d28ef/src/raft.rs:1031\n raft::raw_node::RawNode<T>::tick\n at /rust/git/checkouts/raft-rs-42b8049ef2e3af07/e6d28ef/src/raw_node.rs:343\n 10: raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::on_raft_base_tick\n at /home/jenkins/agent/workspace/optimization-build-tidb-linux-amd/go/src/github.com/pingcap/tikv/components/raftstore/src/store/fsm/peer.rs:1232\n raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::on_tick\n at /home/jenkins/agent/workspace/optimization-build-tidb-linux-amd/go/src/github.com/pingcap/tikv/components/raftstore/src/store/fsm/peer.rs:775\n raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::handle_msgs\n at /home/jenkins/agent/workspace/optimization-build-tidb-linux-amd/go/src/github.com/pingcap/tikv/components/raftstore/src/store/fsm/peer.rs:618\n 11: <raftstore::store::fsm::store::RaftPoller<EK,ER,T> as batch_system::batch::PollHandler<raftstore::store::fsm::peer::PeerFsm<EK,ER>,raftstore::store::fsm::store::StoreFsm<EK>>>::handle_normal\n at /home/jenkins/agent/workspace/optimization-build-tidb-linux-amd/go/src/github.com/pingcap/tikv/components/raftstore/src/store/fsm/store.rs:929\n 12: batch_system::batch::Poller<N,C,Handler>::poll\n at /home/jenkins/agent/workspace/optimization-build-tidb-linux-amd/go/src/github.com/pingcap/tikv/components/batch-system/src/batch.rs:414\n 13: batch_system::batch::BatchSystem<N,C>::start_poller::{{closure}}\n at /home/jenkins/agent/workspace/optimization-build-tidb-linux-amd/go/src/github.com/pingcap/tikv/components/batch-system/src/batch.rs:541\n std::sys_common::backtrace::__rust_begin_short_backtrace\n at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/std/src/sys_common/backtrace.rs:125\n 14: std::thread::Builder::spawn_unchecked::{{closure}}::{{closure}}\n at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/std/src/thread/mod.rs:476\n <std::panic::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/std/src/panic.rs:347\n std::panicking::try::do_call\n at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/std/src/panicking.rs:401\n std::panicking::try\n at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/std/src/panicking.rs:365\n std::panic::catch_unwind\n at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/std/src/panic.rs:434\n std::thread::Builder::spawn_unchecked::{{closure}}\n at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/std/src/thread/mod.rs:475\n core::ops::function::FnOnce::call_once{{vtable.shim}}\n at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/core/src/ops/function.rs:227\n 15: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/alloc/src/boxed.rs:1572\n <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n at /rustc/2faabf579323f5252329264cc53ba9ff803429a3/library/alloc/src/boxed.rs:1572\n std::sys::unix::thread::Thread::new::thread_start\n at library/std/src/sys/unix/thread.rs:91\n 16: start_thread\n 17: __clone\n"] [location=/rust/git/checkouts/raft-rs-42b8049ef2e3af07/e6d28ef/src/raft.rs:1498] [thread_name=raftstore-80512891-1]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment