[2024/01/05 17:43:14.015 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=18]
[2024/01/05 17:43:14.202 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=21] [task="TableScanTask: id=21, startKey=7480000000000000f55f7280000000029eebdf, endKey=7480000000000000f55f728000000002a55b8c"]
[2024/01/05 17:43:14.202 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=19]
[2024/01/05 17:43:14.820 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=20]
[2024/01/05 17:43:14.820 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=22] [task="TableScanTask: id=22, startKey=7480000000000000f55f728000000002a55b8c, endKey=7480000000000000f55f728000000002abc860"]
[2024/01/05 17:43:14.952 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=21]
[2024/01/05 17:43:14.952 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=23] [task="TableScanTask: id=23, startKey=7480000000000000f55f728000000002abc860, endKey=7480000000000000f55f728000000002b23534"]
[2024/01/05 17:43:15.509 +08:00] [INFO] [region_request.go:1532] ["throwing pseudo region error due to no replica available"] [req-ts=446810762894377043] [req-type=Cop] [region="{ region id: 5846, ver: 244, confVer: 17 }"] [replica-read-type=leader] [stale-read=false] [request-sender="{rpcError:<nil>,replicaSelector: replicaSelector{selectorStateStr: invalidStore, cacheRegionIsValid: true, replicaStatus: [peer: 5847, store: 1, isEpochStale: false, attempts: 1, replica-epoch: 0, store-epoch: 0, store-state: resolved, store-liveness-state: reachable peer: 5848, store: 13, isEpochStale: false, attempts: 0, replica-epoch: 0, store-epoch: 0, store-state: resolved, store-liveness-state: reachable peer: 5849, store: 12, isEpochStale: false, attempts: 0, replica-epoch: 0, store-epoch: 0, store-state: resolved, store-liveness-state: reachable]}}"] [retry-times=1] [total-backoff-ms=0] [total-backoff-times=0] [max-exec-timeout-ms=60000] [total-region-errors=5847-not_leader-5849:1]
[2024/01/05 17:43:15.511 +08:00] [INFO] [region_request.go:1532] ["throwing pseudo region error due to no replica available"] [req-ts=446810762894377044] [req-type=Cop] [region="{ region id: 5846, ver: 244, confVer: 17 }"] [replica-read-type=leader] [stale-read=false] [request-sender="{rpcError:<nil>,replicaSelector: replicaSelector{selectorStateStr: invalidStore, cacheRegionIsValid: true, replicaStatus: [peer: 5847, store: 1, isEpochStale: false, attempts: 1, replica-epoch: 0, store-epoch: 0, store-state: resolved, store-liveness-state: reachable peer: 5848, store: 13, isEpochStale: false, attempts: 0, replica-epoch: 0, store-epoch: 0, store-state: resolved, store-liveness-state: reachable peer: 5849, store: 12, isEpochStale: false, attempts: 0, replica-epoch: 0, store-epoch: 0, store-state: resolved, store-liveness-state: reachable]}}"] [retry-times=1] [total-backoff-ms=0] [total-backoff-times=0] [max-exec-timeout-ms=60000] [total-region-errors=5847-not_leader-5849:1]
[2024/01/05 17:43:15.590 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=24] [task="TableScanTask: id=24, startKey=7480000000000000f55f728000000002b23534, endKey=7480000000000000f55f728000000002b8a52c"]
[2024/01/05 17:43:15.590 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=22]
[2024/01/05 17:43:15.776 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=25] [task="TableScanTask: id=25, startKey=7480000000000000f55f728000000002b8a52c, endKey=7480000000000000f55f728000000002bf0957"]
[2024/01/05 17:43:15.776 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=23]
[2024/01/05 17:43:17.861 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=24]
[2024/01/05 17:43:17.861 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=26] [task="TableScanTask: id=26, startKey=7480000000000000f55f728000000002bf0957, endKey=7480000000000000f55f728000000002c5762b"]
[2024/01/05 17:43:18.143 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=25]
[2024/01/05 17:43:18.143 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=27] [task="TableScanTask: id=27, startKey=7480000000000000f55f728000000002c5762b, endKey=7480000000000000f55f728000000002cbe2ff"]
[2024/01/05 17:43:19.364 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=26]
[2024/01/05 17:43:19.373 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=28] [task="TableScanTask: id=28, startKey=7480000000000000f55f728000000002cbe2ff, endKey=7480000000000000f55f728000000002d24fd3"]
[2024/01/05 17:43:19.501 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=29] [task="TableScanTask: id=29, startKey=7480000000000000f55f728000000002d24fd3, endKey=7480000000000000f55f728000000002d8bca7"]
[2024/01/05 17:43:19.503 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=27]
[2024/01/05 17:43:20.182 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=28]
[2024/01/05 17:43:20.182 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=30] [task="TableScanTask: id=30, startKey=7480000000000000f55f728000000002d8bca7, endKey=7480000000000000f55f728000000002df297b"]
[2024/01/05 17:43:20.302 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=31] [task="TableScanTask: id=31, startKey=7480000000000000f55f728000000002df297b, endKey=7480000000000000f55f728000000002e5964f"]
[2024/01/05 17:43:20.305 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=29]
[2024/01/05 17:43:20.477 +08:00] [INFO] [domain.go:2898] ["refreshServerIDTTL succeed"] [serverID=607] ["lease id"=6d68cd8b0585128]
[2024/01/05 17:43:21.117 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=32] [task="TableScanTask: id=32, startKey=7480000000000000f55f728000000002e5964f, endKey=7480000000000000f55f728000000002ec0323"]
[2024/01/05 17:43:21.122 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=30]
[2024/01/05 17:43:21.244 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=33] [task="TableScanTask: id=33, startKey=7480000000000000f55f728000000002ec0323, endKey=7480000000000000f55f728000000002f26ff7"]
[2024/01/05 17:43:21.244 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=31]
[2024/01/05 17:43:22.231 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=32]
[2024/01/05 17:43:22.231 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=34] [task="TableScanTask: id=34, startKey=7480000000000000f55f728000000002f26ff7, endKey=7480000000000000f55f728000000002f8dfa4"]
[2024/01/05 17:43:22.342 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=33]
[2024/01/05 17:43:22.345 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=35] [task="TableScanTask: id=35, startKey=7480000000000000f55f728000000002f8dfa4, endKey=7480000000000000f55f728000000002ff4c78"]
[2024/01/05 17:43:24.698 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=36] [task="TableScanTask: id=36, startKey=7480000000000000f55f728000000002ff4c78, endKey=7480000000000000f55f72800000000305b94c"]
[2024/01/05 17:43:24.702 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=34]
[2024/01/05 17:43:24.716 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=35]
[2024/01/05 17:43:24.720 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=37] [task="TableScanTask: id=37, startKey=7480000000000000f55f72800000000305b94c, endKey=7480000000000000f55f7280000000030c2620"]
[2024/01/05 17:43:25.502 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=38] [task="TableScanTask: id=38, startKey=7480000000000000f55f7280000000030c2620, endKey=7480000000000000f55f7280000000031292f4"]
[2024/01/05 17:43:25.502 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=36]
[2024/01/05 17:43:25.542 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=37]
[2024/01/05 17:43:25.542 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=39] [task="TableScanTask: id=39, startKey=7480000000000000f55f7280000000031292f4, endKey=7480000000000000f55f72800000000318ffc8"]
[2024/01/05 17:43:26.286 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=38]
[2024/01/05 17:43:26.288 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=40] [task="TableScanTask: id=40, startKey=7480000000000000f55f72800000000318ffc8, endKey=7480000000000000f55f7280000000031f6c9c"]
[2024/01/05 17:43:26.300 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=39]
[2024/01/05 17:43:26.300 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=41] [task="TableScanTask: id=41, startKey=7480000000000000f55f7280000000031f6c9c, endKey=7480000000000000f55f72800000000325d970"]
[2024/01/05 17:43:26.419 +08:00] [INFO] [engine.go:1617] ["compact sst"] [fileCount=8] [size=1073742120] [count=6507528] [cost=12.934821151s] [file=/var/lib/tidb-data/tmp_ddl-4000/435/e81a0e89-251e-59cd-9827-5cbe286e40c0.sst/0beddfcb-0e42-4fbb-bdcc-a4d29046fa2a.sst]
[2024/01/05 17:43:26.419 +08:00] [INFO] [engine.go:865] ["write data to local DB"] [size=1073742120] [kvs=6507528] [files=1] [sstFileSize=825193992] [file=/var/lib/tidb-data/tmp_ddl-4000/435/e81a0e89-251e-59cd-9827-5cbe286e40c0.sst/0beddfcb-0e42-4fbb-bdcc-a4d29046fa2a.sst] [firstKey=7480000000000000F55F69800000000000000C013030303030303334FF3932382D30373237FF313839343738382DFF3035373532343730FF3736392D32333635FF343238333530372DFF3532343238343435FF3132352D31343832FF393435313436312DFF3230333830393931FF3433312D33313033FF383435393237312DFF3839373831303336FF3533322D32303935FF3732303635363800FE0380000000025FB508] [lastKey=7480000000000000F55F69800000000000000C013939393939393735FF3334392D33333331FF353533343230392DFF3231393831353433FF3531322D34313539FF393539333537332DFF3930393939343939FF3131362D35303933FF303233313635372DFF3835363737323433FF3835362D33343135FF363839333131322DFF3039323535313135FF3130302D36363939FF3939343835343200FE0380000000026CEA67]
[2024/01/05 17:43:27.243 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=40]
[2024/01/05 17:43:27.244 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=42] [task="TableScanTask: id=42, startKey=7480000000000000f55f72800000000325d970, endKey=7480000000000000f55f7280000000032c4644"]
[2024/01/05 17:43:27.244 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=41]
[2024/01/05 17:43:27.245 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=43] [task="TableScanTask: id=43, startKey=7480000000000000f55f7280000000032c4644, endKey=7480000000000000f55f72800000000332b318"]
[2024/01/05 17:43:30.009 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=43]
[2024/01/05 17:43:30.012 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=44] [task="TableScanTask: id=44, startKey=7480000000000000f55f72800000000332b318, endKey=7480000000000000f55f728000000003391fec"]
[2024/01/05 17:43:30.013 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=42]
[2024/01/05 17:43:30.013 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=45] [task="TableScanTask: id=45, startKey=7480000000000000f55f728000000003391fec, endKey=7480000000000000f55f7280000000033f8cc0"]
[2024/01/05 17:43:30.940 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=46] [task="TableScanTask: id=46, startKey=7480000000000000f55f7280000000033f8cc0, endKey=7480000000000000f55f72800000000345f994"]
[2024/01/05 17:43:30.940 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=45]
[2024/01/05 17:43:30.980 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=47] [task="TableScanTask: id=47, startKey=7480000000000000f55f72800000000345f994, endKey=7480000000000000f55f7280000000034c6668"]
[2024/01/05 17:43:30.982 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=44]
[2024/01/05 17:43:31.935 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=47]
[2024/01/05 17:43:31.935 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=48] [task="TableScanTask: id=48, startKey=7480000000000000f55f7280000000034c6668, endKey=7480000000000000f55f72800000000352d33c"]
[2024/01/05 17:43:31.951 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=49] [task="TableScanTask: id=49, startKey=7480000000000000f55f72800000000352d33c, endKey=7480000000000000f55f728000000003594010"]
[2024/01/05 17:43:31.957 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=46]
[2024/01/05 17:43:32.965 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=48]
[2024/01/05 17:43:32.965 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=50] [task="TableScanTask: id=50, startKey=7480000000000000f55f728000000003594010, endKey=7480000000000000f55f7280000000035fa81a"]
[2024/01/05 17:43:32.977 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=51] [task="TableScanTask: id=51, startKey=7480000000000000f55f7280000000035fa81a, endKey=7480000000000000f55f7280000000036617c7"]
[2024/01/05 17:43:32.982 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=49]
[2024/01/05 17:43:35.483 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=50]
[2024/01/05 17:43:35.483 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=52] [task="TableScanTask: id=52, startKey=7480000000000000f55f7280000000036617c7, endKey=7480000000000000f55f7280000000036c849b"]
[2024/01/05 17:43:35.503 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=51]
[2024/01/05 17:43:35.503 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=53] [task="TableScanTask: id=53, startKey=7480000000000000f55f7280000000036c849b, endKey=7480000000000000f55f72800000000372f16f"]
[2024/01/05 17:43:36.462 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=53]
[2024/01/05 17:43:36.464 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=54] [task="TableScanTask: id=54, startKey=7480000000000000f55f72800000000372f16f, endKey=7480000000000000f55f728000000003795e43"]
[2024/01/05 17:43:36.468 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=52]
[2024/01/05 17:43:36.468 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=55] [task="TableScanTask: id=55, startKey=7480000000000000f55f728000000003795e43, endKey=7480000000000000f55f7280000000037fcb17"]
[2024/01/05 17:43:37.349 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=54]
[2024/01/05 17:43:37.349 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=56] [task="TableScanTask: id=56, startKey=7480000000000000f55f7280000000037fcb17, endKey=7480000000000000f55f7280000000038637eb"]
[2024/01/05 17:43:37.384 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=55]
[2024/01/05 17:43:37.384 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=57] [task="TableScanTask: id=57, startKey=7480000000000000f55f7280000000038637eb, endKey=7480000000000000f55f7280000000038ca4bf"]
[2024/01/05 17:43:38.114 +08:00] [INFO] [engine.go:1617] ["compact sst"] [fileCount=8] [size=1073742120] [count=6507528] [cost=13.552032785s] [file=/var/lib/tidb-data/tmp_ddl-4000/435/e81a0e89-251e-59cd-9827-5cbe286e40c0.sst/5f440a88-ff2e-4b46-84b0-436232db8138.sst]
[2024/01/05 17:43:38.114 +08:00] [INFO] [engine.go:865] ["write data to local DB"] [size=1073742120] [kvs=6507528] [files=1] [sstFileSize=825174821] [file=/var/lib/tidb-data/tmp_ddl-4000/435/e81a0e89-251e-59cd-9827-5cbe286e40c0.sst/5f440a88-ff2e-4b46-84b0-436232db8138.sst] [firstKey=7480000000000000F55F69800000000000000C013030303030303035FF3533372D30313438FF393136333638352DFF3235313833323933FF3937342D36343131FF363232303839382DFF3830383431323134FF3930352D39353931FF333233323736342DFF3739353439353733FF3831312D37313131FF303139393535302DFF3735353539303238FF3735322D37333139FF3534353538343800FE0380000000028C2E99] [lastKey=7480000000000000F55F69800000000000000C013939393939393932FF3338382D30383235FF313039303133302DFF3131393839363639FF3434362D35323138FF373735303032372DFF3932343630343834FF3938312D38373936FF353435323539322DFF3835393039393838FF3733372D31313037FF353533353634352DFF3131373037333232FF3733352D37393230FF3735373734323800FE038000000002C81FFB]
[2024/01/05 17:43:38.493 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=56]
[2024/01/05 17:43:38.493 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=58] [task="TableScanTask: id=58, startKey=7480000000000000f55f7280000000038ca4bf, endKey=7480000000000000f55f728000000003931193"]
[2024/01/05 17:43:38.520 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=57]
[2024/01/05 17:43:38.520 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=59] [task="TableScanTask: id=59, startKey=7480000000000000f55f728000000003931193, endKey=7480000000000000f55f728000000003998216"]
[2024/01/05 17:43:38.825 +08:00] [WARN] [pd.go:156] ["get timestamp too slow"] [conn=1272971362] [session_alias=] ["cost time"=34.890284ms]
[2024/01/05 17:43:38.825 +08:00] [WARN] [pd.go:156] ["get timestamp too slow"] [conn=1272971388] [session_alias=] ["cost time"=31.782232ms]
[2024/01/05 17:43:38.825 +08:00] [WARN] [pd.go:156] ["get timestamp too slow"] [conn=1272971402] [session_alias=] ["cost time"=31.253046ms]
[2024/01/05 17:43:38.825 +08:00] [WARN] [pd.go:156] ["get timestamp too slow"] [conn=1272971406] [session_alias=] ["cost time"=31.286056ms]
[2024/01/05 17:43:39.657 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=59]
[2024/01/05 17:43:39.657 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=58]
[2024/01/05 17:43:39.657 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=60] [task="TableScanTask: id=60, startKey=7480000000000000f55f728000000003998216, endKey=7480000000000000f55f7280000000039feeea"]
[2024/01/05 17:43:39.657 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=61] [task="TableScanTask: id=61, startKey=7480000000000000f55f7280000000039feeea, endKey=7480000000000000f55f728000000003a65bbe"]
[2024/01/05 17:43:41.899 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=61]
[2024/01/05 17:43:41.900 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=62] [task="TableScanTask: id=62, startKey=7480000000000000f55f728000000003a65bbe, endKey=7480000000000000f55f728000000003acc892"]
[2024/01/05 17:43:41.935 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=60]
[2024/01/05 17:43:41.938 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=63] [task="TableScanTask: id=63, startKey=7480000000000000f55f728000000003acc892, endKey=7480000000000000f55f728000000003b33566"]
[2024/01/05 17:43:42.750 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=62]
[2024/01/05 17:43:42.750 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=64] [task="TableScanTask: id=64, startKey=7480000000000000f55f728000000003b33566, endKey=7480000000000000f55f728000000003b9a23a"]
[2024/01/05 17:43:42.766 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=65] [task="TableScanTask: id=65, startKey=7480000000000000f55f728000000003b9a23a, endKey=7480000000000000f55f728000000003c00f0e"]
[2024/01/05 17:43:42.770 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=63]
[2024/01/05 17:43:43.687 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=66] [task="TableScanTask: id=66, startKey=7480000000000000f55f728000000003c00f0e, endKey=7480000000000000f55f728000000003c67ebb"]
[2024/01/05 17:43:43.687 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=64]
[2024/01/05 17:43:43.717 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=65]
[2024/01/05 17:43:43.717 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=67] [task="TableScanTask: id=67, startKey=7480000000000000f55f728000000003c67ebb, endKey=7480000000000000f55f728000000003cceb8f"]
[2024/01/05 17:43:44.810 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=67]
[2024/01/05 17:43:44.810 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=68] [task="TableScanTask: id=68, startKey=7480000000000000f55f728000000003cceb8f, endKey=7480000000000000f55f728000000003d35863"]
[2024/01/05 17:43:44.818 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=66]
[2024/01/05 17:43:44.818 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=69] [task="TableScanTask: id=69, startKey=7480000000000000f55f728000000003d35863, endKey=7480000000000000f55f728000000003d9c537"]
[2024/01/05 17:43:47.303 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=68]
[2024/01/05 17:43:47.303 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=70] [task="TableScanTask: id=70, startKey=7480000000000000f55f728000000003d9c537, endKey=7480000000000000f55f728000000003e03317"]
[2024/01/05 17:43:47.326 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=69]
[2024/01/05 17:43:47.326 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=71] [task="TableScanTask: id=71, startKey=7480000000000000f55f728000000003e03317, endKey=7480000000000000f55f728000000003e69feb"]
[2024/01/05 17:43:48.269 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=72] [task="TableScanTask: id=72, startKey=7480000000000000f55f728000000003e69feb, endKey=7480000000000000f55f728000000003ed0cbf"]
[2024/01/05 17:43:48.269 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=70]
[2024/01/05 17:43:48.293 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=71]
[2024/01/05 17:43:48.294 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=73] [task="TableScanTask: id=73, startKey=7480000000000000f55f728000000003ed0cbf, endKey=7480000000000000f55f728000000003f37993"]
[2024/01/05 17:43:49.148 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=72]
[2024/01/05 17:43:49.150 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=74] [task="TableScanTask: id=74, startKey=7480000000000000f55f728000000003f37993, endKey=7480000000000000f55f728000000003f9e667"]
[2024/01/05 17:43:49.245 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=73]
[2024/01/05 17:43:49.245 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=75] [task="TableScanTask: id=75, startKey=7480000000000000f55f728000000003f9e667, endKey=7480000000000000f55f72800000000400533b"]
[2024/01/05 17:43:49.312 +08:00] [INFO] [engine.go:1617] ["compact sst"] [fileCount=8] [size=1073742120] [count=6507528] [cost=13.383617576s] [file=/var/lib/tidb-data/tmp_ddl-4000/435/e81a0e89-251e-59cd-9827-5cbe286e40c0.sst/e6b792e5-7425-4891-955c-9a58ab6220ed.sst]
[2024/01/05 17:43:49.312 +08:00] [INFO] [engine.go:865] ["write data to local DB"] [size=1073742120] [kvs=6507528] [files=1] [sstFileSize=827386104] [file=/var/lib/tidb-data/tmp_ddl-4000/435/e81a0e89-251e-59cd-9827-5cbe286e40c0.sst/e6b792e5-7425-4891-955c-9a58ab6220ed.sst] [firstKey=7480000000000000F55F69800000000000000C013030303030303031FF3731382D32343230FF363631323631312DFF3639313034313836FF3232312D37363035FF333538323838312DFF3030313236333333FF3031312D30373339FF323739373535322DFF3933393036393638FF3435392D39353132FF323731393531332DFF3932303135373334FF3937302D39343633FF3236393634343500FE03800000000322DF37] [lastKey=7480000000000000F55F69800000000000000C013939393939393836FF3832352D34393139FF373932393639332DFF3436313237333833FF3132382D36333732FF363035313835352DFF3137373434313037FF3131312D37363038FF363638333437352DFF3935353735373437FF3130302D32393931FF353134373630332DFF3939323531383736FF3137392D39323730FF3931343438383100FE038000000002FF731A]
[2024/01/05 17:43:50.075 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=76] [task="TableScanTask: id=76, startKey=7480000000000000f55f72800000000400533b, endKey=7480000000000000f55f72800000000406c00f"]
[2024/01/05 17:43:50.078 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=74]
[2024/01/05 17:43:50.198 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=75]
[2024/01/05 17:43:50.203 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=77] [task="TableScanTask: id=77, startKey=7480000000000000f55f72800000000406c00f, endKey=7480000000000000f55f7280000000040d2ce3"]
[2024/01/05 17:43:50.270 +08:00] [WARN] [pd.go:156] ["get timestamp too slow"] [conn=1272971406] [session_alias=] ["cost time"=37.058117ms]
[2024/01/05 17:43:52.298 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=76]
[2024/01/05 17:43:52.298 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=78] [task="TableScanTask: id=78, startKey=7480000000000000f55f7280000000040d2ce3, endKey=7480000000000000f55f728000000004139c90"]
[2024/01/05 17:43:52.441 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=77]
[2024/01/05 17:43:52.444 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=79] [task="TableScanTask: id=79, startKey=7480000000000000f55f728000000004139c90, endKey=7480000000000000f55f7280000000041a0964"]
[2024/01/05 17:43:53.238 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=78]
[2024/01/05 17:43:53.238 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=80] [task="TableScanTask: id=80, startKey=7480000000000000f55f7280000000041a0964, endKey=7480000000000000f55f728000000004207638"]
[2024/01/05 17:43:53.329 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=79]
[2024/01/05 17:43:53.333 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=81] [task="TableScanTask: id=81, startKey=7480000000000000f55f728000000004207638, endKey=7480000000000000f55f72800000000426e30c"]
[2024/01/05 17:43:53.612 +08:00] [WARN] [pd.go:156] ["get timestamp too slow"] [conn=1272971396] [session_alias=] ["cost time"=40.75993ms]
[2024/01/05 17:43:53.612 +08:00] [WARN] [pd.go:156] ["get timestamp too slow"] [conn=1272971364] [session_alias=] ["cost time"=40.46179ms]
[2024/01/05 17:43:53.612 +08:00] [WARN] [pd.go:156] ["get timestamp too slow"] [conn=1272971376] [session_alias=] ["cost time"=36.641759ms]
[2024/01/05 17:43:53.613 +08:00] [WARN] [pd.go:156] ["get timestamp too slow"] [conn=1272971400] [session_alias=] ["cost time"=34.438386ms]
[2024/01/05 17:43:54.073 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=82] [task="TableScanTask: id=82, startKey=7480000000000000f55f72800000000426e30c, endKey=7480000000000000f55f7280000000042d4fe0"]
[2024/01/05 17:43:54.074 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=80]
[2024/01/05 17:43:54.221 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=81]
[2024/01/05 17:43:54.221 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=83] [task="TableScanTask: id=83, startKey=7480000000000000f55f7280000000042d4fe0, endKey=7480000000000000f55f72800000000433bcb4"]
[2024/01/05 17:43:55.135 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=82]
[2024/01/05 17:43:55.135 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=84] [task="TableScanTask: id=84, startKey=7480000000000000f55f72800000000433bcb4, endKey=7480000000000000f55f7280000000043a2988"]
[2024/01/05 17:43:55.305 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=83]
[2024/01/05 17:43:55.305 +08:00] [INFO] [backfilling_operators.go:456] ["start a table scan task"] [id=85] [task="TableScanTask: id=85, startKey=7480000000000000f55f7280000000043a2988, endKey=7480000000000000f55f72800000000440965c"]
[2024/01/05 17:43:56.343 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=84]
[2024/01/05 17:43:56.434 +08:00] [INFO] [backfilling_operators.go:658] ["finish a index ingest task"] [id=85]
[2024/01/05 17:43:58.525 +08:00] [WARN] [expensivequery.go:145] [expensive_query] [cost_time=60.013419055s] [conn=1272971414] [user=root] [database=sysbench_64_7000w] [txn_start_ts=0] [mem_max="0 Bytes (0 Bytes)"] [sql="alter table sbtest1 add index index_test_1704447778510 (c)"] [session_alias=]
[2024/01/05 17:43:58.890 +08:00] [ERROR] [tso_dispatcher.go:498] ["[tso] getTS error after processing requests"] [dc-location=global] [stream-addr=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetTSO]get TSO failed, %v: rpc error: code = Unavailable desc = error reading from server: read tcp 10.200.63.162:48430->10.200.58.226:2379: read: connection reset by peer"]
[2024/01/05 17:43:58.891 +08:00] [INFO] [tso_dispatcher.go:441] ["[tso] tso stream is not ready"] [dc=global]
[2024/01/05 17:43:58.891 +08:00] [WARN] [txn.go:700] ["wait tso failed"] [error="rpc error: code = Unavailable desc = error reading from server: read tcp 10.200.63.162:48430->10.200.58.226:2379: read: connection reset by peer"]
[2024/01/05 17:43:58.894 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:43:58.895 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:43:59.391 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:43:59.431 +08:00] [ERROR] [pd_service_discovery.go:561] ["[pd] failed to update service mode"] [urls="[http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379,http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379,http://tc-pd-2.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379]"] [error="[PD:client:ErrClientGetClusterInfo]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:43:59.431 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:43:59.632 +08:00] [ERROR] [tso_dispatcher.go:498] ["[tso] getTS error after processing requests"] [dc-location=global] [stream-addr=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetTSO]get TSO failed, %v: [PD:client:ErrClientTSOStreamClosed]encountered TSO stream being closed unexpectedly"]
[2024/01/05 17:43:59.632 +08:00] [ERROR] [pd.go:228] ["updateTS error"] [txnScope=global] [error="[PD:client:ErrClientTSOStreamClosed]encountered TSO stream being closed unexpectedly"]
[2024/01/05 17:43:59.634 +08:00] [ERROR] [pd_service_discovery.go:561] ["[pd] failed to update service mode"] [urls="[http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379,http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379,http://tc-pd-2.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379]"] [error="[PD:client:ErrClientGetClusterInfo]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:43:59.634 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:43:59.634 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:43:59.635 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:43:59.635 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:43:59.652 +08:00] [ERROR] [client.go:217] ["[pd] do http request failed"] [source=lightning] [name=GetMembers] [url=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/members] [method=GET] [caller-id=lightning-pd-http-client-inner] [error="Get \"http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/members\": dial tcp 10.200.58.226:2379: connect: connection refused"]
[2024/01/05 17:43:59.654 +08:00] [ERROR] [client.go:245] ["[pd] request failed with a non-200 status"] [source=lightning] [name=GetMembers] [url=http://tc-pd-2.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/members] [method=GET] [caller-id=lightning-pd-http-client-inner] [status="500 Internal Server Error"] [body="[PD:apiutil:ErrRedirect]redirect failed\n"]
[2024/01/05 17:43:59.656 +08:00] [ERROR] [client.go:245] ["[pd] request failed with a non-200 status"] [source=lightning] [name=GetMembers] [url=http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/members] [method=GET] [caller-id=lightning-pd-http-client-inner] [status="500 Internal Server Error"] [body="[PD:apiutil:ErrRedirect]redirect failed\n"]
[2024/01/05 17:43:59.656 +08:00] [ERROR] [client.go:285] ["[pd] http client get members info failed"] [source=lightning] [error="request pd http api failed with status: '500 Internal Server Error'"]
[2024/01/05 17:43:59.892 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:00.135 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:00.290 +08:00] [INFO] [engine.go:1617] ["compact sst"] [fileCount=8] [size=1073742120] [count=6507528] [cost=13.253589358s] [file=/var/lib/tidb-data/tmp_ddl-4000/435/e81a0e89-251e-59cd-9827-5cbe286e40c0.sst/ae68aecc-6ea8-4f19-8332-40e6ab4cbc78.sst]
[2024/01/05 17:44:00.290 +08:00] [INFO] [engine.go:865] ["write data to local DB"] [size=1073742120] [kvs=6507528] [files=1] [sstFileSize=825201409] [file=/var/lib/tidb-data/tmp_ddl-4000/435/e81a0e89-251e-59cd-9827-5cbe286e40c0.sst/ae68aecc-6ea8-4f19-8332-40e6ab4cbc78.sst] [firstKey=7480000000000000F55F69800000000000000C013030303030303132FF3234352D34343431FF333834323231322DFF3537323835373831FF3135372D33363837FF323336353439302DFF3337323536303733FF3633302D37383130FF383233343039322DFF3931343537313031FF3833322D34313537FF323538343137342DFF3330303339373239FF3837352D33363337FF3136393535333700FE0380000000033CE5F2] [lastKey=7480000000000000F55F69800000000000000C013939393939393930FF3133312D35373334FF373837333835322DFF3333313932333136FF3039322D39383835FF363432363637382DFF3636343633343130FF3937362D38383339FF313435373636352DFF3639373834333836FF3437342D32313536FF313331393238342DFF3739303632343930FF3638342D36323631FF3835373232383600FE0380000000035D7351]
[2024/01/05 17:44:00.391 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:00.433 +08:00] [ERROR] [client.go:217] ["[pd] do http request failed"] [source=tikv-driver] [name=GetMinResolvedTSByStoresIDs] [url=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts] [method=GET] [caller-id=pd-http-client] [error="Get \"http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts\": dial tcp 10.200.58.226:2379: connect: connection refused"]
[2024/01/05 17:44:00.435 +08:00] [ERROR] [client.go:245] ["[pd] request failed with a non-200 status"] [source=tikv-driver] [name=GetMinResolvedTSByStoresIDs] [url=http://tc-pd-2.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts] [method=GET] [caller-id=pd-http-client] [status="500 Internal Server Error"] [body="[PD:apiutil:ErrRedirect]redirect failed\n"]
[2024/01/05 17:44:00.437 +08:00] [ERROR] [client.go:245] ["[pd] request failed with a non-200 status"] [source=tikv-driver] [name=GetMinResolvedTSByStoresIDs] [url=http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts] [method=GET] [caller-id=pd-http-client] [status="500 Internal Server Error"] [body="[PD:apiutil:ErrRedirect]redirect failed\n"]
[2024/01/05 17:44:00.438 +08:00] [ERROR] [client.go:217] ["[pd] do http request failed"] [source=tikv-driver] [name=GetMinResolvedTSByStoresIDs] [url="http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts?scope=1,12,13,14"] [method=GET] [caller-id=pd-http-client] [error="Get \"http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts?scope=1,12,13,14\": dial tcp 10.200.58.226:2379: connect: connection refused"]
[2024/01/05 17:44:00.440 +08:00] [ERROR] [client.go:245] ["[pd] request failed with a non-200 status"] [source=tikv-driver] [name=GetMinResolvedTSByStoresIDs] [url="http://tc-pd-2.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts?scope=1,12,13,14"] [method=GET] [caller-id=pd-http-client] [status="500 Internal Server Error"] [body="[PD:apiutil:ErrRedirect]redirect failed\n"]
[2024/01/05 17:44:00.442 +08:00] [ERROR] [client.go:245] ["[pd] request failed with a non-200 status"] [source=tikv-driver] [name=GetMinResolvedTSByStoresIDs] [url="http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts?scope=1,12,13,14"] [method=GET] [caller-id=pd-http-client] [status="500 Internal Server Error"] [body="[PD:apiutil:ErrRedirect]redirect failed\n"]
[2024/01/05 17:44:00.635 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:00.891 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:01.135 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:01.392 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:01.635 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:01.892 +08:00] [ERROR] [tso_dispatcher.go:562] ["[tso] update connection contexts failed"] [dc=global] [error="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\""]
[2024/01/05 17:44:01.892 +08:00] [ERROR] [tso_dispatcher.go:452] ["[tso] create tso stream error"] [dc-location=global] [error="[PD:client:ErrClientCreateTSOStream]create TSO stream failed, retry timeout"]
[2024/01/05 17:44:01.892 +08:00] [INFO] [tso_dispatcher.go:441] ["[tso] tso stream is not ready"] [dc=global]
[2024/01/05 17:44:01.892 +08:00] [WARN] [txn.go:700] ["wait tso failed"] [error="[PD:client:ErrClientCreateTSOStream]create TSO stream failed, retry timeout"]
[2024/01/05 17:44:01.892 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:01.893 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:02.135 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:02.392 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:02.431 +08:00] [ERROR] [pd_service_discovery.go:561] ["[pd] failed to update service mode"] [urls="[http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379,http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379,http://tc-pd-2.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379]"] [error="[PD:client:ErrClientGetClusterInfo]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:02.431 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:02.433 +08:00] [ERROR] [client.go:217] ["[pd] do http request failed"] [source=tikv-driver] [name=GetMinResolvedTSByStoresIDs] [url=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts] [method=GET] [caller-id=pd-http-client] [error="Get \"http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts\": dial tcp 10.200.58.226:2379: connect: connection refused"]
[2024/01/05 17:44:02.436 +08:00] [ERROR] [client.go:245] ["[pd] request failed with a non-200 status"] [source=tikv-driver] [name=GetMinResolvedTSByStoresIDs] [url=http://tc-pd-2.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts] [method=GET] [caller-id=pd-http-client] [status="500 Internal Server Error"] [body="[PD:apiutil:ErrRedirect]redirect failed\n"]
[2024/01/05 17:44:02.437 +08:00] [ERROR] [client.go:245] ["[pd] request failed with a non-200 status"] [source=tikv-driver] [name=GetMinResolvedTSByStoresIDs] [url=http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts] [method=GET] [caller-id=pd-http-client] [status="500 Internal Server Error"] [body="[PD:apiutil:ErrRedirect]redirect failed\n"]
[2024/01/05 17:44:02.477 +08:00] [INFO] [resource_manager_client.go:348] ["[resource_manager] token request error"] [error=EOF]
[2024/01/05 17:44:02.477 +08:00] [INFO] [controller.go:527] ["[resource group controller] token bucket rpc error: EOF"]
[2024/01/05 17:44:02.477 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:02.632 +08:00] [ERROR] [pd_service_discovery.go:561] ["[pd] failed to update service mode"] [urls="[http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379,http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379,http://tc-pd-2.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379]"] [error="[PD:client:ErrClientGetClusterInfo]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:02.632 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:02.635 +08:00] [ERROR] [tso_dispatcher.go:562] ["[tso] update connection contexts failed"] [dc=global] [error="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\""]
[2024/01/05 17:44:02.635 +08:00] [INFO] [tso_dispatcher.go:441] ["[tso] tso stream is not ready"] [dc=global]
[2024/01/05 17:44:02.635 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:02.811 +08:00] [ERROR] [client.go:217] ["[pd] do http request failed"] [source=tikv-driver] [name=GetMinResolvedTSByStoresIDs] [url="http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts?scope=1,12,13,14"] [method=GET] [caller-id=pd-http-client] [error="Get \"http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts?scope=1,12,13,14\": dial tcp 10.200.58.226:2379: connect: connection refused"]
[2024/01/05 17:44:02.814 +08:00] [ERROR] [client.go:245] ["[pd] request failed with a non-200 status"] [source=tikv-driver] [name=GetMinResolvedTSByStoresIDs] [url="http://tc-pd-2.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts?scope=1,12,13,14"] [method=GET] [caller-id=pd-http-client] [status="500 Internal Server Error"] [body="[PD:apiutil:ErrRedirect]redirect failed\n"]
[2024/01/05 17:44:02.815 +08:00] [ERROR] [client.go:245] ["[pd] request failed with a non-200 status"] [source=tikv-driver] [name=GetMinResolvedTSByStoresIDs] [url="http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts?scope=1,12,13,14"] [method=GET] [caller-id=pd-http-client] [status="500 Internal Server Error"] [body="[PD:apiutil:ErrRedirect]redirect failed\n"]
[2024/01/05 17:44:02.892 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:03.136 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:03.393 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:03.553 +08:00] [INFO] [engine.go:1617] ["compact sst"] [fileCount=6] [size=393746430] [count=2386342] [cost=3.254751967s] [file=/var/lib/tidb-data/tmp_ddl-4000/435/e81a0e89-251e-59cd-9827-5cbe286e40c0.sst/ff2d1d1a-625f-4128-baf1-dce4d646151a.sst]
[2024/01/05 17:44:03.553 +08:00] [INFO] [engine.go:865] ["write data to local DB"] [size=393746430] [kvs=2386342] [files=1] [sstFileSize=303762190] [file=/var/lib/tidb-data/tmp_ddl-4000/435/e81a0e89-251e-59cd-9827-5cbe286e40c0.sst/ff2d1d1a-625f-4128-baf1-dce4d646151a.sst] [firstKey=7480000000000000F55F69800000000000000C013030303030303537FF3539362D37393937FF353335373338332DFF3430323133333934FF3833332D30373630FF353537383234302DFF3437383137343335FF3039362D34383630FF303133363239372DFF3232343230333434FF3332322D38363638FF373130333439382DFF3832313233333735FF3236332D31323538FF3932313537373200FE038000000004303FD8] [lastKey=7480000000000000F55F69800000000000000C013939393939393930FF3033342D34363234FF333131333837302DFF3039313032333335FF3631362D37333939FF353738313035392DFF3433303033353330FF3130322D39313939FF353937323131362DFF3131393239323335FF3439372D30373934FF393136303135372DFF3036333135393138FF3237392D31383930FF3535323233303100FE0380000000041894B7]
[2024/01/05 17:44:03.635 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:03.892 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:04.135 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:04.393 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:04.433 +08:00] [ERROR] [client.go:217] ["[pd] do http request failed"] [source=tikv-driver] [name=GetMinResolvedTSByStoresIDs] [url=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts] [method=GET] [caller-id=pd-http-client] [error="Get \"http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts\": dial tcp 10.200.58.226:2379: connect: connection refused"]
[2024/01/05 17:44:04.435 +08:00] [ERROR] [client.go:245] ["[pd] request failed with a non-200 status"] [source=tikv-driver] [name=GetMinResolvedTSByStoresIDs] [url=http://tc-pd-2.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts] [method=GET] [caller-id=pd-http-client] [status="500 Internal Server Error"] [body="[PD:apiutil:ErrRedirect]redirect failed\n"]
[2024/01/05 17:44:04.437 +08:00] [ERROR] [client.go:245] ["[pd] request failed with a non-200 status"] [source=tikv-driver] [name=GetMinResolvedTSByStoresIDs] [url=http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts] [method=GET] [caller-id=pd-http-client] [status="500 Internal Server Error"] [body="[PD:apiutil:ErrRedirect]redirect failed\n"]
[2024/01/05 17:44:04.438 +08:00] [ERROR] [client.go:217] ["[pd] do http request failed"] [source=tikv-driver] [name=GetMinResolvedTSByStoresIDs] [url="http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts?scope=1,12,13,14"] [method=GET] [caller-id=pd-http-client] [error="Get \"http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts?scope=1,12,13,14\": dial tcp 10.200.58.226:2379: connect: connection refused"]
[2024/01/05 17:44:04.440 +08:00] [ERROR] [client.go:245] ["[pd] request failed with a non-200 status"] [source=tikv-driver] [name=GetMinResolvedTSByStoresIDs] [url="http://tc-pd-2.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts?scope=1,12,13,14"] [method=GET] [caller-id=pd-http-client] [status="500 Internal Server Error"] [body="[PD:apiutil:ErrRedirect]redirect failed\n"]
[2024/01/05 17:44:04.442 +08:00] [ERROR] [client.go:245] ["[pd] request failed with a non-200 status"] [source=tikv-driver] [name=GetMinResolvedTSByStoresIDs] [url="http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts?scope=1,12,13,14"] [method=GET] [caller-id=pd-http-client] [status="500 Internal Server Error"] [body="[PD:apiutil:ErrRedirect]redirect failed\n"]
[2024/01/05 17:44:04.636 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:04.893 +08:00] [ERROR] [tso_dispatcher.go:562] ["[tso] update connection contexts failed"] [dc=global] [error="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\""]
[2024/01/05 17:44:04.893 +08:00] [ERROR] [tso_dispatcher.go:452] ["[tso] create tso stream error"] [dc-location=global] [error="[PD:client:ErrClientCreateTSOStream]create TSO stream failed, retry timeout"]
[2024/01/05 17:44:04.893 +08:00] [INFO] [tso_dispatcher.go:441] ["[tso] tso stream is not ready"] [dc=global]
[2024/01/05 17:44:04.893 +08:00] [WARN] [txn.go:700] ["wait tso failed"] [error="[PD:client:ErrClientCreateTSOStream]create TSO stream failed, retry timeout"]
[2024/01/05 17:44:04.893 +08:00] [WARN] [txn.go:700] ["wait tso failed"] [error="[PD:client:ErrClientCreateTSOStream]create TSO stream failed, retry timeout"]
[2024/01/05 17:44:04.893 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:04.893 +08:00] [WARN] [txn.go:700] ["wait tso failed"] [error="[PD:client:ErrClientCreateTSOStream]create TSO stream failed, retry timeout"]
[2024/01/05 17:44:04.893 +08:00] [WARN] [txn.go:700] ["wait tso failed"] [error="[PD:client:ErrClientCreateTSOStream]create TSO stream failed, retry timeout"]
[2024/01/05 17:44:04.893 +08:00] [WARN] [txn.go:700] ["wait tso failed"] [error="[PD:client:ErrClientCreateTSOStream]create TSO stream failed, retry timeout"]
[2024/01/05 17:44:04.893 +08:00] [WARN] [txn.go:700] ["wait tso failed"] [error="[PD:client:ErrClientCreateTSOStream]create TSO stream failed, retry timeout"]
[2024/01/05 17:44:04.893 +08:00] [ERROR] [pd.go:228] ["updateTS error"] [txnScope=global] [error="[PD:client:ErrClientCreateTSOStream]create TSO stream failed, retry timeout"]
[2024/01/05 17:44:04.893 +08:00] [WARN] [txn.go:700] ["wait tso failed"] [error="[PD:client:ErrClientCreateTSOStream]create TSO stream failed, retry timeout"]
[2024/01/05 17:44:04.893 +08:00] [WARN] [txn.go:700] ["wait tso failed"] [error="[PD:client:ErrClientCreateTSOStream]create TSO stream failed, retry timeout"]
[2024/01/05 17:44:04.894 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:05.136 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:05.393 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:05.431 +08:00] [ERROR] [pd_service_discovery.go:561] ["[pd] failed to update service mode"] [urls="[http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379,http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379,http://tc-pd-2.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379]"] [error="[PD:client:ErrClientGetClusterInfo]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:05.432 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:05.632 +08:00] [ERROR] [pd_service_discovery.go:561] ["[pd] failed to update service mode"] [urls="[http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379,http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379,http://tc-pd-2.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379]"] [error="[PD:client:ErrClientGetClusterInfo]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:05.632 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:05.636 +08:00] [ERROR] [tso_dispatcher.go:562] ["[tso] update connection contexts failed"] [dc=global] [error="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\""]
[2024/01/05 17:44:05.893 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:06.137 +08:00] [INFO] [tso_dispatcher.go:441] ["[tso] tso stream is not ready"] [dc=global]
[2024/01/05 17:44:06.137 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:06.394 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:06.433 +08:00] [ERROR] [client.go:217] ["[pd] do http request failed"] [source=tikv-driver] [name=GetMinResolvedTSByStoresIDs] [url=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts] [method=GET] [caller-id=pd-http-client] [error="Get \"http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts\": dial tcp 10.200.58.226:2379: connect: connection refused"]
[2024/01/05 17:44:06.435 +08:00] [ERROR] [client.go:245] ["[pd] request failed with a non-200 status"] [source=tikv-driver] [name=GetMinResolvedTSByStoresIDs] [url=http://tc-pd-2.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts] [method=GET] [caller-id=pd-http-client] [status="500 Internal Server Error"] [body="[PD:apiutil:ErrRedirect]redirect failed\n"]
[2024/01/05 17:44:06.437 +08:00] [ERROR] [client.go:245] ["[pd] request failed with a non-200 status"] [source=tikv-driver] [name=GetMinResolvedTSByStoresIDs] [url=http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts] [method=GET] [caller-id=pd-http-client] [status="500 Internal Server Error"] [body="[PD:apiutil:ErrRedirect]redirect failed\n"]
[2024/01/05 17:44:06.438 +08:00] [ERROR] [client.go:217] ["[pd] do http request failed"] [source=tikv-driver] [name=GetMinResolvedTSByStoresIDs] [url="http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts?scope=1,12,13,14"] [method=GET] [caller-id=pd-http-client] [error="Get \"http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts?scope=1,12,13,14\": dial tcp 10.200.58.226:2379: connect: connection refused"]
[2024/01/05 17:44:06.440 +08:00] [ERROR] [client.go:245] ["[pd] request failed with a non-200 status"] [source=tikv-driver] [name=GetMinResolvedTSByStoresIDs] [url="http://tc-pd-2.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts?scope=1,12,13,14"] [method=GET] [caller-id=pd-http-client] [status="500 Internal Server Error"] [body="[PD:apiutil:ErrRedirect]redirect failed\n"]
[2024/01/05 17:44:06.442 +08:00] [ERROR] [client.go:245] ["[pd] request failed with a non-200 status"] [source=tikv-driver] [name=GetMinResolvedTSByStoresIDs] [url="http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/min-resolved-ts?scope=1,12,13,14"] [method=GET] [caller-id=pd-http-client] [status="500 Internal Server Error"] [body="[PD:apiutil:ErrRedirect]redirect failed\n"]
[2024/01/05 17:44:06.477 +08:00] [INFO] [resource_manager_client.go:328] ["[resource_manager] token leader may change, try to reconnect the stream"]
[2024/01/05 17:44:06.478 +08:00] [INFO] [controller.go:527] ["[resource group controller] token bucket rpc error: failed to get the stream connection"]
[2024/01/05 17:44:06.478 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:06.637 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:06.859 +08:00] [INFO] [engine.go:1617] ["compact sst"] [fileCount=8] [size=1073742120] [count=6507528] [cost=9.566084562s] [file=/var/lib/tidb-data/tmp_ddl-4000/435/e81a0e89-251e-59cd-9827-5cbe286e40c0.sst/cc6a6424-b525-4e3e-b929-97796078c532.sst]
[2024/01/05 17:44:06.859 +08:00] [INFO] [engine.go:865] ["write data to local DB"] [size=1073742120] [kvs=6507528] [files=1] [sstFileSize=827717749] [file=/var/lib/tidb-data/tmp_ddl-4000/435/e81a0e89-251e-59cd-9827-5cbe286e40c0.sst/cc6a6424-b525-4e3e-b929-97796078c532.sst] [firstKey=7480000000000000F55F69800000000000000C013030303030303037FF3739312D37323234FF323936333131372DFF3430323932383436FF3033372D35363633FF393837373338312DFF3039333434383231FF3634382D36353232FF333436343131322DFF3530353136383936FF3034302D37323633FF323036373038302DFF3130373332313230FF3838362D36383832FF3132343334383600FE038000000003D3289D] [lastKey=7480000000000000F55F69800000000000000C013939393939393939FF3335392D37313731FF333639303834322DFF3432373233373232FF3535342D35333837FF393234373039342DFF3037353039303835FF3733342D31373334FF353338373434352DFF3533373234313436FF3438312D36353538FF303238393735302DFF3539363337323432FF3332302D36313932FF3139393837363000FE038000000003DF6308]
[2024/01/05 17:44:06.863 +08:00] [INFO] [backend.go:204] ["acquire distributed flush lock success"] [jobID=435]
[2024/01/05 17:44:06.864 +08:00] [INFO] [backend.go:226] ["do a partial import data into the storage"] ["index ID"=12] ["usage info"="disk usage: 105661214720/1968786661376, backend usage: 0"]
[2024/01/05 17:44:06.864 +08:00] [INFO] [backend.go:368] ["import start"] [engineUUID=e81a0e89-251e-59cd-9827-5cbe286e40c0] [retryCnt=0]
[2024/01/05 17:44:06.864 +08:00] [WARN] [local.go:1210] ["fail to get region split keys and size"] [error="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\""]
[2024/01/05 17:44:06.864 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:06.865 +08:00] [INFO] [local.go:778] ["split engine key ranges"] [engine=e81a0e89-251e-59cd-9827-5cbe286e40c0] [totalSize=5762457030] [totalCount=34923982] [startKey=7480000000000000F55F69800000000000000C013030303030303031FF3731382D32343230FF363631323631312DFF3639313034313836FF3232312D37363035FF333538323838312DFF3030313236333333FF3031312D30373339FF323739373535322DFF3933393036393638FF3435392D39353132FF323731393531332DFF3932303135373334FF3937302D39343633FF3236393634343500FE03800000000322DF37] [endKey=7480000000000000F55F69800000000000000C013939393939393939FF3335392D37313731FF333639303834322DFF3432373233373232FF3535342D35333837FF393234373039342DFF3037353039303835FF3733342D31373334FF353338373434352DFF3533373234313436FF3438312D36353538FF303238393735302DFF3539363337323432FF3332302D36313932FF3139393837363000FE038000000003DF630800] [ranges=27] []
[2024/01/05 17:44:06.865 +08:00] [INFO] [local.go:1220] ["pause pd scheduler of table scope"]
[2024/01/05 17:44:06.866 +08:00] [ERROR] [client.go:217] ["[pd] do http request failed"] [source=lightning] [name=SetRegionLabelRule] [url=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/config/region-label/rule] [method=POST] [caller-id=pd-http-client] [error="Post \"http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/config/region-label/rule\": dial tcp 10.200.58.226:2379: connect: connection refused"]
[2024/01/05 17:44:06.868 +08:00] [ERROR] [client.go:245] ["[pd] request failed with a non-200 status"] [source=lightning] [name=SetRegionLabelRule] [url=http://tc-pd-2.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/config/region-label/rule] [method=POST] [caller-id=pd-http-client] [status="500 Internal Server Error"] [body="[PD:apiutil:ErrRedirect]redirect failed\n"]
[2024/01/05 17:44:06.893 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:07.064 +08:00] [ERROR] [client.go:245] ["[pd] request failed with a non-200 status"] [source=lightning] [name=SetRegionLabelRule] [url=http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379/pd/api/v1/config/region-label/rule] [method=POST] [caller-id=pd-http-client] [status="500 Internal Server Error"] [body="[PD:apiutil:ErrRedirect]redirect failed\n"]
[2024/01/05 17:44:07.084 +08:00] [ERROR] [backend.go:374] ["import failed"] [engineUUID=e81a0e89-251e-59cd-9827-5cbe286e40c0] [retryCnt=0] [takeTime=220.935813ms] [error="request pd http api failed with status: '500 Internal Server Error'"]
[2024/01/05 17:44:07.085 +08:00] [ERROR] [backend.go:237] ["ingest data into storage error"] ["index ID"=12] ["usage info"="disk usage: 105661214720/1968786661376, backend usage: 0"]
[2024/01/05 17:44:07.086 +08:00] [INFO] [backend.go:210] ["release distributed flush lock success"] [jobID=435]
[2024/01/05 17:44:07.087 +08:00] [ERROR] [backfilling_operators.go:799] ["flush error"] [category=ddl] [error="request pd http api failed with status: '500 Internal Server Error'"]
[2024/01/05 17:44:07.087 +08:00] [ERROR] [task_executor.go:511] [onError] [task-id=210002] [error="request pd http api failed with status: '500 Internal Server Error'"] [stack="github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor.(*BaseTaskExecutor).onError\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/task_executor.go:511\ngithub.com/pingcap/tidb/pkg/disttask/framework/taskexecutor.(*BaseTaskExecutor).runSubtask\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/task_executor.go:310\ngithub.com/pingcap/tidb/pkg/disttask/framework/taskexecutor.(*BaseTaskExecutor).run\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/task_executor.go:290\ngithub.com/pingcap/tidb/pkg/disttask/framework/taskexecutor.(*BaseTaskExecutor).Run\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/task_executor.go:136\ngithub.com/pingcap/tidb/pkg/disttask/framework/taskexecutor.(*Manager).onRunnableTask\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/manager.go:429\ngithub.com/pingcap/tidb/pkg/disttask/framework/taskexecutor.(*Manager).onRunnableTasks.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/manager.go:245\ngithub.com/pingcap/tidb/pkg/resourcemanager/pool/spool.(*Pool).run.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/resourcemanager/pool/spool/spool.go:144"]
[2024/01/05 17:44:07.087 +08:00] [ERROR] [task_executor.go:517] ["taskExecutor met first error"] [task-id=210002] [error="request pd http api failed with status: '500 Internal Server Error'"]
[2024/01/05 17:44:07.087 +08:00] [WARN] [task_executor.go:636] ["subtask failed"] [task-id=210002] [error="request pd http api failed with status: '500 Internal Server Error'"]
[2024/01/05 17:44:07.138 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:07.394 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:07.395 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetLeader]get leader failed, leader address doesn't exist"]
[2024/01/05 17:44:07.395 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-2.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetLeader]get leader failed, leader address doesn't exist"]
[2024/01/05 17:44:07.496 +08:00] [ERROR] [pd_service_discovery.go:534] ["[pd] failed to update member"] [urls="[http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379,http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379,http://tc-pd-2.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379]"] [error="[PD:client:ErrClientGetMember]get member failed"]
[2024/01/05 17:44:07.638 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:07.674 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetLeader]get leader failed, leader address doesn't exist"]
[2024/01/05 17:44:07.675 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-2.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetLeader]get leader failed, leader address doesn't exist"]
[2024/01/05 17:44:07.775 +08:00] [ERROR] [pd_service_discovery.go:534] ["[pd] failed to update member"] [urls="[http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379,http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379,http://tc-pd-2.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379]"] [error="[PD:client:ErrClientGetMember]get member failed"]
[2024/01/05 17:44:07.894 +08:00] [ERROR] [tso_dispatcher.go:562] ["[tso] update connection contexts failed"] [dc=global] [error="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\""]
[2024/01/05 17:44:07.894 +08:00] [ERROR] [tso_dispatcher.go:452] ["[tso] create tso stream error"] [dc-location=global] [error="[PD:client:ErrClientCreateTSOStream]create TSO stream failed, retry timeout"]
[2024/01/05 17:44:07.894 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:07.894 +08:00] [INFO] [tso_dispatcher.go:441] ["[tso] tso stream is not ready"] [dc=global]
[2024/01/05 17:44:07.895 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetLeader]get leader failed, leader address doesn't exist"]
[2024/01/05 17:44:07.896 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-2.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetLeader]get leader failed, leader address doesn't exist"]
[2024/01/05 17:44:08.097 +08:00] [ERROR] [pd_service_discovery.go:534] ["[pd] failed to update member"] [urls="[http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379,http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379,http://tc-pd-2.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379]"] [error="[PD:client:ErrClientGetMember]get member failed"]
[2024/01/05 17:44:08.097 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:08.097 +08:00] [INFO] [tso_client.go:231] ["[tso] switch dc tso global allocator serving address"] [dc-location=global] [new-address=http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379]
[2024/01/05 17:44:08.097 +08:00] [INFO] [pd_service_discovery.go:979] ["[pd] switch leader"] [new-leader=http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [old-leader=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379]
[2024/01/05 17:44:08.137 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:08.138 +08:00] [INFO] [tso_client.go:231] ["[tso] switch dc tso global allocator serving address"] [dc-location=global] [new-address=http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379]
[2024/01/05 17:44:08.138 +08:00] [INFO] [pd_service_discovery.go:979] ["[pd] switch leader"] [new-leader=http://tc-pd-1.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [old-leader=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379]
[2024/01/05 17:44:08.394 +08:00] [INFO] [pd_service_discovery.go:871] ["[pd] cannot update member from this address"] [address=http://tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379] [error="[PD:client:ErrClientGetMember]error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE: error:rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\" target:tc-pd-0.tc-pd-peer.endless-ha-test-add-index-tps-5490547-1-98.svc:2379 status:TRANSIENT_FAILURE"]
[2024/01/05 17:44:08.395 +08:00] [WARN] [pd.go:156] ["get timestamp too slow"] ["cost time"=3.501724671s]
[2024/01/05 17:44:08.395 +08:00] [WARN] [pd.go:156] ["get timestamp too slow"] ["cost time"=3.501665688s]
[2024/01/05 17:44:08.395 +08:00] [WARN] [pd.go:156] ["get timestamp too slow"] ["cost time"=3.501667131s]
[2024/01/05 17:44:08.395 +08:00] [WARN] [pd.go:156] ["get timestamp too slow"] ["cost time"=3.501686273s]
[2024/01/05 17:44:08.395 +08:00] [WARN] [pd.go:156] ["get timestamp too slow"] ["cost time"=3.012462985s]
[2024/01/05 17:44:08.395 +08:00] [WARN] [pd.go:156] ["get timestamp too slow"] ["cost time"=3.501818786s]
[2024/01/05 17:44:08.395 +08:00] [WARN] [pd.go:156] ["get timestamp too slow"] ["cost time"=3.501744329s]
[2024/01/05 17:44:08.395 +08:00] [WARN] [pd.go:156] ["get timestamp too slow"] ["cost time"=3.501945373s]
[2024/01/05 17:44:08.395 +08:00] [WARN] [pd.go:156] ["get timestamp too slow"] ["cost time"=3.501689477s]
[2024/01/05 17:44:08.395 +08:00] [WARN] [pd.go:156] ["get timestamp too slow"] ["cost time"=3.501760751s]
[2024/01/05 17:44:08.395 +08:00] [WARN] [pd.go:156] ["get timestamp too slow"] ["cost time"=213.221255ms]
[2024/01/05 17:44:08.399 +08:00] [INFO] [backfilling_read_index.go:150] ["read index executor cleanup subtask exec env"] [category=ddl]
[2024/01/05 17:44:08.399 +08:00] [INFO] [region_request.go:1532] ["send request failed, err: context canceled"] [req-ts=446810776762581036] [req-type=Cop] [region="{ region id: 38169, ver: 1296, confVer: 59 }"] [replica-read-type=leader] [stale-read=false] [request-sender="{rpcError:context canceled,replicaSelector: replicaSelector{selectorStateStr: accessKnownLeader, cacheRegionIsValid: true, replicaStatus: [peer: 38170, store: 14, isEpochStale: false, attempts: 1, replica-epoch: 0, store-epoch: 0, store-state: resolved, store-liveness-state: reachable peer: 38171, store: 12, isEpochStale: false, attempts: 0, replica-epoch: 0, store-epoch: 0, store-state: resolved, store-liveness-state: reachable peer: 38172, store: 13, isEpochStale: false, attempts: 0, replica-epoch: 0, store-epoch: 0, store-state: resolved, store-liveness-state: reachable]}}"] [retry-times=0] [total-backoff-ms=0] [total-backoff-times=0] [max-exec-timeout-ms=60000] [total-region-errors=]
[2024/01/05 17:44:08.399 +08:00] [INFO] [task_executor.go:107] ["taskExecutor canceled"] [task-id=210002]
[2024/01/05 17:44:08.399 +08:00] [INFO] [task_executor.go:99] ["task executor exits"] [task-id=210002]
[2024/01/05 17:44:08.399 +08:00] [INFO] [task_executor.go:163] ["execute task failed"] [task-id=210002] [step=1] [concurrency=4] [mem-limit-percent=0.7] [server-mem-limit=80%] [takeTime=1m8.437220971s] [error="request pd http api failed with status: '500 Internal Server Error'"]
[2024/01/05 17:44:08.399 +08:00] [ERROR] [manager.go:438] ["failed to handle task"] [error="request pd http api failed with status: '500 Internal Server Error'"]
[2024/01/05 17:44:08.478 +08:00] [INFO] [resource_manager_client.go:328] ["[resource_manager] token leader may change, try to reconnect the stream"]
[2024/01/05 17:44:08.608 +08:00] [INFO] [manager.go:267] [onCanceledTasks] [task-id=210002]
[2024/01/05 17:44:08.608 +08:00] [ERROR] [pd.go:228] ["updateTS error"] [txnScope=global] [error="context canceled"]
[2024/01/05 17:44:08.608 +08:00] [ERROR] [pd.go:228] ["updateTS error"] [txnScope=global] [error="context canceled"]
[2024/01/05 17:44:08.608 +08:00] [ERROR] [tso_dispatcher.go:562] ["[tso] update connection contexts failed"] [dc=global] [error="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 10.200.58.226:2379: connect: connection refused\""]
[2024/01/05 17:44:08.608 +08:00] [INFO] [tso_dispatcher.go:240] ["exit tso dispatcher loop"]
[2024/01/05 17:44:08.608 +08:00] [INFO] [tso_dispatcher.go:344] ["[tso] exit tso dispatcher"] [dc-location=global]
[2024/01/05 17:44:08.608 +08:00] [INFO] [tso_dispatcher.go:186] ["exit tso requests cancel loop"]
[2024/01/05 17:44:08.608 +08:00] [INFO] [resource_manager_client.go:295] ["[resource manager] exit resource token dispatcher"]
[2024/01/05 17:44:08.608 +08:00] [INFO] [pd_service_discovery.go:525] ["[pd] exit member loop due to context canceled"]
[2024/01/05 17:44:08.702 +08:00] [INFO] [task_executor.go:408] ["taskExecutor rollback a step"] [task-id=210002] [step=1]
[2024/01/05 17:44:08.718 +08:00] [INFO] [backfilling_read_index.go:198] ["read index executor rollback backfill add index task"] [category=ddl] [jobID=435]
[2024/01/05 17:44:08.851 +08:00] [INFO] [2pc.go:1215] ["send TxnHeartBeat"] [startTS=446810774258319376] [newTTL=30003]
[2024/01/05 17:44:08.909 +08:00] [INFO] [manager.go:267] [onCanceledTasks] [task-id=210002]
[2024/01/05 17:44:09.222 +08:00] [INFO] [manager.go:267] [onCanceledTasks] [task-id=210002]
[2024/01/05 17:44:09.330 +08:00] [INFO] [manager.go:412] ["onRunnableTask exit"] [task-id=210002] [step=1] [state=reverted]
[2024/01/05 17:44:09.330 +08:00] [INFO] [backend.go:326] ["engine close start"] [engineTag=sbtest1:12] [engineUUID=e81a0e89-251e-59cd-9827-5cbe286e40c0]
[2024/01/05 17:44:09.331 +08:00] [ERROR] [engine.go:109] ["close engine error"] [error="context canceled"] ["job ID"=435] ["index ID"=12]
[2024/01/05 17:44:09.332 +08:00] [INFO] [tso_client.go:134] ["closing tso client"]
[2024/01/05 17:44:09.332 +08:00] [INFO] [tso_client.go:139] ["close tso client"]
[2024/01/05 17:44:09.332 +08:00] [INFO] [tso_client.go:150] ["tso client is closed"]
[2024/01/05 17:44:09.332 +08:00] [INFO] [pd_service_discovery.go:612] ["[pd] close pd service discovery client"]
[2024/01/05 17:44:09.333 +08:00] [INFO] [client.go:268] ["[pd] http client member info updater stopped"] [source=lightning]
[2024/01/05 17:44:09.333 +08:00] [INFO] [client.go:397] ["[pd] http client closed"] [source=lightning]
[2024/01/05 17:44:09.333 +08:00] [INFO] [tso_client.go:134] ["closing tso client"]
[2024/01/05 17:44:09.333 +08:00] [INFO] [tso_client.go:139] ["close tso client"]
[2024/01/05 17:44:09.333 +08:00] [INFO] [tso_client.go:150] ["tso client is closed"]
[2024/01/05 17:44:09.333 +08:00] [INFO] [backend_mgr.go:190] ["close one backend for DDL job"] ["job ID"=435] ["current memory usage"=240] ["max memory quota"=8589934592]
[2024/01/05 17:44:09.590 +08:00] [INFO] [domain.go:272] ["diff load InfoSchema success"] [currentSchemaVersion=155] [neededSchemaVersion=156] ["start time"=2.779511ms] [gotSchemaVersion=156] [phyTblIDs="[245]"] [actionTypes="[7]"] [diffTypes="[\"add index\"]"]
[2024/01/05 17:44:09.611 +08:00] [INFO] [domain.go:873] ["mdl gets lock, update to owner"] [jobID=435] [version=156]
[2024/01/05 17:44:09.699 +08:00] [INFO] [domain.go:272] ["diff load InfoSchema success"] [currentSchemaVersion=156] [neededSchemaVersion=157] ["start time"=2.409255ms] [gotSchemaVersion=157] [phyTblIDs="[245]"] [actionTypes="[7]"] [diffTypes="[\"add index\"]"]
[2024/01/05 17:44:09.761 +08:00] [INFO] [domain.go:873] ["mdl gets lock, update to owner"] [jobID=435] [version=157]
[2024/01/05 17:44:09.839 +08:00] [INFO] [domain.go:272] ["diff load InfoSchema success"] [currentSchemaVersion=157] [neededSchemaVersion=158] ["start time"=2.048098ms] [gotSchemaVersion=158] [phyTblIDs="[245]"] [actionTypes="[7]"] [diffTypes="[\"add index\"]"]
[2024/01/05 17:44:09.861 +08:00] [INFO] [domain.go:873] ["mdl gets lock, update to owner"] [jobID=435] [version=158]
[2024/01/05 17:44:10.539 +08:00] [INFO] [ddl.go:1170] ["DDL job is failed"] [category=ddl] [jobID=435]
[2024/01/05 17:44:10.539 +08:00] [INFO] [tidb.go:286] ["rollbackTxn called due to ddl/autocommit failure"]
[2024/01/05 17:44:10.539 +08:00] [WARN] [session.go:2251] ["run statement failed"] [conn=1272971414] [session_alias=] [schemaVersion=152] [error="[0]request pd http api failed with status: '503 Service Unavailable'"] [session="{\n  \"currDBName\": \"sysbench_64_7000w\",\n  \"id\": 1272971414,\n  \"status\": 2,\n  \"strictMode\": true,\n  \"user\": {\n    \"Username\": \"root\",\n    \"Hostname\": \"10.200.72.82\",\n    \"CurrentUser\": false,\n    \"AuthUsername\": \"root\",\n    \"AuthHostname\": \"%\",\n    \"AuthPlugin\": \"mysql_native_password\"\n  }\n}"]
[2024/01/05 17:44:10.539 +08:00] [INFO] [conn.go:1155] ["command dispatched failed"] [conn=1272971414] [session_alias=] [connInfo="id:1272971414, addr:10.200.72.82:56678 status:10, collation:utf8mb4_general_ci, user:root"] [command=Query] [status="inTxn:0, autocommit:1"] [sql="alter table sbtest1 add index index_test_1704447778510 (c)"] [txn_mode=PESSIMISTIC] [timestamp=446810758437929100] [err="[0]request pd http api failed with status: '503 Service Unavailable'\ngithub.com/pingcap/errors.AddStack\n\t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20231212100244-799fae176cfb/errors.go:174\ngithub.com/pingcap/errors.Trace\n\t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20231212100244-799fae176cfb/juju_adaptor.go:15\ngithub.com/pingcap/tidb/pkg/ddl.(*ddl).DoDDLJob\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/ddl/ddl.go:1171\ngithub.com/pingcap/tidb/pkg/ddl.(*ddl).createIndex\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/ddl/ddl_api.go:7436\ngithub.com/pingcap/tidb/pkg/ddl.(*ddl).AlterTable\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/ddl/ddl_api.go:3750\ngithub.com/pingcap/tidb/pkg/executor.(*DDLExec).executeAlterTable\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/ddl.go:387\ngithub.com/pingcap/tidb/pkg/executor.(*DDLExec).Next\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/ddl.go:151\ngithub.com/pingcap/tidb/pkg/executor/internal/exec.Next\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/internal/exec/executor.go:314\ngithub.com/pingcap/tidb/pkg/executor.(*ExecStmt).next\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/adapter.go:1252\ngithub.com/pingcap/tidb/pkg/executor.(*ExecStmt).handleNoDelayExecutor\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/adapter.go:997\ngithub.com/pingcap/tidb/pkg/executor.(*ExecStmt).handleNoDelay\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/adapter.go:823\ngithub.com/pingcap/tidb/pkg/executor.(*ExecStmt).Exec\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/adapter.go:598\ngithub.com/pingcap/tidb/pkg/session.runStmt\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/session/session.go:2380\ngithub.com/pingcap/tidb/pkg/session.(*session).ExecuteStmt\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/session/session.go:2239\ngithub.com/pingcap/tidb/pkg/server.(*TiDBContext).ExecuteStmt\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/driver_tidb.go:293\ngithub.com/pingcap/tidb/pkg/server.(*clientConn).handleStmt\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/conn.go:2024\ngithub.com/pingcap/tidb/pkg/server.(*clientConn).handleQuery\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/conn.go:1775\ngithub.com/pingcap/tidb/pkg/server.(*clientConn).dispatch\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/conn.go:1349\ngithub.com/pingcap/tidb/pkg/server.(*clientConn).Run\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/conn.go:1122\ngithub.com/pingcap/tidb/pkg/server.(*Server).onConn\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/server.go:713\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"]
[2024/01/05 17:44:10.540 +08:00] [WARN] [terror.go:249] ["Unknown error class"] [class=0]