Created
November 3, 2021 19:47
-
-
Save derekperkins/ac6681eb7c9fa633f234208f4e446faa to your computer and use it in GitHub Desktop.
Deadlock during tablet shutdown
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| I1103 19:42:02.282187 1 snapshot_conn.go:72] Tables unlocked: rankings__searches_listener__msgs_pri | |
| I1103 19:42:02.671255 1 snapshot_conn.go:79] Locking table searches__retriever_dataforseo__msgs for copying | |
| I1103 19:42:02.672326 1 snapshot_conn.go:72] Tables unlocked: searches__retriever_dataforseo__msgs | |
| I1103 19:42:03.073167 1 snapshot_conn.go:79] Locking table searches__requester_dataforseo__msgs_pri for copying | |
| I1103 19:42:03.073282 1 snapshot_conn.go:79] Locking table searches__requester_dataforseo__msgs for copying | |
| I1103 19:42:03.074502 1 snapshot_conn.go:72] Tables unlocked: searches__requester_dataforseo__msgs_pri | |
| I1103 19:42:03.074517 1 snapshot_conn.go:72] Tables unlocked: searches__requester_dataforseo__msgs | |
| I1103 19:42:03.267300 1 snapshot_conn.go:79] Locking table searches__requester_vertifire__msgs_pri for copying | |
| I1103 19:42:03.268367 1 snapshot_conn.go:72] Tables unlocked: searches__requester_vertifire__msgs_pri | |
| I1103 19:42:12.088392 1 rpc_replication.go:364] DemotePrimary | |
| I1103 19:42:12.090068 1 rpc_replication.go:414] DemotePrimary disabling query service | |
| I1103 19:42:12.090089 1 state_manager.go:214] Starting transition to PRIMARY Not Serving, timestamp: 2021-11-03 19:13:28.774398929 +0000 UTC | |
| I1103 19:42:12.090226 1 tablegc.go:193] TableGC: closing | |
| I1103 19:42:14.301091 1 snapshot_conn.go:79] Locking table searches__extractor__msgs for copying | |
| I1103 19:42:14.301836 1 snapshot_conn.go:72] Tables unlocked: searches__extractor__msgs | |
| E1103 19:42:28.819765 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.819792 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.819807 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.819841 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.819859 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.819884 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.819898 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.819911 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.819928 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.819927 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.819942 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.819954 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.819959 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.819958 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.819972 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.819977 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.819983 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.819988 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.819998 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.820035 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.820044 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.820038 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.820098 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.820145 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.820169 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.820179 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:28.820191 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| I1103 19:42:29.658405 1 snapshot_conn.go:79] Locking table searches__extractor__msgs_pri for copying | |
| I1103 19:42:29.659460 1 snapshot_conn.go:72] Tables unlocked: searches__extractor__msgs_pri | |
| I1103 19:42:30.598166 1 snapshot_conn.go:79] Locking table searches__retriever_dataforseo__msgs_pri for copying | |
| I1103 19:42:30.598281 1 snapshot_conn.go:79] Locking table searches__extractor__msgs_low for copying | |
| I1103 19:42:30.598344 1 snapshot_conn.go:79] Locking table searches__retriever_vertifire__msgs_pri for copying | |
| I1103 19:42:30.598344 1 snapshot_conn.go:79] Locking table searches__request_dispatcher__msgs for copying | |
| I1103 19:42:30.598455 1 snapshot_conn.go:79] Locking table searches__request_dispatcher__msgs_pri for copying | |
| I1103 19:42:30.599567 1 snapshot_conn.go:72] Tables unlocked: searches__extractor__msgs_low | |
| I1103 19:42:30.599663 1 snapshot_conn.go:72] Tables unlocked: searches__retriever_dataforseo__msgs_pri | |
| I1103 19:42:30.599773 1 snapshot_conn.go:72] Tables unlocked: searches__request_dispatcher__msgs_pri | |
| I1103 19:42:30.599891 1 snapshot_conn.go:72] Tables unlocked: searches__request_dispatcher__msgs | |
| I1103 19:42:30.599917 1 snapshot_conn.go:72] Tables unlocked: searches__retriever_vertifire__msgs_pri | |
| I1103 19:42:30.608870 1 snapshot_conn.go:79] Locking table searches__retriever_vertifire__msgs for copying | |
| I1103 19:42:30.609841 1 snapshot_conn.go:72] Tables unlocked: searches__retriever_vertifire__msgs | |
| I1103 19:42:30.614790 1 snapshot_conn.go:79] Locking table searches__requester_marketmuse__msgs_pri for copying | |
| I1103 19:42:30.615483 1 snapshot_conn.go:72] Tables unlocked: searches__requester_marketmuse__msgs_pri | |
| I1103 19:42:31.229667 1 snapshot_conn.go:79] Locking table searches__requester_marketmuse__msgs for copying | |
| I1103 19:42:31.231005 1 snapshot_conn.go:72] Tables unlocked: searches__requester_marketmuse__msgs | |
| I1103 19:42:32.153149 1 snapshot_conn.go:79] Locking table rankings__searches_listener__msgs for copying | |
| I1103 19:42:32.153933 1 snapshot_conn.go:72] Tables unlocked: rankings__searches_listener__msgs | |
| I1103 19:42:32.289742 1 snapshot_conn.go:79] Locking table rankings__searches_listener__msgs_pri for copying | |
| I1103 19:42:32.290821 1 snapshot_conn.go:72] Tables unlocked: rankings__searches_listener__msgs_pri | |
| I1103 19:42:32.675836 1 snapshot_conn.go:79] Locking table searches__retriever_dataforseo__msgs for copying | |
| I1103 19:42:32.677506 1 snapshot_conn.go:72] Tables unlocked: searches__retriever_dataforseo__msgs | |
| I1103 19:42:33.077205 1 snapshot_conn.go:79] Locking table searches__requester_dataforseo__msgs_pri for copying | |
| I1103 19:42:33.077205 1 snapshot_conn.go:79] Locking table searches__requester_dataforseo__msgs for copying | |
| I1103 19:42:33.078094 1 snapshot_conn.go:72] Tables unlocked: searches__requester_dataforseo__msgs_pri | |
| I1103 19:42:33.078339 1 snapshot_conn.go:72] Tables unlocked: searches__requester_dataforseo__msgs | |
| I1103 19:42:33.270767 1 snapshot_conn.go:79] Locking table searches__requester_vertifire__msgs_pri for copying | |
| I1103 19:42:33.271771 1 snapshot_conn.go:72] Tables unlocked: searches__requester_vertifire__msgs_pri | |
| I1103 19:42:42.293657 1 vstreamer.go:942] stream (at source tablet) ended @ 04b526f5-e829-11ea-a033-a2de0b053154:1-146054622,2c9813b2-7176-11e9-a434-5aabf7900131:1-53219105,35a340f3-6974-11ea-a310-42c45972a8de:1-194,56ae3c8c-3011-11ea-bbd4-468fd4139175:1-15058935,5b1b7148-716e-11e9-9faf-2ae79d37031e:1-210834,691f19fc-69ab-11ea-9d10-de53fd0769e4:1-37870125,8a9b0af8-696e-11ea-9911-6274707228d6:1-69665,9991e006-6740-11e9-a909-0a580a301c02:1-245981,a559290c-4d03-11e9-b7d2-b2c80c7d3d23:1-7930960,a8f6c7fe-3011-11ea-ae1a-3eaefc605e25:1-37913825,c17cb726-51c8-11e9-a06a-d6f2f792ae0b:1-1469191,cc444cb1-698b-11ea-860e-96131e5ee507:1-51954,cce5005f-6740-11e9-8f99-0a580a301706:1-13,ea9c99ce-69ee-11ea-84f9-62f4dcbfaa3b:1-220294949,ef44ef4b-596f-11e9-9e9a-0a580a30080a:1-5340566 | |
| I1103 19:42:42.293699 1 binlog_connection.go:278] closing binlog socket to unblock reads | |
| I1103 19:42:42.293765 1 binlog_connection.go:152] connection closed during binlog stream (possibly intentional): read unix @->/vtdataroot/tabletdata/mysql.sock: use of closed network connection | |
| io.ReadFull(header size) failed (errno 2013) (sqlstate HY000) | |
| I1103 19:42:42.293779 1 binlog_connection.go:286] waiting for binlog dump thread to end | |
| I1103 19:42:42.293783 1 binlog_connection.go:292] closing binlog MySQL client with serverID 41051618. Will recycle ID. | |
| I1103 19:42:42.293673 1 vstreamer.go:942] stream (at source tablet) ended @ 04b526f5-e829-11ea-a033-a2de0b053154:1-146054622,2c9813b2-7176-11e9-a434-5aabf7900131:1-53219105,35a340f3-6974-11ea-a310-42c45972a8de:1-194,56ae3c8c-3011-11ea-bbd4-468fd4139175:1-15058935,5b1b7148-716e-11e9-9faf-2ae79d37031e:1-210834,691f19fc-69ab-11ea-9d10-de53fd0769e4:1-37870125,8a9b0af8-696e-11ea-9911-6274707228d6:1-69665,9991e006-6740-11e9-a909-0a580a301c02:1-245981,a559290c-4d03-11e9-b7d2-b2c80c7d3d23:1-7930960,a8f6c7fe-3011-11ea-ae1a-3eaefc605e25:1-37913825,c17cb726-51c8-11e9-a06a-d6f2f792ae0b:1-1469191,cc444cb1-698b-11ea-860e-96131e5ee507:1-51954,cce5005f-6740-11e9-8f99-0a580a301706:1-13,ea9c99ce-69ee-11ea-84f9-62f4dcbfaa3b:1-220294949,ef44ef4b-596f-11e9-9e9a-0a580a30080a:1-5340566 | |
| I1103 19:42:42.293791 1 message_manager.go:627] Context canceled, exiting vstream | |
| I1103 19:42:42.293798 1 binlog_connection.go:278] closing binlog socket to unblock reads | |
| I1103 19:42:42.293848 1 binlog_connection.go:152] connection closed during binlog stream (possibly intentional): read unix @->/vtdataroot/tabletdata/mysql.sock: use of closed network connection | |
| io.ReadFull(header size) failed (errno 2013) (sqlstate HY000) | |
| I1103 19:42:42.293919 1 binlog_connection.go:286] waiting for binlog dump thread to end | |
| I1103 19:42:42.294128 1 binlog_connection.go:292] closing binlog MySQL client with serverID 41051622. Will recycle ID. | |
| I1103 19:42:42.294136 1 message_manager.go:627] Context canceled, exiting vstream | |
| I1103 19:42:42.299089 1 vstreamer.go:942] stream (at source tablet) ended @ 04b526f5-e829-11ea-a033-a2de0b053154:1-146054622,2c9813b2-7176-11e9-a434-5aabf7900131:1-53219105,35a340f3-6974-11ea-a310-42c45972a8de:1-194,56ae3c8c-3011-11ea-bbd4-468fd4139175:1-15058935,5b1b7148-716e-11e9-9faf-2ae79d37031e:1-210834,691f19fc-69ab-11ea-9d10-de53fd0769e4:1-37870125,8a9b0af8-696e-11ea-9911-6274707228d6:1-69665,9991e006-6740-11e9-a909-0a580a301c02:1-245981,a559290c-4d03-11e9-b7d2-b2c80c7d3d23:1-7930960,a8f6c7fe-3011-11ea-ae1a-3eaefc605e25:1-37913825,c17cb726-51c8-11e9-a06a-d6f2f792ae0b:1-1469191,cc444cb1-698b-11ea-860e-96131e5ee507:1-51954,cce5005f-6740-11e9-8f99-0a580a301706:1-13,ea9c99ce-69ee-11ea-84f9-62f4dcbfaa3b:1-220294949,ef44ef4b-596f-11e9-9e9a-0a580a30080a:1-5340566 | |
| I1103 19:42:42.299114 1 binlog_connection.go:278] closing binlog socket to unblock reads | |
| I1103 19:42:42.299157 1 vstreamer.go:942] stream (at source tablet) ended @ 04b526f5-e829-11ea-a033-a2de0b053154:1-146054622,2c9813b2-7176-11e9-a434-5aabf7900131:1-53219105,35a340f3-6974-11ea-a310-42c45972a8de:1-194,56ae3c8c-3011-11ea-bbd4-468fd4139175:1-15058935,5b1b7148-716e-11e9-9faf-2ae79d37031e:1-210834,691f19fc-69ab-11ea-9d10-de53fd0769e4:1-37870125,8a9b0af8-696e-11ea-9911-6274707228d6:1-69665,9991e006-6740-11e9-a909-0a580a301c02:1-245981,a559290c-4d03-11e9-b7d2-b2c80c7d3d23:1-7930960,a8f6c7fe-3011-11ea-ae1a-3eaefc605e25:1-37913825,c17cb726-51c8-11e9-a06a-d6f2f792ae0b:1-1469191,cc444cb1-698b-11ea-860e-96131e5ee507:1-51954,cce5005f-6740-11e9-8f99-0a580a301706:1-13,ea9c99ce-69ee-11ea-84f9-62f4dcbfaa3b:1-220294949,ef44ef4b-596f-11e9-9e9a-0a580a30080a:1-5340566 | |
| I1103 19:42:42.299191 1 binlog_connection.go:278] closing binlog socket to unblock reads | |
| I1103 19:42:42.299234 1 binlog_connection.go:152] connection closed during binlog stream (possibly intentional): read unix @->/vtdataroot/tabletdata/mysql.sock: use of closed network connection | |
| io.ReadFull(header size) failed (errno 2013) (sqlstate HY000) | |
| I1103 19:42:42.299254 1 binlog_connection.go:286] waiting for binlog dump thread to end | |
| I1103 19:42:42.299167 1 binlog_connection.go:152] connection closed during binlog stream (possibly intentional): read unix @->/vtdataroot/tabletdata/mysql.sock: use of closed network connection | |
| io.ReadFull(header size) failed (errno 2013) (sqlstate HY000) | |
| I1103 19:42:42.299259 1 binlog_connection.go:292] closing binlog MySQL client with serverID 41051606. Will recycle ID. | |
| I1103 19:42:42.299267 1 message_manager.go:627] Context canceled, exiting vstream | |
| I1103 19:42:42.299176 1 binlog_connection.go:286] waiting for binlog dump thread to end | |
| I1103 19:42:42.299276 1 binlog_connection.go:292] closing binlog MySQL client with serverID 41051613. Will recycle ID. | |
| I1103 19:42:42.299279 1 message_manager.go:627] Context canceled, exiting vstream | |
| I1103 19:42:44.529676 1 snapshot_conn.go:79] Locking table searches__extractor__msgs for copying | |
| I1103 19:42:44.530560 1 snapshot_conn.go:72] Tables unlocked: searches__extractor__msgs | |
| I1103 19:42:48.300087 1 vstreamer.go:942] stream (at source tablet) ended @ 04b526f5-e829-11ea-a033-a2de0b053154:1-146054622,2c9813b2-7176-11e9-a434-5aabf7900131:1-53219105,35a340f3-6974-11ea-a310-42c45972a8de:1-194,56ae3c8c-3011-11ea-bbd4-468fd4139175:1-15058935,5b1b7148-716e-11e9-9faf-2ae79d37031e:1-210834,691f19fc-69ab-11ea-9d10-de53fd0769e4:1-37870125,8a9b0af8-696e-11ea-9911-6274707228d6:1-69665,9991e006-6740-11e9-a909-0a580a301c02:1-245981,a559290c-4d03-11e9-b7d2-b2c80c7d3d23:1-7930960,a8f6c7fe-3011-11ea-ae1a-3eaefc605e25:1-37913825,c17cb726-51c8-11e9-a06a-d6f2f792ae0b:1-1469191,cc444cb1-698b-11ea-860e-96131e5ee507:1-51954,cce5005f-6740-11e9-8f99-0a580a301706:1-13,ea9c99ce-69ee-11ea-84f9-62f4dcbfaa3b:1-220294959,ef44ef4b-596f-11e9-9e9a-0a580a30080a:1-5340566 | |
| I1103 19:42:48.300138 1 binlog_connection.go:278] closing binlog socket to unblock reads | |
| I1103 19:42:48.300209 1 binlog_connection.go:152] connection closed during binlog stream (possibly intentional): read unix @->/vtdataroot/tabletdata/mysql.sock: use of closed network connection | |
| io.ReadFull(header size) failed (errno 2013) (sqlstate HY000) | |
| I1103 19:42:48.300225 1 binlog_connection.go:286] waiting for binlog dump thread to end | |
| I1103 19:42:48.300235 1 binlog_connection.go:292] closing binlog MySQL client with serverID 41051619. Will recycle ID. | |
| I1103 19:42:48.300249 1 message_manager.go:627] Context canceled, exiting vstream | |
| E1103 19:42:58.820155 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820181 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820160 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820168 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820188 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820210 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820222 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820222 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820228 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820231 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820235 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820272 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820272 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820286 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820290 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820292 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820301 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820313 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820274 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820305 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820303 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820330 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820316 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820342 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820351 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820360 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| E1103 19:42:58.820368 1 message_manager.go:804] Unable to delete messages: Code: CLUSTER_EVENT | |
| operation not allowed in state SHUTTING_DOWN | |
| I1103 19:42:59.669743 1 snapshot_conn.go:79] Locking table searches__extractor__msgs_pri for copying | |
| I1103 19:42:59.670809 1 snapshot_conn.go:72] Tables unlocked: searches__extractor__msgs_pri | |
| I1103 19:43:00.602514 1 snapshot_conn.go:79] Locking table searches__extractor__msgs_low for copying | |
| I1103 19:43:00.602547 1 snapshot_conn.go:79] Locking table searches__retriever_dataforseo__msgs_pri for copying | |
| I1103 19:43:00.602519 1 snapshot_conn.go:79] Locking table searches__retriever_vertifire__msgs_pri for copying | |
| I1103 19:43:00.603471 1 snapshot_conn.go:72] Tables unlocked: searches__retriever_dataforseo__msgs_pri | |
| I1103 19:43:00.603521 1 snapshot_conn.go:72] Tables unlocked: searches__extractor__msgs_low |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| goroutine profile: total 2045 | |
| 1539 @ 0x438576 0x44940c 0x4493e6 0x464ec5 0x472f65 0x11ed02f 0x11ed01d 0x1220330 0x12204e2 0x122028b 0x11f2de5 0x469081 | |
| # 0x464ec4 sync.runtime_SemacquireMutex+0x24 runtime/sema.go:71 | |
| # 0x472f64 sync.(*Mutex).lockSlow+0x164 sync/mutex.go:138 | |
| # 0x11ed02e sync.(*Mutex).Lock+0xee sync/mutex.go:81 | |
| # 0x11ed01c vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery+0xdc vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 | |
| # 0x122032f vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1+0x2f vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 | |
| # 0x12204e1 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML+0x181 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 | |
| # 0x122028a vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages+0x6a vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 | |
| # 0x11f2de4 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge+0x1c4 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 | |
| 68 @ 0x438576 0x448312 0x10382b1 0x469081 | |
| # 0x10382b0 vitess.io/vitess/go/timer.(*Timer).run+0xb0 vitess.io/vitess/go/timer/timer.go:98 | |
| 55 @ 0x438576 0x44940c 0x4493e6 0x464ec5 0x472f65 0x11ec571 0x11ec55f 0x12074bf 0x121fcd3 0x12221c3 0x121fb45 0x12f6d2b 0x12ec470 0xea27c9 0xe7d23a 0xe7d0df 0x9f7c84 0x9f9125 0x9f2f18 0x469081 | |
| # 0x464ec4 sync.runtime_SemacquireMutex+0x24 runtime/sema.go:71 | |
| # 0x472f64 sync.(*Mutex).lockSlow+0x164 sync/mutex.go:138 | |
| # 0x11ec570 sync.(*Mutex).Lock+0x110 sync/mutex.go:81 | |
| # 0x11ec55e vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe+0xfe vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 | |
| # 0x12074be vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream+0x21e vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 | |
| # 0x121fcd2 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1+0x112 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 | |
| # 0x12221c2 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest+0x502 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 | |
| # 0x121fb44 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream+0xc4 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 | |
| # 0x12f6d2a vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream+0x1aa vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 | |
| # 0x12ec46f vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler+0xcf vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 | |
| # 0xea27c8 github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1+0x108 github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 | |
| # 0xe7d239 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1+0x39 github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 | |
| # 0xe7d0de github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1+0xbe github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 | |
| # 0x9f7c83 google.golang.org/grpc.(*Server).processStreamingRPC+0xe23 google.golang.org/grpc@v1.39.0/server.go:1548 | |
| # 0x9f9124 google.golang.org/grpc.(*Server).handleStream+0x9e4 google.golang.org/grpc@v1.39.0/server.go:1621 | |
| # 0x9f2f17 google.golang.org/grpc.(*Server).serveStreams.func1.2+0x97 google.golang.org/grpc@v1.39.0/server.go:940 | |
| 39 @ 0x438576 0x430bde 0x4636c9 0x4ed712 0x4eea7a 0x4eea68 0x5b3ac9 0x5c4e05 0x571194 0x4a8c7a 0x98310e 0x9830c8 0x983855 0x9b9fb2 0x9f2b42 0x9f2466 0x469081 | |
| # 0x4636c8 internal/poll.runtime_pollWait+0x88 runtime/netpoll.go:229 | |
| # 0x4ed711 internal/poll.(*pollDesc).wait+0x31 internal/poll/fd_poll_runtime.go:84 | |
| # 0x4eea79 internal/poll.(*pollDesc).waitRead+0x259 internal/poll/fd_poll_runtime.go:89 | |
| # 0x4eea67 internal/poll.(*FD).Read+0x247 internal/poll/fd_unix.go:167 | |
| # 0x5b3ac8 net.(*netFD).Read+0x28 net/fd_posix.go:56 | |
| # 0x5c4e04 net.(*conn).Read+0x44 net/net.go:183 | |
| # 0x571193 bufio.(*Reader).Read+0x1b3 bufio/bufio.go:227 | |
| # 0x4a8c79 io.ReadAtLeast+0x99 io/io.go:328 | |
| # 0x98310d io.ReadFull+0x6d io/io.go:347 | |
| # 0x9830c7 golang.org/x/net/http2.readFrameHeader+0x27 golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 | |
| # 0x983854 golang.org/x/net/http2.(*Framer).ReadFrame+0x94 golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 | |
| # 0x9b9fb1 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams+0xb1 google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 | |
| # 0x9f2b41 google.golang.org/grpc.(*Server).serveStreams+0x141 google.golang.org/grpc@v1.39.0/server.go:924 | |
| # 0x9f2465 google.golang.org/grpc.(*Server).handleRawConn.func1+0x45 google.golang.org/grpc@v1.39.0/server.go:874 | |
| 39 @ 0x438576 0x448312 0x11fd725 0x122416a 0x12f74f4 0x12ed173 0xea27c9 0xe7d23a 0xe7d0df 0x9f7c84 0x9f9125 0x9f2f18 0x469081 | |
| # 0x11fd724 vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream+0x1e4 vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 | |
| # 0x1224169 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth+0x29 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 | |
| # 0x12f74f3 vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth+0x173 vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 | |
| # 0x12ed172 vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler+0xd2 vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 | |
| # 0xea27c8 github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1+0x108 github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 | |
| # 0xe7d239 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1+0x39 github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 | |
| # 0xe7d0de github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1+0xbe github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 | |
| # 0x9f7c83 google.golang.org/grpc.(*Server).processStreamingRPC+0xe23 google.golang.org/grpc@v1.39.0/server.go:1548 | |
| # 0x9f9124 google.golang.org/grpc.(*Server).handleStream+0x9e4 google.golang.org/grpc@v1.39.0/server.go:1621 | |
| # 0x9f2f17 google.golang.org/grpc.(*Server).serveStreams.func1.2+0x97 google.golang.org/grpc@v1.39.0/server.go:940 | |
| 39 @ 0x438576 0x448312 0x9a227b 0x9a2965 0x9b7c26 0x469081 | |
| # 0x9a227a google.golang.org/grpc/internal/transport.(*controlBuffer).get+0x11a google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 | |
| # 0x9a2964 google.golang.org/grpc/internal/transport.(*loopyWriter).run+0x84 google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 | |
| # 0x9b7c25 google.golang.org/grpc/internal/transport.NewServerTransport.func2+0xc5 google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 | |
| 39 @ 0x438576 0x448312 0x9bdd79 0x469081 | |
| # 0x9bdd78 google.golang.org/grpc/internal/transport.(*http2Server).keepalive+0x258 google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 | |
| 27 @ 0x438576 0x46509d 0x46507d 0x47130c 0x11f05e8 0x469081 | |
| # 0x46507c sync.runtime_notifyListWait+0x11c runtime/sema.go:513 | |
| # 0x47130b sync.(*Cond).Wait+0x8b sync/cond.go:56 | |
| # 0x11f05e7 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend+0x427 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:509 | |
| 25 @ 0x438576 0x44940c 0x4493e6 0x464ec5 0x472f65 0x11ecc91 0x11ecc7f 0x12201f8 0x12204e2 0x122014d 0x11f118e 0x11f0c10 0x469081 | |
| # 0x464ec4 sync.runtime_SemacquireMutex+0x24 runtime/sema.go:71 | |
| # 0x472f64 sync.(*Mutex).lockSlow+0x164 sync/mutex.go:138 | |
| # 0x11ecc90 sync.(*Mutex).Lock+0x110 sync/mutex.go:81 | |
| # 0x11ecc7e vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery+0xfe vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:153 | |
| # 0x12201f7 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1+0x37 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1059 | |
| # 0x12204e1 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML+0x181 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 | |
| # 0x122014c vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages+0x8c vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1058 | |
| # 0x11f118d vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone+0x14d vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:596 | |
| # 0x11f0c0f vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send+0x26f vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 | |
| 20 @ 0x438576 0x40640c 0x405e38 0x11efad1 0x469081 | |
| # 0x11efad0 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1+0x30 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:399 | |
| 20 @ 0x438576 0x40640c 0x405e38 0x1207505 0x121fcd3 0x12221c3 0x121fb45 0x12f6d2b 0x12ec470 0xea27c9 0xe7d23a 0xe7d0df 0x9f7c84 0x9f9125 0x9f2f18 0x469081 | |
| # 0x1207504 vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream+0x264 vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 | |
| # 0x121fcd2 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1+0x112 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 | |
| # 0x12221c2 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest+0x502 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 | |
| # 0x121fb44 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream+0xc4 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 | |
| # 0x12f6d2a vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream+0x1aa vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 | |
| # 0x12ec46f vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler+0xcf vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 | |
| # 0xea27c8 github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1+0x108 github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 | |
| # 0xe7d239 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1+0x39 github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 | |
| # 0xe7d0de github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1+0xbe github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 | |
| # 0x9f7c83 google.golang.org/grpc.(*Server).processStreamingRPC+0xe23 google.golang.org/grpc@v1.39.0/server.go:1548 | |
| # 0x9f9124 google.golang.org/grpc.(*Server).handleStream+0x9e4 google.golang.org/grpc@v1.39.0/server.go:1621 | |
| # 0x9f2f17 google.golang.org/grpc.(*Server).serveStreams.func1.2+0x97 google.golang.org/grpc@v1.39.0/server.go:940 | |
| 14 @ 0x438576 0x40640c 0x405e38 0xbdf63b 0x11f10c5 0x11f0c10 0x469081 | |
| # 0xbdf63a vitess.io/vitess/go/sync2.(*Semaphore).Acquire+0x11a vitess.io/vitess/go/sync2/semaphore.go:52 | |
| # 0x11f10c4 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone+0x84 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:589 | |
| # 0x11f0c0f vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send+0x26f vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 | |
| 13 @ 0x438576 0x40640c 0x405e38 0xbe8776 0x469081 | |
| # 0xbe8775 vitess.io/vitess/go/stats.(*Rates).track+0x35 vitess.io/vitess/go/stats/rates.go:99 | |
| 10 @ 0x438576 0x430bde 0x4636c9 0x4ed712 0x4eea7a 0x4eea68 0x5b3ac9 0x5c4e05 0x571194 0x4a8c7a 0xf2274f 0xf2272e 0xf2318d 0xf232e5 0xf2345d 0xf313fd 0x1122605 0x11225ea 0x469081 | |
| # 0x4636c8 internal/poll.runtime_pollWait+0x88 runtime/netpoll.go:229 | |
| # 0x4ed711 internal/poll.(*pollDesc).wait+0x31 internal/poll/fd_poll_runtime.go:84 | |
| # 0x4eea79 internal/poll.(*pollDesc).waitRead+0x259 internal/poll/fd_poll_runtime.go:89 | |
| # 0x4eea67 internal/poll.(*FD).Read+0x247 internal/poll/fd_unix.go:167 | |
| # 0x5b3ac8 net.(*netFD).Read+0x28 net/fd_posix.go:56 | |
| # 0x5c4e04 net.(*conn).Read+0x44 net/net.go:183 | |
| # 0x571193 bufio.(*Reader).Read+0x1b3 bufio/bufio.go:227 | |
| # 0x4a8c79 io.ReadAtLeast+0x99 io/io.go:328 | |
| # 0xf2274e io.ReadFull+0x4e io/io.go:347 | |
| # 0xf2272d vitess.io/vitess/go/mysql.(*Conn).readHeaderFrom+0x2d vitess.io/vitess/go/mysql/conn.go:334 | |
| # 0xf2318c vitess.io/vitess/go/mysql.(*Conn).readOnePacket+0x6c vitess.io/vitess/go/mysql/conn.go:473 | |
| # 0xf232e4 vitess.io/vitess/go/mysql.(*Conn).readPacket+0x24 vitess.io/vitess/go/mysql/conn.go:495 | |
| # 0xf2345c vitess.io/vitess/go/mysql.(*Conn).ReadPacket+0x1c vitess.io/vitess/go/mysql/conn.go:531 | |
| # 0xf313fc vitess.io/vitess/go/mysql.mysqlFlavor.readBinlogEvent+0x1c vitess.io/vitess/go/mysql/flavor_mysql.go:225 | |
| # 0x1122604 vitess.io/vitess/go/mysql.(*Conn).ReadBinlogEvent+0xc4 vitess.io/vitess/go/mysql/flavor.go:237 | |
| # 0x11225e9 vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents.func1+0xa9 vitess.io/vitess/go/vt/binlog/binlog_connection.go:146 | |
| 10 @ 0x438576 0x448312 0x11411c5 0x1140d45 0x1140ab9 0x113f6a5 0x11324bd 0x11f1a13 0x11f1790 0x469081 | |
| # 0x11411c4 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents+0x304 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:309 | |
| # 0x1140d44 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).replicate+0x124 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:178 | |
| # 0x1140ab8 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).Stream+0x158 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:157 | |
| # 0x113f6a4 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream+0x2e4 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:379 | |
| # 0x11324bc vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream+0x1bc vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:227 | |
| # 0x11f1a12 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream+0x132 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:648 | |
| # 0x11f178f vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream+0x6f vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:624 | |
| 10 @ 0x438576 0x448312 0x1141950 0x469081 | |
| # 0x114194f vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents.func2+0xef vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:283 | |
| 5 @ 0x438576 0x448312 0x9fd6f8 0x469081 | |
| # 0x9fd6f7 google.golang.org/grpc.newClientStreamWithParams.func4+0x97 google.golang.org/grpc@v1.39.0/stream.go:363 | |
| 4 @ 0x438576 0x40640c 0x405e38 0x9e77af 0x469081 | |
| # 0x9e77ae google.golang.org/grpc.(*addrConn).resetTransport+0x48e google.golang.org/grpc@v1.39.0/clientconn.go:1219 | |
| 4 @ 0x438576 0x430bde 0x4636c9 0x4ed712 0x4eea7a 0x4eea68 0x5b3ac9 0x5c4e05 0x571194 0x4a8c7a 0x98310e 0x9830c8 0x983855 0x9b4d94 0x469081 | |
| # 0x4636c8 internal/poll.runtime_pollWait+0x88 runtime/netpoll.go:229 | |
| # 0x4ed711 internal/poll.(*pollDesc).wait+0x31 internal/poll/fd_poll_runtime.go:84 | |
| # 0x4eea79 internal/poll.(*pollDesc).waitRead+0x259 internal/poll/fd_poll_runtime.go:89 | |
| # 0x4eea67 internal/poll.(*FD).Read+0x247 internal/poll/fd_unix.go:167 | |
| # 0x5b3ac8 net.(*netFD).Read+0x28 net/fd_posix.go:56 | |
| # 0x5c4e04 net.(*conn).Read+0x44 net/net.go:183 | |
| # 0x571193 bufio.(*Reader).Read+0x1b3 bufio/bufio.go:227 | |
| # 0x4a8c79 io.ReadAtLeast+0x99 io/io.go:328 | |
| # 0x98310d io.ReadFull+0x6d io/io.go:347 | |
| # 0x9830c7 golang.org/x/net/http2.readFrameHeader+0x27 golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 | |
| # 0x983854 golang.org/x/net/http2.(*Framer).ReadFrame+0x94 golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 | |
| # 0x9b4d93 google.golang.org/grpc/internal/transport.(*http2Client).reader+0x413 google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:1451 | |
| 4 @ 0x438576 0x448312 0x9a227b 0x9a2965 0x9ab7e5 0x469081 | |
| # 0x9a227a google.golang.org/grpc/internal/transport.(*controlBuffer).get+0x11a google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 | |
| # 0x9a2964 google.golang.org/grpc/internal/transport.(*loopyWriter).run+0x84 google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 | |
| # 0x9ab7e4 google.golang.org/grpc/internal/transport.newHTTP2Client.func3+0x64 google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:396 | |
| 4 @ 0x438576 0x448312 0x9deac5 0x469081 | |
| # 0x9deac4 google.golang.org/grpc.(*ccBalancerWrapper).watcher+0xa4 google.golang.org/grpc@v1.39.0/balancer_conn_wrappers.go:71 | |
| 3 @ 0x438576 0x40640c 0x405e78 0x1037f6b 0x469081 | |
| # 0x1037f6a vitess.io/vitess/go/timer.(*SuspendableTicker).loop+0x4a vitess.io/vitess/go/timer/suspendable_ticker.go:74 | |
| 2 @ 0x438576 0x40640c 0x405e38 0x1154489 0x469081 | |
| # 0x1154488 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.newRelayLog.func1+0x48 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:58 | |
| 2 @ 0x438576 0x430bde 0x4636c9 0x4ed712 0x4eea7a 0x4eea68 0x5b3ac9 0x5c4e05 0x6a878d 0x570be3 0x5717af 0x571a07 0x63c079 0x6a3c99 0x6a3c9a 0x6a9b45 0x6ade85 0x469081 | |
| # 0x4636c8 internal/poll.runtime_pollWait+0x88 runtime/netpoll.go:229 | |
| # 0x4ed711 internal/poll.(*pollDesc).wait+0x31 internal/poll/fd_poll_runtime.go:84 | |
| # 0x4eea79 internal/poll.(*pollDesc).waitRead+0x259 internal/poll/fd_poll_runtime.go:89 | |
| # 0x4eea67 internal/poll.(*FD).Read+0x247 internal/poll/fd_unix.go:167 | |
| # 0x5b3ac8 net.(*netFD).Read+0x28 net/fd_posix.go:56 | |
| # 0x5c4e04 net.(*conn).Read+0x44 net/net.go:183 | |
| # 0x6a878c net/http.(*connReader).Read+0x16c net/http/server.go:780 | |
| # 0x570be2 bufio.(*Reader).fill+0x102 bufio/bufio.go:101 | |
| # 0x5717ae bufio.(*Reader).ReadSlice+0x2e bufio/bufio.go:360 | |
| # 0x571a06 bufio.(*Reader).ReadLine+0x26 bufio/bufio.go:389 | |
| # 0x63c078 net/textproto.(*Reader).readLineSlice+0x98 net/textproto/reader.go:57 | |
| # 0x6a3c98 net/textproto.(*Reader).ReadLine+0x78 net/textproto/reader.go:38 | |
| # 0x6a3c99 net/http.readRequest+0x79 net/http/request.go:1029 | |
| # 0x6a9b44 net/http.(*conn).readRequest+0x224 net/http/server.go:966 | |
| # 0x6ade84 net/http.(*conn).serve+0x864 net/http/server.go:1855 | |
| 2 @ 0x438576 0x448312 0x1154ea8 0x469081 | |
| # 0x1154ea7 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*relayLog).startTimer.func1+0x87 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:124 | |
| 2 @ 0x438576 0x448312 0x116a505 0x116a0db 0x116de85 0x116d708 0x11493da 0x11484a6 0x469081 | |
| # 0x116a504 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply+0x3e4 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:158 | |
| # 0x116a0da vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).play+0x33a vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:125 | |
| # 0x116de84 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vreplicator).replicate+0x6e4 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go:226 | |
| # 0x116d707 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vreplicator).Replicate+0x27 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go:151 | |
| # 0x11493d9 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*controller).runBlp+0xab9 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/controller.go:271 | |
| # 0x11484a5 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*controller).run+0xa5 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/controller.go:152 | |
| 2 @ 0x438576 0x448312 0x4943a5 0x469081 | |
| # 0x4943a4 context.propagateCancel.func1+0xa4 context/context.go:279 | |
| 2 @ 0x438576 0x448312 0x9b5609 0x469081 | |
| # 0x9b5608 google.golang.org/grpc/internal/transport.(*http2Client).keepalive+0x168 google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:1525 | |
| 2 @ 0x438576 0x448312 0x9c2b8a 0x9c28c7 0x9c3732 0x4a8c7a 0x9c36a5 0x9c367b 0x9ee2e7 0x9eeb46 0x9ef1ee 0xa00850 0x9ffc05 0x9fea96 0x9ff93f 0xdc2bac 0xe6746a 0x469081 | |
| # 0x9c2b89 google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient+0xa9 google.golang.org/grpc@v1.39.0/internal/transport/transport.go:189 | |
| # 0x9c28c6 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read+0x146 google.golang.org/grpc@v1.39.0/internal/transport/transport.go:169 | |
| # 0x9c3731 google.golang.org/grpc/internal/transport.(*transportReader).Read+0x31 google.golang.org/grpc@v1.39.0/internal/transport/transport.go:483 | |
| # 0x4a8c79 io.ReadAtLeast+0x99 io/io.go:328 | |
| # 0x9c36a4 io.ReadFull+0xa4 io/io.go:347 | |
| # 0x9c367a google.golang.org/grpc/internal/transport.(*Stream).Read+0x7a google.golang.org/grpc@v1.39.0/internal/transport/transport.go:467 | |
| # 0x9ee2e6 google.golang.org/grpc.(*parser).recvMsg+0x46 google.golang.org/grpc@v1.39.0/rpc_util.go:559 | |
| # 0x9eeb45 google.golang.org/grpc.recvAndDecompress+0x65 google.golang.org/grpc@v1.39.0/rpc_util.go:690 | |
| # 0x9ef1ed google.golang.org/grpc.recv+0x6d google.golang.org/grpc@v1.39.0/rpc_util.go:758 | |
| # 0xa0084f google.golang.org/grpc.(*csAttempt).recvMsg+0x2af google.golang.org/grpc@v1.39.0/stream.go:957 | |
| # 0x9ffc04 google.golang.org/grpc.(*clientStream).RecvMsg.func1+0x24 google.golang.org/grpc@v1.39.0/stream.go:808 | |
| # 0x9fea95 google.golang.org/grpc.(*clientStream).withRetry+0x2f5 google.golang.org/grpc@v1.39.0/stream.go:662 | |
| # 0x9ff93e google.golang.org/grpc.(*clientStream).RecvMsg+0x11e google.golang.org/grpc@v1.39.0/stream.go:807 | |
| # 0xdc2bab go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchClient).Recv+0x4b go.etcd.io/etcd/api/v3@v3.5.0/etcdserverpb/rpc.pb.go:6714 | |
| # 0xe67469 go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveWatchClient+0x69 go.etcd.io/etcd/client/v3@v3.5.0/watch.go:757 | |
| 2 @ 0x438576 0x448312 0x9c2b8a 0x9c28c7 0x9c3732 0x4a8c7a 0x9c36a5 0x9c367b 0x9ee2e7 0x9eeb46 0x9ef1ee 0xa00850 0x9ffc05 0x9fea96 0x9ff93f 0xea0db7 0x12e888c 0x12ff199 0x1153b29 0x116aab1 0x469081 | |
| # 0x9c2b89 google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient+0xa9 google.golang.org/grpc@v1.39.0/internal/transport/transport.go:189 | |
| # 0x9c28c6 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read+0x146 google.golang.org/grpc@v1.39.0/internal/transport/transport.go:169 | |
| # 0x9c3731 google.golang.org/grpc/internal/transport.(*transportReader).Read+0x31 google.golang.org/grpc@v1.39.0/internal/transport/transport.go:483 | |
| # 0x4a8c79 io.ReadAtLeast+0x99 io/io.go:328 | |
| # 0x9c36a4 io.ReadFull+0xa4 io/io.go:347 | |
| # 0x9c367a google.golang.org/grpc/internal/transport.(*Stream).Read+0x7a google.golang.org/grpc@v1.39.0/internal/transport/transport.go:467 | |
| # 0x9ee2e6 google.golang.org/grpc.(*parser).recvMsg+0x46 google.golang.org/grpc@v1.39.0/rpc_util.go:559 | |
| # 0x9eeb45 google.golang.org/grpc.recvAndDecompress+0x65 google.golang.org/grpc@v1.39.0/rpc_util.go:690 | |
| # 0x9ef1ed google.golang.org/grpc.recv+0x6d google.golang.org/grpc@v1.39.0/rpc_util.go:758 | |
| # 0xa0084f google.golang.org/grpc.(*csAttempt).recvMsg+0x2af google.golang.org/grpc@v1.39.0/stream.go:957 | |
| # 0x9ffc04 google.golang.org/grpc.(*clientStream).RecvMsg.func1+0x24 google.golang.org/grpc@v1.39.0/stream.go:808 | |
| # 0x9fea95 google.golang.org/grpc.(*clientStream).withRetry+0x2f5 google.golang.org/grpc@v1.39.0/stream.go:662 | |
| # 0x9ff93e google.golang.org/grpc.(*clientStream).RecvMsg+0x11e google.golang.org/grpc@v1.39.0/stream.go:807 | |
| # 0xea0db6 github.com/grpc-ecosystem/go-grpc-prometheus.(*monitoredClientStream).RecvMsg+0x36 github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/client_metrics.go:160 | |
| # 0x12e888b vitess.io/vitess/go/vt/proto/queryservice.(*queryVStreamClient).Recv+0x4b vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:379 | |
| # 0x12ff198 vitess.io/vitess/go/vt/vttablet/grpctabletconn.(*gRPCQueryClient).VStream+0x98 vitess.io/vitess/go/vt/vttablet/grpctabletconn/conn.go:634 | |
| # 0x1153b28 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*tabletConnector).VStream+0x68 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/external_connector.go:174 | |
| # 0x116aab0 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func1+0xf0 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:148 | |
| 2 @ 0x438576 0x448312 0xe65034 0x469081 | |
| # 0xe65033 go.etcd.io/etcd/client/v3.(*watchGrpcStream).run+0x373 go.etcd.io/etcd/client/v3@v3.5.0/watch.go:537 | |
| 2 @ 0x438576 0x448312 0xe67845 0x469081 | |
| # 0xe67844 go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream+0x264 go.etcd.io/etcd/client/v3@v3.5.0/watch.go:803 | |
| 2 @ 0x438576 0x448312 0xe7966f 0x469081 | |
| # 0xe7966e vitess.io/vitess/go/vt/topo/etcd2topo.(*Server).Watch.func1+0x1ae vitess.io/vitess/go/vt/topo/etcd2topo/watch.go:78 | |
| 2 @ 0x438576 0x46509d 0x46507d 0x47130c 0x11549fe 0x116b845 0x116a992 0x469081 | |
| # 0x46507c sync.runtime_notifyListWait+0x11c runtime/sema.go:513 | |
| # 0x47130b sync.(*Cond).Wait+0x8b sync/cond.go:56 | |
| # 0x11549fd vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*relayLog).Fetch+0x13d vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:98 | |
| # 0x116b844 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).applyEvents+0x204 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:343 | |
| # 0x116a991 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func2+0x31 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:155 | |
| 1 @ 0x40b8f4 0x465578 0xccac79 0x469081 | |
| # 0x465577 os/signal.signal_recv+0x97 runtime/sigqueue.go:169 | |
| # 0xccac78 os/signal.loop+0x18 os/signal/signal_unix.go:24 | |
| 1 @ 0x438576 0x40640c 0x405e38 0xee5154 0x1d731a6 0x1d7319e 0x4381a7 0x469081 | |
| # 0xee5153 vitess.io/vitess/go/vt/servenv.Run+0x213 vitess.io/vitess/go/vt/servenv/run.go:57 | |
| # 0x1d731a5 vitess.io/vitess/go/vt/servenv.RunDefault+0x745 vitess.io/vitess/go/vt/servenv/servenv.go:226 | |
| # 0x1d7319d main.main+0x73d vitess.io/vitess/go/cmd/vttablet/vttablet.go:121 | |
| # 0x4381a6 runtime.main+0x226 runtime/proc.go:255 | |
| 1 @ 0x438576 0x40640c 0x405e78 0x111f845 0x469081 | |
| # 0x111f844 vitess.io/vitess/go/vt/vttablet/tabletserver/throttle.initThrottleTicker.func1.1+0xa4 vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/client.go:39 | |
| 1 @ 0x438576 0x40640c 0x405e78 0x121afbc 0x469081 | |
| # 0x121afbb vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).InitACL.func1+0x7b vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:325 | |
| 1 @ 0x438576 0x40640c 0x405e78 0x137354a 0x469081 | |
| # 0x1373549 k8s.io/klog.(*loggingT).flushDaemon+0x69 k8s.io/klog@v1.0.0/klog.go:1010 | |
| 1 @ 0x438576 0x40640c 0x405e78 0x7a67ca 0x469081 | |
| # 0x7a67c9 github.com/golang/glog.(*loggingT).flushDaemon+0x69 github.com/golang/glog@v0.0.0-20160126235308-23def4e6c14b/glog.go:882 | |
| 1 @ 0x438576 0x40640c 0x405e78 0xca15fe 0x469081 | |
| # 0xca15fd vitess.io/vitess/go/vt/topo.(*Server).WatchShard.func1+0xbd vitess.io/vitess/go/vt/topo/shard.go:683 | |
| 1 @ 0x438576 0x40640c 0x405e78 0xca7d7e 0x469081 | |
| # 0xca7d7d vitess.io/vitess/go/vt/topo.(*Server).WatchSrvVSchema.func1+0xbd vitess.io/vitess/go/vt/topo/srv_vschema.go:74 | |
| 1 @ 0x438576 0x40640c 0x405e78 0xf965b8 0x469081 | |
| # 0xf965b7 vitess.io/vitess/go/vt/dbconfigs.init.0.func1+0x37 vitess.io/vitess/go/vt/dbconfigs/credentials.go:283 | |
| 1 @ 0x438576 0x40640c 0x405e78 0xfabb0d 0x469081 | |
| # 0xfabb0c vitess.io/vitess/go/vt/srvtopo.NewSrvVSchemaWatcher.func1+0x10c vitess.io/vitess/go/vt/srvtopo/watch_srvvschema.go:49 | |
| 1 @ 0x438576 0x430bde 0x4636c9 0x4ed712 0x4eea7a 0x4eea68 0x5b3ac9 0x5c4e05 0x6a82df 0x469081 | |
| # 0x4636c8 internal/poll.runtime_pollWait+0x88 runtime/netpoll.go:229 | |
| # 0x4ed711 internal/poll.(*pollDesc).wait+0x31 internal/poll/fd_poll_runtime.go:84 | |
| # 0x4eea79 internal/poll.(*pollDesc).waitRead+0x259 internal/poll/fd_poll_runtime.go:89 | |
| # 0x4eea67 internal/poll.(*FD).Read+0x247 internal/poll/fd_unix.go:167 | |
| # 0x5b3ac8 net.(*netFD).Read+0x28 net/fd_posix.go:56 | |
| # 0x5c4e04 net.(*conn).Read+0x44 net/net.go:183 | |
| # 0x6a82de net/http.(*connReader).backgroundRead+0x3e net/http/server.go:672 | |
| 1 @ 0x438576 0x430bde 0x4636c9 0x4ed712 0x4f0a6c 0x4f0a59 0x5b5275 0x5cde88 0x5cd05d 0x6b2bf4 0x6b11f4 0x469081 | |
| # 0x4636c8 internal/poll.runtime_pollWait+0x88 runtime/netpoll.go:229 | |
| # 0x4ed711 internal/poll.(*pollDesc).wait+0x31 internal/poll/fd_poll_runtime.go:84 | |
| # 0x4f0a6b internal/poll.(*pollDesc).waitRead+0x22b internal/poll/fd_poll_runtime.go:89 | |
| # 0x4f0a58 internal/poll.(*FD).Accept+0x218 internal/poll/fd_unix.go:402 | |
| # 0x5b5274 net.(*netFD).accept+0x34 net/fd_unix.go:173 | |
| # 0x5cde87 net.(*TCPListener).accept+0x27 net/tcpsock_posix.go:140 | |
| # 0x5cd05c net.(*TCPListener).Accept+0x3c net/tcpsock.go:262 | |
| # 0x6b2bf3 net/http.(*Server).Serve+0x393 net/http/server.go:3001 | |
| # 0x6b11f3 net/http.Serve+0x73 net/http/server.go:2505 | |
| 1 @ 0x438576 0x430bde 0x4636c9 0x4ed712 0x4f0a6c 0x4f0a59 0x5b5275 0x5cde88 0x5cd05d 0x9f1702 0xee142c 0x469081 | |
| # 0x4636c8 internal/poll.runtime_pollWait+0x88 runtime/netpoll.go:229 | |
| # 0x4ed711 internal/poll.(*pollDesc).wait+0x31 internal/poll/fd_poll_runtime.go:84 | |
| # 0x4f0a6b internal/poll.(*pollDesc).waitRead+0x22b internal/poll/fd_poll_runtime.go:89 | |
| # 0x4f0a58 internal/poll.(*FD).Accept+0x218 internal/poll/fd_unix.go:402 | |
| # 0x5b5274 net.(*netFD).accept+0x34 net/fd_unix.go:173 | |
| # 0x5cde87 net.(*TCPListener).accept+0x27 net/tcpsock_posix.go:140 | |
| # 0x5cd05c net.(*TCPListener).Accept+0x3c net/tcpsock.go:262 | |
| # 0x9f1701 google.golang.org/grpc.(*Server).Serve+0x361 google.golang.org/grpc@v1.39.0/server.go:786 | |
| # 0xee142b vitess.io/vitess/go/vt/servenv.serveGRPC.func1+0x2b vitess.io/vitess/go/vt/servenv/grpc_server.go:242 | |
| 1 @ 0x438576 0x448312 0x10fa9c9 0x469081 | |
| # 0x10fa9c8 github.com/patrickmn/go-cache.(*janitor).Run+0x88 github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 | |
| 1 @ 0x438576 0x448312 0x11e75a8 0x469081 | |
| # 0x11e75a7 vitess.io/vitess/go/vt/vttablet/tabletserver/gc.(*TableGC).Operate+0x1e7 vitess.io/vitess/go/vt/vttablet/tabletserver/gc/tablegc.go:218 | |
| 1 @ 0x438576 0x448312 0x1270719 0x469081 | |
| # 0x1270718 go.opencensus.io/stats/view.(*worker).start+0xb8 go.opencensus.io@v0.23.0/stats/view/worker.go:276 | |
| 1 @ 0x438576 0x448312 0x133b912 0x469081 | |
| # 0x133b911 vitess.io/vitess/go/vt/vttablet/tabletmanager.(*TabletManager).shardSyncLoop+0x1b1 vitess.io/vitess/go/vt/vttablet/tabletmanager/shard_sync.go:67 | |
| 1 @ 0x438576 0x448312 0x9c2b8a 0x9c28c7 0x9c3732 0x4a8c7a 0x9c36a5 0x9c367b 0x9ee2e7 0x9eeb46 0x9ef1ee 0xa00850 0x9ffc05 0x9fea96 0x9ff93f 0xdc328c 0xe57818 0x469081 | |
| # 0x9c2b89 google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient+0xa9 google.golang.org/grpc@v1.39.0/internal/transport/transport.go:189 | |
| # 0x9c28c6 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read+0x146 google.golang.org/grpc@v1.39.0/internal/transport/transport.go:169 | |
| # 0x9c3731 google.golang.org/grpc/internal/transport.(*transportReader).Read+0x31 google.golang.org/grpc@v1.39.0/internal/transport/transport.go:483 | |
| # 0x4a8c79 io.ReadAtLeast+0x99 io/io.go:328 | |
| # 0x9c36a4 io.ReadFull+0xa4 io/io.go:347 | |
| # 0x9c367a google.golang.org/grpc/internal/transport.(*Stream).Read+0x7a google.golang.org/grpc@v1.39.0/internal/transport/transport.go:467 | |
| # 0x9ee2e6 google.golang.org/grpc.(*parser).recvMsg+0x46 google.golang.org/grpc@v1.39.0/rpc_util.go:559 | |
| # 0x9eeb45 google.golang.org/grpc.recvAndDecompress+0x65 google.golang.org/grpc@v1.39.0/rpc_util.go:690 | |
| # 0x9ef1ed google.golang.org/grpc.recv+0x6d google.golang.org/grpc@v1.39.0/rpc_util.go:758 | |
| # 0xa0084f google.golang.org/grpc.(*csAttempt).recvMsg+0x2af google.golang.org/grpc@v1.39.0/stream.go:957 | |
| # 0x9ffc04 google.golang.org/grpc.(*clientStream).RecvMsg.func1+0x24 google.golang.org/grpc@v1.39.0/stream.go:808 | |
| # 0x9fea95 google.golang.org/grpc.(*clientStream).withRetry+0x2f5 google.golang.org/grpc@v1.39.0/stream.go:662 | |
| # 0x9ff93e google.golang.org/grpc.(*clientStream).RecvMsg+0x11e google.golang.org/grpc@v1.39.0/stream.go:807 | |
| # 0xdc328b go.etcd.io/etcd/api/v3/etcdserverpb.(*leaseLeaseKeepAliveClient).Recv+0x4b go.etcd.io/etcd/api/v3@v3.5.0/etcdserverpb/rpc.pb.go:6853 | |
| # 0xe57817 go.etcd.io/etcd/client/v3.(*lessor).recvKeepAliveLoop+0x297 go.etcd.io/etcd/client/v3@v3.5.0/lease.go:447 | |
| 1 @ 0x438576 0x448312 0xbd7fff 0x469081 | |
| # 0xbd7ffe vitess.io/vitess/go/cache/ristretto.(*Cache).processItems+0x13e vitess.io/vitess/go/cache/ristretto/cache.go:442 | |
| 1 @ 0x438576 0x448312 0xbd91ac 0x469081 | |
| # 0xbd91ab vitess.io/vitess/go/cache/ristretto.(*defaultPolicy).processItems+0x8b vitess.io/vitess/go/cache/ristretto/policy.go:107 | |
| 1 @ 0x438576 0x448312 0xe58625 0x469081 | |
| # 0xe58624 go.etcd.io/etcd/client/v3.(*lessor).deadlineLoop+0x84 go.etcd.io/etcd/client/v3@v3.5.0/lease.go:539 | |
| 1 @ 0x438576 0x448312 0xe58999 0x469081 | |
| # 0xe58998 go.etcd.io/etcd/client/v3.(*lessor).sendKeepAliveLoop+0x1f8 go.etcd.io/etcd/client/v3@v3.5.0/lease.go:579 | |
| 1 @ 0x438576 0x44940c 0x4493e6 0x464da5 0x474951 0x9f2b4c 0x9f2466 0x469081 | |
| # 0x464da4 sync.runtime_Semacquire+0x24 runtime/sema.go:56 | |
| # 0x474950 sync.(*WaitGroup).Wait+0x70 sync/waitgroup.go:130 | |
| # 0x9f2b4b google.golang.org/grpc.(*Server).serveStreams+0x14b google.golang.org/grpc@v1.39.0/server.go:950 | |
| # 0x9f2465 google.golang.org/grpc.(*Server).handleRawConn.func1+0x45 google.golang.org/grpc@v1.39.0/server.go:874 | |
| 1 @ 0x438576 0x44940c 0x4493e6 0x464ec5 0x472f65 0x11ec279 0x11ec268 0x12127c2 0x12122c5 0x1210b18 0x12106a5 0x121b025 0x1335978 0x13356bc 0x134e2ac 0x13108d8 0xea2587 0xe7cf1a 0xe7cdbf 0x1310798 0x9f542f 0x9f916a 0x9f2f18 0x469081 | |
| # 0x464ec4 sync.runtime_SemacquireMutex+0x24 runtime/sema.go:71 | |
| # 0x472f64 sync.(*Mutex).lockSlow+0x164 sync/mutex.go:138 | |
| # 0x11ec278 sync.(*Mutex).Lock+0x78 sync/mutex.go:81 | |
| # 0x11ec267 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Close+0x67 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:98 | |
| # 0x12127c1 vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).unserveCommon+0xa1 vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:502 | |
| # 0x12122c4 vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).unservePrimary+0x24 vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:429 | |
| # 0x1210b17 vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).execTransition+0x97 vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:254 | |
| # 0x12106a4 vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).SetServingType+0x284 vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:216 | |
| # 0x121b024 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).SetServingType+0x44 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:348 | |
| # 0x1335977 vitess.io/vitess/go/vt/vttablet/tabletmanager.(*TabletManager).demotePrimary+0x217 vitess.io/vitess/go/vt/vttablet/tabletmanager/rpc_replication.go:415 | |
| # 0x13356bb vitess.io/vitess/go/vt/vttablet/tabletmanager.(*TabletManager).DemotePrimary+0x5b vitess.io/vitess/go/vt/vttablet/tabletmanager/rpc_replication.go:366 | |
| # 0x134e2ab vitess.io/vitess/go/vt/vttablet/grpctmserver.(*server).DemoteMaster+0x1cb vitess.io/vitess/go/vt/vttablet/grpctmserver/server.go:430 | |
| # 0x13108d7 vitess.io/vitess/go/vt/proto/tabletmanagerservice._TabletManager_DemoteMaster_Handler.func1+0x77 vitess.io/vitess/go/vt/proto/tabletmanagerservice/tabletmanagerservice_grpc.pb.go:1591 | |
| # 0xea2586 github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1+0x86 github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:107 | |
| # 0xe7cf19 github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1+0x39 github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:25 | |
| # 0xe7cdbe github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1+0xbe github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:34 | |
| # 0x1310797 vitess.io/vitess/go/vt/proto/tabletmanagerservice._TabletManager_DemoteMaster_Handler+0x137 vitess.io/vitess/go/vt/proto/tabletmanagerservice/tabletmanagerservice_grpc.pb.go:1593 | |
| # 0x9f542e google.golang.org/grpc.(*Server).processUnaryRPC+0xc6e google.golang.org/grpc@v1.39.0/server.go:1292 | |
| # 0x9f9169 google.golang.org/grpc.(*Server).handleStream+0xa29 google.golang.org/grpc@v1.39.0/server.go:1617 | |
| # 0x9f2f17 google.golang.org/grpc.(*Server).serveStreams.func1.2+0x97 google.golang.org/grpc@v1.39.0/server.go:940 | |
| 1 @ 0x438576 0x44940c 0x4493e6 0x464ec5 0x472f65 0x11ed3dd 0x11ed3b6 0x10a6915 0x10a4f37 0x10a431f 0x10a39cd 0x10a39ce 0x1038323 0x469081 | |
| # 0x464ec4 sync.runtime_SemacquireMutex+0x24 runtime/sema.go:71 | |
| # 0x472f64 sync.(*Mutex).lockSlow+0x164 sync/mutex.go:138 | |
| # 0x11ed3dc sync.(*Mutex).Lock+0x9c sync/mutex.go:81 | |
| # 0x11ed3b5 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).schemaChanged+0x75 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:182 | |
| # 0x10a6914 vitess.io/vitess/go/vt/vttablet/tabletserver/schema.(*Engine).broadcast+0x2b4 vitess.io/vitess/go/vt/vttablet/tabletserver/schema/engine.go:528 | |
| # 0x10a4f36 vitess.io/vitess/go/vt/vttablet/tabletserver/schema.(*Engine).reload+0xa36 vitess.io/vitess/go/vt/vttablet/tabletserver/schema/engine.go:396 | |
| # 0x10a431e vitess.io/vitess/go/vt/vttablet/tabletserver/schema.(*Engine).ReloadAt+0x1fe vitess.io/vitess/go/vt/vttablet/tabletserver/schema/engine.go:286 | |
| # 0x10a39cc vitess.io/vitess/go/vt/vttablet/tabletserver/schema.(*Engine).Reload+0x2c vitess.io/vitess/go/vt/vttablet/tabletserver/schema/engine.go:268 | |
| # 0x10a39cd vitess.io/vitess/go/vt/vttablet/tabletserver/schema.(*Engine).Open.func2+0x2d vitess.io/vitess/go/vt/vttablet/tabletserver/schema/engine.go:207 | |
| # 0x1038322 vitess.io/vitess/go/timer.(*Timer).run+0x122 vitess.io/vitess/go/timer/timer.go:112 | |
| 1 @ 0x438576 0x44940c 0x4493e6 0x464ec5 0x472f65 0x11ef6c5 0x11ef697 0x11ec5ef 0x12074bf 0x121fcd3 0x12221c3 0x121fb45 0x12f6d2b 0x12ec470 0xea27c9 0xe7d23a 0xe7d0df 0x9f7c84 0x9f9125 0x9f2f18 0x469081 | |
| # 0x464ec4 sync.runtime_SemacquireMutex+0x24 runtime/sema.go:71 | |
| # 0x472f64 sync.(*Mutex).lockSlow+0x164 sync/mutex.go:138 | |
| # 0x11ef6c4 sync.(*Mutex).Lock+0xa4 sync/mutex.go:81 | |
| # 0x11ef696 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe+0x76 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:372 | |
| # 0x11ec5ee vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe+0x18e vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:127 | |
| # 0x12074be vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream+0x21e vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 | |
| # 0x121fcd2 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1+0x112 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 | |
| # 0x12221c2 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest+0x502 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 | |
| # 0x121fb44 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream+0xc4 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 | |
| # 0x12f6d2a vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream+0x1aa vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 | |
| # 0x12ec46f vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler+0xcf vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 | |
| # 0xea27c8 github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1+0x108 github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 | |
| # 0xe7d239 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1+0x39 github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 | |
| # 0xe7d0de github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1+0xbe github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 | |
| # 0x9f7c83 google.golang.org/grpc.(*Server).processStreamingRPC+0xe23 google.golang.org/grpc@v1.39.0/server.go:1548 | |
| # 0x9f9124 google.golang.org/grpc.(*Server).handleStream+0x9e4 google.golang.org/grpc@v1.39.0/server.go:1621 | |
| # 0x9f2f17 google.golang.org/grpc.(*Server).serveStreams.func1.2+0x97 google.golang.org/grpc@v1.39.0/server.go:940 | |
| 1 @ 0x438576 0x44940c 0x4493e6 0x464ec5 0x472f65 0x11f15f8 0x11f15d2 0x11efd33 0x11efae5 0x469081 | |
| # 0x464ec4 sync.runtime_SemacquireMutex+0x24 runtime/sema.go:71 | |
| # 0x472f64 sync.(*Mutex).lockSlow+0x164 sync/mutex.go:138 | |
| # 0x11f15f7 sync.(*Mutex).Lock+0x57 sync/mutex.go:81 | |
| # 0x11f15d1 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).stopVStream+0x31 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:614 | |
| # 0x11efd32 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).unsubscribe+0x1d2 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:423 | |
| # 0x11efae4 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1+0x44 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:400 | |
| 1 @ 0x438576 0x44940c 0x4493e6 0x464ec5 0x472f65 0x11f1b08 0x11f1ad4 0x113d9fb 0x113ed35 0x113ee3f 0x113f329 0x113f433 0x11324bd 0x11f1a13 0x11f1790 0x469081 | |
| # 0x464ec4 sync.runtime_SemacquireMutex+0x24 runtime/sema.go:71 | |
| # 0x472f64 sync.(*Mutex).lockSlow+0x164 sync/mutex.go:138 | |
| # 0x11f1b07 sync.(*Mutex).Lock+0xa7 sync/mutex.go:81 | |
| # 0x11f1ad3 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream.func1+0x73 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:649 | |
| # 0x113d9fa vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.newUVStreamer.func1+0x5a vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:101 | |
| # 0x113ed34 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).sendEventsForCurrentPos+0x114 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:305 | |
| # 0x113ee3e vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).setStreamStartPosition+0x9e vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:318 | |
| # 0x113f328 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).init+0x28 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:346 | |
| # 0x113f432 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream+0x72 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:364 | |
| # 0x11324bc vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream+0x1bc vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:227 | |
| # 0x11f1a12 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream+0x132 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:648 | |
| # 0x11f178f vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream+0x6f vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:624 | |
| 1 @ 0x438576 0x44940c 0x4493e6 0x464ec5 0x472f65 0x11f25f1 0x11f25db 0x1038323 0x469081 | |
| # 0x464ec4 sync.runtime_SemacquireMutex+0x24 runtime/sema.go:71 | |
| # 0x472f64 sync.(*Mutex).lockSlow+0x164 sync/mutex.go:138 | |
| # 0x11f25f0 sync.(*Mutex).Lock+0x390 sync/mutex.go:81 | |
| # 0x11f25da vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPoller+0x37a vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:758 | |
| # 0x1038322 vitess.io/vitess/go/timer.(*Timer).run+0x122 vitess.io/vitess/go/timer/timer.go:112 | |
| 1 @ 0x4631e5 0xeb2bd5 0xeb29ed 0xeafb6b 0xed791a 0xed84ce 0x6af04f 0x6b0949 0x6b267b 0x6ae128 0x469081 | |
| # 0x4631e4 runtime/pprof.runtime_goroutineProfileWithLabels+0x24 runtime/mprof.go:746 | |
| # 0xeb2bd4 runtime/pprof.writeRuntimeProfile+0xb4 runtime/pprof/pprof.go:724 | |
| # 0xeb29ec runtime/pprof.writeGoroutine+0x4c runtime/pprof/pprof.go:684 | |
| # 0xeafb6a runtime/pprof.(*Profile).WriteTo+0x14a runtime/pprof/pprof.go:331 | |
| # 0xed7919 net/http/pprof.handler.ServeHTTP+0x499 net/http/pprof/pprof.go:253 | |
| # 0xed84cd net/http/pprof.Index+0x12d net/http/pprof/pprof.go:371 | |
| # 0x6af04e net/http.HandlerFunc.ServeHTTP+0x2e net/http/server.go:2046 | |
| # 0x6b0948 net/http.(*ServeMux).ServeHTTP+0x148 net/http/server.go:2424 | |
| # 0x6b267a net/http.serverHandler.ServeHTTP+0x43a net/http/server.go:2878 | |
| # 0x6ae127 net/http.(*conn).serve+0xb07 net/http/server.go:1929 |
This file has been truncated, but you can view the full file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| goroutine 1005452 [running]: | |
| runtime/pprof.writeGoroutineStacks({0x2677180, 0xc001c46700}) | |
| runtime/pprof/pprof.go:693 +0x70 | |
| runtime/pprof.writeGoroutine({0x2677180, 0xc001c46700}, 0x0) | |
| runtime/pprof/pprof.go:682 +0x2b | |
| runtime/pprof.(*Profile).WriteTo(0x20ed280, {0x2677180, 0xc001c46700}, 0xc) | |
| runtime/pprof/pprof.go:331 +0x14b | |
| net/http/pprof.handler.ServeHTTP({0xc002caf9f1, 0x3b07200}, {0x26b1478, 0xc001c46700}, 0xc002caf9e4) | |
| net/http/pprof/pprof.go:253 +0x49a | |
| net/http/pprof.Index({0x26b1478, 0xc001c46700}, 0xc001f8c400) | |
| net/http/pprof/pprof.go:371 +0x12e | |
| net/http.HandlerFunc.ServeHTTP(0x0, {0x26b1478, 0xc001c46700}, 0x0) | |
| net/http/server.go:2046 +0x2f | |
| net/http.(*ServeMux).ServeHTTP(0xc002caf9fb, {0x26b1478, 0xc001c46700}, 0xc001f8c400) | |
| net/http/server.go:2424 +0x149 | |
| net/http.serverHandler.ServeHTTP({0xc0053b9890}, {0x26b1478, 0xc001c46700}, 0xc001f8c400) | |
| net/http/server.go:2878 +0x43b | |
| net/http.(*conn).serve(0xc0034dbb80, {0x26bf4b0, 0xc000730cf0}) | |
| net/http/server.go:1929 +0xb08 | |
| created by net/http.(*Server).Serve | |
| net/http/server.go:3033 +0x4e8 | |
| goroutine 1 [chan receive, 53 minutes]: | |
| vitess.io/vitess/go/vt/servenv.Run(0x3a9a) | |
| vitess.io/vitess/go/vt/servenv/run.go:57 +0x214 | |
| vitess.io/vitess/go/vt/servenv.RunDefault(...) | |
| vitess.io/vitess/go/vt/servenv/servenv.go:226 | |
| main.main() | |
| vitess.io/vitess/go/cmd/vttablet/vttablet.go:121 +0x746 | |
| goroutine 18 [chan receive]: | |
| github.com/golang/glog.(*loggingT).flushDaemon(0x0) | |
| github.com/golang/glog@v0.0.0-20160126235308-23def4e6c14b/glog.go:882 +0x6a | |
| created by github.com/golang/glog.init.0 | |
| github.com/golang/glog@v0.0.0-20160126235308-23def4e6c14b/glog.go:410 +0x1c5 | |
| goroutine 24 [syscall, 53 minutes]: | |
| os/signal.signal_recv() | |
| runtime/sigqueue.go:169 +0x98 | |
| os/signal.loop() | |
| os/signal/signal_unix.go:24 +0x19 | |
| created by os/signal.Notify.func1.1 | |
| os/signal/signal.go:151 +0x2c | |
| goroutine 25 [chan receive, 53 minutes]: | |
| vitess.io/vitess/go/vt/dbconfigs.init.0.func1() | |
| vitess.io/vitess/go/vt/dbconfigs/credentials.go:283 +0x38 | |
| created by vitess.io/vitess/go/vt/dbconfigs.init.0 | |
| vitess.io/vitess/go/vt/dbconfigs/credentials.go:282 +0x15d | |
| goroutine 11 [select]: | |
| github.com/patrickmn/go-cache.(*janitor).Run(0xc00007f930, 0xc0004a1800) | |
| github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 +0x89 | |
| created by github.com/patrickmn/go-cache.runJanitor | |
| github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1099 +0xf2 | |
| goroutine 71 [chan receive]: | |
| k8s.io/klog.(*loggingT).flushDaemon(0xc00017e080) | |
| k8s.io/klog@v1.0.0/klog.go:1010 +0x6a | |
| created by k8s.io/klog.init.0 | |
| k8s.io/klog@v1.0.0/klog.go:411 +0xf4 | |
| goroutine 84 [select, 4 minutes]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000577360, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0003eb4a0) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:396 +0x65 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:394 +0x1ee5 | |
| goroutine 44 [select, 53 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000576eb0) | |
| google.golang.org/grpc@v1.39.0/balancer_conn_wrappers.go:71 +0xa5 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| google.golang.org/grpc@v1.39.0/balancer_conn_wrappers.go:62 +0x246 | |
| goroutine 83 [IO wait, 4 minutes]: | |
| internal/poll.runtime_pollWait(0x7f9483364f30, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc000533b80, 0xc001462000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc000533b80, {0xc001462000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc000533b80, {0xc001462000, 0x1060100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0001393e8, {0xc001462000, 0x988a50, 0x0}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0003eb320, {0xc00147a038, 0x9, 0x18}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0003eb320}, {0xc00147a038, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00147a038, 0x9, 0x3b07ae0}, {0x2673340, 0xc0003eb320}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00147a000) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00000a780) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:1451 +0x414 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:346 +0x188f | |
| goroutine 70 [select]: | |
| go.opencensus.io/stats/view.(*worker).start(0xc0004a1b80) | |
| go.opencensus.io@v0.23.0/stats/view/worker.go:276 +0xb9 | |
| created by go.opencensus.io/stats/view.init.0 | |
| go.opencensus.io@v0.23.0/stats/view/worker.go:34 +0x92 | |
| goroutine 119 [IO wait, 12 minutes]: | |
| internal/poll.runtime_pollWait(0x7f9483364d60, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00056c300, 0xc000308000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00056c300, {0xc000308000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00056c300, {0xc000308000, 0x1060100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc000564290, {0xc000308000, 0x988a50, 0x0}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0003ed1a0, {0xc00147a118, 0x9, 0x18}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0003ed1a0}, {0xc00147a118, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00147a118, 0x9, 0x3b07ae0}, {0x2673340, 0xc0003ed1a0}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00147a0e0) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0000fe000) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:1451 +0x414 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:346 +0x188f | |
| goroutine 45 [chan receive, 53 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc0013de580) | |
| google.golang.org/grpc@v1.39.0/clientconn.go:1219 +0x48f | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| google.golang.org/grpc@v1.39.0/clientconn.go:849 +0x147 | |
| goroutine 104 [select, 5 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/gc.(*TableGC).Operate(0xc0014f23c0, {0x26bf440, 0xc000128000}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/gc/tablegc.go:218 +0x1e8 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/gc.(*TableGC).InitDBConfig | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/gc/tablegc.go:157 +0x175 | |
| goroutine 120 [select, 12 minutes]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000147590, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0003ed260) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:396 +0x65 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:394 +0x1ee5 | |
| goroutine 249 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0023247d0, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002267200) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 113 [chan receive, 53 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc00133e580) | |
| google.golang.org/grpc@v1.39.0/clientconn.go:1219 +0x48f | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| google.golang.org/grpc@v1.39.0/clientconn.go:849 +0x147 | |
| goroutine 80 [select, 53 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000147220) | |
| google.golang.org/grpc@v1.39.0/balancer_conn_wrappers.go:71 +0xa5 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| google.golang.org/grpc@v1.39.0/balancer_conn_wrappers.go:62 +0x246 | |
| goroutine 501 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc0022a3350}, 0xc0022b25d0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00147d260, {0x26bf4b0, 0xc0022a3350}, 0xc0022a33e0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00147d260}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0022b25b8}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce0c0}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce0c0}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce0c0}, 0x1e4d1c0, 0xc00147d250) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0006fa300}, 0xc0022cc120, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0006fa300}, 0xc0022cc120, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 15 [chan receive]: | |
| vitess.io/vitess/go/stats.(*Rates).track(0xc00009bce0) | |
| vitess.io/vitess/go/stats/rates.go:99 +0x36 | |
| created by vitess.io/vitess/go/stats.NewRates | |
| vitess.io/vitess/go/stats/rates.go:91 +0x22f | |
| goroutine 16 [chan receive]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/throttle.initThrottleTicker.func1.1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/client.go:39 +0xa5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/throttle.initThrottleTicker.func1 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/client.go:36 +0x25 | |
| goroutine 97 [select, 5 minutes]: | |
| vitess.io/vitess/go/cache/ristretto.(*defaultPolicy).processItems(0xc0004ad8b0) | |
| vitess.io/vitess/go/cache/ristretto/policy.go:107 +0x8c | |
| created by vitess.io/vitess/go/cache/ristretto.newDefaultPolicy | |
| vitess.io/vitess/go/cache/ristretto/policy.go:91 +0x16f | |
| goroutine 415 [semacquire, 23 minutes]: | |
| sync.runtime_SemacquireMutex(0x203001, 0x1, 0x203001) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).schemaChanged(0xc0004ada40, 0xc0021f9608, {0x0, 0x0, 0x0}, {0x0, 0x0, 0xc001339270}, {0x0, 0x0, ...}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:182 +0x9d | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/schema.(*Engine).broadcast(0xc0000e83c0, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/schema/engine.go:528 +0x2b5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/schema.(*Engine).reload(0xc0000e83c0, {0x26bf4b0, 0xc00151fb90}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/schema/engine.go:396 +0xa37 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/schema.(*Engine).ReloadAt(0xc0000e83c0, {0x26bf4b0, 0xc00151fb90}, {{}, {0x0, 0x0}}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/schema/engine.go:286 +0x1ff | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/schema.(*Engine).Reload(...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/schema/engine.go:268 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/schema.(*Engine).Open.func2() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/schema/engine.go:207 +0x2d | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000497680, 0xc0022603a0) | |
| vitess.io/vitess/go/timer/timer.go:112 +0x123 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 99 [chan receive, 53 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).InitACL.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:325 +0x7c | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).InitACL | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:324 +0x108 | |
| goroutine 100 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc00152fa60, 0xc0000ebf50) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 101 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc00152faa0, 0xc001564170) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 105 [select, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletmanager.(*TabletManager).shardSyncLoop(0xc00024e000, {0x26bf408, 0xc00005f4c0}, 0xc0004e1440, 0x1c) | |
| vitess.io/vitess/go/vt/vttablet/tabletmanager/shard_sync.go:67 +0x1b2 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletmanager.(*TabletManager).startShardSync | |
| vitess.io/vitess/go/vt/vttablet/tabletmanager/shard_sync.go:261 +0x1e5 | |
| goroutine 63833 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0016c3800, 0xc002573bf0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x265 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc0047ec9f0}, 0xc00046ef00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc0047ec9f0}, 0x7f94aa18a108, {0x22c1ddb, 0xc00007c800}, {0x22b51cb, 0x0}, 0x0, 0xc003b025a0, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc0047ec9f0}, 0xc0026c4a90, {0xc0013626f0, 0x40ce34}, 0xc0047ec960) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc00132f860, {0x26ef130, 0xc0041ef310}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc002573bd8}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014f2f00}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014f2f00}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014f2f00}, 0x1e4d1c0, 0xc0041ef300) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0006fa600}, 0xc0013278c0, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0006fa600}, 0xc0013278c0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 481 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc0013f04b0}, 0xc000744b70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc0013e42d0, {0x26bf4b0, 0xc0013f04b0}, 0xc0013f0540) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc0013e42d0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc000744b58}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0013fc000}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0013fc000}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0013fc000}, 0x1e4d1c0, 0xc0013e42c0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc002350300}, 0xc0013e2240, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc002350300}, 0xc0013e2240, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 121 [chan receive, 53 minutes]: | |
| vitess.io/vitess/go/timer.(*SuspendableTicker).loop(0xc0001333f8) | |
| vitess.io/vitess/go/timer/suspendable_ticker.go:74 +0x4b | |
| created by vitess.io/vitess/go/timer.NewSuspendableTicker | |
| vitess.io/vitess/go/timer/suspendable_ticker.go:44 +0xdd | |
| goroutine 122 [chan receive]: | |
| vitess.io/vitess/go/timer.(*SuspendableTicker).loop(0xc000133428) | |
| vitess.io/vitess/go/timer/suspendable_ticker.go:74 +0x4b | |
| created by vitess.io/vitess/go/timer.NewSuspendableTicker | |
| vitess.io/vitess/go/timer/suspendable_ticker.go:44 +0xdd | |
| goroutine 123 [chan receive]: | |
| vitess.io/vitess/go/timer.(*SuspendableTicker).loop(0xc000133458) | |
| vitess.io/vitess/go/timer/suspendable_ticker.go:74 +0x4b | |
| created by vitess.io/vitess/go/timer.NewSuspendableTicker | |
| vitess.io/vitess/go/timer/suspendable_ticker.go:44 +0xdd | |
| goroutine 514 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc0006aa720}, 0xc0006ac288) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00069c1b0, {0x26bf4b0, 0xc0006aa720}, 0xc0006aa7b0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00069c1b0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0006ac270}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba0c0}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba0c0}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba0c0}, 0x1e4d1c0, 0xc00069c1a0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0024d4000}, 0xc0006b2120, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0024d4000}, 0xc0006b2120, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 541 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0006b14f0, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0006fe600) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 515 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc0006aab70}, 0xc0006ac3d8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00069c2b0, {0x26bf4b0, 0xc0006aab70}, 0xc0006aac00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00069c2b0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0006ac3c0}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba180}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba180}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba180}, 0x1e4d1c0, 0xc00069c2a0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc002350480}, 0xc0006b2240, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc002350480}, 0xc0006b2240, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 52 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc00062a330}, 0xc00063a120) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc0006200f0, {0x26bf4b0, 0xc00062a330}, 0xc00062a3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc0006200f0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc00063a108}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c000}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c000}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c000}, 0x1e4d1c0, 0xc0006200e0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0024d4480}, 0xc000644000, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0024d4480}, 0xc000644000, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 401 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483364b90, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0004a1100, 0xc00006c800, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Accept(0xc0004a1100) | |
| internal/poll/fd_unix.go:402 +0x22c | |
| net.(*netFD).accept(0xc0004a1100) | |
| net/fd_unix.go:173 +0x35 | |
| net.(*TCPListener).accept(0xc0021e0318) | |
| net/tcpsock_posix.go:140 +0x28 | |
| net.(*TCPListener).Accept(0xc0021e0318) | |
| net/tcpsock.go:262 +0x3d | |
| net/http.(*Server).Serve(0xc00120e000, {0x26b1268, 0xc0021e0318}) | |
| net/http/server.go:3001 +0x394 | |
| net/http.Serve({0x26b1268, 0xc0021e0318}, {0x0, 0x0}) | |
| net/http/server.go:2505 +0x74 | |
| created by vitess.io/vitess/go/vt/servenv.Run | |
| vitess.io/vitess/go/vt/servenv/run.go:52 +0x174 | |
| goroutine 513 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc0006aa2d0}, 0xc0006ac108) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00069c0b0, {0x26bf4b0, 0xc0006aa2d0}, 0xc0006aa360) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00069c0b0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0006ac0f0}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba000}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba000}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba000}, 0x1e4d1c0, 0xc00069c0a0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc001210300}, 0xc0006b2000, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc001210300}, 0xc0006b2000, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 184 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0003c7980) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 400 [IO wait, 4 minutes]: | |
| internal/poll.runtime_pollWait(0x7f9483364c78, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0004a1080, 0xc00006a000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Accept(0xc0004a1080) | |
| internal/poll/fd_unix.go:402 +0x22c | |
| net.(*netFD).accept(0xc0004a1080) | |
| net/fd_unix.go:173 +0x35 | |
| net.(*TCPListener).accept(0xc0021e02e8) | |
| net/tcpsock_posix.go:140 +0x28 | |
| net.(*TCPListener).Accept(0xc0021e02e8) | |
| net/tcpsock.go:262 +0x3d | |
| google.golang.org/grpc.(*Server).Serve(0xc0004c0fc0, {0x26b1268, 0xc0021e02e8}) | |
| google.golang.org/grpc@v1.39.0/server.go:786 +0x362 | |
| vitess.io/vitess/go/vt/servenv.serveGRPC.func1() | |
| vitess.io/vitess/go/vt/servenv/grpc_server.go:242 +0x2c | |
| created by vitess.io/vitess/go/vt/servenv.serveGRPC | |
| vitess.io/vitess/go/vt/servenv/grpc_server.go:241 +0x2b0 | |
| goroutine 182 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc00222be00}, 0xc0021e1650) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00225cd80, {0x26bf4b0, 0xc00222be00}, 0xc00222be90) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00225cd80}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0021e1638}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002233c80}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002233c80}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002233c80}, 0x1e4d1c0, 0xc00225cd70) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc002350180}, 0xc0013e2120, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc002350180}, 0xc0013e2120, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 242 [select, 2 minutes]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc00157c8a0, 0xc00047f020) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 183 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002221f40, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001385da0) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 91 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000144120, 0xc00147c9c0) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 99176 [semacquire, 26 minutes]: | |
| sync.runtime_SemacquireMutex(0xc003e4cae0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244f60, 0x25}, 0xc0016eec38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0049eb0e0}, 0x0, 0xc0016eee88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881bf2f754450, {0x26bf478, 0xc0049eb0e0}, 0xffffb16b6eb10000, {0xc002244f60, 0x2656fb0}, 0xc002846f18) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244f60, 0x25}, 0x4e94914f0000, 0xc002846f70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 408 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0015f5ae0, {0xc0021e0a90, 0x5, 0x5}) | |
| google.golang.org/grpc@v1.39.0/internal/transport/transport.go:189 +0xaa | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0015f5ae0, {0xc0021e0a90, 0xc0015f0678, 0xc00022da40}) | |
| google.golang.org/grpc@v1.39.0/internal/transport/transport.go:169 +0x147 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00220f740, {0xc0021e0a90, 0xc00022dab8, 0x9ac127}) | |
| google.golang.org/grpc@v1.39.0/internal/transport/transport.go:483 +0x32 | |
| io.ReadAtLeast({0x2676060, 0xc00220f740}, {0xc0021e0a90, 0x5, 0x5}, 0x5) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc002218480, {0xc0021e0a90, 0x5, 0x5}) | |
| google.golang.org/grpc@v1.39.0/internal/transport/transport.go:467 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc0021e0a80, 0x7fffffff) | |
| google.golang.org/grpc@v1.39.0/rpc_util.go:559 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0x3b3b8c8, 0xc002218480, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) | |
| google.golang.org/grpc@v1.39.0/rpc_util.go:690 +0x66 | |
| google.golang.org/grpc.recv(0xc000483a00, {0x7f94833bc520, 0x3b3b8c8}, 0x4058a9, {0x0, 0x0}, {0x21d4300, 0xc001389140}, 0xc00022dd70, 0x0, ...) | |
| google.golang.org/grpc@v1.39.0/rpc_util.go:758 +0x6e | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc0004a1d80, {0x21d4300, 0xc001389140}, 0xc00022dee0) | |
| google.golang.org/grpc@v1.39.0/stream.go:957 +0x2b0 | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x203000) | |
| google.golang.org/grpc@v1.39.0/stream.go:808 +0x25 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc002218240, 0xc00022de90, 0xc00022de60) | |
| google.golang.org/grpc@v1.39.0/stream.go:662 +0x2f6 | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc002218240, {0x21d4300, 0xc001389140}) | |
| google.golang.org/grpc@v1.39.0/stream.go:807 +0x11f | |
| go.etcd.io/etcd/api/v3/etcdserverpb.(*leaseLeaseKeepAliveClient).Recv(0xc0021f6850) | |
| go.etcd.io/etcd/api/v3@v3.5.0/etcdserverpb/rpc.pb.go:6853 +0x4c | |
| go.etcd.io/etcd/client/v3.(*lessor).recvKeepAliveLoop(0xc0000e8000) | |
| go.etcd.io/etcd/client/v3@v3.5.0/lease.go:447 +0x298 | |
| created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive.func1 | |
| go.etcd.io/etcd/client/v3@v3.5.0/lease.go:299 +0x65 | |
| goroutine 409 [select]: | |
| go.etcd.io/etcd/client/v3.(*lessor).deadlineLoop(0xc0000e8000) | |
| go.etcd.io/etcd/client/v3@v3.5.0/lease.go:539 +0x85 | |
| created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive.func1 | |
| go.etcd.io/etcd/client/v3@v3.5.0/lease.go:300 +0xa7 | |
| goroutine 87240 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x44d072, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022558c0, 0x23}, 0xc004ea0c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003eda840}, 0x0, 0xc004ea0e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881832f2ea097, {0x26bf478, 0xc003eda840}, 0xffffb16b6eb10000, {0xc0022558c0, 0x2656fb0}, 0xc000051830) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022558c0, 0x23}, 0x4e94914f0000, 0xc00166d770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 411 [select, 53 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc@v1.39.0/stream.go:363 +0x98 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc@v1.39.0/stream.go:362 +0x1066 | |
| goroutine 412 [select]: | |
| go.etcd.io/etcd/client/v3.(*lessor).sendKeepAliveLoop(0xc0000e8000, {0x26ff940, 0xc0021f6850}) | |
| go.etcd.io/etcd/client/v3@v3.5.0/lease.go:579 +0x1f9 | |
| created by go.etcd.io/etcd/client/v3.(*lessor).resetRecv | |
| go.etcd.io/etcd/client/v3@v3.5.0/lease.go:489 +0x2d1 | |
| goroutine 455 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002438280, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00116e960) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 243 [select, 2 minutes]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc00157c8e0, 0xc00047f060) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 416 [select, 2 minutes]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0022603e0, 0xc00225c940) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 417 [select, 2 minutes]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc002260420, 0xc00225c980) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 419 [select, 2 minutes]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc002260580, 0xc00225ca10) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 420 [select, 2 minutes]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0022605c0, 0xc00225ca50) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 66105 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x80, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc00399b830}, {0xc002245bc0, 0x26}, 0x7f948321ed50) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002f15680, 0xc0015de960) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc00399b830}, 0xc0042ef600) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc00399b830}, 0x7f94aa18c1d8, {0x22c1ddb, 0xc0006b9c00}, {0x22b51cb, 0x0}, 0x0, 0xc001918960, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc00399b830}, 0xc001b66a90, {0xc003b904e0, 0x40ce34}, 0xc00399b7a0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc0004adbd0, {0x26ef130, 0xc001c543b0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0015de948}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014f3200}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014f3200}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014f3200}, 0x1e4d1c0, 0xc001c543a0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc001210480}, 0xc002e626c0, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc001210480}, 0xc002e626c0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 99178 [semacquire, 26 minutes]: | |
| sync.runtime_SemacquireMutex(0xc002303e00, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223afc0, 0x35}, 0xc0007c8c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0049eb260}, 0x0, 0xc0007c8e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881bf2f758834, {0x26bf478, 0xc0049eb260}, 0xffffb16b6eb10000, {0xc00223afc0, 0x2656fb0}, 0xc0048a6fc0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223afc0, 0x35}, 0x4e94914f0000, 0xc003e76770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63379 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc001173358, 0x40) | |
| runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc000576780) | |
| sync/cond.go:56 +0x8c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc0011732c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:509 +0x428 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:334 +0xfb | |
| goroutine 63810 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00074b380, 0xc000133f08) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x265 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc0027e7b60}, 0xc00191fd00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc0027e7b60}, 0x7f94aa18af18, {0x22c1ddb, 0xc000720000}, {0x22b51cb, 0x0}, 0x0, 0xc0002eacc0, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc0027e7b60}, 0xc003e0ea90, {0xc00434ca60, 0x40ce34}, 0xc0027e7ad0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc00166e320, {0x26ef130, 0xc0000ad050}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc000133ef0}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014243c0}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014243c0}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014243c0}, 0x1e4d1c0, 0xc0000ad040) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0000c4600}, 0xc0016bc120, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0000c4600}, 0xc0016bc120, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 95 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc0022a2b40}, 0xc0022b2360) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00147d090, {0x26bf4b0, 0xc0022a2b40}, 0xc0022a2bd0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00147d090}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0022b2348}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce000}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce000}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce000}, 0x1e4d1c0, 0xc00147d080) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0000c4480}, 0xc0022cc000, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0000c4480}, 0xc0022cc000, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 250 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002350000) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 251 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483328660, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc002326280, 0xc002336000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc002326280, {0xc002336000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc002326280, {0xc002336000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00000ed28, {0xc002336000, 0xc0001036c0, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc00231b1a0, {0xc00234e038, 0x9, 0xc0013d5d30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc00231b1a0}, {0xc00234e038, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00234e038, 0x9, 0x2676000}, {0x2673340, 0xc00231b1a0}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00234e000) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002350000, 0x10246c8b48282444, 0x58d48c318c48348) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc002350000}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 429 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc00222bb00}, 0xc0021e15a8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00225ccd0, {0x26bf4b0, 0xc00222bb00}, 0xc00222bb90) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00225ccd0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0021e1590}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002233bc0}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002233bc0}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002233bc0}, 0x1e4d1c0, 0xc00225ccc0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc002350000}, 0xc002386480, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc002350000}, 0xc002386480, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 253 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0023249b0, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0022b8660) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 254 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002350180) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 255 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483328578, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc002326300, 0xc00235c000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc002326300, {0xc00235c000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc002326300, {0xc00235c000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00000ed30, {0xc00235c000, 0xc002352340, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc00231b440, {0xc00234e118, 0x9, 0xc0013d3d30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc00231b440}, {0xc00234e118, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00234e118, 0x9, 0x2676000}, {0x2673340, 0xc00231b440}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00234e0e0) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002350180, 0xf736e8240c894828, 0x840824448b48ffff) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc002350180}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 538 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00063e280, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00251aa80) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 185 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483328490, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc002223200, 0xc0023b2000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc002223200, {0xc0023b2000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc002223200, {0xc0023b2000, 0x7f9482cbf0a0, 0x6}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc000499840, {0xc0023b2000, 0x7f94aa18af18, 0x6}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc002267500, {0xc00024e658, 0x9, 0xc0013d4d30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc002267500}, {0xc00024e658, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00024e658, 0x9, 0x2676000}, {0x2673340, 0xc002267500}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00024e620) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0003c7980, 0x0, 0x0) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc0003c7980}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 190 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002324b90, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0013fa1e0) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 126 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc0005788d0}, 0xc000133578) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc0000ada00, {0x26bf4b0, 0xc0005788d0}, 0xc000578960) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc0000ada00}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc000133560}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014f2180}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014f2180}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014f2180}, 0x1e4d1c0, 0xc0000ad9f0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0003c7980}, 0xc000279680, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0003c7980}, 0xc000279680, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 431 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0023d2280, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0022b86c0) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 187 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000147950, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001385f20) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 432 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0003c7b00) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 188 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0000c4480) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 433 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f94833282c0, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00074ba80, 0xc0023d4000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00074ba80, {0xc0023d4000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00074ba80, {0xc0023d4000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00073c530, {0xc0023d4000, 0xc0022796c0, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0022679e0, {0xc00024e738, 0x9, 0xc0013ced30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0022679e0}, {0xc00024e738, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00024e738, 0x9, 0x2676000}, {0x2673340, 0xc0022679e0}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00024e700) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0003c7b00, 0xc001453f78, 0x405780) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc0003c7b00}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 189 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f94833283a8, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00017f480, 0xc002400000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00017f480, {0xc002400000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00017f480, {0xc002400000, 0x60100000000, 0x413b05}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0005642e0, {0xc002400000, 0x7f948308aa18, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0003edbc0, {0xc00147a1f8, 0x9, 0xc00142bd30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0003edbc0}, {0xc00147a1f8, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00147a1f8, 0x9, 0x2676000}, {0x2673340, 0xc0003edbc0}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00147a1c0) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0000c4480, 0x0, 0x0) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc0000c4480}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 451 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc000578f00}, 0xc000133728) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc0000adb60, {0x26bf4b0, 0xc000578f00}, 0xc000578f90) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc0000adb60}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc000133710}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014f2240}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014f2240}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014f2240}, 0x1e4d1c0, 0xc0000adb50) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0003c7b00}, 0xc0002797a0, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0003c7b00}, 0xc0002797a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 128 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000147d60, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0003edf80) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 449 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0000c4600) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 450 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f94833281d8, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00017f500, 0xc002418000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00017f500, {0xc002418000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00017f500, {0xc002418000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0005642e8, {0xc002418000, 0xc00116c4e0, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0003edf20, {0xc00147a2d8, 0x9, 0xc0023cdd30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0003edf20}, {0xc00147a2d8, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00147a2d8, 0x9, 0x2676000}, {0x2673340, 0xc0003edf20}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00147a2a0) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0000c4600, 0x0, 0x0) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc0000c4600}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 465 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc002322cc0}, 0xc001563a58) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00047f760, {0x26bf4b0, 0xc002322cc0}, 0xc002322d50) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00047f760}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc001563a40}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e000}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e000}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e000}, 0x1e4d1c0, 0xc00047f750) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0000c4600}, 0xc00231c480, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0000c4600}, 0xc00231c480, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 191 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002350300) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 192 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f94833280f0, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc002326380, 0xc002480000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc002326380, {0xc002480000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc002326380, {0xc002480000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00000ed38, {0xc002480000, 0xc00116c000, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc00231b740, {0xc00234e1f8, 0x9, 0xc00237bd30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc00231b740}, {0xc00234e1f8, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00234e1f8, 0x9, 0x2676000}, {0x2673340, 0xc00231b740}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00234e1c0) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002350300, 0x0, 0x0) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc002350300}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 452 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0022d00f0, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00116e5a0) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 453 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001210180) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 454 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483328008, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00061f180, 0xc0022d2000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00061f180, {0xc0022d2000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00061f180, {0xc0022d2000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc000139438, {0xc0022d2000, 0xc00116c9c0, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0022b8840, {0xc00120e1f8, 0x9, 0xc0013d1d30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0022b8840}, {0xc00120e1f8, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00120e1f8, 0x9, 0x2676000}, {0x2673340, 0xc0022b8840}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00120e1c0) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001210180, 0x0, 0x0) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc001210180}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 498 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0022d01e0, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0022b8900) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 499 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001210300) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 500 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483327f20, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00061f200, 0xc0022ea000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00061f200, {0xc0022ea000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00061f200, {0xc0022ea000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc000139440, {0xc0022ea000, 0xc000723520, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0022b88a0, {0xc00120e2d8, 0x9, 0xc00135ad30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0022b88a0}, {0xc00120e2d8, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00120e2d8, 0x9, 0x2676000}, {0x2673340, 0xc0022b88a0}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00120e2a0) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001210300, 0x9a4fe5, 0x9a298f) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc001210300}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 466 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc002323020}, 0xc001563ba8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00047f850, {0x26bf4b0, 0xc002323020}, 0xc0023230b0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00047f850}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc001563b90}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e0c0}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e0c0}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e0c0}, 0x1e4d1c0, 0xc00047f840) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc001210180}, 0xc00231c5a0, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc001210180}, 0xc00231c5a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 533 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0023256d0, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0006a1f20) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 483 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00074cf00, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0006a0300) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 484 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0024d4000) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 485 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483327e38, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00074bc00, 0xc0024ba000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00074bc00, {0xc0024ba000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00074bc00, {0xc0024ba000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00073c538, {0xc0024ba000, 0xc0024b81a0, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0013fa7e0, {0xc0024d2038, 0x9, 0xc000624d30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0013fa7e0}, {0xc0024d2038, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc0024d2038, 0x9, 0x2676000}, {0x2673340, 0xc0013fa7e0}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0024d2000) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0024d4000, 0x0, 0x0) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc0024d4000}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 471 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc002323590}, 0xc001563d58) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00047f9a0, {0x26bf4b0, 0xc002323590}, 0xc002323620) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00047f9a0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc001563d40}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e180}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e180}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e180}, 0x1e4d1c0, 0xc00047f990) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0006fa000}, 0xc00231c6c0, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0006fa000}, 0xc00231c6c0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 468 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0023250e0, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0006a0660) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 469 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002350480) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 470 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483327d50, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc002326400, 0xc002500000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc002326400, {0xc002500000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc002326400, {0xc002500000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00000ed40, {0xc002500000, 0xc002352d00, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc00231bce0, {0xc00234e2d8, 0x9, 0xc0023cad30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc00231bce0}, {0xc00234e2d8, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00234e2d8, 0x9, 0x2676000}, {0x2673340, 0xc00231bce0}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00234e2a0) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002350480, 0xcad6e82404894810, 0x44110fc0570f0002) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc002350480}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 524 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc0006ab4a0}, 0xc0006ac5d0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00069c4a0, {0x26bf4b0, 0xc0006ab4a0}, 0xc0006ab530) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00069c4a0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0006ac5b8}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba240}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba240}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba240}, 0x1e4d1c0, 0xc00069c490) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0024d4180}, 0xc0006b2360, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0024d4180}, 0xc0006b2360, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 517 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0006b06e0, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0006a0ae0) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 518 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0006fa000) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 519 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483327c68, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0006c0100, 0xc0006de000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0006c0100, {0xc0006de000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0006c0100, {0xc0006de000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0006da000, {0xc0006de000, 0xc0006dc1a0, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0006a0a80, {0xc0006f6038, 0x9, 0xc000625d30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0006a0a80}, {0xc0006f6038, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc0006f6038, 0x9, 0x2676000}, {0x2673340, 0xc0006a0a80}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0006f6000) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0006fa000, 0x0, 0x0) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc0006fa000}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 487 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00074d1d0, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0013fad20) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 488 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0024d4180) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 489 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483327b80, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00074bd00, 0xc0024d6000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00074bd00, {0xc0024d6000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00074bd00, {0xc0024d6000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00073c540, {0xc0024d6000, 0xc0024b8680, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0013facc0, {0xc0024d2118, 0x9, 0xc002379d30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0013facc0}, {0xc0024d2118, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc0024d2118, 0x9, 0x2676000}, {0x2673340, 0xc0013facc0}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0024d20e0) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0024d4180, 0x0, 0x0) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc0024d4180}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 521 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0006b07d0, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0006a0c60) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 522 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0006fa180) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 523 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483327a98, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0006c0200, 0xc002588000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0006c0200, {0xc002588000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0006c0200, {0xc002588000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0006da010, {0xc002588000, 0xc0006dc680, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0006a0c00, {0xc0006f6118, 0x9, 0xc000626d30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0006a0c00}, {0xc0006f6118, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc0006f6118, 0x9, 0x2676000}, {0x2673340, 0xc0006a0c00}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0006f60e0) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0006fa180, 0x9a4fe5, 0x9a298f) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc0006fa180}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 472 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00074d2c0, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0013faea0) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 473 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0024d4300) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 474 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f94833279b0, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00074be00, 0xc0024f6000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00074be00, {0xc0024f6000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00074be00, {0xc0024f6000, 0x7f9482c32500, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00073c548, {0xc0024f6000, 0x7f94aa18b878, 0x6}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0013fae40, {0xc0024d21f8, 0x9, 0xc00237ad30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0013fae40}, {0xc0024d21f8, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc0024d21f8, 0x9, 0x2676000}, {0x2673340, 0xc0013fae40}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0024d21c0) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0024d4300, 0xcad6e82404894810, 0x44110fc0570f0002) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc0024d4300}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 525 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc0006ab800}, 0xc0006ac708) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00069c580, {0x26bf4b0, 0xc0006ab800}, 0xc0006ab890) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00069c580}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0006ac6f0}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba300}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba300}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba300}, 0x1e4d1c0, 0xc00069c570) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0006fa180}, 0xc0006b2480, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0006fa180}, 0xc0006b2480, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 530 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0006b1040, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0006a1860) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 527 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0006b0e60, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0006a1500) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 528 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0006fa300) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 529 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f94833278c8, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0006c0300, 0xc0025c8000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0006c0300, {0xc0025c8000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0006c0300, {0xc0025c8000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0006da018, {0xc0025c8000, 0xc0006dcea0, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0006a14a0, {0xc0006f61f8, 0x9, 0xc000628d30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0006a14a0}, {0xc0006f61f8, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc0006f61f8, 0x9, 0x2676000}, {0x2673340, 0xc0006a14a0}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0006f61c0) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0006fa300, 0x9a4fe5, 0x9a298f) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc0006fa300}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 491 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc0013f0f60}, 0xc000744ea0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc0013e4510, {0x26bf4b0, 0xc0013f0f60}, 0xc0013f0ff0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc0013e4510}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc000744e88}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0013fc0c0}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0013fc0c0}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0013fc0c0}, 0x1e4d1c0, 0xc0013e4500) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0024d4300}, 0xc0013e2360, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0024d4300}, 0xc0013e2360, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 495 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc0013f1500}, 0xc000745050) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc0013e4680, {0x26bf4b0, 0xc0013f1500}, 0xc0013f1590) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc0013e4680}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc000745038}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0013fc180}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0013fc180}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0013fc180}, 0x1e4d1c0, 0xc0013e4670) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0006fa780}, 0xc0013e2480, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0006fa780}, 0xc0013e2480, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 546 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f94833293f8, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc002326600, 0xc002524000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc002326600, {0xc002524000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc002326600, {0xc002524000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00000ed60, {0xc002524000, 0xc0024b8000, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc00251a780, {0xc00234e3b8, 0x9, 0xc002374d30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc00251a780}, {0xc00234e3b8, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00234e3b8, 0x9, 0x2676000}, {0x2673340, 0xc00251a780}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00234e380) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002350600, 0x8d4800006cbae810, 0x48948003d8ba505) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc002350600}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 547 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0023258b0, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00062e000) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 531 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0006fa480) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 532 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f94833277e0, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc002326480, 0xc0025f0000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc002326480, {0xc0025f0000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc002326480, {0xc0025f0000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00000ed48, {0xc0025f0000, 0xc0006dcd00, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0006a1800, {0xc0006f62d8, 0x9, 0xc002520d30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0006a1800}, {0xc0006f62d8, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc0006f62d8, 0x9, 0x2676000}, {0x2673340, 0xc0006a1800}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0006f62a0) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0006fa480, 0x9a4fe5, 0x9a298f) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc0006fa480}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 492 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00074d5e0, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0013fb320) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 493 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0024d4480) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 494 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f94833295c8, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc002326500, 0xc002608000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc002326500, {0xc002608000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc002326500, {0xc002608000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00000ed50, {0xc002608000, 0xc0024b8d00, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0013fb2c0, {0xc0024d22d8, 0x9, 0xc0025efd30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0013fb2c0}, {0xc0024d22d8, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc0024d22d8, 0x9, 0x2676000}, {0x2673340, 0xc0013fb2c0}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0024d22a0) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0024d4480, 0x2, 0xa) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc0024d4480}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 479 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0023257c0, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00116e7e0) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 480 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002350780) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 534 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002350600) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 545 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f94833294e0, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc002326580, 0xc00253c000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc002326580, {0xc00253c000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc002326580, {0xc00253c000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00000ed58, {0xc00253c000, 0xc002353ba0, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc00251a7e0, {0xc00234e498, 0x9, 0xc002375d30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc00251a7e0}, {0xc00234e498, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00234e498, 0x9, 0x2676000}, {0x2673340, 0xc00251a7e0}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00234e460) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002350780, 0x8d4800006cbae810, 0x48948003d8ba505) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc002350780}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 502 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc0022a36e0}, 0xc0022b2720) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00147d340, {0x26bf4b0, 0xc0022a36e0}, 0xc0022a3770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00147d340}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0022b2708}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce180}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce180}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce180}, 0x1e4d1c0, 0xc00147d330) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0006fa480}, 0xc0022cc240, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0006fa480}, 0xc0022cc240, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 550 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc00256e0c0}, 0xc002572030) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00047fbe0, {0x26bf4b0, 0xc00256e0c0}, 0xc00256e150) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00047fbe0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc002572018}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e240}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e240}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e240}, 0x1e4d1c0, 0xc00047fbd0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc002350600}, 0xc00231c7e0, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc002350600}, 0xc00231c7e0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 440 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc0023ec6c0}, 0xc0021e1968) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00225cf40, {0x26bf4b0, 0xc0023ec6c0}, 0xc0023ec750) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00225cf40}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0021e1950}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002233d40}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002233d40}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002233d40}, 0x1e4d1c0, 0xc00225cf30) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc002350780}, 0xc0023865a0, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc002350780}, 0xc0023865a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 437 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0023d2550, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0022b8fc0) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 438 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0003c7c80) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 439 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483329140, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc002223380, 0xc0023f0000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc002223380, {0xc0023f0000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc002223380, {0xc0023f0000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc000499858, {0xc0023f0000, 0xc0023f8000, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0023ee120, {0xc00024e818, 0x9, 0xc0025ebd30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0023ee120}, {0xc00024e818, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00024e818, 0x9, 0x2676000}, {0x2673340, 0xc0023ee120}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00024e7e0) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0003c7c80, 0xc00156bf78, 0x405780) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc0003c7c80}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 548 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002350900) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 549 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483329228, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc002223300, 0xc002556000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc002223300, {0xc002556000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc002223300, {0xc002556000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc000499850, {0xc002556000, 0xc002353a00, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc00251a840, {0xc00234e578, 0x9, 0xc002376d30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc00251a840}, {0xc00234e578, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00234e578, 0x9, 0x2676000}, {0x2673340, 0xc00251a840}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00234e540) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002350900, 0x0, 0x0) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc002350900}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 503 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc0022a3c50}, 0xc0022b2900) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00147d4b0, {0x26bf4b0, 0xc0022a3c50}, 0xc0022a3ce0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00147d4b0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0022b28e8}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce240}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce240}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce240}, 0x1e4d1c0, 0xc00147d4a0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc002350900}, 0xc0022cc360, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc002350900}, 0xc0022cc360, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 504 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc002678000}, 0xc0022b2a50) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00147d590, {0x26bf4b0, 0xc002678000}, 0xc002678090) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00147d590}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0022b2a38}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce300}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce300}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce300}, 0x1e4d1c0, 0xc00147d580) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0003c7c80}, 0xc0022cc480, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0003c7c80}, 0xc0022cc480, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 535 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0022d0820, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0023ee660) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 536 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001210480) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 537 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483329310, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc002223280, 0xc002638000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc002223280, {0xc002638000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc002223280, {0xc002638000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc000499848, {0xc002638000, 0xc0006dd860, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0022b8f60, {0xc00120e3b8, 0x9, 0xc001428d30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0022b8f60}, {0xc00120e3b8, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00120e3b8, 0x9, 0x2676000}, {0x2673340, 0xc0022b8f60}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00120e380) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001210480, 0x9a4fe5, 0x9a298f) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc001210480}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 54 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc00062a870}, 0xc00063a2b8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc000620230, {0x26bf4b0, 0xc00062a870}, 0xc00062a900) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc000620230}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc00063a2a0}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c0c0}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c0c0}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c0c0}, 0x1e4d1c0, 0xc000620220) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc001210480}, 0xc000644120, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc001210480}, 0xc000644120, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 539 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00066e000) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 509 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc002678540}, 0xc0022b2c30) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00147d6d0, {0x26bf4b0, 0xc002678540}, 0xc0026785d0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00147d6d0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0022b2c18}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce3c0}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce3c0}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce3c0}, 0x1e4d1c0, 0xc00147d6c0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc001210600}, 0xc0022cc5a0, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc001210600}, 0xc0022cc5a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 540 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483329058, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00062c080, 0xc000652000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00062c080, {0xc000652000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00062c080, {0xc000652000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc000650000, {0xc000652000, 0xc002279520, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc00062e360, {0xc00066a038, 0x9, 0xc000629d30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc00062e360}, {0xc00066a038, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00066a038, 0x9, 0x2676000}, {0x2673340, 0xc00062e360}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00066a000) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00066e000, 0x9a4fe5, 0x9a298f) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc00066e000}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 506 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0022d0e60, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0023ee780) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 507 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001210600) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 508 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483328f70, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00061f380, 0xc002700000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00061f380, {0xc002700000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00061f380, {0xc002700000, 0x60100000000, 0x413b05}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc000139448, {0xc002700000, 0x7f9483095150, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0022b9800, {0xc00120e498, 0x9, 0xc0025e8d30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0022b9800}, {0xc00120e498, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00120e498, 0x9, 0x2676000}, {0x2673340, 0xc0022b9800}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00120e460) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001210600, 0x2, 0xa) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc001210600}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 441 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc0023ecbd0}, 0xc0021e1ad0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00225d070, {0x26bf4b0, 0xc0023ecbd0}, 0xc0023ecc60) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00225d070}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0021e1ab8}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002233e00}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002233e00}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002233e00}, 0x1e4d1c0, 0xc00225d060) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc00066e000}, 0xc0023866c0, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc00066e000}, 0xc0023866c0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 555 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc0023ec510}, 0xc002572180) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00047e090, {0x26bf4b0, 0xc0023ec510}, 0xc0023ec720) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00047e090}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc002572120}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e300}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e300}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e300}, 0x1e4d1c0, 0xc00047e080) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc00066e180}, 0xc002386000, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc00066e180}, 0xc002386000, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 552 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002325bd0, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0023eeae0) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 553 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002350a80) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 554 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483328e88, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc002326700, 0xc002720000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc002326700, {0xc002720000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc002326700, {0xc002720000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00000ed68, {0xc002720000, 0xc002554680, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc00251aba0, {0xc00234e658, 0x9, 0xc00266bd30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc00251aba0}, {0xc00234e658, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00234e658, 0x9, 0x2676000}, {0x2673340, 0xc00251aba0}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00234e620) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002350a80, 0x0, 0xfe7b0000) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc002350a80}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 458 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc000579710}, 0xc0001339f8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc0000add30, {0x26bf4b0, 0xc000579710}, 0xc0005797a0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc0000add30}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0001339e0}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014f2300}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014f2300}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014f2300}, 0x1e4d1c0, 0xc0000add20) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0000c4780}, 0xc0002798c0, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0000c4780}, 0xc0002798c0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 511 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc002678960}, 0xc0022b2db0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00147d7b0, {0x26bf4b0, 0xc002678960}, 0xc0026789f0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00147d7b0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0022b2d98}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce480}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce480}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce480}, 0x1e4d1c0, 0xc00147d7a0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc002350a80}, 0xc0022cc6c0, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc002350a80}, 0xc0022cc6c0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 456 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0000c4780) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 457 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483328da0, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00061f400, 0xc00244c000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00061f400, {0xc00244c000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00061f400, {0xc00244c000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc000139450, {0xc00244c000, 0xc000682340, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc00116e900, {0xc00147a3b8, 0x9, 0xc0023d0d30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc00116e900}, {0xc00147a3b8, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00147a3b8, 0x9, 0x2676000}, {0x2673340, 0xc00116e900}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00147a380) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0000c4780, 0x34c7d6aac, 0x3b07ae0) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc0000c4780}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 63386 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646500, 0xc0038f5a60) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 542 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0006fa600) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 543 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483328cb8, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00017f900, 0xc00269e000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00017f900, {0xc00269e000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00017f900, {0xc00269e000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0005642f0, {0xc00269e000, 0xc00025a9c0, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0006fe5a0, {0xc0006f63b8, 0x9, 0xc002669d30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0006fe5a0}, {0xc0006f63b8, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc0006f63b8, 0x9, 0x2676000}, {0x2673340, 0xc0006fe5a0}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0006f6380) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0006fa600, 0x9a4fe5, 0x9a298f) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc0006fa600}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 445 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc0023ed170}, 0xc0021e1c98) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00225d1d0, {0x26bf4b0, 0xc0023ed170}, 0xc0023ed200) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00225d1d0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0021e1c80}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002233ec0}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002233ec0}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002233ec0}, 0x1e4d1c0, 0xc00225d1c0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0006fa600}, 0xc0023867e0, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0006fa600}, 0xc0023867e0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 442 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002438690, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0023eecc0) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 443 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0000c4900) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 444 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483328bd0, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00017f980, 0xc00246e000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00017f980, {0xc00246e000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00017f980, {0xc00246e000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0005642f8, {0xc00246e000, 0xc0023f8680, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc00116ef60, {0xc00147a498, 0x9, 0xc002694d30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc00116ef60}, {0xc00147a498, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00147a498, 0x9, 0x2676000}, {0x2673340, 0xc00116ef60}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00147a460) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0000c4900, 0x9a4fe5, 0x9a298f) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc0000c4900}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 446 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc0023ed500}, 0xc0021e1de8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00225d2b0, {0x26bf4b0, 0xc0023ed500}, 0xc0023ed590) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00225d2b0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0021e1dd0}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002764000}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002764000}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002764000}, 0x1e4d1c0, 0xc00225d2a0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0000c4900}, 0xc002386900, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0000c4900}, 0xc002386900, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 561 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0006b16d0, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0006fe960) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 562 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0006fa780) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 563 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483328ae8, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0006c0600, 0xc0026c6000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0006c0600, {0xc0026c6000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0006c0600, {0xc0026c6000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0006da020, {0xc0026c6000, 0xc00269a4e0, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0006fe900, {0xc0006f6498, 0x9, 0xc0026c5d30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0006fe900}, {0xc0006f6498, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc0006f6498, 0x9, 0x2676000}, {0x2673340, 0xc0006fe900}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0006f6460) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0006fa780, 0x9a4fe5, 0x9a298f) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc0006fa780}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 597 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc000578b40}, 0xc0006ac528) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc0000acc20, {0x26bf4b0, 0xc000578b40}, 0xc000578cc0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc0000acc20}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0006ac480}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce600}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce600}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce600}, 0x1e4d1c0, 0xc0000acc10) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc001210900}, 0xc00231c900, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc001210900}, 0xc00231c900, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 55 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0023d30e0, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00062e600) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 56 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0003c7e00) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 57 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483328a00, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc002223600, 0xc002766000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc002223600, {0xc002766000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc002223600, {0xc002766000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc000499860, {0xc002766000, 0xc0006701a0, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0023ef500, {0xc00024e8f8, 0x9, 0xc0025eed30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0023ef500}, {0xc00024e8f8, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00024e8f8, 0x9, 0x2676000}, {0x2673340, 0xc0023ef500}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00024e8c0) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0003c7e00, 0xc000525750, 0xee1515) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc0003c7e00}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 461 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002438780, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00116f020) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 462 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0000c4a80) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 463 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483328918, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0006c0700, 0xc002476000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0006c0700, {0xc002476000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0006c0700, {0xc002476000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0006da028, {0xc002476000, 0xc00116d520, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc00116efc0, {0xc00147a578, 0x9, 0xc002695d30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc00116efc0}, {0xc00147a578, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00147a578, 0x9, 0x2676000}, {0x2673340, 0xc00116efc0}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00147a540) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0000c4a80, 0x34c7d6aac, 0x3b07ae0) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc0000c4a80}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 569 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc002698b40}, 0xc0006acc78) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00069c980, {0x26bf4b0, 0xc002698b40}, 0xc002698bd0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00069c980}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0006acc60}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba3c0}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba3c0}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba3c0}, 0x1e4d1c0, 0xc00069c970) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0000c4a80}, 0xc0006b25a0, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0000c4a80}, 0xc0006b25a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 566 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0006b18b0, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0006fed80) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 567 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0006fa900) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 568 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483328830, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0006c0780, 0xc0026ee000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0006c0780, {0xc0026ee000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0006c0780, {0xc0026ee000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0006da030, {0xc0026ee000, 0xc00269ab60, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0006fed20, {0xc0006f6578, 0x9, 0xc002692d30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0006fed20}, {0xc0006f6578, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc0006f6578, 0x9, 0x2676000}, {0x2673340, 0xc0006fed20}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0006f6540) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0006fa900, 0x1d5591f, 0xc00069c000) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc0006fa900}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 59 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00063e690, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00062ea20) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 60 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00066e180) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 61 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483328748, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00062c180, 0xc000672000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00062c180, {0xc000672000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00062c180, {0xc000672000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc000650010, {0xc000672000, 0xc000670680, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc00062e9c0, {0xc00066a118, 0x9, 0xc00135ed30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc00062e9c0}, {0xc00066a118, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00066a118, 0x9, 0x2676000}, {0x2673340, 0xc00062e9c0}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00066a0e0) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00066e180, 0xc000525750, 0xee1515) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc00066e180}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 496 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc0013f1860}, 0xc000745188) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc0013e4760, {0x26bf4b0, 0xc0013f1860}, 0xc0013f18f0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc0013e4760}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc000745170}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0013fc240}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0013fc240}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0013fc240}, 0x1e4d1c0, 0xc0013e4750) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0003c7e00}, 0xc0013e25a0, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0003c7e00}, 0xc0013e25a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 448 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc00256e480}, 0xc0021e0120) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00225c080, {0x26bf4b0, 0xc00256e480}, 0xc00256e540) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00225c080}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0021e0108}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0027640c0}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0027640c0}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0027640c0}, 0x1e4d1c0, 0xc00225c070) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0006fa900}, 0xc0007ac000, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0006fa900}, 0xc0007ac000, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 560 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc002678c00}, 0xc000133158) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00147c170, {0x26bf4b0, 0xc002678c00}, 0xc002678c90) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00147c170}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0001330b0}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014f2480}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014f2480}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014f2480}, 0x1e4d1c0, 0xc00147c160) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0000c4300}, 0xc002386240, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0000c4300}, 0xc002386240, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 512 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0023d2910, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00062e1e0) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 577 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0003c7380) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 578 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f948332a618, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc002222080, 0xc000200000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc002222080, {0xc000200000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc002222080, {0xc000200000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc000498090, {0xc000200000, 0xc0022fcd00, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0023ef620, {0xc00024e9d8, 0x9, 0xc0026bfd30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0023ef620}, {0xc00024e9d8, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00024e9d8, 0x9, 0x2676000}, {0x2673340, 0xc0023ef620}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00024e9a0) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0003c7380, 0x0, 0x0) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc0003c7380}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 593 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc000578330}, 0xc0006ac1e0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc0000aca70, {0x26bf4b0, 0xc000578330}, 0xc0005783c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc0000aca70}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0006ac180}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce540}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce540}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce540}, 0x1e4d1c0, 0xc0000aca60) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0003c7380}, 0xc00231c240, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0003c7380}, 0xc00231c240, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 557 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0022d04b0, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0023ef680) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 558 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002350c00) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 559 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f948332a530, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00061e000, 0xc000208000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00061e000, {0xc000208000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00061e000, {0xc000208000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00000e010, {0xc000208000, 0xc002554d00, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc00116f260, {0xc00234e738, 0x9, 0xc0027b5d30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc00116f260}, {0xc00234e738, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00234e738, 0x9, 0x2676000}, {0x2673340, 0xc00116f260}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00234e700) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002350c00, 0x8d5c8d5be1bc8d5a, 0x8d608d5f8d5e8d5d) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc002350c00}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 580 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc0026789c0}, 0xc000132708) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00147c100, {0x26bf4b0, 0xc0026789c0}, 0xc002678b40) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc00147c100}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0001326f0}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014f2000}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014f2000}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014f2000}, 0x1e4d1c0, 0xc00147c0f0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc002350c00}, 0xc0022cc7e0, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc002350c00}, 0xc0022cc7e0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 594 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00063e050, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0023ef980) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 595 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001210780) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 596 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f948332a448, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00017e380, 0xc000630000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00017e380, {0xc000630000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00017e380, {0xc000630000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc000138258, {0xc000630000, 0xc00116dba0, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0006ff380, {0xc00120e578, 0x9, 0xc00283dd30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0006ff380}, {0xc00120e578, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00120e578, 0x9, 0x2676000}, {0x2673340, 0xc0006ff380}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00120e540) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001210780, 0x240489480027a265, 0x20082444c748) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc001210780}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 611 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002438d70, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0006ff560) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 64 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00063e1e0, 0x1) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0006ff440) | |
| google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f | |
| goroutine 609 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001210900) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 610 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f948332a360, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00017eb80, 0xc001154000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00017eb80, {0xc001154000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00017eb80, {0xc001154000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0001382f0, {0xc001154000, 0xc00269b1e0, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0006ff3e0, {0xc00120e658, 0x9, 0xc00283ed30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0006ff3e0}, {0xc00120e658, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00120e658, 0x9, 0x2676000}, {0x2673340, 0xc0006ff3e0}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00120e620) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001210900, 0x9a4fe5, 0x9a298f) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc001210900}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 598 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0000d9180, {0x26bf4b0, 0xc0005794a0}, 0xc0006ac768) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc0000acdc0, {0x26bf4b0, 0xc0005794a0}, 0xc000579530) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1464 +0x2a | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0000ad750, 0x214fb60, {0x26ef270, 0xc0000acdc0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0006ac750}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce6c0}, 0x8, 0x23e15f0) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce6c0}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ce6c0}, 0x1e4d1c0, 0xc0000acd90) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc001210780}, 0xc00231ca20, 0xc000572690, 0x3ace2a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc001210780}, 0xc00231ca20, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 85497 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004b735c0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a540, 0x21}, 0xc0058bfc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00298f6e0}, 0x0, 0xc0058bfe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058817baf277810, {0x26bf478, 0xc00298f6e0}, 0xffffb16b6eb10000, {0xc00225a540, 0x2656fb0}, 0x12ec4eb) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a540, 0x21}, 0x4e94914f0000, 0xc002844f70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 612 [select, 53 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0000c4300) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577 | |
| goroutine 613 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f948332a278, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc002326880, 0xc00284a000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc002326880, {0xc00284a000, 0x8000, 0x8000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc002326880, {0xc00284a000, 0x60100000000, 0x8}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0005640f0, {0xc00284a000, 0xc00269b040, 0x4}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc00062f620, {0xc00147a658, 0x9, 0xc0027b0d30}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc00062f620}, {0xc00147a658, 0x9, 0x9}, 0x9) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00147a658, 0x9, 0x2676000}, {0x2673340, 0xc00062f620}) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00147a620) | |
| golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0000c4300, 0xc00156bf78, 0x405780) | |
| google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0004c0fc0, {0x27013d0, 0xc0000c4300}) | |
| google.golang.org/grpc@v1.39.0/server.go:924 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc@v1.39.0/server.go:874 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc@v1.39.0/server.go:873 +0x4af | |
| goroutine 63403 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc001173e98, 0x50) | |
| runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc000576780) | |
| sync/cond.go:56 +0x8c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc001173e00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:509 +0x428 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:334 +0xfb | |
| goroutine 904654 [semacquire, 11 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004a29320, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002248720, 0x1d}, 0xc0054ccc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004cebda0}, 0x0, 0xc0054cce88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05882a0305f2a57, {0x26bf478, 0xc004cebda0}, 0xffffb16b6eb10000, {0xc002248720, 0x2656fb0}, 0xc005b82540) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002248720, 0x1d}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63381 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646360, 0xc0038f58d0) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 104137 [semacquire, 23 minutes]: | |
| sync.runtime_SemacquireMutex(0xc002a8c600, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225ad80, 0x24}, 0xc004943c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0038cb320}, 0x0, 0xc004943e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881ec2fb523ba, {0x26bf478, 0xc0038cb320}, 0xffffb16b6eb10000, {0xc00225ad80, 0x2656fb0}, 0xc004756900) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225ad80, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63825 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0016c2900, 0xc002573290) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x265 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc002801620}, 0xc00046e900) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc002801620}, 0x7f94aa18a108, {0x22c1ddb, 0xc00007c800}, {0x22b51cb, 0x0}, 0x0, 0xc0037db620, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc002801620}, 0xc001b3aa90, {0xc00389e200, 0x40ce34}, 0xc002801590) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc00132e050, {0x26ef130, 0xc0041eeab0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc002573278}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014f2b40}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014f2b40}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014f2b40}, 0x1e4d1c0, 0xc0041eeaa0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc001210600}, 0xc004523c20, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc001210600}, 0xc004523c20, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 107266 [semacquire, 22 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a540, 0x21}, 0xc0018f7c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002d0a540}, 0x0, 0xc0018f7e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588202afdae5f7, {0x26bf478, 0xc002d0a540}, 0xffffb16b6eb10000, {0xc00225a540, 0x2656fb0}, 0xc003ff41e0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a540, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 107267 [semacquire, 22 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244f60, 0x25}, 0xc0018f8c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002d0a660}, 0x0, 0xc0018f8e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588202afdb0857, {0x26bf478, 0xc002d0a660}, 0xffffb16b6eb10000, {0xc002244f60, 0x2656fb0}, 0xc003ff4210) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244f60, 0x25}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63389 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646620, 0xc0038f5b30) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 97252 [semacquire, 27 minutes]: | |
| sync.runtime_SemacquireMutex(0x3036333236363030, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a120, 0x2c}, 0xc003388c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002cc65a0}, 0x0, 0xc003388e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881b02f691c18, {0x26bf478, 0xc002cc65a0}, 0xffffb16b6eb10000, {0xc00225a120, 0x2656fb0}, 0xc000051830) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a120, 0x2c}, 0x4e94914f0000, 0xc002210770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 78997 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x4f, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002248720, 0x1d}, 0xc003b4cc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00267f620}, 0x0, 0xc003b4ce88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058815daf020697, {0x26bf478, 0xc00267f620}, 0xffffb16b6eb10000, {0xc002248720, 0x2656fb0}, 0x12ec4eb) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002248720, 0x1d}, 0x4e94914f0000, 0xc001b11770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 711 [chan receive]: | |
| vitess.io/vitess/go/stats.(*Rates).track(0xc0022b8240) | |
| vitess.io/vitess/go/stats/rates.go:99 +0x36 | |
| created by vitess.io/vitess/go/stats.NewRates | |
| vitess.io/vitess/go/stats/rates.go:91 +0x22f | |
| goroutine 109509 [semacquire, 20 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x80, 0xc001385980) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254420, 0x28}, 0xc0063b9c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00471e2a0}, 0x0, 0xc0063b9e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05882193000e0f5, {0x26bf478, 0xc00471e2a0}, 0xffffb16b6eb10000, {0xc002254420, 0x2656fb0}, 0xc005989ce0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254420, 0x28}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 80984 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244300, 0x21}, 0xc0058a1c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003dd1ec0}, 0x0, 0xc0058a1e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881652efcb8d1, {0x26bf478, 0xc003dd1ec0}, 0xffffb16b6eb10000, {0xc002244300, 0x2656fb0}, 0xc004c29fb0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244300, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 710 [chan receive]: | |
| vitess.io/vitess/go/stats.(*Rates).track(0xc0022b81e0) | |
| vitess.io/vitess/go/stats/rates.go:99 +0x36 | |
| created by vitess.io/vitess/go/stats.NewRates | |
| vitess.io/vitess/go/stats/rates.go:91 +0x22f | |
| goroutine 713 [chan receive]: | |
| vitess.io/vitess/go/stats.(*Rates).track(0xc0022b8360) | |
| vitess.io/vitess/go/stats/rates.go:99 +0x36 | |
| created by vitess.io/vitess/go/stats.NewRates | |
| vitess.io/vitess/go/stats/rates.go:91 +0x22f | |
| goroutine 63388 [sync.Cond.Wait, 12 minutes]: | |
| sync.runtime_notifyListWait(0xc001173718, 0x34) | |
| runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc000576780) | |
| sync/cond.go:56 +0x8c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc001173680) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:509 +0x428 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:334 +0xfb | |
| goroutine 90288 [semacquire, 29 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022489e0, 0x1d}, 0xc0049edc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0042a2d80}, 0x0, 0xc0049ede88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881922f3f5ed3, {0x26bf478, 0xc0042a2d80}, 0xffffb16b6eb10000, {0xc0022489e0, 0x2656fb0}, 0xc004a8eba0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022489e0, 0x1d}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 712 [chan receive]: | |
| vitess.io/vitess/go/stats.(*Rates).track(0xc0022b8300) | |
| vitess.io/vitess/go/stats/rates.go:99 +0x36 | |
| created by vitess.io/vitess/go/stats.NewRates | |
| vitess.io/vitess/go/stats/rates.go:91 +0x22f | |
| goroutine 65968 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x2656fb0, 0x30, 0x1211fac) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc0004ada40, {0xc002244f60, 0x25}, {0xc0038f5400, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:153 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1059 +0x38 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00251b1a0}, 0x0, 0xc004ccbde0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0x26bf4b0, {0x26bf478, 0xc00251b1a0}, 0x2656fb0, {0xc002244f60, 0x2656fb0}, {0xc0038f5400, 0x0, 0x7f94aa18af18}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1058 +0x8d | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc00016e780, {0x26ef630, 0xc0002fd860}, {0xc002244f60, 0x25}, 0x22b658d, {0xc0038f5400, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:596 +0x14e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc00016e780, 0xc004008950, 0xc00251b0e0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 97241 [semacquire, 27 minutes]: | |
| sync.runtime_SemacquireMutex(0x2d313a3133313030, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254420, 0x28}, 0xc0055b9c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002dc73e0}, 0x0, 0xc0055b9e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881b02f6928a2, {0x26bf478, 0xc002dc73e0}, 0xffffb16b6eb10000, {0xc002254420, 0x2656fb0}, 0xc002678480) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254420, 0x28}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 66103 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x2656fb0, 0x30, 0x1211fac) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc0004ada40, {0xc002244720, 0x25}, {0xc001c541a0, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:153 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1059 +0x38 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0019188a0}, 0x0, 0xc001b65de0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0x26bf4b0, {0x26bf478, 0xc0019188a0}, 0x2656fb0, {0xc002244720, 0x2656fb0}, {0xc001c541a0, 0x0, 0x7f94aa18c1d8}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1058 +0x8d | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc001173680, {0x26ef630, 0xc0002fd860}, {0xc002244720, 0x25}, 0xc0005d8618, {0xc001c541a0, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:596 +0x14e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc001173680, 0xc003fa2e10, 0xc0019187e0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 63854 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents(0xc00234eb60, {0x26bf408, 0xc00005e680}, 0xc00247fa40) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:309 +0x305 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).replicate(0xc00234eb60, {0x2c1, 0xc001bd7070}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:178 +0x125 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).Stream(0xc00234eb60) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:157 +0x159 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream(0xc0016bc7e0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:379 +0x2e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream(0xc0000e06c0, {0x26bf408, 0xc0007ab8c0}, {0x22b658d, 0x216e900}, {0x0, 0xc001d1ff68, 0x12f6f44}, 0x0, 0xc0027e7c80) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:227 +0x1bd | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream(0xc0011732c0, {0x26bf408, 0xc0007ab8c0}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:648 +0x133 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream(0xc0011732c0, {0x26bf408, 0xc0007ab8c0}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:624 +0x70 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).startVStream | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:610 +0x168 | |
| goroutine 80973 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225ad80, 0x24}, 0xc00589ec38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0028c7c20}, 0x0, 0xc00589ee88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881652ef96dd0, {0x26bf478, 0xc0028c7c20}, 0xffffb16b6eb10000, {0xc00225ad80, 0x2656fb0}, 0xc004ab7710) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225ad80, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 65944 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x2656fb0, 0x30, 0x1211fac) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc0004ada40, {0xc002248460, 0x19}, {0xc00431cfa0, 0xa, 0xa}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:153 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1059 +0x38 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003911260}, 0x0, 0xc001962de0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0x26bf4b0, {0x26bf478, 0xc003911260}, 0x2656fb0, {0xc002248460, 0x2656fb0}, {0xc00431cfa0, 0x0, 0x7f94aa18b3c8}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1058 +0x8d | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc00016e500, {0x26ef630, 0xc0002fd860}, {0xc002248460, 0x19}, 0x22b658d, {0xc00431cfa0, 0xa, 0xa}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:596 +0x14e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc00016e500, 0xc003a690a0, 0xc003910de0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 981 [select, 51 minutes]: | |
| context.propagateCancel.func1() | |
| context/context.go:279 +0xa5 | |
| created by context.propagateCancel | |
| context/context.go:278 +0x1d0 | |
| goroutine 63384 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646440, 0xc0038f59a0) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 64635 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:399 +0x31 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:398 +0x3b3 | |
| goroutine 63599 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents.func2() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:283 +0xf0 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:270 +0x1f1 | |
| goroutine 92177 [semacquire, 29 minutes]: | |
| sync.runtime_SemacquireMutex(0x44d072, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022481a0, 0x1f}, 0xc003c23c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0029bca80}, 0x0, 0xc003c23e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588199af4d6b4c, {0x26bf478, 0xc0029bca80}, 0xffffb16b6eb10000, {0xc0022481a0, 0x2656fb0}, 0xc002323ec0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022481a0, 0x1f}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 93003 [semacquire, 28 minutes]: | |
| sync.runtime_SemacquireMutex(0x2d656539362d6563, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244b40, 0x21}, 0xc004939c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004375c20}, 0x0, 0xc004939e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881a12f5e2fdd, {0x26bf478, 0xc004375c20}, 0xffffb16b6eb10000, {0xc002244b40, 0x2656fb0}, 0xc00062b470) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244b40, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 79046 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004be2600, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a120, 0x2c}, 0xc004172c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0014523c0}, 0x0, 0xc004172e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058815daef07189, {0x26bf478, 0xc0014523c0}, 0xffffb16b6eb10000, {0xc00225a120, 0x2656fb0}, 0x0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a120, 0x2c}, 0x4e94914f0000, 0xc003e71f70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63707 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents(0xc0024d2380, {0x26bf408, 0xc000742280}, 0xc00247fb60) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:309 +0x305 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).replicate(0xc0024d2380, {0x2c1, 0xc001450250}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:178 +0x125 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).Stream(0xc0024d2380) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:157 +0x159 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream(0xc002386ea0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:379 +0x2e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream(0xc0000e06c0, {0x26bf408, 0xc0006a2900}, {0x22b658d, 0x216e900}, {0x0, 0xc00327f768, 0x12f6f44}, 0x0, 0xc0027e6f00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:227 +0x1bd | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream(0xc00016edc0, {0x26bf408, 0xc0006a2900}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:648 +0x133 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream(0xc00016edc0, {0x26bf408, 0xc0006a2900}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:624 +0x70 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).startVStream | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:610 +0x168 | |
| goroutine 78046 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x0, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc004707c80}, {0xc002248720, 0x1d}, 0x7f948325d568) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc003ddf500, 0xc003dfd9c8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc004707c80}, 0xc003d9eb00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc004707c80}, 0x7f94aa18bd28, {0x22c1ddb, 0x3b092c0}, {0x22b51cb, 0x0}, 0x0, 0xc0003ed080, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc004707c80}, 0xc0038cfa90, {0xc002249760, 0x40ce34}, 0xc004707bf0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc00412c0a0, {0x26ef130, 0xc003f11b30}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc003dfd9b0}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002764240}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002764240}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002764240}, 0x1e4d1c0, 0xc003f11b20) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc001210900}, 0xc0041246c0, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc001210900}, 0xc0041246c0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 97219 [semacquire, 27 minutes]: | |
| sync.runtime_SemacquireMutex(0x3530623065643261, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002255080, 0x24}, 0xc002cdfc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0049bbda0}, 0x0, 0xc002cdfe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881b02f696970, {0x26bf478, 0xc0049bbda0}, 0xffffb16b6eb10000, {0xc002255080, 0x2656fb0}, 0xc004d45500) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002255080, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 104524 [semacquire, 23 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004e88780, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223bbc0, 0x36}, 0xc003933c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004152660}, 0x0, 0xc003933e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881f3afc4cab7, {0x26bf478, 0xc004152660}, 0xffffb16b6eb10000, {0xc00223bbc0, 0x2656fb0}, 0xc00216c330) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223bbc0, 0x36}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 88690 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0044043c0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022489e0, 0x1d}, 0xc00168ec38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004896060}, 0x0, 0xc00168ee88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058818aaf323f41, {0x26bf478, 0xc004896060}, 0xffffb16b6eb10000, {0xc0022489e0, 0x2656fb0}, 0xc004ab7e90) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022489e0, 0x1d}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63419 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646f00, 0xc001836390) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 103133 [semacquire, 24 minutes]: | |
| sync.runtime_SemacquireMutex(0x7272457473726966, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223afc0, 0x35}, 0xc0039f0c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc005222660}, 0x0, 0xc0039f0e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881e4afaf7de9, {0x26bf478, 0xc005222660}, 0xffffb16b6eb10000, {0xc00223afc0, 0x2656fb0}, 0xc004706ab0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223afc0, 0x35}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63413 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646d40, 0xc0018361f0) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 82063 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0xa, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225ad80, 0x24}, 0xc003122c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00471f1a0}, 0x0, 0xc003122e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058816caf09ed7c, {0x26bf478, 0xc00471f1a0}, 0xffffb16b6eb10000, {0xc00225ad80, 0x2656fb0}, 0xc0046e2d50) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225ad80, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63376 [sync.Cond.Wait, 33 minutes]: | |
| sync.runtime_notifyListWait(0xc001173218, 0x0) | |
| runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc001bf57a0) | |
| sync/cond.go:56 +0x8c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc001173180) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:509 +0x428 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:334 +0xfb | |
| goroutine 101672 [semacquire, 25 minutes]: | |
| sync.runtime_SemacquireMutex(0x109, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254420, 0x28}, 0xc0043ebc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004520fc0}, 0x0, 0xc0043ebe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881d5af8f4933, {0x26bf478, 0xc004520fc0}, 0xffffb16b6eb10000, {0xc002254420, 0x2656fb0}, 0xc000051830) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254420, 0x28}, 0x4e94914f0000, 0xc001e5c770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 911091 [semacquire, 11 minutes]: | |
| sync.runtime_SemacquireMutex(0x3131343038333234, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254c60, 0x27}, 0xc003a45c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc001a488a0}, 0x0, 0xc003a45e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05882a7b066f509, {0x26bf478, 0xc001a488a0}, 0xffffb16b6eb10000, {0xc002254c60, 0x2656fb0}, 0xc004dbe3c8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254c60, 0x27}, 0x4e94914f0000, 0xc005306f70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63832 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002222e80, 0xc001efcc30) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x265 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc005d9a4b0}, 0xc001b75d00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc005d9a4b0}, 0x7f94aa18a5b8, {0x22c1ddb, 0xc000720000}, {0x22b51cb, 0x0}, 0x0, 0xc001385b00, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc005d9a4b0}, 0xc001722a90, {0xc0005e2540, 0x40ce34}, 0xc005d9a420) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc00141c5f0, {0x26ef130, 0xc001dedd90}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc001efcc18}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0001ece40}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0001ece40}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0001ece40}, 0x1e4d1c0, 0xc001dedd80) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0006fa600}, 0xc0013277a0, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0006fa600}, 0xc0013277a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 79271 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x80, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc003bc4ab0}, {0xc002248460, 0x19}, 0x7f9483303b98) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc004384480, 0xc0043807e0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc003bc4ab0}, 0xc003a52f00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc003bc4ab0}, 0x7f94aa18af18, {0x22c1ddb, 0xc000580000}, {0x22b51cb, 0x0}, 0x0, 0xc004520480, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc003bc4ab0}, 0xc0027b2a90, {0xc0005945a0, 0x40ce34}, 0xc003bc4a20) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc003ffe550, {0x26ef130, 0xc003e35760}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0043807c8}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e540}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e540}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e540}, 0x1e4d1c0, 0xc003e35750) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0000c4780}, 0xc00167f0e0, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0000c4780}, 0xc00167f0e0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 1015900 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*relayLog).startTimer.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:124 +0x88 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*relayLog).startTimer | |
| vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:123 +0xb6 | |
| goroutine 968325 [semacquire, 6 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00460ef60, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244300, 0x21}, 0xc003c18c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002dedb00}, 0x0, 0xc003c18e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05882eb30b8c7de, {0x26bf478, 0xc002dedb00}, 0xffffb16b6eb10000, {0xc002244300, 0x2656fb0}, 0xc0049c1290) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244300, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 65659 [chan receive, 20 minutes]: | |
| vitess.io/vitess/go/sync2.(*Semaphore).Acquire(0xc00153f520) | |
| vitess.io/vitess/go/sync2/semaphore.go:52 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc00016e500, {0x26ef630, 0xc0002fd860}, {0xc002248460, 0x19}, 0xc0013650c8, {0xc00431d9a0, 0xa, 0xa}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:589 +0x85 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc00016e500, 0xc003fc3a50, 0xc004344600) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 83807 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x80, 0xc004834420) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244720, 0x25}, 0xc005670c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002d0b680}, 0x0, 0xc005670e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881742f192192, {0x26bf478, 0xc002d0b680}, 0xffffb16b6eb10000, {0xc002244720, 0x2656fb0}, 0xc005d9a690) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244720, 0x25}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 85500 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0x2816, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002248460, 0x19}, 0xc0007e8c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00298f9e0}, 0x0, 0xc0007e8e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058817baf28308e, {0x26bf478, 0xc00298f9e0}, 0xffffb16b6eb10000, {0xc002248460, 0x2656fb0}, 0xc004c00750) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002248460, 0x19}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63408 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646ba0, 0xc001836040) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 90328 [semacquire, 29 minutes]: | |
| sync.runtime_SemacquireMutex(0x2816, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254840, 0x23}, 0xc0007e4c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0042a35c0}, 0x0, 0xc0007e4e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881922f40c679, {0x26bf478, 0xc0042a35c0}, 0xffffb16b6eb10000, {0xc002254840, 0x2656fb0}, 0xc0038efc80) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254840, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 79067 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x11, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254420, 0x28}, 0xc003967c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00417cae0}, 0x0, 0xc003967e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058815daf0230bd, {0x26bf478, 0xc00417cae0}, 0xffffb16b6eb10000, {0xc002254420, 0x2656fb0}, 0x4064c5) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254420, 0x28}, 0x4e94914f0000, 0xc002842770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63402 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0006469c0, 0xc0038f5ea0) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 97253 [semacquire, 27 minutes]: | |
| sync.runtime_SemacquireMutex(0x3130383831323637, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002248720, 0x1d}, 0xc003389c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002cc6660}, 0x0, 0xc003389e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881b02f693f36, {0x26bf478, 0xc002cc6660}, 0xffffb16b6eb10000, {0xc002248720, 0x2656fb0}, 0xc002678600) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002248720, 0x1d}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 82100 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x312d313a34353133, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a540, 0x21}, 0xc00303cc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00471f740}, 0x0, 0xc00303ce88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058816caf0a6a23, {0x26bf478, 0xc00471f740}, 0xffffb16b6eb10000, {0xc00225a540, 0x2656fb0}, 0xc0046e2e40) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a540, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 83874 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022489e0, 0x1d}, 0xc0058bdc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc005d0d5c0}, 0x0, 0xc0058bde88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881742f1890c4, {0x26bf478, 0xc005d0d5c0}, 0xffffb16b6eb10000, {0xc0022489e0, 0x2656fb0}, 0xc005d9a720) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022489e0, 0x1d}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 103155 [semacquire, 24 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0027e6d80, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002245380, 0x23}, 0xc004150c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc001fff260}, 0x0, 0xc004150e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881e4afb02b27, {0x26bf478, 0xc001fff260}, 0xffffb16b6eb10000, {0xc002245380, 0x2656fb0}, 0xc00455d740) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002245380, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63370 [sync.Cond.Wait, 33 minutes]: | |
| sync.runtime_notifyListWait(0xc001172f98, 0x0) | |
| runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc000358fa0) | |
| sync/cond.go:56 +0x8c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc001172f00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:509 +0x428 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:334 +0xfb | |
| goroutine 63415 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00016ebd8, 0x4e) | |
| runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc000576780) | |
| sync/cond.go:56 +0x8c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc00016eb40) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:509 +0x428 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:334 +0xfb | |
| goroutine 88733 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x2816, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002255080, 0x24}, 0xc0037fac38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc001721aa0}, 0x0, 0xc0037fae88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058818aaf348053, {0x26bf478, 0xc001721aa0}, 0xffffb16b6eb10000, {0xc002255080, 0x2656fb0}, 0xc004af3a10) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002255080, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63416 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646e20, 0xc0018362c0) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 63866 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:399 +0x31 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:398 +0x3b3 | |
| goroutine 98181 [semacquire, 27 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0039e3230, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022558c0, 0x23}, 0xc005390c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0049db320}, 0x0, 0xc005390e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881b7af6f86f1, {0x26bf478, 0xc0049db320}, 0xffffb16b6eb10000, {0xc0022558c0, 0x2656fb0}, 0xc0028833e0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022558c0, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 87248 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x14, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022457a0, 0x22}, 0xc004a45c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003eda9c0}, 0x0, 0xc004a45e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881832f303781, {0x26bf478, 0xc003eda9c0}, 0xffffb16b6eb10000, {0xc0022457a0, 0x2656fb0}, 0xc0042672f0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022457a0, 0x22}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 88698 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x100000000002816, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223bbc0, 0x36}, 0xc0037fcc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0048962a0}, 0x0, 0xc0037fce88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058818aaf32b589, {0x26bf478, 0xc0048962a0}, 0xffffb16b6eb10000, {0xc00223bbc0, 0x2656fb0}, 0xc004962150) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223bbc0, 0x36}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 101712 [semacquire, 25 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x80, 0xc001ab29c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244300, 0x21}, 0xc0038d7c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002ddc6c0}, 0x0, 0xc0038d7e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881d5af8fd385, {0x26bf478, 0xc002ddc6c0}, 0xffffb16b6eb10000, {0xc002244300, 0x2656fb0}, 0xc004922c30) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244300, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63420 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646f20, 0xc0018363a0) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 64646 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f94832f2c98, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc002e65a00, 0xc000798000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc002e65a00, {0xc000798000, 0x4000, 0x4000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc002e65a00, {0xc000798000, 0xc001b37c88, 0xc001b37c28}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0006da2e0, {0xc000798000, 0x42c94c, 0x2}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0041077a0, {0xc002fd5f20, 0x4, 0x112268e}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0041077a0}, {0xc002fd5f20, 0x4, 0x4}, 0x4) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| vitess.io/vitess/go/mysql.(*Conn).readHeaderFrom(0xc002fd5e60, {0x2673340, 0xc0041077a0}) | |
| vitess.io/vitess/go/mysql/conn.go:334 +0x4f | |
| vitess.io/vitess/go/mysql.(*Conn).readOnePacket(0x447fa8) | |
| vitess.io/vitess/go/mysql/conn.go:473 +0x6d | |
| vitess.io/vitess/go/mysql.(*Conn).readPacket(0x11fe3b130e022) | |
| vitess.io/vitess/go/mysql/conn.go:495 +0x25 | |
| vitess.io/vitess/go/mysql.(*Conn).ReadPacket(0xc0015d1ba0) | |
| vitess.io/vitess/go/mysql/conn.go:531 +0x1d | |
| vitess.io/vitess/go/mysql.mysqlFlavor.readBinlogEvent({}, 0x0) | |
| vitess.io/vitess/go/mysql/flavor_mysql.go:225 +0x1d | |
| vitess.io/vitess/go/mysql.(*Conn).ReadBinlogEvent(...) | |
| vitess.io/vitess/go/mysql/flavor.go:237 | |
| vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents.func1() | |
| vitess.io/vitess/go/vt/binlog/binlog_connection.go:146 +0xc5 | |
| created by vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents | |
| vitess.io/vitess/go/vt/binlog/binlog_connection.go:140 +0xdd | |
| goroutine 80379 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x0, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc002303c20}, {0xc002248460, 0x19}, 0x7f9483059970) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005511700, 0xc0055505d0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc002303c20}, 0xc0044e5800) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc002303c20}, 0x7f94aa18aa68, {0x22c1ddb, 0xc000480000}, {0x22b51cb, 0x0}, 0x0, 0xc00554a4e0, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc002303c20}, 0xc000793a90, {0xc000595960, 0x40ce34}, 0xc002303b90) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc00401dc20, {0x26ef130, 0xc003d36710}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0055505b8}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0013fc480}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0013fc480}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0013fc480}, 0x1e4d1c0, 0xc003d36700) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0006fa600}, 0xc0022cd0e0, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0006fa600}, 0xc0022cd0e0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 63421 [sync.Cond.Wait, 33 minutes]: | |
| sync.runtime_notifyListWait(0xc00016ef98, 0x1) | |
| runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xea2a01) | |
| sync/cond.go:56 +0x8c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc00016ef00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:509 +0x428 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:334 +0xfb | |
| goroutine 388279 [semacquire, 16 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0016f3aa0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244f60, 0x25}, 0xc00564cc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc005a59f80}, 0x0, 0xc00564ce88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058824db0381b5f, {0x26bf478, 0xc005a59f80}, 0xffffb16b6eb10000, {0xc002244f60, 0x2656fb0}, 0xc0015e6228) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244f60, 0x25}, 0x4e94914f0000, 0x0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 79049 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x11, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254000, 0x24}, 0xc003b2ec38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc001452660}, 0x0, 0xc003b2ee88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058815daef115f3, {0x26bf478, 0xc001452660}, 0xffffb16b6eb10000, {0xc002254000, 0x2656fb0}, 0x12ec4eb) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254000, 0x24}, 0x4e94914f0000, 0xc000604770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 79392 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x0, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc001bd9fb0}, {0xc002248720, 0x1d}, 0x7f9483313010) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc004801700, 0xc0043d3ef0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc001bd9fb0}, 0xc00046f400) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc001bd9fb0}, 0x7f94aa18a108, {0x22c1ddb, 0xc003f24400}, {0x22b51cb, 0x0}, 0x0, 0xc0038ca6c0, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc001bd9fb0}, 0xc003dd5a90, {0xc004244600, 0x40ce34}, 0xc001bd9f20) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc004485590, {0x26ef130, 0xc004610d50}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0043d3ed8}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022cee40}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022cee40}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022cee40}, 0x1e4d1c0, 0xc004610d40) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0006fa000}, 0xc003c350e0, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0006fa000}, 0xc003c350e0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 63636 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/topo.(*Server).WatchShard.func1() | |
| vitess.io/vitess/go/vt/topo/shard.go:683 +0xbe | |
| created by vitess.io/vitess/go/vt/topo.(*Server).WatchShard | |
| vitess.io/vitess/go/vt/topo/shard.go:680 +0x230 | |
| goroutine 64608 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:399 +0x31 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:398 +0x3b3 | |
| goroutine 92158 [semacquire, 29 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223afc0, 0x35}, 0xc0057e8c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0029bc840}, 0x0, 0xc0057e8e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588199af4cf734, {0x26bf478, 0xc0029bc840}, 0xffffb16b6eb10000, {0xc00223afc0, 0x2656fb0}, 0xc002323e30) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223afc0, 0x35}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 96658 [semacquire, 28 minutes]: | |
| sync.runtime_SemacquireMutex(0x3530623065643261, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002248720, 0x1d}, 0xc002cd5c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002cad7a0}, 0x0, 0xc002cd5e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881a8af64cb2e, {0x26bf478, 0xc002cad7a0}, 0xffffb16b6eb10000, {0xc002248720, 0x2656fb0}, 0xc0048a7110) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002248720, 0x1d}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 323030 [semacquire, 17 minutes]: | |
| sync.runtime_SemacquireMutex(0x3634343033, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244f60, 0x25}, 0xc005792c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0039140c0}, 0x0, 0xc005792e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05882463031de21, {0x26bf478, 0xc0039140c0}, 0xffffb16b6eb10000, {0xc002244f60, 0x2656fb0}, 0xc0043b6060) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244f60, 0x25}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 92962 [semacquire, 28 minutes]: | |
| sync.runtime_SemacquireMutex(0xc001382b10, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254c60, 0x27}, 0xc0050c1c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00407b4a0}, 0x0, 0xc0050c1e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881a12f5bb54b, {0x26bf478, 0xc00407b4a0}, 0xffffb16b6eb10000, {0xc002254c60, 0x2656fb0}, 0xc004733890) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254c60, 0x27}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 199758 [semacquire, 18 minutes]: | |
| sync.runtime_SemacquireMutex(0xc003b7b6b0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002245380, 0x23}, 0xc005a0ec38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003bc9020}, 0x0, 0xc005a0ee88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588237302802b9, {0x26bf478, 0xc003bc9020}, 0xffffb16b6eb10000, {0xc002245380, 0x2656fb0}, 0xc0042d94d0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002245380, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 64892 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:399 +0x31 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:398 +0x3b3 | |
| goroutine 92991 [semacquire, 28 minutes]: | |
| sync.runtime_SemacquireMutex(0x7272457473726966, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223bbc0, 0x36}, 0xc0048aec38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004375620}, 0x0, 0xc0048aee88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881a12f5d2ee9, {0x26bf478, 0xc004375620}, 0xffffb16b6eb10000, {0xc00223bbc0, 0x2656fb0}, 0xc003a63b60) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223bbc0, 0x36}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 82130 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x44d072, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022558c0, 0x23}, 0xc003e09c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004025380}, 0x0, 0xc003e09e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058816caf0c5940, {0x26bf478, 0xc004025380}, 0xffffb16b6eb10000, {0xc0022558c0, 0x2656fb0}, 0xc002068720) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022558c0, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63380 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646340, 0xc0038f58c0) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 79387 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x80, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc001bd90e0}, {0xc0022489e0, 0x1d}, 0x7f9483313010) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc004800c80, 0xc0043d2fd8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc001bd90e0}, 0xc00046f000) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc001bd90e0}, 0x7f94aa18a108, {0x22c1ddb, 0x3b092c0}, {0x22b51cb, 0x0}, 0x0, 0xc0043df920, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc001bd90e0}, 0xc003e0ca90, {0xc004244400, 0x40ce34}, 0xc001bd9050) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc004484cd0, {0x26ef130, 0xc004610870}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0043d2fc0}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ced80}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ced80}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022ced80}, 0x1e4d1c0, 0xc004610860) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0006fa000}, 0xc003c34c60, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0006fa000}, 0xc003c34c60, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 63382 [sync.Cond.Wait, 33 minutes]: | |
| sync.runtime_notifyListWait(0xc001173498, 0x0) | |
| runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0x0) | |
| sync/cond.go:56 +0x8c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc001173400) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:509 +0x428 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:334 +0xfb | |
| goroutine 63373 [sync.Cond.Wait, 33 minutes]: | |
| sync.runtime_notifyListWait(0xc0011730d8, 0x1) | |
| runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc00035bf01) | |
| sync/cond.go:56 +0x8c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc001173040) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:509 +0x428 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:334 +0xfb | |
| goroutine 83846 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0x109, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002248720, 0x1d}, 0xc0007bbc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00180f800}, 0x0, 0xc0007bbe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881742f2b3fb2, {0x26bf478, 0xc00180f800}, 0xffffb16b6eb10000, {0xc002248720, 0x2656fb0}, 0xc005d9a930) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002248720, 0x1d}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63375 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0006461a0, 0xc0038f5710) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 63620 [select, 33 minutes]: | |
| context.propagateCancel.func1() | |
| context/context.go:279 +0xa5 | |
| created by context.propagateCancel | |
| context/context.go:278 +0x1d0 | |
| goroutine 66055 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x0, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc002323c50}, {0xc002244b40, 0x21}, 0x7f94832f39a8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc001f69e00, 0xc0046cd308) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc002323c50}, 0xc00171f400) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc002323c50}, 0x7f94aa18bd28, {0x22c1ddb, 0xc000480000}, {0x22b51cb, 0x0}, 0x0, 0xc00298ec00, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc002323c50}, 0xc004c73a90, {0xc0045c6060, 0x40ce34}, 0xc0023239b0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc0023245f0, {0x26ef130, 0xc003408c00}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0046cd2f0}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c900}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c900}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c900}, 0x1e4d1c0, 0xc003408bf0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc001210480}, 0xc001b7f560, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc001210480}, 0xc001b7f560, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 102463 [semacquire, 24 minutes]: | |
| sync.runtime_SemacquireMutex(0x3a37303565653565, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223afc0, 0x35}, 0xc003d30c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00510c4e0}, 0x0, 0xc003d30e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881dd2f9f4539, {0x26bf478, 0xc00510c4e0}, 0xffffb16b6eb10000, {0xc00223afc0, 0x2656fb0}, 0xc0044b4480) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223afc0, 0x35}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 97209 [semacquire, 27 minutes]: | |
| sync.runtime_SemacquireMutex(0x44d072, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022554a0, 0x28}, 0xc005394c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002cc6120}, 0x0, 0xc005394e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881b02f67d272, {0x26bf478, 0xc002cc6120}, 0xffffb16b6eb10000, {0xc0022554a0, 0x2656fb0}, 0xc005989bc0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022554a0, 0x28}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 388235 [semacquire, 16 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0039e2d20, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254c60, 0x27}, 0xc003f35c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004a6bf20}, 0x0, 0xc003f35e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058824db0389643, {0x26bf478, 0xc004a6bf20}, 0xffffb16b6eb10000, {0xc002254c60, 0x2656fb0}, 0xc00163abe8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254c60, 0x27}, 0x4e94914f0000, 0x0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 66024 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x2656fb0, 0x30, 0x1211fac) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc0004ada40, {0xc002244720, 0x25}, {0xc0039131a0, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:153 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1059 +0x38 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0034f1da0}, 0x0, 0xc001963de0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0x26bf4b0, {0x26bf478, 0xc0034f1da0}, 0x2656fb0, {0xc002244720, 0x2656fb0}, {0xc0039131a0, 0x0, 0x7f94aa18b3c8}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1058 +0x8d | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc001173680, {0x26ef630, 0xc0002fd860}, {0xc002244720, 0x25}, 0x22b658d, {0xc0039131a0, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:596 +0x14e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc001173680, 0xc0041048e0, 0xc0034f1ce0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 63425 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0006470c0, 0xc001836530) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 79047 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x3a37303565653565, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225b1a0, 0x28}, 0xc00396cc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0014524e0}, 0x0, 0xc00396ce88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058815daef0a343, {0x26bf478, 0xc0014524e0}, 0xffffb16b6eb10000, {0xc00225b1a0, 0x2656fb0}, 0x12ec4eb) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225b1a0, 0x28}, 0x4e94914f0000, 0xc001e5ef70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63411 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646c80, 0xc001836130) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 63414 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646d60, 0xc001836200) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 105621 [semacquire, 22 minutes]: | |
| sync.runtime_SemacquireMutex(0x759113d48f46d4bb, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244b40, 0x21}, 0xc00315bc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004ba28a0}, 0x0, 0xc00315be88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881fb2fd13537, {0x26bf478, 0xc004ba28a0}, 0xffffb16b6eb10000, {0xc002244b40, 0x2656fb0}, 0xc001b4a8a0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244b40, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63377 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646260, 0xc0038f57d0) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 88931 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x80, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc0044f83c0}, {0xc0022489e0, 0x1d}, 0x7f9482c87bb8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc003f16480, 0xc0046f1ef0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc0044f83c0}, 0xc0044e4700) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc0044f83c0}, 0x7f94aa18af18, {0x22c1ddb, 0xc000480800}, {0x22b51cb, 0x0}, 0x0, 0xc003edb980, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc0044f83c0}, 0xc00501da90, {0xc0040bb500, 0x40ce34}, 0xc0044f82d0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc003b7f950, {0x26ef130, 0xc001ded8f0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0046f1ed8}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064ca80}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064ca80}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064ca80}, 0x1e4d1c0, 0xc001ded8e0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc00066e180}, 0xc0020aea20, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc00066e180}, 0xc0020aea20, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 63741 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f94830624e8, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc002b1a800, 0xc001d38000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc002b1a800, {0xc001d38000, 0x4000, 0x4000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc002b1a800, {0xc001d38000, 0xc004175c88, 0xc004175c28}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc000564620, {0xc001d38000, 0x42c94c, 0x2}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc00471ef60, {0xc0016bd3e0, 0x4, 0x112268e}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc00471ef60}, {0xc0016bd3e0, 0x4, 0x4}, 0x4) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| vitess.io/vitess/go/mysql.(*Conn).readHeaderFrom(0xc0016bd320, {0x2673340, 0xc00471ef60}) | |
| vitess.io/vitess/go/mysql/conn.go:334 +0x4f | |
| vitess.io/vitess/go/mysql.(*Conn).readOnePacket(0x447fa8) | |
| vitess.io/vitess/go/mysql/conn.go:473 +0x6d | |
| vitess.io/vitess/go/mysql.(*Conn).readPacket(0x11fe3b12f51d0) | |
| vitess.io/vitess/go/mysql/conn.go:495 +0x25 | |
| vitess.io/vitess/go/mysql.(*Conn).ReadPacket(0xc00163ba00) | |
| vitess.io/vitess/go/mysql/conn.go:531 +0x1d | |
| vitess.io/vitess/go/mysql.mysqlFlavor.readBinlogEvent({}, 0x0) | |
| vitess.io/vitess/go/mysql/flavor_mysql.go:225 +0x1d | |
| vitess.io/vitess/go/mysql.(*Conn).ReadBinlogEvent(...) | |
| vitess.io/vitess/go/mysql/flavor.go:237 | |
| vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents.func1() | |
| vitess.io/vitess/go/vt/binlog/binlog_connection.go:146 +0xc5 | |
| created by vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents | |
| vitess.io/vitess/go/vt/binlog/binlog_connection.go:140 +0xdd | |
| goroutine 63817 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002b1a300, 0xc00045a8e8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x265 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc0028be4e0}, 0xc001b14000) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc0028be4e0}, 0x7f94aa18af18, {0x22c1ddb, 0xc000720000}, {0x22b51cb, 0x0}, 0x0, 0xc00471e060, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc0028be4e0}, 0xc001689a90, {0xc00434cae0, 0x40ce34}, 0xc0028be450) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc00166e820, {0x26ef130, 0xc0000ad470}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc00045a8d0}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc001424540}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc001424540}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc001424540}, 0x1e4d1c0, 0xc0000ad450) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc002350a80}, 0xc0016bcc60, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc002350a80}, 0xc0016bcc60, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 81994 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x80, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc003c32d80}, {0xc0022489e0, 0x1d}, 0x7f9482fb9800) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00387d680, 0xc0045aa0f0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc003c32d80}, 0xc004586200) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc003c32d80}, 0x7f94aa18a108, {0x22c1ddb, 0xc000480000}, {0x22b51cb, 0x0}, 0x0, 0xc004025020, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc003c32d80}, 0xc003b4aa90, {0xc0015e5d00, 0x40ce34}, 0xc003c32cf0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc003deb090, {0x26ef130, 0xc003408e90}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0045aa0d8}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0013fc540}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0013fc540}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0013fc540}, 0x1e4d1c0, 0xc003408e80) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0000c4600}, 0xc004124d80, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0000c4600}, 0xc004124d80, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 79048 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x312d313a34353133, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254c60, 0x27}, 0xc00396dc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0014525a0}, 0x0, 0xc00396de88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058815daef0e3c1, {0x26bf478, 0xc0014525a0}, 0xffffb16b6eb10000, {0xc002254c60, 0x2656fb0}, 0x12ec4eb) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254c60, 0x27}, 0x4e94914f0000, 0xc000604770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63365 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0017c4d20, 0xc8, 0xec3ef5400ce99dd6) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc001172a78) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPoller(0xc001172a00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:758 +0x391 | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc00157de60, 0xc0038f5490) | |
| vitess.io/vitess/go/timer/timer.go:112 +0x123 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 83722 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00179d560, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a960, 0x26}, 0xc00592ac38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002c3c660}, 0x0, 0xc00592ae88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881742f2a8842, {0x26bf478, 0xc002c3c660}, 0xffffb16b6eb10000, {0xc00225a960, 0x2656fb0}, 0xc0013833e0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a960, 0x26}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63338 [chan receive]: | |
| vitess.io/vitess/go/stats.(*Rates).track(0xc004ba25a0) | |
| vitess.io/vitess/go/stats/rates.go:99 +0x36 | |
| created by vitess.io/vitess/go/stats.NewRates | |
| vitess.io/vitess/go/stats/rates.go:91 +0x22f | |
| goroutine 107859 [semacquire, 21 minutes]: | |
| sync.runtime_SemacquireMutex(0x3530623065643261, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244b40, 0x21}, 0xc0030f1c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003a26f00}, 0x0, 0xc0030f1e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058820a2fde94d3, {0x26bf478, 0xc003a26f00}, 0xffffb16b6eb10000, {0xc002244b40, 0x2656fb0}, 0xc0059898f0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244b40, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63850 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:399 +0x31 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:398 +0x3b3 | |
| goroutine 92994 [semacquire, 28 minutes]: | |
| sync.runtime_SemacquireMutex(0x6972547473726966, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254420, 0x28}, 0xc00493dc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0043758c0}, 0x0, 0xc00493de88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881a12f5da833, {0x26bf478, 0xc0043758c0}, 0xffffb16b6eb10000, {0xc002254420, 0x2656fb0}, 0xc003a63a70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254420, 0x28}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 79059 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x3a37303565653565, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254840, 0x23}, 0xc000790c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00417c120}, 0x0, 0xc000790e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058815daeefc4cd, {0x26bf478, 0xc00417c120}, 0xffffb16b6eb10000, {0xc002254840, 0x2656fb0}, 0x12ec4eb) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254840, 0x23}, 0x4e94914f0000, 0xc0017f3f70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 100060 [semacquire, 26 minutes]: | |
| sync.runtime_SemacquireMutex(0x100000000000000, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002245380, 0x23}, 0xc005edac38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004a8acc0}, 0x0, 0xc005edae88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881c6af860ff1, {0x26bf478, 0xc004a8acc0}, 0xffffb16b6eb10000, {0xc002245380, 0x2656fb0}, 0xc002b16de0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002245380, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 929 [chan receive, 51 minutes]: | |
| vitess.io/vitess/go/vt/srvtopo.NewSrvVSchemaWatcher.func1({0x26bf440, 0xc000128008}, 0xc0001b02c0) | |
| vitess.io/vitess/go/vt/srvtopo/watch_srvvschema.go:49 +0x10d | |
| created by vitess.io/vitess/go/vt/srvtopo.(*watchEntry).ensureWatchingLocked | |
| vitess.io/vitess/go/vt/srvtopo/watch.go:113 +0x133 | |
| goroutine 66163 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x2656fb0, 0x30, 0x1211fac) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc0004ada40, {0xc002244b40, 0x21}, {0xc0035e53f0, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:153 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1059 +0x38 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0049eac00}, 0x0, 0xc004c69de0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0x26bf4b0, {0x26bf478, 0xc0049eac00}, 0x2656fb0, {0xc002244b40, 0x2656fb0}, {0xc0035e53f0, 0x0, 0x7f94aa18b3c8}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1058 +0x8d | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc001173e00, {0x26ef630, 0xc0002fd860}, {0xc002244b40, 0x21}, 0x26bf408, {0xc0035e53f0, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:596 +0x14e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc001173e00, 0xc003b71670, 0xc0049eab40) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 98176 [semacquire, 27 minutes]: | |
| sync.runtime_SemacquireMutex(0xc001ccacc0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254000, 0x24}, 0xc002cd3c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0041c2d80}, 0x0, 0xc002cd3e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881b7af70a6f4, {0x26bf478, 0xc0041c2d80}, 0xffffb16b6eb10000, {0xc002254000, 0x2656fb0}, 0xc0028833b0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254000, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 87287 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x80, 0xc0049ea060) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225b1a0, 0x28}, 0xc004a4ec38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003edb140}, 0x0, 0xc004a4ee88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881832f3166e7, {0x26bf478, 0xc003edb140}, 0xffffb16b6eb10000, {0xc00225b1a0, 0x2656fb0}, 0xc004656fc0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225b1a0, 0x28}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 92984 [semacquire, 28 minutes]: | |
| sync.runtime_SemacquireMutex(0x2d313a3133313030, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223afc0, 0x35}, 0xc0031d5c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004375080}, 0x0, 0xc0031d5e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881a12f5c8a57, {0x26bf478, 0xc004375080}, 0xffffb16b6eb10000, {0xc00223afc0, 0x2656fb0}, 0xc003a63920) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223afc0, 0x35}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 110000 [semacquire, 20 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254420, 0x28}, 0xc003071c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0046d9e60}, 0x0, 0xc003071e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588220b014f750, {0x26bf478, 0xc0046d9e60}, 0xffffb16b6eb10000, {0xc002254420, 0x2656fb0}, 0xc000051830) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254420, 0x28}, 0x4e94914f0000, 0xc001df5770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 80406 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x0, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc0021ce900}, {0xc002248720, 0x1d}, 0x7f948305d508) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005730000, 0xc005703650) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc0021ce900}, 0xc001d00700) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc0021ce900}, 0x7f94aa18a5b8, {0x22c1ddb, 0xc000480000}, {0x22b51cb, 0x0}, 0x0, 0xc002b791a0, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc0021ce900}, 0xc000795a90, {0xc0057182e0, 0x40ce34}, 0xc002163d70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc003e559a0, {0x26ef130, 0xc002f84a00}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc005703638}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c300}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c300}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c300}, 0x1e4d1c0, 0xc002f849f0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0006fa600}, 0xc002272240, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0006fa600}, 0xc002272240, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 80982 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004b433b0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223b5c0, 0x35}, 0xc00589fc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003dd1d40}, 0x0, 0xc00589fe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881652efc3650, {0x26bf478, 0xc003dd1d40}, 0xffffb16b6eb10000, {0xc00223b5c0, 0x2656fb0}, 0xc002ad8030) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223b5c0, 0x35}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63371 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0006460a0, 0xc0038f5630) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 79001 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x2656fb0, 0x30, 0x1211fac) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc0004ada40, {0xc0022489e0, 0x1d}, {0xc003368780, 0xa, 0xa}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:153 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1059 +0x38 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0031116e0}, 0x0, 0xc003b4dde0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0x26bf4b0, {0x26bf478, 0xc0031116e0}, 0x2656fb0, {0xc0022489e0, 0x2656fb0}, {0xc003368780, 0x0, 0x7f94aa18aa68}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1058 +0x8d | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc00016eb40, {0x26ef630, 0xc0002fd860}, {0xc0022489e0, 0x1d}, 0x1000000026bf408, {0xc003368780, 0xa, 0xa}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:596 +0x14e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc00016eb40, 0xc0021f6eb0, 0xc003110f60) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 97200 [semacquire, 27 minutes]: | |
| sync.runtime_SemacquireMutex(0xc001bd89c0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022558c0, 0x23}, 0xc002ce5c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0049bbb60}, 0x0, 0xc002ce5e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881b02f67697c, {0x26bf478, 0xc0049bbb60}, 0xffffb16b6eb10000, {0xc0022558c0, 0x2656fb0}, 0xc003bc4ff0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022558c0, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 78718 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x0, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc004862810}, {0xc002248460, 0x19}, 0x7f948323ac90) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0040d8500, 0xc0041a5a28) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc004862810}, 0xc0044e4000) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc004862810}, 0x7f94aa18af18, {0x22c1ddb, 0xc0006b9c00}, {0x22b51cb, 0x0}, 0x0, 0xc002e78de0, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc004862810}, 0xc004c62a90, {0xc00346edc0, 0x40ce34}, 0xc004862780) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc004b93ae0, {0x26ef130, 0xc004333dc0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0041a5a10}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e480}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e480}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e480}, 0x1e4d1c0, 0xc004333db0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc001210780}, 0xc001f72240, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc001210780}, 0xc001f72240, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 107196 [semacquire, 22 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004a77e90, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223b5c0, 0x35}, 0xc004944c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc001ec3560}, 0x0, 0xc004944e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588202afde6673, {0x26bf478, 0xc001ec3560}, 0xffffb16b6eb10000, {0xc00223b5c0, 0x2656fb0}, 0xc001e5ff18) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223b5c0, 0x35}, 0x4e94914f0000, 0xc001e5ffb8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 81030 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004976a20, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002245380, 0x23}, 0xc004174c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0013fb860}, 0x0, 0xc004174e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881652efe837d, {0x26bf478, 0xc0013fb860}, 0xffffb16b6eb10000, {0xc002245380, 0x2656fb0}, 0xc003e4d6e0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002245380, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 83878 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0x109, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223b5c0, 0x35}, 0xc0007c5c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc005d0d920}, 0x0, 0xc0007c5e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881742f18fb04, {0x26bf478, 0xc005d0d920}, 0xffffb16b6eb10000, {0xc00223b5c0, 0x2656fb0}, 0xc005d9a840) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223b5c0, 0x35}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 97251 [semacquire, 27 minutes]: | |
| sync.runtime_SemacquireMutex(0x109, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a960, 0x26}, 0xc00338fc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002cc64e0}, 0x0, 0xc00338fe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881b02f68fdc8, {0x26bf478, 0xc002cc64e0}, 0xffffb16b6eb10000, {0xc00225a960, 0x2656fb0}, 0xc0026784b0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a960, 0x26}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 103156 [semacquire, 24 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004e895f0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244b40, 0x21}, 0xc00414ac38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc001fff320}, 0x0, 0xc00414ae88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881e4afb03fa9, {0x26bf478, 0xc001fff320}, 0xffffb16b6eb10000, {0xc002244b40, 0x2656fb0}, 0xc00455d770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244b40, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63374 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646180, 0xc0038f5700) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 63860 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:399 +0x31 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:398 +0x3b3 | |
| goroutine 101677 [semacquire, 25 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x80, 0xc001ab29c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244f60, 0x25}, 0xc0044d0c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0045215c0}, 0x0, 0xc0044d0e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881d5af90071f, {0x26bf478, 0xc0045215c0}, 0xffffb16b6eb10000, {0xc002244f60, 0x2656fb0}, 0xc004922c60) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244f60, 0x25}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 90356 [semacquire, 29 minutes]: | |
| sync.runtime_SemacquireMutex(0x109, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244b40, 0x21}, 0xc0007e7c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0036e4d20}, 0x0, 0xc0007e7e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881922f4061e3, {0x26bf478, 0xc0036e4d20}, 0xffffb16b6eb10000, {0xc002244b40, 0x2656fb0}, 0xc0038efb30) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244b40, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 97199 [semacquire, 27 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022481a0, 0x1f}, 0xc002ce3c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0049bba40}, 0x0, 0xc002ce3e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881b02f674ed8, {0x26bf478, 0xc0049bba40}, 0xffffb16b6eb10000, {0xc0022481a0, 0x2656fb0}, 0xc003bc4fc0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022481a0, 0x1f}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 92075 [semacquire, 29 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00220f7d0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223b5c0, 0x35}, 0xc003c24c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0049ea2a0}, 0x0, 0xc003c24e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588199af4ebaf6, {0x26bf478, 0xc0049ea2a0}, 0xffffb16b6eb10000, {0xc00223b5c0, 0x2656fb0}, 0xc004bf0120) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223b5c0, 0x35}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 83818 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00179da40, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022457a0, 0x22}, 0xc003b59c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0039cd1a0}, 0x0, 0xc003b59e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881742f19c9da, {0x26bf478, 0xc0039cd1a0}, 0xffffb16b6eb10000, {0xc0022457a0, 0x2656fb0}, 0xc005d9a660) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022457a0, 0x22}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 80970 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0046e2330, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002245bc0, 0x26}, 0xc00589bc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0028c7980}, 0x0, 0xc00589be88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881652ef8e689, {0x26bf478, 0xc0028c7980}, 0xffffb16b6eb10000, {0xc002245bc0, 0x2656fb0}, 0xc004ab7740) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002245bc0, 0x26}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63422 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646fe0, 0xc001836460) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 100010 [semacquire, 26 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00494a6f0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254c60, 0x27}, 0xc00293ac38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0019e6f60}, 0x0, 0xc00293ae88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881c6af83e7c1, {0x26bf478, 0xc0019e6f60}, 0xffffb16b6eb10000, {0xc002254c60, 0x2656fb0}, 0xc0029a6f18) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254c60, 0x27}, 0x4e94914f0000, 0xc0029a6f70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 92157 [semacquire, 29 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002248460, 0x19}, 0xc0057e7c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0029bc720}, 0x0, 0xc0057e7e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588199af4cb4b8, {0x26bf478, 0xc0029bc720}, 0xffffb16b6eb10000, {0xc002248460, 0x2656fb0}, 0xc002323dd0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002248460, 0x19}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 982 [select, 51 minutes]: | |
| go.etcd.io/etcd/client/v3.(*watchGrpcStream).run(0xc00048dc70) | |
| go.etcd.io/etcd/client/v3@v3.5.0/watch.go:537 +0x374 | |
| created by go.etcd.io/etcd/client/v3.(*watcher).newWatcherGrpcStream | |
| go.etcd.io/etcd/client/v3@v3.5.0/watch.go:285 +0x352 | |
| goroutine 985 [select, 51 minutes]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc001a58230, {0xc001a2ba08, 0x5, 0x5}) | |
| google.golang.org/grpc@v1.39.0/internal/transport/transport.go:189 +0xaa | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001a58230, {0xc001a2ba08, 0xc0001277d0, 0xc0027b7a60}) | |
| google.golang.org/grpc@v1.39.0/internal/transport/transport.go:169 +0x147 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001a35a10, {0xc001a2ba08, 0xc0027b7ad8, 0x9ac127}) | |
| google.golang.org/grpc@v1.39.0/internal/transport/transport.go:483 +0x32 | |
| io.ReadAtLeast({0x2676060, 0xc001a35a10}, {0xc001a2ba08, 0x5, 0x5}, 0x5) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc002218000, {0xc001a2ba08, 0x5, 0x5}) | |
| google.golang.org/grpc@v1.39.0/internal/transport/transport.go:467 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc001a2b9f8, 0x7fffffff) | |
| google.golang.org/grpc@v1.39.0/rpc_util.go:559 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0x3b3b8c8, 0xc002218000, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) | |
| google.golang.org/grpc@v1.39.0/rpc_util.go:690 +0x66 | |
| google.golang.org/grpc.recv(0x0, {0x7f94833bc520, 0x3b3b8c8}, 0x1fbe140, {0x0, 0x0}, {0x2226aa0, 0xc0002acd20}, 0x10b3746, 0x0, ...) | |
| google.golang.org/grpc@v1.39.0/rpc_util.go:758 +0x6e | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc001a36c80, {0x2226aa0, 0xc0002acd20}, 0x448b2f) | |
| google.golang.org/grpc@v1.39.0/stream.go:957 +0x2b0 | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x203000) | |
| google.golang.org/grpc@v1.39.0/stream.go:808 +0x25 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc000645d40, 0xc0027b7eb0, 0xc0027b7e80) | |
| google.golang.org/grpc@v1.39.0/stream.go:662 +0x2f6 | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc000645d40, {0x2226aa0, 0xc0002acd20}) | |
| google.golang.org/grpc@v1.39.0/stream.go:807 +0x11f | |
| go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchClient).Recv(0xc001837b60) | |
| go.etcd.io/etcd/api/v3@v3.5.0/etcdserverpb/rpc.pb.go:6714 +0x4c | |
| go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveWatchClient(0xc00048dc70, {0x26ffa48, 0xc001837b60}) | |
| go.etcd.io/etcd/client/v3@v3.5.0/watch.go:757 +0x6a | |
| created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).newWatchClient | |
| go.etcd.io/etcd/client/v3@v3.5.0/watch.go:912 +0x4e5 | |
| goroutine 891868 [semacquire, 11 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00584f800, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002248720, 0x1d}, 0xc001704c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0059f4360}, 0x0, 0xc001704e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588298b05d8d9a, {0x26bf478, 0xc0059f4360}, 0xffffb16b6eb10000, {0xc002248720, 0x2656fb0}, 0xc003841410) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002248720, 0x1d}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 984 [select, 51 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc@v1.39.0/stream.go:363 +0x98 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc@v1.39.0/stream.go:362 +0x1066 | |
| goroutine 986 [select, 51 minutes]: | |
| go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc00048dc70, 0xc00021fe40, 0xc0019ff5c0) | |
| go.etcd.io/etcd/client/v3@v3.5.0/watch.go:803 +0x265 | |
| created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run | |
| go.etcd.io/etcd/client/v3@v3.5.0/watch.go:554 +0xed1 | |
| goroutine 63423 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000647000, 0xc001836470) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 82120 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x616e003a00005305, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002255ce0, 0x27}, 0xc004c8ac38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00471ff80}, 0x0, 0xc004c8ae88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058816caf1eea57, {0x26bf478, 0xc00471ff80}, 0xffffb16b6eb10000, {0xc002255ce0, 0x2656fb0}, 0xc004d8e4b0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002255ce0, 0x27}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 87281 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0049014a0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a540, 0x21}, 0xc0050c7c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003edaae0}, 0x0, 0xc0050c7e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881832f3050a9, {0x26bf478, 0xc003edaae0}, 0xffffb16b6eb10000, {0xc00225a540, 0x2656fb0}, 0xc004267320) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a540, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 88631 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0xc003d4dd40, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022554a0, 0x28}, 0xc004a4dc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc001be7a40}, 0x0, 0xc004a4de88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058818aaf32967b, {0x26bf478, 0xc001be7a40}, 0xffffb16b6eb10000, {0xc0022554a0, 0x2656fb0}, 0x12ec4eb) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022554a0, 0x28}, 0x4e94914f0000, 0xc001e61fb8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 79065 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x662e6d6720726570, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244f60, 0x25}, 0xc003b51c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00417c840}, 0x0, 0xc003b51e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058815daef0f86b, {0x26bf478, 0xc00417c840}, 0xffffb16b6eb10000, {0xc002244f60, 0x2656fb0}, 0x12ec4eb) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244f60, 0x25}, 0x4e94914f0000, 0xc00183e770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 66295 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x2656fb0, 0x30, 0x1211fac) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc0004ada40, {0xc002244f60, 0x25}, {0xc0043329e0, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:153 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1059 +0x38 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002c69a40}, 0x0, 0xc004c5dde0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0x26bf4b0, {0x26bf478, 0xc002c69a40}, 0x2656fb0, {0xc002244f60, 0x2656fb0}, {0xc0043329e0, 0x0, 0x7f94aa18af18}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1058 +0x8d | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc00016e780, {0x26ef630, 0xc0002fd860}, {0xc002244f60, 0x25}, 0x488897, {0xc0043329e0, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:596 +0x14e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc00016e780, 0xc003b71520, 0xc002c69980) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 79063 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a540, 0x21}, 0xc003b33c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00417c600}, 0x0, 0xc003b33e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058815daef0944d, {0x26bf478, 0xc00417c600}, 0xffffb16b6eb10000, {0xc00225a540, 0x2656fb0}, 0x4064c5) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a540, 0x21}, 0x4e94914f0000, 0xc0029ab770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 92139 [semacquire, 29 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x80, 0xc0021dec60) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022457a0, 0x22}, 0xc005ca9c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004025980}, 0x0, 0xc005ca9e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588199af4e9d8c, {0x26bf478, 0xc004025980}, 0xffffb16b6eb10000, {0xc0022457a0, 0x2656fb0}, 0xc0023239b0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022457a0, 0x22}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 82129 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0028003c0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022457a0, 0x22}, 0xc000796c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0040252c0}, 0x0, 0xc000796e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058816caf0c3f1f, {0x26bf478, 0xc0040252c0}, 0xffffb16b6eb10000, {0xc0022457a0, 0x2656fb0}, 0xc002068ab0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022457a0, 0x22}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63418 [sync.Cond.Wait, 33 minutes]: | |
| sync.runtime_notifyListWait(0xc00016ee58, 0x1) | |
| runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc00052a7a0) | |
| sync/cond.go:56 +0x8c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc00016edc0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:509 +0x428 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:334 +0xfb | |
| goroutine 83888 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254000, 0x24}, 0xc00566ec38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc005d0dd40}, 0x0, 0xc00566ee88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881742f199ee2, {0x26bf478, 0xc005d0dd40}, 0xffffb16b6eb10000, {0xc002254000, 0x2656fb0}, 0xc003c32f60) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254000, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 80994 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0021dcb10, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022558c0, 0x23}, 0xc004cccc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0003f5d40}, 0x0, 0xc004ccce88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881652efe4e7c, {0x26bf478, 0xc0003f5d40}, 0xffffb16b6eb10000, {0xc0022558c0, 0x2656fb0}, 0xc0003402a0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022558c0, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 78995 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004b03950, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002255ce0, 0x27}, 0xc004cc6c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00267f4a0}, 0x0, 0xc004cc6e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058815daf01a20b, {0x26bf478, 0xc00267f4a0}, 0xffffb16b6eb10000, {0xc002255ce0, 0x2656fb0}, 0x12ec4eb) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002255ce0, 0x27}, 0x4e94914f0000, 0xc004e06f70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 83880 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0x109, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244f60, 0x25}, 0xc0007c7c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc005d0daa0}, 0x0, 0xc0007c7e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881742f1924bc, {0x26bf478, 0xc005d0daa0}, 0xffffb16b6eb10000, {0xc002244f60, 0x2656fb0}, 0xc005d9a8a0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244f60, 0x25}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 78795 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x80, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc002678ed0}, {0xc002248720, 0x1d}, 0x7f9483210758) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002df8280, 0xc00228e4f8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc002678ed0}, 0xc001d00200) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc002678ed0}, 0x7f94aa18c1d8, {0x22c1ddb, 0xc0006b9c00}, {0x22b51cb, 0x0}, 0x0, 0xc0037a5560, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc002678ed0}, 0xc00168aa90, {0xc003be9300, 0x40ce34}, 0xc002678e40) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc00141c690, {0x26ef130, 0xc003918ab0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc00228e4e0}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002764300}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002764300}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002764300}, 0x1e4d1c0, 0xc003918aa0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc001210780}, 0xc004125e60, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc001210780}, 0xc004125e60, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 63345 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc00157d840, 0xc0038f4eb0) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 63596 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483337800, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00062c480, 0xc001e0a000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00062c480, {0xc001e0a000, 0x4000, 0x4000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00062c480, {0xc001e0a000, 0xc0026c4c88, 0xc0026c4c28}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00277e190, {0xc001e0a000, 0x42c94c, 0x2}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc004f06c00, {0xc0006452c0, 0x4, 0x112268e}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc004f06c00}, {0xc0006452c0, 0x4, 0x4}, 0x4) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| vitess.io/vitess/go/mysql.(*Conn).readHeaderFrom(0xc000645200, {0x2673340, 0xc004f06c00}) | |
| vitess.io/vitess/go/mysql/conn.go:334 +0x4f | |
| vitess.io/vitess/go/mysql.(*Conn).readOnePacket(0x447fa8) | |
| vitess.io/vitess/go/mysql/conn.go:473 +0x6d | |
| vitess.io/vitess/go/mysql.(*Conn).readPacket(0x11fe3b1311f97) | |
| vitess.io/vitess/go/mysql/conn.go:495 +0x25 | |
| vitess.io/vitess/go/mysql.(*Conn).ReadPacket(0xc001ba6680) | |
| vitess.io/vitess/go/mysql/conn.go:531 +0x1d | |
| vitess.io/vitess/go/mysql.mysqlFlavor.readBinlogEvent({}, 0x0) | |
| vitess.io/vitess/go/mysql/flavor_mysql.go:225 +0x1d | |
| vitess.io/vitess/go/mysql.(*Conn).ReadBinlogEvent(...) | |
| vitess.io/vitess/go/mysql/flavor.go:237 | |
| vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents.func1() | |
| vitess.io/vitess/go/vt/binlog/binlog_connection.go:146 +0xc5 | |
| created by vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents | |
| vitess.io/vitess/go/vt/binlog/binlog_connection.go:140 +0xdd | |
| goroutine 79064 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x270c0005002200, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002255080, 0x24}, 0xc003b50c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00417c6c0}, 0x0, 0xc003b50e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058815daef0bc89, {0x26bf478, 0xc00417c6c0}, 0xffffb16b6eb10000, {0xc002255080, 0x2656fb0}, 0x4064c5) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002255080, 0x24}, 0x4e94914f0000, 0xc0029ab770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 100054 [semacquire, 26 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a540, 0x21}, 0xc005edcc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004a8a600}, 0x0, 0xc005edce88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881c6af834479, {0x26bf478, 0xc004a8a600}, 0xffffb16b6eb10000, {0xc00225a540, 0x2656fb0}, 0xc002b16e40) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a540, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63362 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc00157dd80, 0xc0038f53a0) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 82101 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004d44f30, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022489e0, 0x1d}, 0xc004c6fc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00471f800}, 0x0, 0xc004c6fe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058816caf0a848b, {0x26bf478, 0xc00471f800}, 0xffffb16b6eb10000, {0xc0022489e0, 0x2656fb0}, 0xc0046e2e70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022489e0, 0x1d}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63417 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646e40, 0xc0018362d0) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 97240 [semacquire, 27 minutes]: | |
| sync.runtime_SemacquireMutex(0xc002801b90, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244300, 0x21}, 0xc0055b8c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002dc7320}, 0x0, 0xc0055b8e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881b02f690eda, {0x26bf478, 0xc002dc7320}, 0xffffb16b6eb10000, {0xc002244300, 0x2656fb0}, 0xc002678630) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244300, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63752 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents(0xc00024eb60, {0x26bf408, 0xc00226b740}, 0xc0017ad920) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:309 +0x305 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).replicate(0xc00024eb60, {0x2c1, 0xc0000ad8c0}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:178 +0x125 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).Stream(0xc00024eb60) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:157 +0x159 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream(0xc0013e37a0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:379 +0x2e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream(0xc0000e06c0, {0x26bf408, 0xc0007aa600}, {0x22b658d, 0x216e900}, {0x0, 0xc00166af68, 0x12f6f44}, 0x0, 0xc003ee9f80) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:227 +0x1bd | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream(0xc001173cc0, {0x26bf408, 0xc0007aa600}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:648 +0x133 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream(0xc001173cc0, {0x26bf408, 0xc0007aa600}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:624 +0x70 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).startVStream | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:610 +0x168 | |
| goroutine 88738 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x109, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225b1a0, 0x28}, 0xc0037f7c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004b82000}, 0x0, 0xc0037f7e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058818aaf34fe9d, {0x26bf478, 0xc004b82000}, 0xffffb16b6eb10000, {0xc00225b1a0, 0x2656fb0}, 0xc004af3b60) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225b1a0, 0x28}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 66354 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x2656fb0, 0x30, 0x1211fac) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc0004ada40, {0xc002244720, 0x25}, {0xc0039b8740, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:153 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1059 +0x38 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004395260}, 0x0, 0xc004c63de0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0x26bf4b0, {0x26bf478, 0xc004395260}, 0x2656fb0, {0xc002244720, 0x2656fb0}, {0xc0039b8740, 0x0, 0x7f94aa18b878}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1058 +0x8d | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc001173680, {0x26ef630, 0xc0002fd860}, {0xc002244720, 0x25}, 0xc0005d8618, {0xc0039b8740, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:596 +0x14e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc001173680, 0xc00477ff40, 0xc0043951a0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 84716 [chan receive, 31 minutes]: | |
| vitess.io/vitess/go/sync2.(*Semaphore).Acquire(0xc0019e96c0) | |
| vitess.io/vitess/go/sync2/semaphore.go:52 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc001173cc0, {0x26ef630, 0xc0002fd860}, {0xc00225ad80, 0x24}, 0xc002212f40, {0xc001f574a0, 0xa, 0xa}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:589 +0x85 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc001173cc0, 0xc001ded060, 0xc00554aa80) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 63864 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:399 +0x31 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:398 +0x3b3 | |
| goroutine 63742 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents.func2() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:283 +0xf0 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:270 +0x1f1 | |
| goroutine 63819 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:399 +0x31 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:398 +0x3b3 | |
| goroutine 79058 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x202c223030323131, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022457a0, 0x22}, 0xc003b35c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00417c000}, 0x0, 0xc003b35e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058815daeefabcd, {0x26bf478, 0xc00417c000}, 0xffffb16b6eb10000, {0xc0022457a0, 0x2656fb0}, 0xc001c714c8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022457a0, 0x22}, 0x4e94914f0000, 0xc0038f5040) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 66140 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x2656fb0, 0x30, 0x1211fac) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc0004ada40, {0xc002244f60, 0x25}, {0xc00147dea0, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:153 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1059 +0x38 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0042a2ea0}, 0x0, 0xc001961de0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0x26bf4b0, {0x26bf478, 0xc0042a2ea0}, 0x2656fb0, {0xc002244f60, 0x2656fb0}, {0xc00147dea0, 0x0, 0x7f94aa18aa68}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1058 +0x8d | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc00016e780, {0x26ef630, 0xc0002fd860}, {0xc002244f60, 0x25}, 0xc00062fb60, {0xc00147dea0, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:596 +0x14e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc00016e780, 0xc0035e4c60, 0xc0042a2de0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 67340 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x2656fb0, 0x30, 0x1211fac) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc0004ada40, {0xc002244720, 0x25}, {0xc003a682d0, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:153 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1059 +0x38 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00047b7a0}, 0x0, 0xc001b68de0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0x26bf4b0, {0x26bf478, 0xc00047b7a0}, 0x2656fb0, {0xc002244720, 0x2656fb0}, {0xc003a682d0, 0x0, 0x7f94aa18a5b8}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1058 +0x8d | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc001173680, {0x26ef630, 0xc0002fd860}, {0xc002244720, 0x25}, 0x26bf408, {0xc003a682d0, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:596 +0x14e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc001173680, 0xc00305d7b0, 0xc00047b560) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 80989 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x2656fb0, 0x30, 0x1211fac) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc0004ada40, {0xc0022489e0, 0x1d}, {0xc003004780, 0xa, 0xa}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:153 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1059 +0x38 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0048974a0}, 0x0, 0xc00303fde0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0x26bf4b0, {0x26bf478, 0xc0048974a0}, 0x2656fb0, {0xc0022489e0, 0x2656fb0}, {0xc003004780, 0x0, 0x7f94aa18c1d8}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1058 +0x8d | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc00016eb40, {0x26ef630, 0xc0002fd860}, {0xc0022489e0, 0x1d}, 0x26bf478, {0xc003004780, 0xa, 0xa}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:596 +0x14e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc00016eb40, 0xc001ded5d0, 0xc004896840) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 64650 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents(0xc001c461c0, {0x26bf408, 0xc005f06880}, 0xc002b0f2c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:309 +0x305 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).replicate(0xc001c461c0, {0x2c1, 0xc00318fd40}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:178 +0x125 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).Stream(0xc001c461c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:157 +0x159 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream(0xc0006b2900) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:379 +0x2e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream(0xc0000e06c0, {0x26bf408, 0xc0027e3bc0}, {0x22b658d, 0x216e900}, {0x0, 0xc00183ff68, 0x12f6f44}, 0x0, 0xc004657fb0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:227 +0x1bd | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream(0xc001173040, {0x26bf408, 0xc0027e3bc0}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:648 +0x133 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream(0xc001173040, {0x26bf408, 0xc0027e3bc0}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:624 +0x70 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).startVStream | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:610 +0x168 | |
| goroutine 64890 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002327d00, 0xc00484a738) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x265 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc004d23d10}, 0xc002935500) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc004d23d10}, 0x7f94aa18bd28, {0x22c1ddb, 0xc003f24400}, {0x22b51cb, 0x0}, 0x0, 0xc0031943c0, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc004d23d10}, 0xc001b66a90, {0xc00400e240, 0x40ce34}, 0xc004d23c20) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc002dbd680, {0x26ef130, 0xc0040088c0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc00484a720}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022cf2c0}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022cf2c0}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022cf2c0}, 0x1e4d1c0, 0xc0040088b0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0006fa600}, 0xc002e62c60, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0006fa600}, 0xc002e62c60, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 63216 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc00157d800, 0xc0038f4e70) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 92987 [semacquire, 28 minutes]: | |
| sync.runtime_SemacquireMutex(0x3a37303565653565, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022481a0, 0x1f}, 0xc0031d0c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0043752c0}, 0x0, 0xc0031d0e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881a12f5cd59d, {0x26bf478, 0xc0043752c0}, 0xffffb16b6eb10000, {0xc0022481a0, 0x2656fb0}, 0xc003a63a10) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022481a0, 0x1f}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 79003 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x2656fb0, 0x30, 0x1211fac) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc0004ada40, {0xc0022489e0, 0x1d}, {0xc003b04dc0, 0xa, 0xa}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:153 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1059 +0x38 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00417d080}, 0x0, 0xc003b5ede0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0x26bf4b0, {0x26bf478, 0xc00417d080}, 0x2656fb0, {0xc0022489e0, 0x2656fb0}, {0xc003b04dc0, 0x0, 0x7f94aa18c1d8}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1058 +0x8d | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc00016eb40, {0x26ef630, 0xc0002fd860}, {0xc0022489e0, 0x1d}, 0x26bf408, {0xc003b04dc0, 0xa, 0xa}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:596 +0x14e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc00016eb40, 0xc002357d50, 0xc0031110e0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 1048 [select, 51 minutes]: | |
| vitess.io/vitess/go/vt/topo/etcd2topo.(*Server).Watch.func1() | |
| vitess.io/vitess/go/vt/topo/etcd2topo/watch.go:78 +0x1af | |
| created by vitess.io/vitess/go/vt/topo/etcd2topo.(*Server).Watch | |
| vitess.io/vitess/go/vt/topo/etcd2topo/watch.go:72 +0x4ef | |
| goroutine 102459 [semacquire, 24 minutes]: | |
| sync.runtime_SemacquireMutex(0x617300a580a998f, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254c60, 0x27}, 0xc004946c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00510c180}, 0x0, 0xc004946e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881dd2f9e5737, {0x26bf478, 0xc00510c180}, 0xffffb16b6eb10000, {0xc002254c60, 0x2656fb0}, 0xc002ad8d50) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254c60, 0x27}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 79002 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x2656fb0, 0x30, 0x1211fac) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc0004ada40, {0xc0022489e0, 0x1d}, {0xc003368820, 0xa, 0xa}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:153 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1059 +0x38 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003111b60}, 0x0, 0xc003b4ede0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0x26bf4b0, {0x26bf478, 0xc003111b60}, 0x2656fb0, {0xc0022489e0, 0x2656fb0}, {0xc003368820, 0x0, 0x7f94aa18aa68}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1058 +0x8d | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc00016eb40, {0x26ef630, 0xc0002fd860}, {0xc0022489e0, 0x1d}, 0x26bf408, {0xc003368820, 0xa, 0xa}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:596 +0x14e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc00016eb40, 0xc0021f71c0, 0xc003111020) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 93001 [semacquire, 28 minutes]: | |
| sync.runtime_SemacquireMutex(0x109, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a120, 0x2c}, 0xc0048abc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004375aa0}, 0x0, 0xc0048abe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881a12f5e0963, {0x26bf478, 0xc004375aa0}, 0xffffb16b6eb10000, {0xc00225a120, 0x2656fb0}, 0xc00062b410) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a120, 0x2c}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 90323 [semacquire, 29 minutes]: | |
| sync.runtime_SemacquireMutex(0x6972547473726966, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225ad80, 0x24}, 0xc00079fc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0042a3080}, 0x0, 0xc00079fe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881922f3fe62d, {0x26bf478, 0xc0042a3080}, 0xffffb16b6eb10000, {0xc00225ad80, 0x2656fb0}, 0xc0038efb90) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225ad80, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 87269 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x80, 0xc005d0c000) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002245380, 0x23}, 0xc004dcfc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0051950e0}, 0x0, 0xc004dcfe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881832f319e0f, {0x26bf478, 0xc0051950e0}, 0xffffb16b6eb10000, {0xc002245380, 0x2656fb0}, 0xc0046572f0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002245380, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 66123 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x2656fb0, 0x30, 0x1211fac) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc0004ada40, {0xc002244300, 0x21}, {0xc002f45d70, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:153 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1059 +0x38 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0042f09c0}, 0x0, 0xc004cc7de0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0x26bf4b0, {0x26bf478, 0xc0042f09c0}, 0x2656fb0, {0xc002244300, 0x2656fb0}, {0xc002f45d70, 0x0, 0x7f94aa18a108}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1058 +0x8d | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc001172500, {0x26ef630, 0xc0002fd860}, {0xc002244300, 0x21}, 0x26bf408, {0xc002f45d70, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:596 +0x14e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc001172500, 0xc00318fab0, 0xc0042f0900) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 101692 [semacquire, 25 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0028006f0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002245380, 0x23}, 0xc0054e5c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002892660}, 0x0, 0xc0054e5e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881d5af8ef947, {0x26bf478, 0xc002892660}, 0xffffb16b6eb10000, {0xc002245380, 0x2656fb0}, 0xc004275590) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002245380, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 1049 [chan receive, 51 minutes]: | |
| vitess.io/vitess/go/vt/topo.(*Server).WatchSrvVSchema.func1() | |
| vitess.io/vitess/go/vt/topo/srv_vschema.go:74 +0xbe | |
| created by vitess.io/vitess/go/vt/topo.(*Server).WatchSrvVSchema | |
| vitess.io/vitess/go/vt/topo/srv_vschema.go:71 +0x1ef | |
| goroutine 63428 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0006471a0, 0xc001836600) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 923287 [semacquire, 10 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x80, 0xc00417d4a0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254420, 0x28}, 0xc003171c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002cc67e0}, 0x0, 0xc003171e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05882b6b07ad13e, {0x26bf478, 0xc002cc67e0}, 0xffffb16b6eb10000, {0xc002254420, 0x2656fb0}, 0xc002e28be8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254420, 0x28}, 0x4e94914f0000, 0xc00327cf70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 323029 [semacquire, 17 minutes]: | |
| sync.runtime_SemacquireMutex(0x4c5153794d02a51a, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225ad80, 0x24}, 0xc005791c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003dd1f20}, 0x0, 0xc005791e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058824630315e47, {0x26bf478, 0xc003dd1f20}, 0xffffb16b6eb10000, {0xc00225ad80, 0x2656fb0}, 0xc0043b6030) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225ad80, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 79446 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x0, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc003bc5da0}, {0xc002248460, 0x19}, 0x7f948330bc48) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00413a600, 0xc0042231e8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc003bc5da0}, 0xc00384c500) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc003bc5da0}, 0x7f94aa18af18, {0x22c1ddb, 0xc000680000}, {0x22b51cb, 0x0}, 0x0, 0xc0039567e0, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc003bc5da0}, 0xc004c6ca90, {0xc0040ba280, 0x40ce34}, 0xc003bc5d10) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc0042094a0, {0x26ef130, 0xc0034d6a10}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0042231d0}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc001424600}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc001424600}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc001424600}, 0x1e4d1c0, 0xc0034d6a00) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0006fa000}, 0xc00167ea20, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0006fa000}, 0xc00167ea20, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 63385 [sync.Cond.Wait, 33 minutes]: | |
| sync.runtime_notifyListWait(0xc0011735d8, 0x2) | |
| runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xea2a01) | |
| sync/cond.go:56 +0x8c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc001173540) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:509 +0x428 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:334 +0xfb | |
| goroutine 911082 [semacquire, 11 minutes]: | |
| sync.runtime_SemacquireMutex(0x203000, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a540, 0x21}, 0xc005271c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0006189c0}, 0x0, 0xc005271e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05882a7b0652d0f, {0x26bf478, 0xc0006189c0}, 0xffffb16b6eb10000, {0xc00225a540, 0x2656fb0}, 0xc00455ca80) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a540, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 88739 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x2816, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254000, 0x24}, 0xc0036f2c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004b820c0}, 0x0, 0xc0036f2e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058818aaf351031, {0x26bf478, 0xc004b820c0}, 0xffffb16b6eb10000, {0xc002254000, 0x2656fb0}, 0xc004af3b90) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254000, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 101533 [semacquire, 25 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004624330, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223afc0, 0x35}, 0xc00414cc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002ddc540}, 0x0, 0xc00414ce88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881d5af8ddef9, {0x26bf478, 0xc002ddc540}, 0xffffb16b6eb10000, {0xc00223afc0, 0x2656fb0}, 0xc004c00570) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223afc0, 0x35}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 88576 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x44d072, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002255ce0, 0x27}, 0xc004a4cc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00395eb40}, 0x0, 0xc004a4ce88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058818aaf329289, {0x26bf478, 0xc00395eb40}, 0xffffb16b6eb10000, {0xc002255ce0, 0x2656fb0}, 0xc004ab7e60) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002255ce0, 0x27}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63843 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:399 +0x31 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:398 +0x3b3 | |
| goroutine 63890 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00206cf60, 0x2f, 0x1) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc001172aec) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).stopVStream(0xc001172a00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:614 +0x58 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).unsubscribe(0xc001172a00, 0x103816a) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:423 +0x1d3 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:400 +0x45 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:398 +0x3b3 | |
| goroutine 83877 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0x2816, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254c60, 0x27}, 0xc0007c4c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc005d0d800}, 0x0, 0xc0007c4e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881742f18dc32, {0x26bf478, 0xc005d0d800}, 0xffffb16b6eb10000, {0xc002254c60, 0x2656fb0}, 0xc005d9a810) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254c60, 0x27}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 82060 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0047edf20, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002245bc0, 0x26}, 0xc00311ec38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00471eb40}, 0x0, 0xc00311ee88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058816caf09589f, {0x26bf478, 0xc00471eb40}, 0xffffb16b6eb10000, {0xc002245bc0, 0x2656fb0}, 0xc0046e2ea0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002245bc0, 0x26}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 107231 [semacquire, 22 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022554a0, 0x28}, 0xc0030cac38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002225020}, 0x0, 0xc0030cae88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588202afdb1e9b, {0x26bf478, 0xc002225020}, 0xffffb16b6eb10000, {0xc0022554a0, 0x2656fb0}, 0xc003ff41b0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022554a0, 0x28}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63339 [chan receive]: | |
| vitess.io/vitess/go/stats.(*Rates).track(0xc004ba2660) | |
| vitess.io/vitess/go/stats/rates.go:99 +0x36 | |
| created by vitess.io/vitess/go/stats.NewRates | |
| vitess.io/vitess/go/stats/rates.go:91 +0x22f | |
| goroutine 87286 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x312d313a34353133, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254840, 0x23}, 0xc004a43c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003edb080}, 0x0, 0xc004a43e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881832f31474d, {0x26bf478, 0xc003edb080}, 0xffffb16b6eb10000, {0xc002254840, 0x2656fb0}, 0xc004656f90) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254840, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 83805 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0048a9620, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002248460, 0x19}, 0xc003b58c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002d0b500}, 0x0, 0xc003b58e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881742f18f9ce, {0x26bf478, 0xc002d0b500}, 0xffffb16b6eb10000, {0xc002248460, 0x2656fb0}, 0xc005d9a570) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002248460, 0x19}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 90329 [semacquire, 29 minutes]: | |
| sync.runtime_SemacquireMutex(0x109, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225b1a0, 0x28}, 0xc0007e5c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0042a3680}, 0x0, 0xc0007e5e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881922f40e857, {0x26bf478, 0xc0042a3680}, 0xffffb16b6eb10000, {0xc00225b1a0, 0x2656fb0}, 0xc0038efcb0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225b1a0, 0x28}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 64625 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002f14d80, 0xc004617968) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x265 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc0005ca870}, 0xc00026f600) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc0005ca870}, 0x7f94aa18b3c8, {0x22c1ddb, 0xc00007c800}, {0x22b51cb, 0x0}, 0x0, 0xc003915d40, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc0005ca870}, 0xc001b37a90, {0xc0016d6e70, 0x40ce34}, 0xc0005ca6f0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc0026e9630, {0x26ef130, 0xc00441acd0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc004617950}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba600}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba600}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba600}, 0x1e4d1c0, 0xc00441acc0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0024d4000}, 0xc00297cea0, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0024d4000}, 0xc00297cea0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 82145 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0xc001383200, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022481a0, 0x1f}, 0xc003e0dc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003668fc0}, 0x0, 0xc003e0de88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058816caf0cc790, {0x26bf478, 0xc003668fc0}, 0xffffb16b6eb10000, {0xc0022481a0, 0x2656fb0}, 0xc002068750) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022481a0, 0x1f}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 82131 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a120, 0x2c}, 0xc004cc3c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004025440}, 0x0, 0xc004cc3e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058816caf0c70a7, {0x26bf478, 0xc004025440}, 0xffffb16b6eb10000, {0xc00225a120, 0x2656fb0}, 0xc002068720) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a120, 0x2c}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 79044 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0xc003e4d440, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002245380, 0x23}, 0xc004c6dc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0014520c0}, 0x0, 0xc004c6de88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058815daeeffe7f, {0x26bf478, 0xc0014520c0}, 0xffffb16b6eb10000, {0xc002245380, 0x2656fb0}, 0x25) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002245380, 0x23}, 0x4e94914f0000, 0xc003e73770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 88728 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x44d072, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a120, 0x2c}, 0xc0049efc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc001721680}, 0x0, 0xc0049efe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058818aaf340e61, {0x26bf478, 0xc001721680}, 0xffffb16b6eb10000, {0xc00225a120, 0x2656fb0}, 0xc004af3920) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a120, 0x2c}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 87267 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x80, 0xc0048790e0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244300, 0x21}, 0xc004dcdc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc005194f60}, 0x0, 0xc004dcde88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881832f314d1f, {0x26bf478, 0xc005194f60}, 0xffffb16b6eb10000, {0xc002244300, 0x2656fb0}, 0xc004657080) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244300, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 88736 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x746e756f432e6663, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223afc0, 0x35}, 0xc0036f7c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc001721e60}, 0x0, 0xc0036f7e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058818aaf34d4bd, {0x26bf478, 0xc001721e60}, 0xffffb16b6eb10000, {0xc00223afc0, 0x2656fb0}, 0xc004af3aa0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223afc0, 0x35}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63889 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x8, 0xde, 0x0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc001172aec) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream.func1({0xc00225d5c0, 0x2, 0x98}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:649 +0xa8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.newUVStreamer.func1({0xc00225d5c0, 0xc001319cb0, 0x2}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:101 +0x5b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).sendEventsForCurrentPos(0xc0007acfc0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:305 +0x115 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).setStreamStartPosition(0xc0007acfc0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:318 +0x9f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).init(0xc0007acfc0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:346 +0x29 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream(0xc0007acfc0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:364 +0x73 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream(0xc0000e06c0, {0x26bf408, 0xc00226abc0}, {0x22b658d, 0x216e900}, {0x0, 0xc001d1a768, 0x12f6f44}, 0x0, 0xc00437e690) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:227 +0x1bd | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream(0xc001172a00, {0x26bf408, 0xc00226abc0}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:648 +0x133 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream(0xc001172a00, {0x26bf408, 0xc00226abc0}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:624 +0x70 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).startVStream | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:610 +0x168 | |
| goroutine 65021 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc003b76000, 0xc001b639f8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x265 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc00494aa20}, 0xc001b14300) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc00494aa20}, 0x7f94aa18b878, {0x22c1ddb, 0xc000580000}, {0x22b51cb, 0x0}, 0x0, 0xc004194900, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc00494aa20}, 0xc001723a90, {0xc003d6d710, 0x40ce34}, 0xc00494a930) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc0048fc1e0, {0x26ef130, 0xc004192690}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc001b639e0}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002764c00}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002764c00}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002764c00}, 0x1e4d1c0, 0xc004192680) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0006fa600}, 0xc002838120, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0006fa600}, 0xc002838120, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 100058 [semacquire, 26 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022558c0, 0x23}, 0xc005ed8c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004a8aa80}, 0x0, 0xc005ed8e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881c6af83c6a1, {0x26bf478, 0xc004a8aa80}, 0xffffb16b6eb10000, {0xc0022558c0, 0x2656fb0}, 0xc002b16d80) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022558c0, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63383 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646420, 0xc0038f5990) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 65919 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x0, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc003d4d6b0}, {0xc00225b1a0, 0x28}, 0x7f94832f39a8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc001f68f00, 0xc0047b95a8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc003d4d6b0}, 0xc00171ee00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc003d4d6b0}, 0x7f94aa18bd28, {0x22c1ddb, 0xc000480000}, {0x22b51cb, 0x0}, 0x0, 0xc004201020, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc003d4d6b0}, 0xc001964a90, {0xc0029e9530, 0x40ce34}, 0xc003d4d5f0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc001c6a1e0, {0x26ef130, 0xc0013e59a0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0047b9590}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c6c0}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c6c0}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c6c0}, 0x1e4d1c0, 0xc0013e5990) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc001210480}, 0xc001b7f0e0, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc001210480}, 0xc001b7f0e0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 90286 [semacquire, 29 minutes]: | |
| sync.runtime_SemacquireMutex(0xc002322a20, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002255ce0, 0x27}, 0xc0049f1c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0042a2ba0}, 0x0, 0xc0049f1e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881922f3ee7c3, {0x26bf478, 0xc0042a2ba0}, 0xffffb16b6eb10000, {0xc002255ce0, 0x2656fb0}, 0xc000051830) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002255ce0, 0x27}, 0x4e94914f0000, 0xc00327e7b8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 78748 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x80, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc004d45da0}, {0xc0022489e0, 0x1d}, 0x7f948330be68) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc003341580, 0xc0028a4078) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc004d45da0}, 0xc0043ab500) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc004d45da0}, 0x7f94aa18bd28, {0x22c1ddb, 0x3b092c0}, {0x22b51cb, 0x0}, 0x0, 0xc00379aea0, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc004d45da0}, 0xc004c6aa90, {0xc00379cc20, 0x40ce34}, 0xc004d45d10) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc0048fc6e0, {0x26ef130, 0xc0034096c0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0028a4060}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022cecc0}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022cecc0}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022cecc0}, 0x1e4d1c0, 0xc0034096b0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc001210780}, 0xc001d686c0, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc001210780}, 0xc001d686c0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 65961 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x0, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc004656210}, {0xc0022558c0, 0x23}, 0x7f948312a9c8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00061e900, 0xc00415cd80) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc004656210}, 0xc00046f100) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc004656210}, 0x7f94aa18af18, {0x22c1ddb, 0xc000480000}, {0x22b51cb, 0x0}, 0x0, 0xc0045211a0, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc004656210}, 0xc001b38a90, {0xc002e30ea0, 0x40ce34}, 0xc004656180) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc004640460, {0x26ef130, 0xc0038f4d60}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc00415cd68}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022cf500}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022cf500}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022cf500}, 0x1e4d1c0, 0xc0038f4d50) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc001210480}, 0xc0026ebb00, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc001210480}, 0xc0026ebb00, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 90308 [semacquire, 29 minutes]: | |
| sync.runtime_SemacquireMutex(0x44d072, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002245bc0, 0x26}, 0xc004dd1c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0036e4720}, 0x0, 0xc004dd1e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881922f3e8c6f, {0x26bf478, 0xc0036e4720}, 0xffffb16b6eb10000, {0xc002245bc0, 0x2656fb0}, 0xc002a8c570) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002245bc0, 0x26}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 65918 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x0, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc003d4d170}, {0xc002255ce0, 0x27}, 0x7f94832f39a8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc001f68e00, 0xc0047b9440) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc003d4d170}, 0xc00171eb00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc003d4d170}, 0x7f94aa18bd28, {0x22c1ddb, 0xc00007c800}, {0x22b51cb, 0x0}, 0x0, 0xc004200f60, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc003d4d170}, 0xc00168ca90, {0xc0029e9500, 0x40ce34}, 0xc003d4d050) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc0028ebea0, {0x26ef130, 0xc0013e58b0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0047b9428}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c600}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c600}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c600}, 0x1e4d1c0, 0xc0013e58a0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc001210480}, 0xc001b7eea0, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc001210480}, 0xc001b7eea0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 65958 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x0, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc0000ddd70}, {0xc002244720, 0x25}, 0x7f948312a9c8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00061e600, 0xc00415cb28) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc0000ddd70}, 0xc00046ec00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc0000ddd70}, 0x7f94aa18af18, {0x22c1ddb, 0xc00007c800}, {0x22b51cb, 0x0}, 0x0, 0xc004520ea0, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc0000ddd70}, 0xc001b36a90, {0xc002e30e40, 0x40ce34}, 0xc0000ddc80) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc004640140, {0x26ef130, 0xc0038f4b60}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc00415cb10}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022cf440}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022cf440}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022cf440}, 0x1e4d1c0, 0xc0038f4b50) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc001210480}, 0xc0026eb7a0, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc001210480}, 0xc0026eb7a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 63625 [select, 33 minutes]: | |
| go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc0034b8a90, 0xc0000bd080, 0xc000649f20) | |
| go.etcd.io/etcd/client/v3@v3.5.0/watch.go:803 +0x265 | |
| created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run | |
| go.etcd.io/etcd/client/v3@v3.5.0/watch.go:554 +0xed1 | |
| goroutine 96656 [semacquire, 28 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0038eed20, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244300, 0x21}, 0xc003c20c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002cad4a0}, 0x0, 0xc003c20e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881a8af6451a8, {0x26bf478, 0xc002cad4a0}, 0xffffb16b6eb10000, {0xc002244300, 0x2656fb0}, 0xc0048a7050) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244300, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 85375 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0007beb70, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254840, 0x23}, 0xc0007bec38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002bfede0}, 0x0, 0xc0007bee88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058817baf2837a0, {0x26bf478, 0xc002bfede0}, 0xffffb16b6eb10000, {0xc002254840, 0x2656fb0}, 0x12ec4eb) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254840, 0x23}, 0x4e94914f0000, 0xc001df0f70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 79004 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x2656fb0, 0x30, 0x1211fac) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc0004ada40, {0xc0022489e0, 0x1d}, {0xc003a930e0, 0xa, 0xa}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:153 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1059 +0x38 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003b56780}, 0x0, 0xc000792de0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0x26bf4b0, {0x26bf478, 0xc003b56780}, 0x2656fb0, {0xc0022489e0, 0x2656fb0}, {0xc003a930e0, 0x0, 0x7f94aa18b878}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1058 +0x8d | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc00016eb40, {0x26ef630, 0xc0002fd860}, {0xc0022489e0, 0x1d}, 0x26bf478, {0xc003a930e0, 0xa, 0xa}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:596 +0x14e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc00016eb40, 0xc001decf90, 0xc0031111a0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 63399 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0006468e0, 0xc0038f5db0) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 111214 [semacquire, 19 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254c60, 0x27}, 0xc003daec38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003000d20}, 0x0, 0xc003daee88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05882283022f5ba, {0x26bf478, 0xc003000d20}, 0xffffb16b6eb10000, {0xc002254c60, 0x2656fb0}, 0xc000051830) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254c60, 0x27}, 0x4e94914f0000, 0xc003280770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63407 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646b80, 0xc001836030) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 66028 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x80, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc00481a2a0}, {0xc002244300, 0x21}, 0x7f948305b158) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002ad1c80, 0xc004140648) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc00481a2a0}, 0xc002d0fa00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc00481a2a0}, 0x7f94aa18b3c8, {0x22c1ddb, 0xc000480000}, {0x22b51cb, 0x0}, 0x0, 0xc001b6c3c0, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc00481a2a0}, 0xc0038cea90, {0xc002ec8a20, 0x40ce34}, 0xc00481a210) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc00166e0a0, {0x26ef130, 0xc003913470}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc004140630}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237ed80}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237ed80}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237ed80}, 0x1e4d1c0, 0xc003913460) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc001210480}, 0xc0006b39e0, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc001210480}, 0xc0006b39e0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 88727 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004d45d10, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a960, 0x26}, 0xc0049eec38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0017215c0}, 0x0, 0xc0049eee88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058818aaf33f741, {0x26bf478, 0xc0017215c0}, 0xffffb16b6eb10000, {0xc00225a960, 0x2656fb0}, 0xc004af38f0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a960, 0x26}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63623 [select, 33 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc@v1.39.0/stream.go:363 +0x98 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc@v1.39.0/stream.go:362 +0x1066 | |
| goroutine 66207 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x2656fb0, 0x30, 0x1211fac) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc0004ada40, {0xc002244b40, 0x21}, {0xc002f842a0, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:153 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1059 +0x38 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002c3d3e0}, 0x0, 0xc001688de0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0x26bf4b0, {0x26bf478, 0xc002c3d3e0}, 0x2656fb0, {0xc002244b40, 0x2656fb0}, {0xc002f842a0, 0x0, 0x7f94aa18c1d8}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1058 +0x8d | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc001173e00, {0x26ef630, 0xc0002fd860}, {0xc002244b40, 0x21}, 0xc0005d8618, {0xc002f842a0, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:596 +0x14e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc001173e00, 0xc00441ad60, 0xc002c3d320) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 65026 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:399 +0x31 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:398 +0x3b3 | |
| goroutine 85439 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004267020, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225ad80, 0x24}, 0xc0007efc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00298f2c0}, 0x0, 0xc0007efe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058817baf266736, {0x26bf478, 0xc00298f2c0}, 0xffffb16b6eb10000, {0xc00225ad80, 0x2656fb0}, 0xc00427b800) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225ad80, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 81058 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x2656fb0, 0x30, 0x1211fac) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc0004ada40, {0xc0022489e0, 0x1d}, {0xc0030046e0, 0xa, 0xa}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:153 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1059 +0x38 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004897080}, 0x0, 0xc00303ede0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0x26bf4b0, {0x26bf478, 0xc004897080}, 0x2656fb0, {0xc0022489e0, 0x2656fb0}, {0xc0030046e0, 0x0, 0x7f94aa18c1d8}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1058 +0x8d | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc00016eb40, {0x26ef630, 0xc0002fd860}, {0xc0022489e0, 0x1d}, 0x100000000000000, {0xc0030046e0, 0xa, 0xa}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:596 +0x14e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc00016eb40, 0xc0046102a0, 0xc004896c00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 81032 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004d23980, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254c60, 0x27}, 0xc004173c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0013fbc80}, 0x0, 0xc004173e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881652efed1a8, {0x26bf478, 0xc0013fbc80}, 0xffffb16b6eb10000, {0xc002254c60, 0x2656fb0}, 0xc003e4d680) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254c60, 0x27}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 904655 [semacquire, 11 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00476a870, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002255080, 0x24}, 0xc0057bec38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004cebe60}, 0x0, 0xc0057bee88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05882a0305f4465, {0x26bf478, 0xc004cebe60}, 0xffffb16b6eb10000, {0xc002255080, 0x2656fb0}, 0xc005b82540) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002255080, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 102455 [semacquire, 24 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004706090, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225b1a0, 0x28}, 0xc00439dc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00379be00}, 0x0, 0xc00439de88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881dd2f9daa7b, {0x26bf478, 0xc00379be00}, 0xffffb16b6eb10000, {0xc00225b1a0, 0x2656fb0}, 0xc002ad8de0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225b1a0, 0x28}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 65925 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x2656fb0, 0x30, 0x1211fac) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc0004ada40, {0xc002244f60, 0x25}, {0xc002f44590, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:153 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1059 +0x38 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0037dab40}, 0x0, 0xc003dd8de0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0x26bf4b0, {0x26bf478, 0xc0037dab40}, 0x2656fb0, {0xc002244f60, 0x2656fb0}, {0xc002f44590, 0x0, 0x7f94aa18a108}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1058 +0x8d | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc00016e780, {0x26ef630, 0xc0002fd860}, {0xc002244f60, 0x25}, 0x26bf408, {0xc002f44590, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:596 +0x14e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc00016e780, 0xc00441ac20, 0xc0037daa80) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 80997 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254000, 0x24}, 0xc003039c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002d48180}, 0x0, 0xc003039e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881652eff14cd, {0x26bf478, 0xc002d48180}, 0xffffb16b6eb10000, {0xc002254000, 0x2656fb0}, 0xc003e4d6b0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254000, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 64606 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002f14c80, 0xc004617788) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x265 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc003c33c50}, 0xc00026f200) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc003c33c50}, 0x7f94aa18b3c8, {0x22c1ddb, 0xc00007c800}, {0x22b51cb, 0x0}, 0x0, 0xc003915c20, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc003c33c50}, 0xc001b37a90, {0xc0016d6e40, 0x40ce34}, 0xc003c33bc0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc0026e93b0, {0x26ef130, 0xc00441ab90}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc004617770}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba540}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba540}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba540}, 0x1e4d1c0, 0xc00441ab80) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0003c7c80}, 0xc00297cd80, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0003c7c80}, 0xc00297cd80, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 63350 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc00157d900, 0xc0038f5040) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 64975 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents.func2() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:283 +0xf0 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:270 +0x1f1 | |
| goroutine 96660 [semacquire, 28 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254420, 0x28}, 0xc0055b3c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002cad9e0}, 0x0, 0xc0055b3e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881a8af6504f4, {0x26bf478, 0xc002cad9e0}, 0xffffb16b6eb10000, {0xc002254420, 0x2656fb0}, 0xc0048a7170) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254420, 0x28}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 104565 [semacquire, 23 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022489e0, 0x1d}, 0xc0048ddc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003017620}, 0x0, 0xc0048dde88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881f3afc4a8d8, {0x26bf478, 0xc003017620}, 0xffffb16b6eb10000, {0xc0022489e0, 0x2656fb0}, 0xc002a8c9c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022489e0, 0x1d}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 96659 [semacquire, 28 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254000, 0x24}, 0xc0055b2c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002cad920}, 0x0, 0xc0055b2e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881a8af64ee06, {0x26bf478, 0xc002cad920}, 0xffffb16b6eb10000, {0xc002254000, 0x2656fb0}, 0xc0048a7140) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254000, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 83847 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0x2816, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254420, 0x28}, 0xc0007bcc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00180f8c0}, 0x0, 0xc0007bce88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881742f2b5704, {0x26bf478, 0xc00180f8c0}, 0xffffb16b6eb10000, {0xc002254420, 0x2656fb0}, 0xc000051830) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254420, 0x28}, 0x4e94914f0000, 0xc001bf6770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 80996 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a120, 0x2c}, 0xc000797c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002d480c0}, 0x0, 0xc000797e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881652efec5f0, {0x26bf478, 0xc002d480c0}, 0xffffb16b6eb10000, {0xc00225a120, 0x2656fb0}, 0xc003e4d620) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a120, 0x2c}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 100461 [semacquire, 25 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x80, 0xc00116e540) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002248460, 0x19}, 0xc0031eec38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002ebcfc0}, 0x0, 0xc0031eee88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881ce2f8c9e2d, {0x26bf478, 0xc002ebcfc0}, 0xffffb16b6eb10000, {0xc002248460, 0x2656fb0}, 0xc004e88de0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002248460, 0x19}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 92950 [semacquire, 28 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002245380, 0x23}, 0xc0057ebc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004374c60}, 0x0, 0xc0057ebe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881a12f5b819d, {0x26bf478, 0xc004374c60}, 0xffffb16b6eb10000, {0xc002245380, 0x2656fb0}, 0xc0047337a0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002245380, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 90317 [semacquire, 29 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0044b49f0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223bbc0, 0x36}, 0xc0007c9c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0036e48a0}, 0x0, 0xc0007c9e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881922f3f2bd9, {0x26bf478, 0xc0036e48a0}, 0xffffb16b6eb10000, {0xc00223bbc0, 0x2656fb0}, 0xc004a8eb70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223bbc0, 0x36}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 90224 [semacquire, 29 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0039e3e60, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223b5c0, 0x35}, 0xc004ea6c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003444720}, 0x0, 0xc004ea6e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881922f3f56a9, {0x26bf478, 0xc003444720}, 0xffffb16b6eb10000, {0xc00223b5c0, 0x2656fb0}, 0xc001bf8f18) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223b5c0, 0x35}, 0x4e94914f0000, 0xc001bf8fb8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 92992 [semacquire, 28 minutes]: | |
| sync.runtime_SemacquireMutex(0x61502d7061476465, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002248720, 0x1d}, 0xc0048afc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0043756e0}, 0x0, 0xc0048afe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881a12f5d4b45, {0x26bf478, 0xc0043756e0}, 0xffffb16b6eb10000, {0xc002248720, 0x2656fb0}, 0xc003a63bf0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002248720, 0x1d}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 199760 [semacquire, 18 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254000, 0x24}, 0xc004599c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003bc9320}, 0x0, 0xc004599e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05882373029477d, {0x26bf478, 0xc003bc9320}, 0xffffb16b6eb10000, {0xc002254000, 0x2656fb0}, 0xc0054c9080) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254000, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63372 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0006460c0, 0xc0038f5640) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 63597 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents.func2() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:283 +0xf0 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:270 +0x1f1 | |
| goroutine 82105 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x44d072, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002245380, 0x23}, 0xc004c70c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00471f8c0}, 0x0, 0xc004c70e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058816caf0acad0, {0x26bf478, 0xc00471f8c0}, 0xffffb16b6eb10000, {0xc002245380, 0x2656fb0}, 0xc004976ff0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002245380, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63412 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00016e818, 0x8a) | |
| runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc000576780) | |
| sync/cond.go:56 +0x8c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc00016e780) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:509 +0x428 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:334 +0xfb | |
| goroutine 78996 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004274a20, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a960, 0x26}, 0xc004cc8c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00267f560}, 0x0, 0xc004cc8e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058815daf01ecbb, {0x26bf478, 0xc00267f560}, 0xffffb16b6eb10000, {0xc00225a960, 0x2656fb0}, 0x12ec4eb) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a960, 0x26}, 0x4e94914f0000, 0xc004e06f70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63613 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc001bc4080, 0xc00063b308) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x265 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc004bc76e0}, 0xc00026fd00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc004bc76e0}, 0x7f94aa18c1d8, {0x22c1ddb, 0xc000580000}, {0x22b51cb, 0x0}, 0x0, 0xc00388f440, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc004bc76e0}, 0xc00168ea90, {0xc0006a49a0, 0x40ce34}, 0xc004bc7650) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc001a59860, {0x26ef130, 0xc00434b950}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc00063b2f0}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022cef00}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022cef00}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022cef00}, 0x1e4d1c0, 0xc00434b940) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0003c7e00}, 0xc002219320, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0003c7e00}, 0xc002219320, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 84943 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x80, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc004f0b6e0}, {0xc0022489e0, 0x1d}, 0x7f948325d568) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc003dde880, 0xc003fe56c8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc004f0b6e0}, 0xc00384ce00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc004f0b6e0}, 0x7f94aa18b878, {0x22c1ddb, 0xc000480000}, {0x22b51cb, 0x0}, 0x0, 0xc0034f0ba0, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc004f0b6e0}, 0xc00592ba90, {0xc00576d980, 0x40ce34}, 0xc004f0b650) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc003b7eaa0, {0x26ef130, 0xc003409da0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc003fe56b0}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e900}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e900}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e900}, 0x1e4d1c0, 0xc003409d90) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0024d4180}, 0xc0007ad8c0, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0024d4180}, 0xc0007ad8c0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 63858 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002222600, 0xc001efca50) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x265 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc005d9a1e0}, 0xc001b75c00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc005d9a1e0}, 0x7f94aa18a5b8, {0x22c1ddb, 0xc000480000}, {0x22b51cb, 0x0}, 0x0, 0xc001385860, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc005d9a1e0}, 0xc004172a90, {0xc005992940, 0x40ce34}, 0xc005d9a150) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc00141c2d0, {0x26ef130, 0xc001dedc50}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc001efca38}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0001ecd80}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0001ecd80}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0001ecd80}, 0x1e4d1c0, 0xc001dedc40) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0000c4480}, 0xc001eec900, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0000c4480}, 0xc001eec900, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 80992 [chan receive, 32 minutes]: | |
| vitess.io/vitess/go/sync2.(*Semaphore).Acquire(0xc001c71a00) | |
| vitess.io/vitess/go/sync2/semaphore.go:52 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc00016eb40, {0x26ef630, 0xc0002fd860}, {0xc0022489e0, 0x1d}, 0x100000000000000, {0xc002bb2460, 0xa, 0xa}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:589 +0x85 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc00016eb40, 0xc00310cfd0, 0xc004896a80) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 65920 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x0, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc003d4dce0}, {0xc00225ad80, 0x24}, 0x7f94832f39a8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc001f69000, 0xc0047b9728) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc003d4dce0}, 0xc00171ef00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc003d4dce0}, 0x7f94aa18bd28, {0x22c1ddb, 0xc00007c800}, {0x22b51cb, 0x0}, 0x0, 0xc0042010e0, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc003d4dce0}, 0xc001965a90, {0xc0029e9560, 0x40ce34}, 0xc003d4db90) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc001c6a550, {0x26ef130, 0xc0013e5ae0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0047b9710}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c780}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c780}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c780}, 0x1e4d1c0, 0xc0013e5ad0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc001210480}, 0xc001b7f200, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc001210480}, 0xc001b7f200, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 603344 [semacquire, 14 minutes]: | |
| sync.runtime_SemacquireMutex(0x72705f7369202c64, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a960, 0x26}, 0xc0013ecc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002f20120}, 0x0, 0xc0013ece88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058826bb04cdd7b, {0x26bf478, 0xc002f20120}, 0xffffb16b6eb10000, {0xc00225a960, 0x2656fb0}, 0xc001d3d748) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a960, 0x26}, 0x4e94914f0000, 0x0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 1015875 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*relayLog).startTimer.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:124 +0x88 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*relayLog).startTimer | |
| vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:123 +0xb6 | |
| goroutine 107812 [semacquire, 21 minutes]: | |
| sync.runtime_SemacquireMutex(0x44d072, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244f60, 0x25}, 0xc00392ec38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003a26900}, 0x0, 0xc00392ee88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058820a2fdcf67d, {0x26bf478, 0xc003a26900}, 0xffffb16b6eb10000, {0xc002244f60, 0x2656fb0}, 0xc005989620) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244f60, 0x25}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 99070 [semacquire, 26 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x80, 0xc002d77980) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022558c0, 0x23}, 0xc00541fc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc005195a40}, 0x0, 0xc00541fe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881bf2f75c614, {0x26bf478, 0xc005195a40}, 0xffffb16b6eb10000, {0xc0022558c0, 0x2656fb0}, 0xc001c41170) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022558c0, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 99184 [semacquire, 26 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022489e0, 0x1d}, 0xc004132c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0049eb7a0}, 0x0, 0xc004132e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881bf2f763496, {0x26bf478, 0xc0049eb7a0}, 0xffffb16b6eb10000, {0xc0022489e0, 0x2656fb0}, 0xc000051830) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022489e0, 0x1d}, 0x4e94914f0000, 0xc00162cf70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63378 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646280, 0xc0038f57e0) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 108678 [semacquire, 21 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022489e0, 0x1d}, 0xc002f80c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002c3c240}, 0x0, 0xc002f80e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588211afefe0ec, {0x26bf478, 0xc002c3c240}, 0xffffb16b6eb10000, {0xc0022489e0, 0x2656fb0}, 0xc005d9bef0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022489e0, 0x1d}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63424 [sync.Cond.Wait, 33 minutes]: | |
| sync.runtime_notifyListWait(0xc00016f0d8, 0x2) | |
| runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0x2444894800000301) | |
| sync/cond.go:56 +0x8c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc00016f040) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:509 +0x428 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:334 +0xfb | |
| goroutine 80995 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0xc003ac2f30, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022481a0, 0x1f}, 0xc004c64c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002d48000}, 0x0, 0xc004c64e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881652efea669, {0x26bf478, 0xc002d48000}, 0xffffb16b6eb10000, {0xc0022481a0, 0x2656fb0}, 0xc003e4d5f0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022481a0, 0x1f}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 88700 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x31333136392d6530, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002248720, 0x1d}, 0xc00501cc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004896420}, 0x0, 0xc00501ce88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058818aaf32f76f, {0x26bf478, 0xc004896420}, 0xffffb16b6eb10000, {0xc002248720, 0x2656fb0}, 0xc000051830) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002248720, 0x1d}, 0x4e94914f0000, 0xc00156d770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 96673 [semacquire, 28 minutes]: | |
| sync.runtime_SemacquireMutex(0x6c616605746e756f, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254c60, 0x27}, 0xc002cd2c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00460d1a0}, 0x0, 0xc002cd2e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881a8af659266, {0x26bf478, 0xc00460d1a0}, 0xffffb16b6eb10000, {0xc002254c60, 0x2656fb0}, 0xc0049019e0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254c60, 0x27}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 64651 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:399 +0x31 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:398 +0x3b3 | |
| goroutine 82113 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x39, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223b5c0, 0x35}, 0xc004c8ec38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00471fc20}, 0x0, 0xc004c8ee88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058816caf0b9b2c, {0x26bf478, 0xc00471fc20}, 0xffffb16b6eb10000, {0xc00223b5c0, 0x2656fb0}, 0xc004b437a0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223b5c0, 0x35}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63356 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc00157dac0, 0xc0038f5200) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 79005 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x2656fb0, 0x30, 0x1211fac) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc0004ada40, {0xc0022489e0, 0x1d}, {0xc003bc34c0, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:153 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1059 +0x38 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0031112c0}, 0x0, 0xc00168bde0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0x26bf4b0, {0x26bf478, 0xc0031112c0}, 0x2656fb0, {0xc0022489e0, 0x2656fb0}, {0xc003bc34c0, 0x0, 0x7f94aa18aa68}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1058 +0x8d | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc00016eb40, {0x26ef630, 0xc0002fd860}, {0xc0022489e0, 0x1d}, 0x26bf408, {0xc003bc34c0, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:596 +0x14e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc00016eb40, 0xc001ded320, 0xc003111200) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 87282 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0xc001b4b350, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002255080, 0x24}, 0xc0050c5c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003edaba0}, 0x0, 0xc0050c5e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881832f307ecb, {0x26bf478, 0xc003edaba0}, 0xffffb16b6eb10000, {0xc002255080, 0x2656fb0}, 0xc004267380) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002255080, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 78928 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004030ba0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223b5c0, 0x35}, 0xc004c60c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00490b8c0}, 0x0, 0xc004c60e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058815daeef3f53, {0x26bf478, 0xc00490b8c0}, 0xffffb16b6eb10000, {0xc00223b5c0, 0x2656fb0}, 0x4000) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223b5c0, 0x35}, 0x4e94914f0000, 0xc00162bf70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63354 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc00157da00, 0xc0038f5120) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 108721 [semacquire, 21 minutes]: | |
| sync.runtime_SemacquireMutex(0xc003a63e00, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254c60, 0x27}, 0xc0030dcc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0037a47e0}, 0x0, 0xc0030dce88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588211aff0dcfe, {0x26bf478, 0xc0037a47e0}, 0xffffb16b6eb10000, {0xc002254c60, 0x2656fb0}, 0xc0047b57a0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254c60, 0x27}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 797263 [semacquire, 12 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0046c7c50, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244f60, 0x25}, 0xc003186c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc005e16180}, 0x0, 0xc003186e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588289b059a718, {0x26bf478, 0xc005e16180}, 0xffffb16b6eb10000, {0xc002244f60, 0x2656fb0}, 0xc004c35268) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244f60, 0x25}, 0x4e94914f0000, 0xc004e06f70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 64014 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002b1ba00, 0xc002145e48) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x265 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc001ac99e0}, 0xc001b14d00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc001ac99e0}, 0x7f94aa18af18, {0x22c1ddb, 0xc000480000}, {0x22b51cb, 0x0}, 0x0, 0xc0034d3bc0, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc001ac99e0}, 0xc001689a90, {0xc003954ea0, 0x40ce34}, 0xc001ac98f0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc001d6eaf0, {0x26ef130, 0xc0035e4bd0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc002145e30}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc001424840}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc001424840}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc001424840}, 0x1e4d1c0, 0xc0035e4bc0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc00066e000}, 0xc00231d0e0, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc00066e000}, 0xc00231d0e0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 107306 [semacquire, 22 minutes]: | |
| sync.runtime_SemacquireMutex(0xa, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022457a0, 0x22}, 0xc0030dac38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002d0b080}, 0x0, 0xc0030dae88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588202afde694d, {0x26bf478, 0xc002d0b080}, 0xffffb16b6eb10000, {0xc0022457a0, 0x2656fb0}, 0xc001ac80c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022457a0, 0x22}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 82107 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x50746f6e70616c72, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254c60, 0x27}, 0xc004c8dc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00471fa40}, 0x0, 0xc004c8de88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058816caf0af637, {0x26bf478, 0xc00471fa40}, 0xffffb16b6eb10000, {0xc002254c60, 0x2656fb0}, 0xc004976fc0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254c60, 0x27}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 81009 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002255080, 0x24}, 0xc00589ac38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0028c7f80}, 0x0, 0xc00589ae88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881652efce5a0, {0x26bf478, 0xc0028c7f80}, 0xffffb16b6eb10000, {0xc002255080, 0x2656fb0}, 0xc004c29f80) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002255080, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63624 [select, 33 minutes]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0006b0af0, {0xc00000c790, 0x5, 0x5}) | |
| google.golang.org/grpc@v1.39.0/internal/transport/transport.go:189 +0xaa | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0006b0af0, {0xc00000c790, 0xc00069e888, 0xc00283ba60}) | |
| google.golang.org/grpc@v1.39.0/internal/transport/transport.go:169 +0x147 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc004732ed0, {0xc00000c790, 0xc00283bad8, 0x9ac127}) | |
| google.golang.org/grpc@v1.39.0/internal/transport/transport.go:483 +0x32 | |
| io.ReadAtLeast({0x2676060, 0xc004732ed0}, {0xc00000c790, 0x5, 0x5}, 0x5) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001b7f9e0, {0xc00000c790, 0x5, 0x5}) | |
| google.golang.org/grpc@v1.39.0/internal/transport/transport.go:467 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc00000c780, 0x7fffffff) | |
| google.golang.org/grpc@v1.39.0/rpc_util.go:559 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0x3b3b8c8, 0xc001b7f9e0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) | |
| google.golang.org/grpc@v1.39.0/rpc_util.go:690 +0x66 | |
| google.golang.org/grpc.recv(0xc00156f4f0, {0x7f94833bc520, 0x3b3b8c8}, 0x2224480, {0x0, 0x0}, {0x2226aa0, 0xc000029c00}, 0x7f94833c1ac0, 0x0, ...) | |
| google.golang.org/grpc@v1.39.0/rpc_util.go:758 +0x6e | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc003341080, {0x2226aa0, 0xc000029c00}, 0x448b2f) | |
| google.golang.org/grpc@v1.39.0/stream.go:957 +0x2b0 | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x203000) | |
| google.golang.org/grpc@v1.39.0/stream.go:808 +0x25 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc001b7f7a0, 0xc00283beb0, 0xc00283be80) | |
| google.golang.org/grpc@v1.39.0/stream.go:662 +0x2f6 | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc001b7f7a0, {0x2226aa0, 0xc000029c00}) | |
| google.golang.org/grpc@v1.39.0/stream.go:807 +0x11f | |
| go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchClient).Recv(0xc001889a10) | |
| go.etcd.io/etcd/api/v3@v3.5.0/etcdserverpb/rpc.pb.go:6714 +0x4c | |
| go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveWatchClient(0xc0034b8a90, {0x26ffa48, 0xc001889a10}) | |
| go.etcd.io/etcd/client/v3@v3.5.0/watch.go:757 +0x6a | |
| created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).newWatchClient | |
| go.etcd.io/etcd/client/v3@v3.5.0/watch.go:912 +0x4e5 | |
| goroutine 96633 [semacquire, 28 minutes]: | |
| sync.runtime_SemacquireMutex(0x1e03379de72aaf9f, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225ad80, 0x24}, 0xc002cd9c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004dbd5c0}, 0x0, 0xc002cd9e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881a8af658d7a, {0x26bf478, 0xc004dbd5c0}, 0xffffb16b6eb10000, {0xc00225ad80, 0x2656fb0}, 0xc004901a10) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225ad80, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63355 [sync.Cond.Wait, 12 minutes]: | |
| sync.runtime_notifyListWait(0xc001172598, 0x32) | |
| runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc000576780) | |
| sync/cond.go:56 +0x8c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc001172500) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:509 +0x428 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:334 +0xfb | |
| goroutine 78113 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x80, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc003840990}, {0xc0022489e0, 0x1d}, 0x7f948309ba00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0042e4080, 0xc0042a4d98) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc003840990}, 0xc00384de00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc003840990}, 0x7f94aa18af18, {0x22c1ddb, 0x3b092c0}, {0x22b51cb, 0x0}, 0x0, 0xc001b16000, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc003840990}, 0xc001b69a90, {0xc004c80320, 0x40ce34}, 0xc003840900) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc0042a0c80, {0x26ef130, 0xc003b709b0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0042a4d80}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022cf140}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022cf140}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0022cf140}, 0x1e4d1c0, 0xc003b709a0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc001210180}, 0xc003a4ed80, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc001210180}, 0xc003a4ed80, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 78988 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x2d343334612d3965, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225ad80, 0x24}, 0xc00396ac38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc005222540}, 0x0, 0xc00396ae88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058815daef10f9f, {0x26bf478, 0xc005222540}, 0xffffb16b6eb10000, {0xc00225ad80, 0x2656fb0}, 0x12ec4eb) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225ad80, 0x24}, 0x4e94914f0000, 0xc001844f70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63598 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483337c88, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc002223100, 0xc001f80000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc002223100, {0xc001f80000, 0x4000, 0x4000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc002223100, {0xc001f80000, 0xc003e0bc88, 0xc003e0bc28}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0006da1e8, {0xc001f80000, 0x42c94c, 0x2}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc00326e5a0, {0xc001eed860, 0x4, 0x112268e}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc00326e5a0}, {0xc001eed860, 0x4, 0x4}, 0x4) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| vitess.io/vitess/go/mysql.(*Conn).readHeaderFrom(0xc001eed7a0, {0x2673340, 0xc00326e5a0}) | |
| vitess.io/vitess/go/mysql/conn.go:334 +0x4f | |
| vitess.io/vitess/go/mysql.(*Conn).readOnePacket(0x447fa8) | |
| vitess.io/vitess/go/mysql/conn.go:473 +0x6d | |
| vitess.io/vitess/go/mysql.(*Conn).readPacket(0x11fe3b12f4f5a) | |
| vitess.io/vitess/go/mysql/conn.go:495 +0x25 | |
| vitess.io/vitess/go/mysql.(*Conn).ReadPacket(0xc00206a4e0) | |
| vitess.io/vitess/go/mysql/conn.go:531 +0x1d | |
| vitess.io/vitess/go/mysql.mysqlFlavor.readBinlogEvent({}, 0x0) | |
| vitess.io/vitess/go/mysql/flavor_mysql.go:225 +0x1d | |
| vitess.io/vitess/go/mysql.(*Conn).ReadBinlogEvent(...) | |
| vitess.io/vitess/go/mysql/flavor.go:237 | |
| vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents.func1() | |
| vitess.io/vitess/go/vt/binlog/binlog_connection.go:146 +0xc5 | |
| created by vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents | |
| vitess.io/vitess/go/vt/binlog/binlog_connection.go:140 +0xdd | |
| goroutine 64868 [chan receive, 21 minutes]: | |
| vitess.io/vitess/go/sync2.(*Semaphore).Acquire(0xc00206a820) | |
| vitess.io/vitess/go/sync2/semaphore.go:52 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc00016e500, {0x26ef630, 0xc0002fd860}, {0xc002248460, 0x19}, 0xc002572780, {0xc002c646e0, 0xa, 0xa}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:589 +0x85 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc00016e500, 0xc003b717e0, 0xc002893560) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 63337 [chan receive]: | |
| vitess.io/vitess/go/stats.(*Rates).track(0xc004ba2540) | |
| vitess.io/vitess/go/stats/rates.go:99 +0x36 | |
| created by vitess.io/vitess/go/stats.NewRates | |
| vitess.io/vitess/go/stats/rates.go:91 +0x22f | |
| goroutine 63361 [sync.Cond.Wait, 29 minutes]: | |
| sync.runtime_notifyListWait(0xc001172958, 0x2) | |
| runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc000576780) | |
| sync/cond.go:56 +0x8c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc0011728c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:509 +0x428 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:334 +0xfb | |
| goroutine 92137 [semacquire, 29 minutes]: | |
| sync.runtime_SemacquireMutex(0x100000000000000, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254c60, 0x27}, 0xc005ca7c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004025800}, 0x0, 0xc005ca7e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588199af4e2578, {0x26bf478, 0xc004025800}, 0xffffb16b6eb10000, {0xc002254c60, 0x2656fb0}, 0xc0023236e0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254c60, 0x27}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 92178 [semacquire, 29 minutes]: | |
| sync.runtime_SemacquireMutex(0xc001a8ce70, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254000, 0x24}, 0xc003c22c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0029bcb40}, 0x0, 0xc003c22e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588199af4d849c, {0x26bf478, 0xc0029bcb40}, 0xffffb16b6eb10000, {0xc002254000, 0x2656fb0}, 0xc002323ef0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254000, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 64125 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:399 +0x31 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:398 +0x3b3 | |
| goroutine 82132 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x663836342d346462, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254000, 0x24}, 0xc004cc4c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004025500}, 0x0, 0xc004cc4e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058816caf0c95b8, {0x26bf478, 0xc004025500}, 0xffffb16b6eb10000, {0xc002254000, 0x2656fb0}, 0xc002068a80) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254000, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 80972 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002248460, 0x19}, 0xc00589dc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0028c7b60}, 0x0, 0xc00589de88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881652ef95804, {0x26bf478, 0xc0028c7b60}, 0xffffb16b6eb10000, {0xc002248460, 0x2656fb0}, 0xc004ab76e0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002248460, 0x19}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63394 [sync.Cond.Wait, 33 minutes]: | |
| sync.runtime_notifyListWait(0xc001173998, 0x2) | |
| runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xea2a01) | |
| sync/cond.go:56 +0x8c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc001173900) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:509 +0x428 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:334 +0xfb | |
| goroutine 64571 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:399 +0x31 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:398 +0x3b3 | |
| goroutine 65056 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:399 +0x31 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:398 +0x3b3 | |
| goroutine 87268 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x80, 0xc005d0c000) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022489e0, 0x1d}, 0xc004dcec38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc005195020}, 0x0, 0xc004dcee88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881832f31769b, {0x26bf478, 0xc005195020}, 0xffffb16b6eb10000, {0xc0022489e0, 0x2656fb0}, 0xc0046572c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022489e0, 0x1d}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63340 [chan receive]: | |
| vitess.io/vitess/go/stats.(*Rates).track(0xc004ba26c0) | |
| vitess.io/vitess/go/stats/rates.go:99 +0x36 | |
| created by vitess.io/vitess/go/stats.NewRates | |
| vitess.io/vitess/go/stats/rates.go:91 +0x22f | |
| goroutine 81031 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x44d072, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225b1a0, 0x28}, 0xc004170c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0013fbb60}, 0x0, 0xc004170e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881652efeacf0, {0x26bf478, 0xc0013fbb60}, 0xffffb16b6eb10000, {0xc00225b1a0, 0x2656fb0}, 0x12ec4eb) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225b1a0, 0x28}, 0x4e94914f0000, 0xc002210770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 109543 [semacquire, 20 minutes]: | |
| sync.runtime_SemacquireMutex(0xc001bd9170, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a120, 0x2c}, 0xc00312cc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002e58600}, 0x0, 0xc00312ce88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05882192fff9f15, {0x26bf478, 0xc002e58600}, 0xffffb16b6eb10000, {0xc00225a120, 0x2656fb0}, 0xc001bd9320) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a120, 0x2c}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63368 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc00157df40, 0xc0038f5560) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 79045 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004b91da0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022481a0, 0x1f}, 0xc004c71c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0014522a0}, 0x0, 0xc004c71e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058815daef03d4f, {0x26bf478, 0xc0014522a0}, 0xffffb16b6eb10000, {0xc0022481a0, 0x2656fb0}, 0x25) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022481a0, 0x1f}, 0x4e94914f0000, 0xc003e73770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 99182 [semacquire, 26 minutes]: | |
| sync.runtime_SemacquireMutex(0x6e756f63656d616e, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002248460, 0x19}, 0xc002936c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0049eb620}, 0x0, 0xc002936e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881bf2f7607c8, {0x26bf478, 0xc0049eb620}, 0xffffb16b6eb10000, {0xc002248460, 0x2656fb0}, 0xc001c41350) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002248460, 0x19}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63426 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0006470e0, 0xc001836540) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 64632 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002f15200, 0xc004617d58) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x265 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc003d4c6f0}, 0xc0004e6400) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc003d4c6f0}, 0x7f94aa18b3c8, {0x22c1ddb, 0xc000580000}, {0x22b51cb, 0x0}, 0x0, 0xc003194060, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc003d4c6f0}, 0xc001b37a90, {0xc0016d6fc0, 0x40ce34}, 0xc003d4c660) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc0026e9bd0, {0x26ef130, 0xc00441afa0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc004617d40}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba780}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba780}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0006ba780}, 0x1e4d1c0, 0xc00441af90) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0024d4000}, 0xc00297d320, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0024d4000}, 0xc00297d320, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 99957 [semacquire, 26 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004d23cb0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002248720, 0x1d}, 0xc0038fdc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0029f8c00}, 0x0, 0xc0038fde88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881c6af86733d, {0x26bf478, 0xc0029f8c00}, 0xffffb16b6eb10000, {0xc002248720, 0x2656fb0}, 0x25) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002248720, 0x1d}, 0x4e94914f0000, 0xc0029a77b8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 151196 [semacquire, 18 minutes]: | |
| sync.runtime_SemacquireMutex(0x6130383561302d66, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002248460, 0x19}, 0xc0005ebc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0059435c0}, 0x0, 0xc0005ebe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058822fb0254823, {0x26bf478, 0xc0059435c0}, 0xffffb16b6eb10000, {0xc002248460, 0x2656fb0}, 0xc00206b268) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002248460, 0x19}, 0x4e94914f0000, 0x0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 90324 [semacquire, 29 minutes]: | |
| sync.runtime_SemacquireMutex(0x3531333538393232, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022457a0, 0x22}, 0xc0007a0c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0042a3140}, 0x0, 0xc0007a0e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881922f400801, {0x26bf478, 0xc0042a3140}, 0xffffb16b6eb10000, {0xc0022457a0, 0x2656fb0}, 0xc0038efb60) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022457a0, 0x22}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 107813 [semacquire, 21 minutes]: | |
| sync.runtime_SemacquireMutex(0xa, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022489e0, 0x1d}, 0xc0030e4c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003a269c0}, 0x0, 0xc0030e4e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058820a2fdd2b6b, {0x26bf478, 0xc003a269c0}, 0xffffb16b6eb10000, {0xc0022489e0, 0x2656fb0}, 0xc0059895f0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022489e0, 0x1d}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 104607 [semacquire, 23 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022554a0, 0x28}, 0xc00393dc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004a6b680}, 0x0, 0xc00393de88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881f3afc3a74f, {0x26bf478, 0xc004a6b680}, 0xffffb16b6eb10000, {0xc0022554a0, 0x2656fb0}, 0xc002a8c900) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022554a0, 0x28}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 893104 [semacquire, 11 minutes]: | |
| sync.runtime_SemacquireMutex(0xc001ccaa50, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225ad80, 0x24}, 0xc002c93c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0059f5140}, 0x0, 0xc002c93e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588298b05f1f52, {0x26bf478, 0xc0059f5140}, 0xffffb16b6eb10000, {0xc00225ad80, 0x2656fb0}, 0xc0028bfa40) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225ad80, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 82147 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0044b4060, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a960, 0x26}, 0xc005929c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0036690e0}, 0x0, 0xc005929e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058816caf1e4cb8, {0x26bf478, 0xc0036690e0}, 0xffffb16b6eb10000, {0xc00225a960, 0x2656fb0}, 0xc002068ae0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a960, 0x26}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 85479 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a120, 0x2c}, 0xc0007ecc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003b57980}, 0x0, 0xc0007ece88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058817baf2866c6, {0x26bf478, 0xc003b57980}, 0xffffb16b6eb10000, {0xc00225a120, 0x2656fb0}, 0x25) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a120, 0x2c}, 0x4e94914f0000, 0xc0029a6f70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 108663 [semacquire, 21 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022558c0, 0x23}, 0xc0030fbc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0028c7380}, 0x0, 0xc0030fbe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588211afebe320, {0x26bf478, 0xc0028c7380}, 0xffffb16b6eb10000, {0xc0022558c0, 0x2656fb0}, 0xc004b61830) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022558c0, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 64627 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:399 +0x31 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:398 +0x3b3 | |
| goroutine 90318 [semacquire, 29 minutes]: | |
| sync.runtime_SemacquireMutex(0x3039343137303033, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002248720, 0x1d}, 0xc0007a2c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0036e4960}, 0x0, 0xc0007a2e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881922f3f4afb, {0x26bf478, 0xc0036e4960}, 0xffffb16b6eb10000, {0xc002248720, 0x2656fb0}, 0xc004a8ebd0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002248720, 0x1d}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 85323 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0xc002302de0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254c60, 0x27}, 0xc001b3cc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002a59b00}, 0x0, 0xc001b3ce88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058817baf29f5a4, {0x26bf478, 0xc002a59b00}, 0xffffb16b6eb10000, {0xc002254c60, 0x2656fb0}, 0xc0046c75f0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254c60, 0x27}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63393 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646720, 0xc0038f5c10) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 92138 [semacquire, 29 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225ad80, 0x24}, 0xc005ca8c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0040258c0}, 0x0, 0xc005ca8e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588199af4e81da, {0x26bf478, 0xc0040258c0}, 0xffffb16b6eb10000, {0xc00225ad80, 0x2656fb0}, 0xc002323920) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225ad80, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 83873 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0046c7050, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002245bc0, 0x26}, 0xc0058bcc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc005d0d500}, 0x0, 0xc0058bce88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881742f1857da, {0x26bf478, 0xc005d0d500}, 0xffffb16b6eb10000, {0xc002245bc0, 0x2656fb0}, 0xc005d9a900) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002245bc0, 0x26}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 96627 [semacquire, 28 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0047fa960, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022457a0, 0x22}, 0xc005ca2c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004dbd320}, 0x0, 0xc005ca2e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881a8af650ef4, {0x26bf478, 0xc004dbd320}, 0xffffb16b6eb10000, {0xc0022457a0, 0x2656fb0}, 0xc0048a6e10) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022457a0, 0x22}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 78994 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0xc001a8cba0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223bbc0, 0x36}, 0xc004ccdc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00267f380}, 0x0, 0xc004ccde88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058815daef11bb1, {0x26bf478, 0xc00267f380}, 0xffffb16b6eb10000, {0xc00223bbc0, 0x2656fb0}, 0x12ec4eb) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223bbc0, 0x36}, 0x4e94914f0000, 0xc0022107b8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 904585 [semacquire, 11 minutes]: | |
| sync.runtime_SemacquireMutex(0xc005950f90, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022457a0, 0x22}, 0xc005c43c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0029f83c0}, 0x0, 0xc005c43e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05882a0305fe979, {0x26bf478, 0xc0029f83c0}, 0xffffb16b6eb10000, {0xc0022457a0, 0x2656fb0}, 0xc0056b21b0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022457a0, 0x22}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 104562 [semacquire, 23 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004b73e30, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002245380, 0x23}, 0xc0040c9c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0030173e0}, 0x0, 0xc0040c9e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881f3afc0c827, {0x26bf478, 0xc0030173e0}, 0xffffb16b6eb10000, {0xc002245380, 0x2656fb0}, 0xc003f78ba0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002245380, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 80987 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x2d656539362d6563, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223bbc0, 0x36}, 0xc00303dc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002f20060}, 0x0, 0xc00303de88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881652efd7339, {0x26bf478, 0xc002f20060}, 0xffffb16b6eb10000, {0xc00223bbc0, 0x2656fb0}, 0xc001a8ced0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223bbc0, 0x36}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 904669 [semacquire, 11 minutes]: | |
| sync.runtime_SemacquireMutex(0xc005a56460, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244720, 0x25}, 0xc005a52c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0021df3e0}, 0x0, 0xc005a52e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05882a03060a4db, {0x26bf478, 0xc0021df3e0}, 0xffffb16b6eb10000, {0xc002244720, 0x2656fb0}, 0xc0056b2270) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244720, 0x25}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 65913 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x11edb5c, 0x0, 0x1) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc001172a78) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe(0xc001172a00, {0x26bf4b0, 0xc0047ed890}, 0x23) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:372 +0xa5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc0047ed890}, {0xc002254840, 0x23}, 0x7f94832f39a8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:127 +0x18f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc001f68580, 0xc0016cdd10) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc0047ed890}, 0xc00171e800) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc0047ed890}, 0x7f94aa18bd28, {0x22c1ddb, 0x3b092c0}, {0x22b51cb, 0x0}, 0x0, 0xc0039cdec0, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc0047ed890}, 0xc00168ca90, {0xc00288d860, 0x40ce34}, 0xc0047ed800) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc0028eb220, {0x26ef130, 0xc0013e50d0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0016cdcf8}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c540}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c540}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c540}, 0x1e4d1c0, 0xc0013e50c0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0006fa000}, 0xc001b7e7e0, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0006fa000}, 0xc001b7e7e0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 322968 [semacquire, 17 minutes]: | |
| sync.runtime_SemacquireMutex(0x3634343033, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022554a0, 0x28}, 0xc0056c0c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003729260}, 0x0, 0xc0056c0e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05882463030580d, {0x26bf478, 0xc003729260}, 0xffffb16b6eb10000, {0xc0022554a0, 0x2656fb0}, 0xc0027e6a50) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022554a0, 0x28}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 935711 [semacquire, 8 minutes]: | |
| sync.runtime_SemacquireMutex(0xc003f7b2c0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002255ce0, 0x27}, 0xc0018a2c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003405080}, 0x0, 0xc0018a2e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05882c5b0862bb8, {0x26bf478, 0xc003405080}, 0xffffb16b6eb10000, {0xc002255ce0, 0x2656fb0}, 0xc00256e750) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002255ce0, 0x27}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63396 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646800, 0xc0038f5ce0) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 83875 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0x37303565653565, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002245380, 0x23}, 0xc0058c0c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc005d0d680}, 0x0, 0xc0058c0e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881742f18a8b6, {0x26bf478, 0xc005d0d680}, 0xffffb16b6eb10000, {0xc002245380, 0x2656fb0}, 0xc005d9a750) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002245380, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63405 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646aa0, 0xc0038f5f70) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 92183 [semacquire, 29 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x80, 0xc00510d620) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244300, 0x21}, 0xc005ca3c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0029bd020}, 0x0, 0xc005ca3e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588199af4e5ebc, {0x26bf478, 0xc0029bd020}, 0xffffb16b6eb10000, {0xc002244300, 0x2656fb0}, 0xc00151fe60) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244300, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 109840 [semacquire, 20 minutes]: | |
| sync.runtime_SemacquireMutex(0xc002068030, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002245380, 0x23}, 0xc003dd9c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0013e1da0}, 0x0, 0xc003dd9e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588220b01290d2, {0x26bf478, 0xc0013e1da0}, 0xffffb16b6eb10000, {0xc002245380, 0x2656fb0}, 0xc000051830) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002245380, 0x23}, 0x4e94914f0000, 0xc001841f70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 88729 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0013823f0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022558c0, 0x23}, 0xc004dd2c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc001721740}, 0x0, 0xc004dd2e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058818aaf3422d9, {0x26bf478, 0xc001721740}, 0xffffb16b6eb10000, {0xc0022558c0, 0x2656fb0}, 0xc004af3950) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022558c0, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 92993 [semacquire, 28 minutes]: | |
| sync.runtime_SemacquireMutex(0x3037363532363035, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254000, 0x24}, 0xc00493cc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004375800}, 0x0, 0xc00493ce88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881a12f5d76d3, {0x26bf478, 0xc004375800}, 0xffffb16b6eb10000, {0xc002254000, 0x2656fb0}, 0xc003a63aa0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254000, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 83881 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223bbc0, 0x36}, 0xc00566cc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc005d0db60}, 0x0, 0xc00566ce88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881742f193812, {0x26bf478, 0xc005d0db60}, 0xffffb16b6eb10000, {0xc00223bbc0, 0x2656fb0}, 0xc005d9a8d0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223bbc0, 0x36}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63358 [sync.Cond.Wait, 33 minutes]: | |
| sync.runtime_notifyListWait(0xc001172818, 0x0) | |
| runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc001df6fa0) | |
| sync/cond.go:56 +0x8c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc001172780) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:509 +0x428 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:334 +0xfb | |
| goroutine 88604 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004a8e7b0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244300, 0x21}, 0xc004ea5c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003bc9f80}, 0x0, 0xc004ea5e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058818aaf31e901, {0x26bf478, 0xc003bc9f80}, 0xffffb16b6eb10000, {0xc002244300, 0x2656fb0}, 0xc000051830) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244300, 0x21}, 0x4e94914f0000, 0xc000609f70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 80985 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244f60, 0x25}, 0xc005924c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003dd1f80}, 0x0, 0xc005924e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881652efd12bd, {0x26bf478, 0xc003dd1f80}, 0xffffb16b6eb10000, {0xc002244f60, 0x2656fb0}, 0xc000340540) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244f60, 0x25}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 82097 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0xc005184750, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254840, 0x23}, 0xc0038ccc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00471f500}, 0x0, 0xc0038cce88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058816caf0a297c, {0x26bf478, 0xc00471f500}, 0xffffb16b6eb10000, {0xc002254840, 0x2656fb0}, 0xc0046e2db0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254840, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63364 [sync.Cond.Wait, 33 minutes]: | |
| sync.runtime_notifyListWait(0xc001172a98, 0x1) | |
| runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc00035d7a0) | |
| sync/cond.go:56 +0x8c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc001172a00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:509 +0x428 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:334 +0xfb | |
| goroutine 63404 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646a80, 0xc0038f5f60) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 111243 [semacquire, 19 minutes]: | |
| sync.runtime_SemacquireMutex(0xe46907fd53de109d, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244b40, 0x21}, 0xc003db5c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004b82f60}, 0x0, 0xc003db5e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058822830220736, {0x26bf478, 0xc004b82f60}, 0xffffb16b6eb10000, {0xc002244b40, 0x2656fb0}, 0xc001b4ab10) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244b40, 0x21}, 0x4e94914f0000, 0xc0027bbf70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 100506 [semacquire, 25 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x80, 0xc002ab1bc0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002255080, 0x24}, 0xc002ce1c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00395f740}, 0x0, 0xc002ce1e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881ce2f8c10d9, {0x26bf478, 0xc00395f740}, 0xffffb16b6eb10000, {0xc002255080, 0x2656fb0}, 0xc0021e3aa0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002255080, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63390 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646640, 0xc0038f5b40) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 888186 [semacquire, 11 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x80, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc004901a40}, {0xc002244300, 0x21}, 0x7f9482cc5a10) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005a84a80, 0xc004e65e90) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc004901a40}, 0xc00171e700) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc004901a40}, 0x7f94aa18a108, {0x22c1ddb, 0xc000580000}, {0x22b51cb, 0x0}, 0x0, 0xc001d333e0, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc004901a40}, 0xc001aafa90, {0xc005955530, 0x40ce34}, 0xc004901770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc005082fa0, {0x26ef130, 0xc0041ee9f0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc004e65e78}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc004ce2f00}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc004ce2f00}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc004ce2f00}, 0x1e4d1c0, 0xc0041ee9e0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0003c7c80}, 0xc004124fc0, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0003c7c80}, 0xc004124fc0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 64647 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents.func2() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:283 +0xf0 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:270 +0x1f1 | |
| goroutine 266067 [semacquire, 17 minutes]: | |
| sync.runtime_SemacquireMutex(0xc001c40870, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022558c0, 0x23}, 0xc0053c9c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003001380}, 0x0, 0xc0053c9e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058823eb00792de, {0x26bf478, 0xc003001380}, 0xffffb16b6eb10000, {0xc0022558c0, 0x2656fb0}, 0xc00437eea0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022558c0, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 97141 [semacquire, 27 minutes]: | |
| sync.runtime_SemacquireMutex(0x44d072, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244720, 0x25}, 0xc0036f1c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0003edce0}, 0x0, 0xc0036f1e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881b02f68fe54, {0x26bf478, 0xc0003edce0}, 0xffffb16b6eb10000, {0xc002244720, 0x2656fb0}, 0xc003bc5020) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244720, 0x25}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 85538 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0xa, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254420, 0x28}, 0xc004dd3c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00298fc80}, 0x0, 0xc004dd3e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058817baf300db8, {0x26bf478, 0xc00298fc80}, 0xffffb16b6eb10000, {0xc002254420, 0x2656fb0}, 0xc0018112f0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254420, 0x28}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63479 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents.func2() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:283 +0xf0 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:270 +0x1f1 | |
| goroutine 80974 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x10000c004a9a500, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244b40, 0x21}, 0xc003123c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0028c7d40}, 0x0, 0xc003123e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881652efc6864, {0x26bf478, 0xc0028c7d40}, 0xffffb16b6eb10000, {0xc002244b40, 0x2656fb0}, 0xc004c29e60) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244b40, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 92951 [semacquire, 28 minutes]: | |
| sync.runtime_SemacquireMutex(0x31333136392d6530, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022457a0, 0x22}, 0xc0031d2c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004374d20}, 0x0, 0xc0031d2e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881a12f5b9f11, {0x26bf478, 0xc004374d20}, 0xffffb16b6eb10000, {0xc0022457a0, 0x2656fb0}, 0xc004733980) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022457a0, 0x22}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 87239 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004e884b0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a120, 0x2c}, 0xc0007c0c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003eda720}, 0x0, 0xc0007c0e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881832f2e8e27, {0x26bf478, 0xc003eda720}, 0xffffb16b6eb10000, {0xc00225a120, 0x2656fb0}, 0xc002303080) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a120, 0x2c}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 98991 [semacquire, 26 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x80, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc0042fce40}, {0xc002248720, 0x1d}, 0x7f948326b930) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc004823280, 0xc0035dced0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc0042fce40}, 0xc003ee4800) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc0042fce40}, 0x7f94aa18a108, {0x22c1ddb, 0xc000720000}, {0x22b51cb, 0x0}, 0x0, 0xc0041066c0, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc0042fce40}, 0xc0016eda90, {0xc004497560, 0x40ce34}, 0xc0042fcd50) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc001c9f810, {0x26ef130, 0xc0037593e0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0035dceb8}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc001424cc0}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc001424cc0}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc001424cc0}, 0x1e4d1c0, 0xc0037593d0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0000c4a80}, 0xc0003b9b00, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0000c4a80}, 0xc0003b9b00, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 90354 [semacquire, 29 minutes]: | |
| sync.runtime_SemacquireMutex(0x2d313a3133313030, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a120, 0x2c}, 0xc0057e5c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0036e4ba0}, 0x0, 0xc0057e5e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881922f400cc5, {0x26bf478, 0xc0036e4ba0}, 0xffffb16b6eb10000, {0xc00225a120, 0x2656fb0}, 0xc0038efa70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a120, 0x2c}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 64181 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483062230, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0006c1200, 0xc0020b2000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0006c1200, {0xc0020b2000, 0x4000, 0x4000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0006c1200, {0xc0020b2000, 0xc0027b1c88, 0xc0027b1c28}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc000564778, {0xc0020b2000, 0x42c94c, 0x2}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc001b6d380, {0xc0020ae1e0, 0x4, 0x112268e}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc001b6d380}, {0xc0020ae1e0, 0x4, 0x4}, 0x4) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| vitess.io/vitess/go/mysql.(*Conn).readHeaderFrom(0xc0020ae120, {0x2673340, 0xc001b6d380}) | |
| vitess.io/vitess/go/mysql/conn.go:334 +0x4f | |
| vitess.io/vitess/go/mysql.(*Conn).readOnePacket(0x447fa8) | |
| vitess.io/vitess/go/mysql/conn.go:473 +0x6d | |
| vitess.io/vitess/go/mysql.(*Conn).readPacket(0x11fe3b1387d32) | |
| vitess.io/vitess/go/mysql/conn.go:495 +0x25 | |
| vitess.io/vitess/go/mysql.(*Conn).ReadPacket(0xc001aefba0) | |
| vitess.io/vitess/go/mysql/conn.go:531 +0x1d | |
| vitess.io/vitess/go/mysql.mysqlFlavor.readBinlogEvent({}, 0x0) | |
| vitess.io/vitess/go/mysql/flavor_mysql.go:225 +0x1d | |
| vitess.io/vitess/go/mysql.(*Conn).ReadBinlogEvent(...) | |
| vitess.io/vitess/go/mysql/flavor.go:237 | |
| vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents.func1() | |
| vitess.io/vitess/go/vt/binlog/binlog_connection.go:146 +0xc5 | |
| created by vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents | |
| vitess.io/vitess/go/vt/binlog/binlog_connection.go:140 +0xdd | |
| goroutine 63348 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc00152f7c0, 0xc0038f4f40) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 63395 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0006467e0, 0xc0038f5cd0) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 85498 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004462d20, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002255080, 0x24}, 0xc004cbfc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00298f7a0}, 0x0, 0xc004cbfe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058817baf27a4a2, {0x26bf478, 0xc00298f7a0}, 0xffffb16b6eb10000, {0xc002255080, 0x2656fb0}, 0xc004c007e0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002255080, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 80990 [chan receive, 32 minutes]: | |
| vitess.io/vitess/go/sync2.(*Semaphore).Acquire(0xc001655380) | |
| vitess.io/vitess/go/sync2/semaphore.go:52 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc00016eb40, {0x26ef630, 0xc0002fd860}, {0xc0022489e0, 0x1d}, 0x26bf408, {0xc002bb2500, 0xa, 0xa}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:589 +0x85 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc00016eb40, 0xc001ded6e0, 0xc004896900) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 64675 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents.func2() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:283 +0xf0 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:270 +0x1f1 | |
| goroutine 97216 [semacquire, 27 minutes]: | |
| sync.runtime_SemacquireMutex(0x13, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223b5c0, 0x35}, 0xc00338cc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002cc61e0}, 0x0, 0xc00338ce88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881b02f683e88, {0x26bf478, 0xc002cc61e0}, 0xffffb16b6eb10000, {0xc00223b5c0, 0x2656fb0}, 0xc000051830) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223b5c0, 0x35}, 0x4e94914f0000, 0xc0029a9770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 84718 [chan receive, 31 minutes]: | |
| vitess.io/vitess/go/sync2.(*Semaphore).Acquire(0xc0016d96c0) | |
| vitess.io/vitess/go/sync2/semaphore.go:52 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc001173cc0, {0x26ef630, 0xc0002fd860}, {0xc00225ad80, 0x24}, 0x488897, {0xc0035c6140, 0xa, 0xa}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:589 +0x85 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc001173cc0, 0xc0000ad7d0, 0xc00554ac00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 87270 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022554a0, 0x28}, 0xc004a4ac38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0051951a0}, 0x0, 0xc004a4ae88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881832f31c99d, {0x26bf478, 0xc0051951a0}, 0xffffb16b6eb10000, {0xc0022554a0, 0x2656fb0}, 0xc004657050) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022554a0, 0x28}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 83845 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0xc001810fc0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002255ce0, 0x27}, 0xc0007bac38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00180f740}, 0x0, 0xc0007bae88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881742f2b1adc, {0x26bf478, 0xc00180f740}, 0xffffb16b6eb10000, {0xc002255ce0, 0x2656fb0}, 0xc005d9a990) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002255ce0, 0x27}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 107265 [semacquire, 22 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0059891a0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a120, 0x2c}, 0xc0018f5c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002d0a420}, 0x0, 0xc0018f5e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588202afda9e53, {0x26bf478, 0xc002d0a420}, 0xffffb16b6eb10000, {0xc00225a120, 0x2656fb0}, 0xc003ff40c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a120, 0x2c}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 107832 [semacquire, 21 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002255ce0, 0x27}, 0xc0030efc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003f4f200}, 0x0, 0xc0030efe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058820a2fdf09bd, {0x26bf478, 0xc003f4f200}, 0xffffb16b6eb10000, {0xc002255ce0, 0x2656fb0}, 0xc0059898c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002255ce0, 0x27}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 98203 [semacquire, 27 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002248720, 0x1d}, 0xc005422c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002bff920}, 0x0, 0xc005422e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881b7af713145, {0x26bf478, 0xc002bff920}, 0xffffb16b6eb10000, {0xc002248720, 0x2656fb0}, 0xc003e4dad0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002248720, 0x1d}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 79272 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x80, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc003bc4de0}, {0xc002248720, 0x1d}, 0x7f9483303b98) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc004384580, 0xc004380948) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc003bc4de0}, 0xc003a53000) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc003bc4de0}, 0x7f94aa18af18, {0x22c1ddb, 0xc0006b9c00}, {0x22b51cb, 0x0}, 0x0, 0xc004520540, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc003bc4de0}, 0xc00396ba90, {0xc000594660, 0x40ce34}, 0xc003bc4d20) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc003ffe6e0, {0x26ef130, 0xc003e35840}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc004380930}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e600}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e600}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00237e600}, 0x1e4d1c0, 0xc003e35830) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0000c4780}, 0xc00167f200, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0000c4780}, 0xc00167f200, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 955210 [semacquire, 7 minutes]: | |
| sync.runtime_SemacquireMutex(0xa, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254c60, 0x27}, 0xc0018a9c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc000735800}, 0x0, 0xc0018a9e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05882dc30aa0b0e, {0x26bf478, 0xc000735800}, 0xffffb16b6eb10000, {0xc002254c60, 0x2656fb0}, 0xc002ad9470) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254c60, 0x27}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63324 [select, 12 minutes]: | |
| vitess.io/vitess/go/cache/ristretto.(*Cache).processItems(0xc00056d780) | |
| vitess.io/vitess/go/cache/ristretto/cache.go:442 +0x13f | |
| created by vitess.io/vitess/go/cache/ristretto.(*Cache).Clear | |
| vitess.io/vitess/go/cache/ristretto/cache.go:362 +0x11c | |
| goroutine 82062 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0046e3c50, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002248460, 0x19}, 0xc003121c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00471f0e0}, 0x0, 0xc003121e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058816caf09d29c, {0x26bf478, 0xc00471f0e0}, 0xffffb16b6eb10000, {0xc002248460, 0x2656fb0}, 0xc0046e2d20) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002248460, 0x19}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63351 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc00157d920, 0xc0038f5050) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 88691 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0023ed1d0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002245380, 0x23}, 0xc003b5cc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004896120}, 0x0, 0xc003b5ce88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058818aaf3256d9, {0x26bf478, 0xc004896120}, 0xffffb16b6eb10000, {0xc002245380, 0x2656fb0}, 0xc004ab7ec0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002245380, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 893091 [semacquire, 11 minutes]: | |
| sync.runtime_SemacquireMutex(0x2673900, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002255080, 0x24}, 0xc005d2ac38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0059f47e0}, 0x0, 0xc005d2ae88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588298b05e06ee, {0x26bf478, 0xc0059f47e0}, 0xffffb16b6eb10000, {0xc002255080, 0x2656fb0}, 0xc0047b4540) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002255080, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 97249 [semacquire, 27 minutes]: | |
| sync.runtime_SemacquireMutex(0x4bab5b4b410316c, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a540, 0x21}, 0xc00338dc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002cc62a0}, 0x0, 0xc00338de88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881b02f685dbe, {0x26bf478, 0xc002cc62a0}, 0xffffb16b6eb10000, {0xc00225a540, 0x2656fb0}, 0xc000051830) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a540, 0x21}, 0x4e94914f0000, 0xc0029a9770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 64574 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:399 +0x31 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:398 +0x3b3 | |
| goroutine 65524 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x2656fb0, 0x30, 0x1211fac) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc0004ada40, {0xc002248460, 0x19}, {0xc0030c1900, 0xa, 0xa}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:153 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1059 +0x38 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc001be6480}, 0x0, 0xc004b09de0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0x26bf4b0, {0x26bf478, 0xc001be6480}, 0x2656fb0, {0xc002248460, 0x2656fb0}, {0xc0030c1900, 0x2c06a8e, 0xc0000d89a0}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1058 +0x8d | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc00016e500, {0x26ef630, 0xc0002fd860}, {0xc002248460, 0x19}, 0x26bf408, {0xc0030c1900, 0xa, 0xa}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:596 +0x14e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc00016e500, 0xc0047cc880, 0xc002807f80) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 77952 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x0, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc0043b75f0}, {0xc002248460, 0x19}, 0x7f948330ed68) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc004083300, 0xc004093650) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc0043b75f0}, 0xc0040c0000) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc0043b75f0}, 0x7f94aa18aa68, {0x22c1ddb, 0x3b092c0}, {0x22b51cb, 0x0}, 0x0, 0xc001657f20, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc0043b75f0}, 0xc0038d1a90, {0xc0042457c0, 0x40ce34}, 0xc0043b7500) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc0040ae910, {0x26ef130, 0xc00318ebd0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc004093638}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014240c0}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014240c0}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0014240c0}, 0x1e4d1c0, 0xc00318ebc0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc001210900}, 0xc003d117a0, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc001210900}, 0xc003d117a0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 64113 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:399 +0x31 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:398 +0x3b3 | |
| goroutine 79057 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0042fdb00, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244b40, 0x21}, 0xc004c5cc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00490b980}, 0x0, 0xc004c5ce88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058815daeef75d1, {0x26bf478, 0xc00490b980}, 0xffffb16b6eb10000, {0xc002244b40, 0x2656fb0}, 0x25) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244b40, 0x21}, 0x4e94914f0000, 0xc0029a7770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 97142 [semacquire, 27 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0027e6c30, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223bbc0, 0x36}, 0xc004cc1c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0003ede00}, 0x0, 0xc004cc1e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881b02f6926cc, {0x26bf478, 0xc0003ede00}, 0xffffb16b6eb10000, {0xc00223bbc0, 0x2656fb0}, 0xc001b4bf50) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223bbc0, 0x36}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 104182 [semacquire, 23 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004b2abd0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244300, 0x21}, 0xc0040cbc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0030c3560}, 0x0, 0xc0040cbe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881ec2fb5f272, {0x26bf478, 0xc0030c3560}, 0xffffb16b6eb10000, {0xc002244300, 0x2656fb0}, 0xc0047fb050) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244300, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 66307 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x2656fb0, 0x30, 0x1211fac) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc0004ada40, {0xc002244300, 0x21}, {0xc002f845d0, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:153 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1059 +0x38 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002c3dbc0}, 0x0, 0xc00251dde0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0x26bf4b0, {0x26bf478, 0xc002c3dbc0}, 0x2656fb0, {0xc002244300, 0x2656fb0}, {0xc002f845d0, 0x0, 0x7f94aa18c1d8}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1058 +0x8d | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc001172500, {0x26ef630, 0xc0002fd860}, {0xc002244300, 0x21}, 0xc0005d8618, {0xc002f845d0, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:596 +0x14e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc001172500, 0xc00441ae90, 0xc002c3db00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 66675 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x2656fb0, 0x30, 0x1211fac) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc0004ada40, {0xc002244b40, 0x21}, {0xc003ebb410, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:153 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1059 +0x38 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002bff6e0}, 0x0, 0xc004c6bde0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0x26bf4b0, {0x26bf478, 0xc002bff6e0}, 0x2656fb0, {0xc002244b40, 0x2656fb0}, {0xc003ebb410, 0x0, 0x7f94aa18b3c8}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1058 +0x8d | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc001173e00, {0x26ef630, 0xc0002fd860}, {0xc002244b40, 0x21}, 0x0, {0xc003ebb410, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:596 +0x14e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc001173e00, 0xc004192720, 0xc002bff620) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 104604 [semacquire, 23 minutes]: | |
| sync.runtime_SemacquireMutex(0x617300a580a998f, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225ad80, 0x24}, 0xc0033a8c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004a6b140}, 0x0, 0xc0033a8e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881f3afc35443, {0x26bf478, 0xc004a6b140}, 0xffffb16b6eb10000, {0xc00225ad80, 0x2656fb0}, 0xc002a8ca80) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225ad80, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 64182 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents.func2() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:283 +0xf0 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:270 +0x1f1 | |
| goroutine 63392 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646700, 0xc0038f5c00) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 81010 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0xa, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a540, 0x21}, 0xc00311fc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0047f2180}, 0x0, 0xc00311fe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881652efd045d, {0x26bf478, 0xc0047f2180}, 0xffffb16b6eb10000, {0xc00225a540, 0x2656fb0}, 0xc000340270) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a540, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 85468 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x80, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc001b4a810}, {0xc00225ad80, 0x24}, 0x7f9482fbacb8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00377a380, 0xc0049fc588) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc001b4a810}, 0xc00026e800) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc001b4a810}, 0x7f94aa18b3c8, {0x22c1ddb, 0xc000720000}, {0x22b51cb, 0x0}, 0x0, 0xc004344ae0, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc001b4a810}, 0xc0050c4a90, {0xc0045c6450, 0x40ce34}, 0xc001b4a720) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc004d9e500, {0x26ef130, 0xc004004260}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0049fc570}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0013fc600}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0013fc600}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0013fc600}, 0x1e4d1c0, 0xc004004250) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc001210480}, 0xc004876360, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc001210480}, 0xc004876360, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 81057 [chan receive, 32 minutes]: | |
| vitess.io/vitess/go/sync2.(*Semaphore).Acquire(0xc0047841a0) | |
| vitess.io/vitess/go/sync2/semaphore.go:52 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc00016eb40, {0x26ef630, 0xc0002fd860}, {0xc0022489e0, 0x1d}, 0x100000000000000, {0xc003004820, 0xa, 0xa}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:589 +0x85 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc00016eb40, 0xc00225da40, 0xc004896b40) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 108722 [semacquire, 21 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0046568a0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225ad80, 0x24}, 0xc004945c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0037a48a0}, 0x0, 0xc004945e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588211aff0f14e, {0x26bf478, 0xc0037a48a0}, 0xffffb16b6eb10000, {0xc00225ad80, 0x2656fb0}, 0xc0047b5800) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225ad80, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 84858 [chan receive, 31 minutes]: | |
| vitess.io/vitess/go/sync2.(*Semaphore).Acquire(0xc0047844e0) | |
| vitess.io/vitess/go/sync2/semaphore.go:52 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc0011732c0, {0x26ef630, 0xc0002fd860}, {0xc00225b1a0, 0x28}, 0x26bf408, {0xc002ad0b80, 0x8, 0x8}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:589 +0x85 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc0011732c0, 0xc001dede20, 0xc0042a3980) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 92134 [semacquire, 29 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004a76960, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244f60, 0x25}, 0xc004cc0c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0040253e0}, 0x0, 0xc004cc0e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588199af4d34b0, {0x26bf478, 0xc0040253e0}, 0xffffb16b6eb10000, {0xc002244f60, 0x2656fb0}, 0xc002323500) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244f60, 0x25}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63366 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc00157de80, 0xc0038f54a0) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 63387 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000646560, 0xc0038f5a70) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 87250 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x80, 0xc005d0c000) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223bbc0, 0x36}, 0xc00283fc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00009ae40}, 0x0, 0xc00283fe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881832f315da5, {0x26bf478, 0xc00009ae40}, 0xffffb16b6eb10000, {0xc00223bbc0, 0x2656fb0}, 0xc002ad9c80) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223bbc0, 0x36}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 80405 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x1207cd7, 0x0, 0x1ebd4c0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0004ada40, {0x26bf4b0, 0xc001319e00}, {0xc0022489e0, 0x1d}, 0x7f9482fb9f70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:121 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005343f00, 0xc0057034e8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc001319e00}, 0xc001d00600) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc001319e00}, 0x7f94aa18a5b8, {0x22c1ddb, 0xc000480000}, {0x22b51cb, 0x0}, 0x0, 0xc002b790e0, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc001319e00}, 0xc000794a90, {0xc0057182a0, 0x40ce34}, 0xc001319c20) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc003e55680, {0x26ef130, 0xc002f84920}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0057034d0}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c240}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c240}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c240}, 0x1e4d1c0, 0xc002f84910) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0006fa600}, 0xc002272120, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0006fa600}, 0xc002272120, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 93004 [semacquire, 28 minutes]: | |
| sync.runtime_SemacquireMutex(0x6465697254747365, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002255080, 0x24}, 0xc00493ac38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004375ce0}, 0x0, 0xc00493ae88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881a12f5e41cb, {0x26bf478, 0xc004375ce0}, 0xffffb16b6eb10000, {0xc002255080, 0x2656fb0}, 0xc00062b4a0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002255080, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63409 [sync.Cond.Wait, 33 minutes]: | |
| sync.runtime_notifyListWait(0xc00016e6d8, 0x1) | |
| runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xea2a33) | |
| sync/cond.go:56 +0x8c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc00016e640) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:509 +0x428 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:334 +0xfb | |
| goroutine 63367 [sync.Cond.Wait, 33 minutes]: | |
| sync.runtime_notifyListWait(0xc001172d18, 0x1) | |
| runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc0022127a0) | |
| sync/cond.go:56 +0x8c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc001172c80) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:509 +0x428 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:334 +0xfb | |
| goroutine 63363 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc00157dda0, 0xc0038f53b0) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 90316 [semacquire, 29 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004a49b70, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244300, 0x21}, 0xc004a49c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0036e47e0}, 0x0, 0xc004a49e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881922f3f07e9, {0x26bf478, 0xc0036e47e0}, 0xffffb16b6eb10000, {0xc002244300, 0x2656fb0}, 0xc004a8ec60) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244300, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 66700 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x2656fb0, 0x30, 0x1211fac) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc0004ada40, {0xc002244b40, 0x21}, {0xc004663630, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:153 +0x111 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1059 +0x38 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002f20180}, 0x0, 0xc004c72de0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0x26bf4b0, {0x26bf478, 0xc002f20180}, 0x2656fb0, {0xc002244b40, 0x2656fb0}, {0xc004663630, 0x0, 0x7f94aa18a108}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1058 +0x8d | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc001173e00, {0x26ef630, 0xc0002fd860}, {0xc002244b40, 0x21}, 0x26bf478, {0xc004663630, 0x1, 0x1}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:596 +0x14e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc001173e00, 0xc0035e52f0, 0xc002f200c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:584 +0x270 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:541 +0xf3 | |
| goroutine 65074 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f9483252dc8, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc001674780, 0xc003c62000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc001674780, {0xc003c62000, 0x4000, 0x4000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc001674780, {0xc003c62000, 0xc001966c88, 0xc001966c28}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00073cc88, {0xc003c62000, 0x42c94c, 0x2}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc002d49ec0, {0xc002ebf860, 0x4, 0x112268e}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc002d49ec0}, {0xc002ebf860, 0x4, 0x4}, 0x4) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| vitess.io/vitess/go/mysql.(*Conn).readHeaderFrom(0xc002ebf7a0, {0x2673340, 0xc002d49ec0}) | |
| vitess.io/vitess/go/mysql/conn.go:334 +0x4f | |
| vitess.io/vitess/go/mysql.(*Conn).readOnePacket(0x447fa8) | |
| vitess.io/vitess/go/mysql/conn.go:473 +0x6d | |
| vitess.io/vitess/go/mysql.(*Conn).readPacket(0x11fe3b12e9aea) | |
| vitess.io/vitess/go/mysql/conn.go:495 +0x25 | |
| vitess.io/vitess/go/mysql.(*Conn).ReadPacket(0xc002b8e4e0) | |
| vitess.io/vitess/go/mysql/conn.go:531 +0x1d | |
| vitess.io/vitess/go/mysql.mysqlFlavor.readBinlogEvent({}, 0x0) | |
| vitess.io/vitess/go/mysql/flavor_mysql.go:225 +0x1d | |
| vitess.io/vitess/go/mysql.(*Conn).ReadBinlogEvent(...) | |
| vitess.io/vitess/go/mysql/flavor.go:237 | |
| vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents.func1() | |
| vitess.io/vitess/go/vt/binlog/binlog_connection.go:146 +0xc5 | |
| created by vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents | |
| vitess.io/vitess/go/vt/binlog/binlog_connection.go:140 +0xdd | |
| goroutine 82116 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x5d0e190000ffffff, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223bbc0, 0x36}, 0xc004c89c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00471fe60}, 0x0, 0xc004c89e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058816caf0bd9cb, {0x26bf478, 0xc00471fe60}, 0xffffb16b6eb10000, {0xc00223bbc0, 0x2656fb0}, 0xc004b43770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223bbc0, 0x36}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 64927 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc001674480, 0xc004353ec0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x265 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc001ccbb90}, 0xc00171fc00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc001ccbb90}, 0x7f94aa18a108, {0x22c1ddb, 0xc003f24400}, {0x22b51cb, 0x0}, 0x0, 0xc002a76ba0, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc001ccbb90}, 0xc003e08a90, {0xc003c15470, 0x40ce34}, 0xc001ccbb00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc0015d4a50, {0x26ef130, 0xc0037f3030}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc004353ea8}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c3c0}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c3c0}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc00064c3c0}, 0x1e4d1c0, 0xc0037f3020) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0006fa600}, 0xc002ebf440, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0006fa600}, 0xc002ebf440, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 88731 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x2816, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022457a0, 0x22}, 0xc0037f8c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0017218c0}, 0x0, 0xc0037f8e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058818aaf34528b, {0x26bf478, 0xc0017218c0}, 0xffffb16b6eb10000, {0xc0022457a0, 0x2656fb0}, 0xc004af39b0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022457a0, 0x22}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 104605 [semacquire, 23 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002248460, 0x19}, 0xc0033a9c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004a6b2c0}, 0x0, 0xc0033a9e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881f3afc36e28, {0x26bf478, 0xc004a6b2c0}, 0xffffb16b6eb10000, {0xc002248460, 0x2656fb0}, 0xc002a8c8a0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002248460, 0x19}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 80983 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x80, 0xc004d07e60) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022554a0, 0x28}, 0xc0058a0c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003dd1e00}, 0x0, 0xc0058a0e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881652efc8a55, {0x26bf478, 0xc003dd1e00}, 0xffffb16b6eb10000, {0xc0022554a0, 0x2656fb0}, 0xc002ad8030) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022554a0, 0x28}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 88737 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x2816, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254840, 0x23}, 0xc0037f6c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc001721f20}, 0x0, 0xc0037f6e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058818aaf34e8d1, {0x26bf478, 0xc001721f20}, 0xffffb16b6eb10000, {0xc002254840, 0x2656fb0}, 0xc004af3ad0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254840, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 87284 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x13f010200010100, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002248460, 0x19}, 0xc004a41c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003edae40}, 0x0, 0xc004a41e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881832f30d803, {0x26bf478, 0xc003edae40}, 0xffffb16b6eb10000, {0xc002248460, 0x2656fb0}, 0xc0042673e0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002248460, 0x19}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 81995 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x44d072, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244300, 0x21}, 0xc003b4fc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0040251a0}, 0x0, 0xc003b4fe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058816caf0bc30f, {0x26bf478, 0xc0040251a0}, 0xffffb16b6eb10000, {0xc002244300, 0x2656fb0}, 0xc004b436b0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244300, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 93000 [semacquire, 28 minutes]: | |
| sync.runtime_SemacquireMutex(0x2816, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a960, 0x26}, 0xc0048aac38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0043759e0}, 0x0, 0xc0048aae88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881a12f5df2a7, {0x26bf478, 0xc0043759e0}, 0xffffb16b6eb10000, {0xc00225a960, 0x2656fb0}, 0xc00062b4d0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a960, 0x26}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 83876 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0x3530623065643261, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225b1a0, 0x28}, 0xc0058c1c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc005d0d740}, 0x0, 0xc0058c1e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881742f18c4e0, {0x26bf478, 0xc005d0d740}, 0xffffb16b6eb10000, {0xc00225b1a0, 0x2656fb0}, 0xc005d9a7e0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225b1a0, 0x28}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 65023 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:399 +0x31 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:398 +0x3b3 | |
| goroutine 955249 [semacquire, 7 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00399aed0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a960, 0x26}, 0xc001829c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc001ab3f80}, 0x0, 0xc001829e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05882dc30abbaf7, {0x26bf478, 0xc001ab3f80}, 0xffffb16b6eb10000, {0xc00225a960, 0x2656fb0}, 0xc004ecc5a0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a960, 0x26}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 64572 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00061fb00, 0xc0025724c8) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x265 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc002068150}, 0xc002d0eb00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc002068150}, 0x7f94aa18a5b8, {0x22c1ddb, 0x3b092c0}, {0x22b51cb, 0x0}, 0x0, 0xc0049da480, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc002068150}, 0xc001426a90, {0xc004b0fd70, 0x40ce34}, 0xc002068030) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc002d6f450, {0x26ef130, 0xc003b715e0}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc0025724b0}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002764540}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002764540}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc002764540}, 0x1e4d1c0, 0xc003b715d0) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0003c7c80}, 0xc002838fc0, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0003c7c80}, 0xc002838fc0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 83905 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022481a0, 0x1f}, 0xc00566fc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc005d0de00}, 0x0, 0xc00566fe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881742f19e212, {0x26bf478, 0xc005d0de00}, 0xffffb16b6eb10000, {0xc0022481a0, 0x2656fb0}, 0xc003c32f60) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022481a0, 0x1f}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 83804 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x80, 0xc00326e960) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223afc0, 0x35}, 0xc003b5fc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002d0b440}, 0x0, 0xc003b5fe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881742f18dbb0, {0x26bf478, 0xc002d0b440}, 0xffffb16b6eb10000, {0xc00223afc0, 0x2656fb0}, 0xc005d9a540) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223afc0, 0x35}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 83808 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002255080, 0x24}, 0xc005671c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002d0b740}, 0x0, 0xc005671e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881742f193d76, {0x26bf478, 0xc002d0b740}, 0xffffb16b6eb10000, {0xc002255080, 0x2656fb0}, 0xc005d9a6c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002255080, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63468 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00061f900, 0xc0029870b0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x265 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc002c1cf30}, 0xc00060ce00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc002c1cf30}, 0x7f94aa18b878, {0x22c1ddb, 0xc000480800}, {0x22b51cb, 0x0}, 0x0, 0xc001d322a0, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc002c1cf30}, 0xc002840a90, {0xc00041a5e0, 0x40ce34}, 0xc002c1cea0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc0015f5c20, {0x26ef130, 0xc0021f7130}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc002987098}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0013fc3c0}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0013fc3c0}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0013fc3c0}, 0x1e4d1c0, 0xc0021f7120) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0000c4300}, 0xc002ad3560, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0000c4300}, 0xc002ad3560, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 78987 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x2230303231313835, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002248460, 0x19}, 0xc003969c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc005222480}, 0x0, 0xc003969e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058815daef0f5f5, {0x26bf478, 0xc005222480}, 0xffffb16b6eb10000, {0xc002248460, 0x2656fb0}, 0x12ec4eb) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002248460, 0x19}, 0x4e94914f0000, 0xc001df2770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 98204 [semacquire, 27 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244720, 0x25}, 0xc005423c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002bff9e0}, 0x0, 0xc005423e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881b7af715bf8, {0x26bf478, 0xc002bff9e0}, 0xffffb16b6eb10000, {0xc002244720, 0x2656fb0}, 0xc003e4db00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244720, 0x25}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 82098 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0xc001ccaed0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244720, 0x25}, 0xc0038d0c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00471f5c0}, 0x0, 0xc0038d0e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058816caf0a425f, {0x26bf478, 0xc00471f5c0}, 0xffffb16b6eb10000, {0xc002244720, 0x2656fb0}, 0xc0046e2de0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244720, 0x25}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 64569 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00061f980, 0xc002572198) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x265 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26bf4b0, 0xc0029a4ed0}, 0xc002d0ea00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1033 +0x113 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc0002fd860, {0x26bf4b0, 0xc0029a4ed0}, 0x7f94aa18a5b8, {0x22c1ddb, 0x3b092c0}, {0x22b51cb, 0x0}, 0x0, 0xc0049da360, ...) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1256 +0x503 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26bf4b0, {0x26bf4b0, 0xc0029a4ed0}, 0xc001426a90, {0xc004b0fd40, 0x40ce34}, 0xc0029a4e40) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1017 +0xc5 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0000ad750, 0xc002d6f220, {0x26ef130, 0xc003b71490}) | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2238ee0, 0xc0000ad750}, {0x26e5748, 0xc002572108}) | |
| vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0027643c0}, 0x8, 0x23e1598) | |
| github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0027643c0}) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2238ee0, 0xc0000ad750}, {0x26e5d78, 0xc0027643c0}, 0x1e4d1c0, 0xc003b71480) | |
| github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004c0fc0, {0x27013d0, 0xc0024d4000}, 0xc002838ea0, 0xc000572690, 0x3ace280, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0004c0fc0, {0x27013d0, 0xc0024d4000}, 0xc002838ea0, 0x0) | |
| google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc@v1.39.0/server.go:940 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc@v1.39.0/server.go:938 +0x294 | |
| goroutine 65055 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents(0xc0006f6ee0, {0x26bf408, 0xc00421d8c0}, 0xc003845500) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:309 +0x305 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).replicate(0xc0006f6ee0, {0x2c1, 0xc003c5e0b0}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:178 +0x125 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).Stream(0xc0006f6ee0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:157 +0x159 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream(0xc002981b00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:379 +0x2e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream(0xc0000e06c0, {0x26bf408, 0xc002329f80}, {0x22b658d, 0x216e900}, {0x0, 0xc002214768, 0x12f6f44}, 0x0, 0xc003f79b90) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:227 +0x1bd | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream(0xc001173540, {0x26bf408, 0xc002329f80}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:648 +0x133 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream(0xc001173540, {0x26bf408, 0xc002329f80}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:624 +0x70 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).startVStream | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:610 +0x168 | |
| goroutine 79066 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x25, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022489e0, 0x1d}, 0xc003966c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00417c960}, 0x0, 0xc003966e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058815daef14519, {0x26bf478, 0xc00417c960}, 0xffffb16b6eb10000, {0xc0022489e0, 0x2656fb0}, 0x12ec4eb) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022489e0, 0x1d}, 0x4e94914f0000, 0xc00183e770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 87247 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x55e, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225ad80, 0x24}, 0xc004a44c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003eda900}, 0x0, 0xc004a44e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881832f3016ed, {0x26bf478, 0xc003eda900}, 0xffffb16b6eb10000, {0xc00225ad80, 0x2656fb0}, 0xc004267410) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225ad80, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 85374 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0x109, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223afc0, 0x35}, 0xc0007cbc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002bfed20}, 0x0, 0xc0007cbe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058817baf28172a, {0x26bf478, 0xc002bfed20}, 0xffffb16b6eb10000, {0xc00223afc0, 0x2656fb0}, 0xc004c00720) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223afc0, 0x35}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 82106 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0xc004d44330, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225b1a0, 0x28}, 0xc0038d3c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00471f980}, 0x0, 0xc0038d3e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058816caf0ae057, {0x26bf478, 0xc00471f980}, 0xffffb16b6eb10000, {0xc00225b1a0, 0x2656fb0}, 0xc004976f90) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225b1a0, 0x28}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 101670 [semacquire, 25 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0023ec780, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a120, 0x2c}, 0xc0031efc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004520cc0}, 0x0, 0xc0031efe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881d5af8ee0d3, {0x26bf478, 0xc004520cc0}, 0xffffb16b6eb10000, {0xc00225a120, 0x2656fb0}, 0xc004275530) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a120, 0x2c}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 81044 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002248720, 0x1d}, 0xc005926c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002f9a2a0}, 0x0, 0xc005926e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881652f11a9df, {0x26bf478, 0xc002f9a2a0}, 0xffffb16b6eb10000, {0xc002248720, 0x2656fb0}, 0xc001a8cf30) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002248720, 0x1d}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 88726 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0038ef6b0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254c60, 0x27}, 0xc004ea2c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc001721500}, 0x0, 0xc004ea2e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058818aaf33d6c1, {0x26bf478, 0xc001721500}, 0xffffb16b6eb10000, {0xc002254c60, 0x2656fb0}, 0xc004af3bf0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254c60, 0x27}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 83803 [semacquire, 31 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x80, 0xc0019e7ec0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a540, 0x21}, 0xc003b5dc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc002d0b320}, 0x0, 0xc003b5de88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881742f18a5a0, {0x26bf478, 0xc002d0b320}, 0xffffb16b6eb10000, {0xc00225a540, 0x2656fb0}, 0x4064c5) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a540, 0x21}, 0x4e94914f0000, 0xc001643770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 98210 [semacquire, 27 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a540, 0x21}, 0xc0038f7c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0041c2f00}, 0x0, 0xc0038f7e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881b7af70ed94, {0x26bf478, 0xc0041c2f00}, 0xffffb16b6eb10000, {0xc00225a540, 0x2656fb0}, 0xc003e4d9b0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a540, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 99181 [semacquire, 26 minutes]: | |
| sync.runtime_SemacquireMutex(0x3baabfdcf462f984, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022457a0, 0x22}, 0xc004137c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0049eb560}, 0x0, 0xc004137e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881bf2f75eee6, {0x26bf478, 0xc0049eb560}, 0xffffb16b6eb10000, {0xc0022457a0, 0x2656fb0}, 0xc001c41320) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022457a0, 0x22}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 79061 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0x312d313a34353133, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc0022554a0, 0x28}, 0xc003b31c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc00417c360}, 0x0, 0xc003b31e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058815daef00325, {0x26bf478, 0xc00417c360}, 0xffffb16b6eb10000, {0xc0022554a0, 0x2656fb0}, 0x12ec4eb) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc0022554a0, 0x28}, 0x4e94914f0000, 0xc001d1a770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63400 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc001173d58, 0x40) | |
| runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc001173d38) | |
| sync/cond.go:56 +0x8c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc001173cc0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:509 +0x428 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:334 +0xfb | |
| goroutine 64115 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f948309a9a0, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc002ad0400, 0xc00257c000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc002ad0400, {0xc00257c000, 0x4000, 0x4000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc002ad0400, {0xc00257c000, 0xc001689c88, 0xc001689c28}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00277e2f8, {0xc00257c000, 0x42c94c, 0x2}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc00038e300, {0xc0028ab3e0, 0x4, 0x112268e}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc00038e300}, {0xc0028ab3e0, 0x4, 0x4}, 0x4) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| vitess.io/vitess/go/mysql.(*Conn).readHeaderFrom(0xc0028ab320, {0x2673340, 0xc00038e300}) | |
| vitess.io/vitess/go/mysql/conn.go:334 +0x4f | |
| vitess.io/vitess/go/mysql.(*Conn).readOnePacket(0x447fa8) | |
| vitess.io/vitess/go/mysql/conn.go:473 +0x6d | |
| vitess.io/vitess/go/mysql.(*Conn).readPacket(0x11fe3b12f1029) | |
| vitess.io/vitess/go/mysql/conn.go:495 +0x25 | |
| vitess.io/vitess/go/mysql.(*Conn).ReadPacket(0xc00309fd40) | |
| vitess.io/vitess/go/mysql/conn.go:531 +0x1d | |
| vitess.io/vitess/go/mysql.mysqlFlavor.readBinlogEvent({}, 0x0) | |
| vitess.io/vitess/go/mysql/flavor_mysql.go:225 +0x1d | |
| vitess.io/vitess/go/mysql.(*Conn).ReadBinlogEvent(...) | |
| vitess.io/vitess/go/mysql/flavor.go:237 | |
| vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents.func1() | |
| vitess.io/vitess/go/vt/binlog/binlog_connection.go:146 +0xc5 | |
| created by vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents | |
| vitess.io/vitess/go/vt/binlog/binlog_connection.go:140 +0xdd | |
| goroutine 63406 [sync.Cond.Wait, 3 minutes]: | |
| sync.runtime_notifyListWait(0xc00016e598, 0x105) | |
| runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc000576780) | |
| sync/cond.go:56 +0x8c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc00016e500) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:509 +0x428 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:334 +0xfb | |
| goroutine 88699 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x109, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244f60, 0x25}, 0xc0037fdc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004896360}, 0x0, 0xc0037fde88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058818aaf32e473, {0x26bf478, 0xc004896360}, 0xffffb16b6eb10000, {0xc002244f60, 0x2656fb0}, 0xc000051830) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244f60, 0x25}, 0x4e94914f0000, 0xc00156d770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 923276 [semacquire, 10 minutes]: | |
| sync.runtime_SemacquireMutex(0x3732363634363434, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a960, 0x26}, 0xc003322c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003a308a0}, 0x0, 0xc003322e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05882b6b07a741e, {0x26bf478, 0xc003a308a0}, 0xffffb16b6eb10000, {0xc00225a960, 0x2656fb0}, 0xc005addce0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a960, 0x26}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 923307 [semacquire, 10 minutes]: | |
| sync.runtime_SemacquireMutex(0xa, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244300, 0x21}, 0xc003220c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0030b1bc0}, 0x0, 0xc003220e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05882b6b079f4b2, {0x26bf478, 0xc0030b1bc0}, 0xffffb16b6eb10000, {0xc002244300, 0x2656fb0}, 0xc005addf50) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244300, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 105584 [semacquire, 22 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002244300, 0x21}, 0xc00315ec38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004ba2300}, 0x0, 0xc00315ee88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881fb2fd0bacb, {0x26bf478, 0xc004ba2300}, 0xffffb16b6eb10000, {0xc002244300, 0x2656fb0}, 0xc001b4a4e0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002244300, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 88732 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x109, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a540, 0x21}, 0xc0037f9c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0017219e0}, 0x0, 0xc0037f9e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058818aaf346cf3, {0x26bf478, 0xc0017219e0}, 0xffffb16b6eb10000, {0xc00225a540, 0x2656fb0}, 0xc004af39e0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a540, 0x21}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 104610 [semacquire, 23 minutes]: | |
| sync.runtime_SemacquireMutex(0x3a37303565653565, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225b1a0, 0x28}, 0xc0033a6c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004a6bc80}, 0x0, 0xc0033a6e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881f3afc47acb, {0x26bf478, 0xc004a6bc80}, 0xffffb16b6eb10000, {0xc00225b1a0, 0x2656fb0}, 0xc002a8c990) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225b1a0, 0x28}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 87288 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x80, 0xc005d0c000) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254000, 0x24}, 0xc004a4fc38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc003edb200}, 0x0, 0xc004a4fe88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881832f3189d3, {0x26bf478, 0xc003edb200}, 0xffffb16b6eb10000, {0xc002254000, 0x2656fb0}, 0xc0046570b0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254000, 0x24}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 110020 [semacquire, 20 minutes]: | |
| sync.runtime_SemacquireMutex(0x2100000054, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002248460, 0x19}, 0xc00305ec38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc005414960}, 0x0, 0xc00305ee88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588220b01225ca, {0x26bf478, 0xc005414960}, 0xffffb16b6eb10000, {0xc002248460, 0x2656fb0}, 0xc004be2690) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002248460, 0x19}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 90325 [semacquire, 29 minutes]: | |
| sync.runtime_SemacquireMutex(0x6972547473726966, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00225a540, 0x21}, 0xc0007a1c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0042a32c0}, 0x0, 0xc0007a1e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881922f404fc3, {0x26bf478, 0xc0042a32c0}, 0xffffb16b6eb10000, {0xc00225a540, 0x2656fb0}, 0xc0006a2900) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00225a540, 0x21}, 0x4e94914f0000, 0xc00327f770) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 100534 [semacquire, 25 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002248720, 0x1d}, 0xc003932c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004897e00}, 0x0, 0xc003932e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881ce2f8cdd1b, {0x26bf478, 0xc004897e00}, 0xffffb16b6eb10000, {0xc002248720, 0x2656fb0}, 0xc004e88c90) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002248720, 0x1d}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 88697 [semacquire, 30 minutes]: | |
| sync.runtime_SemacquireMutex(0x0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc00223b5c0, 0x35}, 0xc0007eec38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0048961e0}, 0x0, 0xc0007eee88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058818aaf32a247, {0x26bf478, 0xc0048961e0}, 0xffffb16b6eb10000, {0xc00223b5c0, 0x2656fb0}, 0xc0049622d0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc00223b5c0, 0x35}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63352 [sync.Cond.Wait, 33 minutes]: | |
| sync.runtime_notifyListWait(0xc001172458, 0x1) | |
| runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc00166cfa0) | |
| sync/cond.go:56 +0x8c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc0011723c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:509 +0x428 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:334 +0xfb | |
| goroutine 63635 [select, 33 minutes]: | |
| vitess.io/vitess/go/vt/topo/etcd2topo.(*Server).Watch.func1() | |
| vitess.io/vitess/go/vt/topo/etcd2topo/watch.go:78 +0x1af | |
| created by vitess.io/vitess/go/vt/topo/etcd2topo.(*Server).Watch | |
| vitess.io/vitess/go/vt/topo/etcd2topo/watch.go:72 +0x4ef | |
| goroutine 108662 [semacquire, 21 minutes]: | |
| sync.runtime_SemacquireMutex(0xc0059892c0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254840, 0x23}, 0xc0030fac38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0028c7200}, 0x0, 0xc0030fae88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588211afeb9690, {0x26bf478, 0xc0028c7200}, 0xffffb16b6eb10000, {0xc002254840, 0x2656fb0}, 0xc004b61830) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254840, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63848 [chan receive, 33 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:399 +0x31 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:398 +0x3b3 | |
| goroutine 80975 [semacquire, 32 minutes]: | |
| sync.runtime_SemacquireMutex(0xa, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254840, 0x23}, 0xc003124c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0028c7e00}, 0x0, 0xc003124e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc05881652efca4c8, {0x26bf478, 0xc0028c7e00}, 0xffffb16b6eb10000, {0xc002254840, 0x2656fb0}, 0xc000076f18) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254840, 0x23}, 0x4e94914f0000, 0xc000076f70) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 78985 [semacquire, 33 minutes]: | |
| sync.runtime_SemacquireMutex(0xc000473fb0, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002245bc0, 0x26}, 0xc001967c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc0052221e0}, 0x0, 0xc001967e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc058815daef00e97, {0x26bf478, 0xc0052221e0}, 0xffffb16b6eb10000, {0xc002245bc0, 0x2656fb0}, 0xc00163b808) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002245bc0, 0x26}, 0x4e94914f0000, 0xc0038f57d0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 893073 [semacquire, 11 minutes]: | |
| sync.runtime_SemacquireMutex(0x44d072, 0x0, 0x2656fb0) | |
| runtime/sema.go:71 +0x25 | |
| sync.(*Mutex).lockSlow(0xc0004ada44) | |
| sync/mutex.go:138 +0x165 | |
| sync.(*Mutex).Lock(...) | |
| sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc0004ada40, {0xc002254840, 0x23}, 0xc005068c38) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:170 +0xef | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1() | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1067 +0x30 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc0002fd860, {0x26bf478, 0xc004374120}, 0x0, 0xc005068e88) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078 +0x182 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc0588298b04a27be, {0x26bf478, 0xc004374120}, 0xffffb16b6eb10000, {0xc002254840, 0x2656fb0}, 0xc0007a9b00) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1066 +0x6b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge({0x26ef630, 0xc0002fd860}, {0xc002254840, 0x23}, 0x4e94914f0000, 0x3b3b3c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:801 +0x1c5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:789 +0xd8 | |
| goroutine 63478 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f948309ac58, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc001a36d80, 0xc0007a4000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc001a36d80, {0xc0007a4000, 0x4000, 0x4000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc001a36d80, {0xc0007a4000, 0xc00283ac88, 0xc00283ac28}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0006500e8, {0xc0007a4000, 0x42c94c, 0x2}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc004b83980, {0xc00043af60, 0x4, 0x112268e}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc004b83980}, {0xc00043af60, 0x4, 0x4}, 0x4) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| vitess.io/vitess/go/mysql.(*Conn).readHeaderFrom(0xc00043aea0, {0x2673340, 0xc004b83980}) | |
| vitess.io/vitess/go/mysql/conn.go:334 +0x4f | |
| vitess.io/vitess/go/mysql.(*Conn).readOnePacket(0x447fa8) | |
| vitess.io/vitess/go/mysql/conn.go:473 +0x6d | |
| vitess.io/vitess/go/mysql.(*Conn).readPacket(0x11fe3b130b3d6) | |
| vitess.io/vitess/go/mysql/conn.go:495 +0x25 | |
| vitess.io/vitess/go/mysql.(*Conn).ReadPacket(0xc003d46d00) | |
| vitess.io/vitess/go/mysql/conn.go:531 +0x1d | |
| vitess.io/vitess/go/mysql.mysqlFlavor.readBinlogEvent({}, 0x0) | |
| vitess.io/vitess/go/mysql/flavor_mysql.go:225 +0x1d | |
| vitess.io/vitess/go/mysql.(*Conn).ReadBinlogEvent(...) | |
| vitess.io/vitess/go/mysql/flavor.go:237 | |
| vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents.func1() | |
| vitess.io/vitess/go/vt/binlog/binlog_connection.go:146 +0xc5 | |
| created by vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents | |
| vitess.io/vitess/go/vt/binlog/binlog_connection.go:140 +0xdd | |
| goroutine 63349 [sync.Cond.Wait, 33 minutes]: | |
| sync.runtime_notifyListWait(0xc001172318, 0x0) | |
| runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc001840fa0) | |
| sync/cond.go:56 +0x8c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc001172280) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:509 +0x428 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:334 +0xfb | |
| goroutine 64124 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents(0xc00024ec40, {0x26bf408, 0xc002f02c80}, 0xc0026e40c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:309 +0x305 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).replicate(0xc00024ec40, {0x2c1, 0xc0035e5450}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:178 +0x125 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).Stream(0xc00024ec40) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:157 +0x159 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream(0xc00231d8c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:379 +0x2e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream(0xc0000e06c0, {0x26bf408, 0xc003f6b800}, {0x22b658d, 0x216e900}, {0x0, 0xc00035c768, 0x12f6f44}, 0x0, 0xc001811020) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:227 +0x1bd | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream(0xc001173e00, {0x26bf408, 0xc003f6b800}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:648 +0x133 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream(0xc001173e00, {0x26bf408, 0xc003f6b800}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:624 +0x70 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).startVStream | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:610 +0x168 | |
| goroutine 64674 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f94832f33d8, 0x72) | |
| runtime/netpoll.go:229 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00377b300, 0xc00120a000, 0x0) | |
| internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00377b300, {0xc00120a000, 0x4000, 0x4000}) | |
| internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00377b300, {0xc00120a000, 0xc0026c3c88, 0xc0026c3c28}) | |
| net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0005643f0, {0xc00120a000, 0x42c94c, 0x2}) | |
| net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0003ecd20, {0xc002e63860, 0x4, 0x112268e}) | |
| bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x2673340, 0xc0003ecd20}, {0xc002e63860, 0x4, 0x4}, 0x4) | |
| io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| io/io.go:347 | |
| vitess.io/vitess/go/mysql.(*Conn).readHeaderFrom(0xc002e637a0, {0x2673340, 0xc0003ecd20}) | |
| vitess.io/vitess/go/mysql/conn.go:334 +0x4f | |
| vitess.io/vitess/go/mysql.(*Conn).readOnePacket(0x448b2f) | |
| vitess.io/vitess/go/mysql/conn.go:473 +0x6d | |
| vitess.io/vitess/go/mysql.(*Conn).readPacket(0xc00007c800) | |
| vitess.io/vitess/go/mysql/conn.go:495 +0x25 | |
| vitess.io/vitess/go/mysql.(*Conn).ReadPacket(0xc00346e3c1) | |
| vitess.io/vitess/go/mysql/conn.go:531 +0x1d | |
| vitess.io/vitess/go/mysql.mysqlFlavor.readBinlogEvent({}, 0x2) | |
| vitess.io/vitess/go/mysql/flavor_mysql.go:225 +0x1d | |
| vitess.io/vitess/go/mysql.(*Conn).ReadBinlogEvent(...) | |
| vitess.io/vitess/go/mysql/flavor.go:237 | |
| vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents.func1() | |
| vitess.io/vitess/go/vt/binlog/binlog_connection.go:146 +0xc5 | |
| created by vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents | |
| vitess.io/vitess/go/vt/binlog/binlog_connection.go:140 +0xdd | |
| goroutine 63401 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0006469a0, 0xc0038f5e90) | |
| vitess.io/vitess/go/timer/timer.go:98 +0xb1 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| vitess.io/vitess/go/timer/timer.go:86 +0xf2 | |
| goroutine 64634 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents(0xc00147ab60, {0x26bf408, 0xc0027e3880}, 0xc0029e38c0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:309 +0x305 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).replicate(0xc00147ab60, {0x2c1, 0xc004104b90}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:178 +0x125 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).Stream(0xc00147ab60) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:157 +0x159 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream(0xc00297d560) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:379 +0x2e5 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream(0xc0000e06c0, {0x26bf408, 0xc0041a25c0}, {0x22b658d, 0x216e900}, {0x0, 0xc001d43f68, 0x12f6f44}, 0x0, 0xc003d4c8a0) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:227 +0x1bd | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream(0xc001173900, {0x26bf408, 0xc0041a25c0}) | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:648 +0x133 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment