Created: 06 Jan 2023
Last modified: 26 Mar 2023
Recv-Q vs Send-Q
socket statistics
首先从 man ss
中可以找到相关信息:
Recv-Q Established: The count of bytes not copied by the user program connected to this socket. Listening: Since Kernel 2.6.18 this column contains the current syn backlog. Send-Q Established: The count of bytes not acknowledged by the remote host. Listening: Since Kernel 2.6.18 this column contains the maximum size of the syn backlog.
也许这样的排列方式有些不便于理解和数据对照参考,我来根据 Socket 状态稍微进行调整:
- Listening:
- Recv-Q: Since Kernel 2.6.18 this column contains the current syn backlog.
- Send-Q: Since Kernel 2.6.18 this column contains the maximum size of the syn backlog.
- Established:
- Recv-Q: The count of bytes not copied by the user program connected to this socket.
- Send-Q: The count of bytes not acknowledged by the remote host.
还有一点来自 https://www.ibm.com/support/pages/node/6537582 提到,高 Send-Q 的原因之一是数据堆积在 send buffer
中:
High Send-Q means the data is put on TCP/IP send buffer, but it is not sent or it is sent but not ACKed.
Coding
来编写一个单一链接的 tcp server,并且客户端由 nc
直接连接验证,服务端代码如下:
#include <errno.h> #include <netinet/in.h> #include <stdio.h> #include <string.h> #include <sys/socket.h> #include <sys/types.h> #include <unistd.h> #define MAXLNE 4096 #define PORT 8080 #define DO_IGNORE_REVC 0 #define DO_FLOOD_SEND 1 #define FLOOD_SEND_CONCURRENCY 100000 int main(int argc, char **argv) { int listenfd, connfd, n; char buff[MAXLNE]; if ((listenfd = socket(AF_INET, SOCK_STREAM, 0)) == -1) { printf("create socket error: %s(errno: %d)\n", strerror(errno), errno); return 0; } printf("create listenfd %d\n", listenfd); struct sockaddr_in servaddr; memset(&servaddr, 0, sizeof(servaddr)); servaddr.sin_family = AF_INET; servaddr.sin_addr.s_addr = htonl(INADDR_ANY); servaddr.sin_port = htons(PORT); if (bind(listenfd, (struct sockaddr *)&servaddr, sizeof(servaddr)) == -1) { printf("bind socket error: %s(errno: %d)\n", strerror(errno), errno); return 0; } if (listen(listenfd, 10) == -1) { printf("listen socket error: %s(errno: %d)\n", strerror(errno), errno); return 0; } struct sockaddr_in client; socklen_t len = sizeof(client); if ((connfd = accept(listenfd, (struct sockaddr *)&client, &len)) == -1) { printf("accept socket error: %s(errno: %d)\n", strerror(errno), errno); return 0; } printf("== waiting for client's request ==\n"); while (1) { #if DO_IGNORE_REVC sleep(10); printf("ignore recv, do nothing"); #else n = recv(connfd, buff, MAXLNE, 0); if (n > 0) { buff[n] = '\0'; printf("recv msg from client: %s\n", buff); if (DO_FLOOD_SEND) { printf("flood send\n"); for (int i = 1; i < FLOOD_SEND_CONCURRENCY; i++) { send(connfd, buff, n, 0); } printf("done\n"); } else { send(connfd, buff, n, 0); } } else if (n == 0) { close(connfd); } #endif } close(listenfd); return 0; }
由 DO_IGNORE_REVC
和 DO_FLOOD_SEND
分别控制服务端,不调用 recv
和接收数据后成十万倍回复客户端。
Recv-Q 堆积
客户端通过 nc 连接后,发送任意数据都会使服务端堆积的效果显著:
## client # nc -v localhost 8080 ## server # ss -tnap State Recv-Q Send-Q Local Address:Port Peer Address:Port Process LISTEN 0 10 0.0.0.0:8080 0.0.0.0:* users:(("a.out",pid=123979,fd=3)) ESTAB 1630 0 127.0.0.1:8080 127.0.0.1:44896 users:(("a.out",pid=123979,fd=4)) ESTAB 0 0 127.0.0.1:44896 127.0.0.1:8080
Send-Q 堆积
开启加倍回复之后,server 端 Send-Q 因为 send buffer 数据过多而增长:
## client # nc -v 192.168.64.4 8080 ## server # ss -tnap State Recv-Q Send-Q Local Address:Port Peer Address:Port Process LISTEN 0 10 0.0.0.0:8080 0.0.0.0:* users:(("a.out",pid=697458,fd=3)) ESTAB 0 3208667 192.168.64.4:8080 192.168.64.4:20030 users:(("a.out",pid=697458,fd=4)) ESTAB 5976309 0 192.168.64.4:20030 192.168.64.4:8080 users:(("nc",pid=697462,fd=3))
深入源码
参考代码提示来自:https://www.cxyxiaowu.com/10962.html
file: linux-5.15.86/net/ipv4/tcpdiag.c
static void tcp_diag_get_info(struct sock *sk, struct inet_diag_msg *r, void *_info) { struct tcp_info *info = _info; if (inet_sk_state_load(sk) == TCP_LISTEN) { r->idiag_rqueue = READ_ONCE(sk->sk_ack_backlog); r->idiag_wqueue = READ_ONCE(sk->sk_max_ack_backlog); } else if (sk->sk_type == SOCK_STREAM) { const struct tcp_sock *tp = tcp_sk(sk); r->idiag_rqueue = max_t(int, READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->copied_seq), 0); r->idiag_wqueue = READ_ONCE(tp->write_seq) - tp->snd_una; } if (info) tcp_get_info(sk, info); }
renyddd by Renyddd is licensed under a Creative Commons Attribution-ShareAlike 3.0 Unported License.