Renyddd Site
Created: 06 Jan 2023

Recv-Q vs Send-Q

socket statistics

首先从 man ss 中可以找到相关信息:

Recv-Q
    Established: The count of bytes not copied by the user program connected to this socket.  Listening: Since Kernel 2.6.18 this column contains the current syn backlog.

Send-Q
    Established: The count of bytes not acknowledged by the remote host.  Listening: Since Kernel 2.6.18 this column contains the maximum size of the syn backlog.

也许这样的排列方式有些不便于理解和数据对照参考,我来根据 Socket 状态稍微进行调整:

  1. Listening:
    • Recv-Q: Since Kernel 2.6.18 this column contains the current syn backlog.
    • Send-Q: Since Kernel 2.6.18 this column contains the maximum size of the syn backlog.
  2. Established:
    • Recv-Q: The count of bytes not copied by the user program connected to this socket.
    • Send-Q: The count of bytes not acknowledged by the remote host.

还有一点来自 https://www.ibm.com/support/pages/node/6537582 提到,高 Send-Q 的原因之一是数据堆积在 send buffer 中:

High Send-Q means the data is put on TCP/IP send buffer, but it is not sent or it is sent but not ACKed.

Coding

来编写一个单一链接的 tcp server,并且客户端由 nc 直接连接验证,服务端代码如下:

#include <errno.h>
#include <netinet/in.h>
#include <stdio.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <unistd.h>

#define MAXLNE 4096
#define PORT 8080
#define DO_IGNORE_REVC 0
#define DO_FLOOD_SEND 1
#define FLOOD_SEND_CONCURRENCY 100000

int main(int argc, char **argv) {
  int listenfd, connfd, n;
  char buff[MAXLNE];

  if ((listenfd = socket(AF_INET, SOCK_STREAM, 0)) == -1) {
    printf("create socket error: %s(errno: %d)\n", strerror(errno), errno);
    return 0;
  }

  printf("create listenfd %d\n", listenfd);

  struct sockaddr_in servaddr;
  memset(&servaddr, 0, sizeof(servaddr));
  servaddr.sin_family = AF_INET;
  servaddr.sin_addr.s_addr = htonl(INADDR_ANY);
  servaddr.sin_port = htons(PORT);

  if (bind(listenfd, (struct sockaddr *)&servaddr, sizeof(servaddr)) == -1) {
    printf("bind socket error: %s(errno: %d)\n", strerror(errno), errno);
    return 0;
  }

  if (listen(listenfd, 10) == -1) {
    printf("listen socket error: %s(errno: %d)\n", strerror(errno), errno);
    return 0;
  }

  struct sockaddr_in client;
  socklen_t len = sizeof(client);
  if ((connfd = accept(listenfd, (struct sockaddr *)&client, &len)) == -1) {
    printf("accept socket error: %s(errno: %d)\n", strerror(errno), errno);
    return 0;
  }

  printf("== waiting for client's request ==\n");

  while (1) {

#if DO_IGNORE_REVC

    sleep(10);
    printf("ignore recv, do nothing");

#else

    n = recv(connfd, buff, MAXLNE, 0);
    if (n > 0) {
      buff[n] = '\0';
      printf("recv msg from client: %s\n", buff);

      if (DO_FLOOD_SEND) {
        printf("flood send\n");
        for (int i = 1; i < FLOOD_SEND_CONCURRENCY; i++) {
          send(connfd, buff, n, 0);
        }
        printf("done\n");
      } else {
        send(connfd, buff, n, 0);
      }

    } else if (n == 0) {
      close(connfd);
    }

#endif
  }

  close(listenfd);
  return 0;
}

DO_IGNORE_REVCDO_FLOOD_SEND 分别控制服务端,不调用 recv 和接收数据后成十万倍回复客户端。

Recv-Q 堆积

客户端通过 nc 连接后,发送任意数据都会使服务端堆积的效果显著:

## client
# nc -v localhost 8080

## server
# ss -tnap
State     Recv-Q    Send-Q       Local Address:Port        Peer Address:Port    Process

LISTEN    0         10                 0.0.0.0:8080             0.0.0.0:*        users:(("a.out",pid=123979,fd=3))

ESTAB     1630      0                127.0.0.1:8080           127.0.0.1:44896    users:(("a.out",pid=123979,fd=4))

ESTAB     0         0                127.0.0.1:44896          127.0.0.1:8080

Send-Q 堆积

开启加倍回复之后,server 端 Send-Q 因为 send buffer 数据过多而增长:

## client
# nc -v 192.168.64.4 8080

## server
# ss -tnap
State         Recv-Q     Send-Q        Local Address:Port         Peer Address:Port     Process
LISTEN        0          10                  0.0.0.0:8080              0.0.0.0:*         users:(("a.out",pid=697458,fd=3))
ESTAB         0          3208667        192.168.64.4:8080         192.168.64.4:20030     users:(("a.out",pid=697458,fd=4))
ESTAB         5976309    0              192.168.64.4:20030        192.168.64.4:8080      users:(("nc",pid=697462,fd=3))

深入源码

参考代码提示来自:https://www.cxyxiaowu.com/10962.html

file: linux-5.15.86/net/ipv4/tcpdiag.c

static void tcp_diag_get_info(struct sock *sk, struct inet_diag_msg *r,
                              void *_info)
{
        struct tcp_info *info = _info;

        if (inet_sk_state_load(sk) == TCP_LISTEN) {
                r->idiag_rqueue = READ_ONCE(sk->sk_ack_backlog);
                r->idiag_wqueue = READ_ONCE(sk->sk_max_ack_backlog);
        } else if (sk->sk_type == SOCK_STREAM) {
                const struct tcp_sock *tp = tcp_sk(sk);

                r->idiag_rqueue = max_t(int, READ_ONCE(tp->rcv_nxt) -
                                             READ_ONCE(tp->copied_seq), 0);
                r->idiag_wqueue = READ_ONCE(tp->write_seq) - tp->snd_una;
        }
        if (info)
                tcp_get_info(sk, info);
}
Creative Commons License
renyddd by Renyddd is licensed under a Creative Commons Attribution-ShareAlike 3.0 Unported License.