web 缓存服务器 HTTP2 性能测试: nuster vs nginx

简单的用 h2load 测试了 nuster 和 nginx 的 http/2 下的缓存性能，结果显示 RPS（每秒请求数）nuster 差不多是 nginx 的 3 倍

jiangwenyuan/nuster A high performance HTTP proxy cache server and RESTful NoSQL cache server based on HAProxy - jiangwenyuan/nuster GitHub - github.com

data size       | CONN | nuster  | nginx

---------       | ---- | ------  | -----

12(hello world) | 1000 | 338924  | 110419

测试环境

服务器

两台 linux 服务器：一台测试服务器 server129, 一台缓存服务器 server130.

Server     | port | app

--         | --   | ---

10.0.0.129 |      | wrk

10.0.0.129 | 8080 | nginx, origin web server

10.0.0.130 | 8081 | nuster, all cores

10.0.0.130 | 8083 | nginx, all cores

硬件

• Intel(R) Xeon(R) CPU X5650 @ 2.67GHz(12 cores)

• RAM 32GB

• 1Gbps ethernet card

软件

• CentOS: 7.4.1708 (Core)

• h2load: nghttp2/1.30.0

• nginx: nginx/1.14.0

* nuster: nuster/1.8.8.2

系统设置

/etc/sysctl.conf

fs.file-max                    = 9999999
fs.nr_open                     = 9999999
net.core.netdev\_max\_backlog    = 4096
net.core.rmem_max              = 16777216
net.core.somaxconn             = 65535
net.core.wmem_max              = 16777216
net.ipv4.ip_forward            = 0
net.ipv4.ip\_local\_port_range   = 1025       65535
net.ipv4.tcp\_fin\_timeout       = 30
net.ipv4.tcp\_keepalive\_time    = 30
net.ipv4.tcp\_max\_syn_backlog   = 20480
net.ipv4.tcp\_max\_tw_buckets    = 400000
net.ipv4.tcp\_no\_metrics_save   = 1
net.ipv4.tcp\_syn\_retries       = 2
net.ipv4.tcp\_synack\_retries    = 2
net.ipv4.tcp\_tw\_recycle        = 1
net.ipv4.tcp\_tw\_reuse          = 1
net.ipv4.tcp_timestamps        = 1
vm.min\_free\_kbytes             = 65536
vm.overcommit_memory           = 1

/etc/security/limits.conf

* soft nofile 1000000
* hard nofile 1000000
* soft nproc  1000000
* hard nproc  1000000

配置文件

nuster, all cores

global
    maxconn 1000000
    nuster cache on data-size 1g
    daemon
    nbproc 12
    tune.maxaccept -1
    tune.ssl.default-dh-param 2048
    tune.h2.max-concurrent-streams 1000

defaults
    retries 3
    maxconn 1000000
    option redispatch
    option dontlognull
    timeout client  300s
    timeout connect 300s
    timeout server  300s
    http-reuse always

frontend web1
    bind *:8081 ssl crt asdf.pem alpn h2,http/1.1
    mode http
    default_backend app1

backend app1
    balance roundrobin
    mode http
    nuster cache on
    nuster rule all ttl 0
    server a2 10.0.0.129:8080

nginx, all cores

user  nginx;
worker_processes  auto;
worker\_rlimit\_nofile 1000000;
error_log  /var/log/nginx/errorall.log warn;
pid        /var/run/nginxall.pid;
events {
  worker_connections  1000000;
  use epoll;
  multi_accept on;
}

http {
  include                     /etc/nginx/mime.types;
  default_type                application/octet-stream;
  access_log                  off;
  sendfile                    on;
  server_tokens               off;
  keepalive_timeout           300;
  keepalive_requests          100000;
  tcp_nopush                  on;
  tcp_nodelay                 on;
  client\_body\_buffer_size     128k;
  client\_header\_buffer_size   1m;
  large\_client\_header_buffers 4 4k;
  output_buffers              1 32k;
  postpone_output             1460;
  open\_file\_cache             max=200000 inactive=20s;
  open\_file\_cache_valid       30s;
  open\_file\_cache\_min\_uses    2;
  open\_file\_cache_errors      on;

  proxy\_cache\_path /tmp/cache\_all levels=1:2 keys\_zone=STATIC:10m inactive=24h max_size=1g;
  server {
    listen 8083 ssl http2;
    ssl on;
    ssl\_session\_cache shared:SSL:5m;
    ssl\_session\_timeout 1h;
    http2\_max\_concurrent_streams 1000;
    http2\_max\_requests 10000000;
    ssl_certificate     a.pem;
    ssl\_certificate\_key a.key;
  
    location / {
      proxy_pass        http://10.0.0.129:8080/;
      proxy_cache       STATIC;
      proxy\_cache\_valid any 1d;
    }
  }
}

测试

nuster

$ h2load -n 10000000 -c 1000 -m 1000 -t 1000 https://10.0.0.130:8081/helloworld
  

finished in 29.51s, 338924.15 req/s, 48.81MB/s
requests: 10000000 total, 10000000 started, 10000000 done, 10000000 succeeded, 0 failed, 0 errored, 0 timeout
status codes: 10000000 2xx, 0 3xx, 0 4xx, 0 5xx
traffic: 1.41GB (1510024000) total, 1.13GB (1210000000) headers (space savings 34.24%), 114.44MB (120000000) data
                     min         max         mean         sd        +/- sd
time for request:    14.51ms       3.21s       2.53s    500.02ms    74.50%
time for connect:    61.12ms       1.26s    672.73ms    332.30ms    63.50%
time to 1st byte:    95.03ms       3.74s       2.68s    777.51ms    74.00%
req/s           :     339.11      459.68      358.92       17.35    76.20%

nginx

$ h2load -n 10000000 -c 1000 -m 1000 -t 1000 https://10.0.0.130:8083/helloworld

  
finished in 90.56s, 110419.16 req/s, 15.62MB/s
requests: 10000000 total, 10000000 started, 10000000 done, 10000000 succeeded, 0 failed, 0 errored, 0 timeout
status codes: 10000000 2xx, 0 3xx, 0 4xx, 0 5xx
traffic: 1.38GB (1482955210) total, 1.10GB (1182906210) headers (space savings 35.01%), 114.44MB (120000000) data
                     min         max         mean         sd        +/- sd
time for request:     3.98ms      14.65s       5.14s       3.67s    70.30%
time for connect:    69.96ms       6.74s       1.35s       1.08s    88.90%
time to 1st byte:   114.92ms      15.46s       5.67s       4.06s    63.40%
req/s           :     110.43     5143.28      378.40      590.47    92.50%