Есть сервер:
- Supermicro 6016T-URF
- CPU: 2 x Intel(R) Xeon(R) CPU E5620@2.40GHz (2 x 4 cores, hyper threading выключен)
- RAM: 6Gb (3x2Gb DDR3)
- HDD: Flash 8Gb
- OS: NanoBSD 8.2 p4 amd64
- NETWORK: 2 x Intel ET Dual Port Server Adapter
igb0 - vlan пользователей
igb1 - vlan национальный трафик
igb2 - vlan пользователей
igb3 - vlan мировой трафик
При нагрузке в 530M симметричного трафика 81 kpps сервер целиком загружен прерываниями:
Код: Выделить всё
# top -aSP
last pid: 4616; load averages: 1.91, 1.19, 0.67 up 0+12:49:46 15:44:28
188 processes: 36 running, 118 sleeping, 34 waiting
CPU 0: 0.0% user, 0.0% nice, 2.7% system, 97.3% interrupt, 0.0% idle
CPU 1: 0.0% user, 0.0% nice, 6.0% system, 93.3% interrupt, 0.7% idle
CPU 2: 0.0% user, 0.0% nice, 8.1% system, 91.9% interrupt, 0.0% idle
CPU 3: 0.0% user, 0.0% nice, 2.0% system, 98.0% interrupt, 0.0% idle
CPU 4: 0.0% user, 0.0% nice, 17.4% system, 77.2% interrupt, 5.4% idle
CPU 5: 2.0% user, 0.0% nice, 26.8% system, 63.8% interrupt, 7.4% idle
CPU 6: 0.0% user, 0.0% nice, 8.1% system, 87.2% interrupt, 4.7% idle
CPU 7: 2.7% user, 0.0% nice, 30.9% system, 55.0% interrupt, 11.4% idle
Mem: 38M Active, 13M Inact, 519M Wired, 1128K Cache, 77M Buf, 5338M Free
Swap:
PID USERNAME THR PRI NICE SIZE RES STATE C TIME WCPU COMMAND
12 root 58 -60 - 0K 928K WAIT 0 21.3H 686.23% [intr]
0 root 40 -68 0 0K 624K - 7 26:11 36.52% [kernel]
11 root 8 171 ki31 0K 128K RUN 0 80.1H 36.04% [idle]
...
Код: Выделить всё
#cat /boot/loader.conf
loader_logo="none"
autoboot_delay="2"
hw.ata.atapi_dma="0"
hw.ata.ata_dma="0"
hw.ata.wc="0"
net.fibs=16
kern.cam.boot_delay=10000
hw.igb.rxd=4096
hw.igb.txd=4096
hw.igb.max_interrupt_rate=1000
Код: Выделить всё
#cat /etc/sysctl.conf
net.inet.tcp.blackhole=2
net.inet.udp.blackhole=1
net.inet.icmp.drop_redirect=1
net.inet.icmp.log_redirect=1
net.inet.ip.redirect=0
net.inet.tcp.drop_synfin=1
net.inet.ip.forwarding=1
net.inet.flowtable.enable=0
kern.coredump=0
kern.ipc.nmbclusters=512000
dev.igb.0.rx_processing_limit=4096
dev.igb.0.enable_aim=0
dev.igb.0.flow_control=0
dev.igb.1.rx_processing_limit=4096
dev.igb.1.enable_aim=0
dev.igb.1.flow_control=0
dev.igb.2.rx_processing_limit=4096
dev.igb.2.enable_aim=0
dev.igb.2.flow_control=0
dev.igb.3.rx_processing_limit=4096
dev.igb.3.enable_aim=0
dev.igb.3.flow_control=0
Код: Выделить всё
1# cat /etc/start_if.igb0
ifconfig igb0 -rxcsum -txcsum -lro -tso up
ifconfig igb1 -rxcsum -txcsum -lro -tso up
ifconfig igb2 -rxcsum -txcsum -lro -tso up
ifconfig igb3 -rxcsum -txcsum -lro -tso up
....
Код: Выделить всё
# cat /usr/local/etc/rc.d/cpuset-igb
#!/bin/sh
# PROVIDE: cpuset-igb
# REQUIRE: FILESYSTEMS
# BEFORE: netif
# KEYWORD: nojail
case "$1" in
*start)
echo "Binding igb(4) IRQs to CPUs"
cpus=`sysctl -n kern.smp.cpus`
vmstat -ai | sed -E '/^irq.*que/!d; s/^irq([0-9]+): igb([0-9]+):que ([0-9]+).*/\1 \2 \3/' |\
while read irq igb que
do
cpuset -l $(( ($igb+$que) % $cpus )) -x $irq
done
;;
esac
Код: Выделить всё
# sysctl -a | grep igb.0
dev.igb.0.%desc: Intel(R) PRO/1000 Network Connection version - 2.0.7
dev.igb.0.%driver: igb
dev.igb.0.%location: slot=0 function=0
dev.igb.0.%pnpinfo: vendor=0x8086 device=0x10c9 subvendor=0x15d9 subdevice=0x0600 class=0x020000
dev.igb.0.%parent: pci1
dev.igb.0.nvm: -1
dev.igb.0.flow_control: 0
dev.igb.0.enable_aim: 0
dev.igb.0.rx_processing_limit: 4096
dev.igb.0.link_irq: 2
dev.igb.0.dropped: 0
dev.igb.0.tx_dma_fail: 0
dev.igb.0.rx_overruns: 0
dev.igb.0.watchdog_timeouts: 0
dev.igb.0.device_control: 1087373889
dev.igb.0.rx_control: 67141634
dev.igb.0.interrupt_mask: 4
dev.igb.0.extended_int_mask: 2147483941
dev.igb.0.tx_buf_alloc: 0
dev.igb.0.rx_buf_alloc: 0
dev.igb.0.fc_high_water: 58976
dev.igb.0.fc_low_water: 58960
dev.igb.0.queue0.interrupt_rate: 1000
dev.igb.0.queue0.txd_head: 2334
dev.igb.0.queue0.txd_tail: 2334
dev.igb.0.queue0.no_desc_avail: 0
dev.igb.0.queue0.tx_packets: 48850063
dev.igb.0.queue0.rxd_head: 3402
dev.igb.0.queue0.rxd_tail: 3401
dev.igb.0.queue0.rx_packets: 34090314
dev.igb.0.queue0.rx_bytes: 14927930908
dev.igb.0.queue0.lro_queued: 0
dev.igb.0.queue0.lro_flushed: 0
dev.igb.0.queue1.interrupt_rate: 1000
dev.igb.0.queue1.txd_head: 2948
dev.igb.0.queue1.txd_tail: 2948
dev.igb.0.queue1.no_desc_avail: 0
dev.igb.0.queue1.tx_packets: 54633922
dev.igb.0.queue1.rxd_head: 170
dev.igb.0.queue1.rxd_tail: 167
dev.igb.0.queue1.rx_packets: 31957162
dev.igb.0.queue1.rx_bytes: 16608051512
dev.igb.0.queue1.lro_queued: 0
dev.igb.0.queue1.lro_flushed: 0
dev.igb.0.queue2.interrupt_rate: 1000
dev.igb.0.queue2.txd_head: 592
dev.igb.0.queue2.txd_tail: 592
dev.igb.0.queue2.no_desc_avail: 0
dev.igb.0.queue2.tx_packets: 34478376
dev.igb.0.queue2.rxd_head: 2247
dev.igb.0.queue2.rxd_tail: 2246
dev.igb.0.queue2.rx_packets: 29161671
dev.igb.0.queue2.rx_bytes: 14408816351
dev.igb.0.queue2.lro_queued: 0
dev.igb.0.queue2.lro_flushed: 0
dev.igb.0.queue3.interrupt_rate: 1000
dev.igb.0.queue3.txd_head: 1648
dev.igb.0.queue3.txd_tail: 1648
dev.igb.0.queue3.no_desc_avail: 0
dev.igb.0.queue3.tx_packets: 29875000
dev.igb.0.queue3.rxd_head: 2972
dev.igb.0.queue3.rxd_tail: 2971
dev.igb.0.queue3.rx_packets: 67283868
dev.igb.0.queue3.rx_bytes: 39140146916
dev.igb.0.queue3.lro_queued: 0
dev.igb.0.queue3.lro_flushed: 0
dev.igb.0.queue4.interrupt_rate: 1000
dev.igb.0.queue4.txd_head: 700
dev.igb.0.queue4.txd_tail: 700
dev.igb.0.queue4.no_desc_avail: 0
dev.igb.0.queue4.tx_packets: 69646686
dev.igb.0.queue4.rxd_head: 1781
dev.igb.0.queue4.rxd_tail: 1780
dev.igb.0.queue4.rx_packets: 31586037
dev.igb.0.queue4.rx_bytes: 15191217027
dev.igb.0.queue4.lro_queued: 0
dev.igb.0.queue4.lro_flushed: 0
dev.igb.0.queue5.interrupt_rate: 1000
dev.igb.0.queue5.txd_head: 1440
dev.igb.0.queue5.txd_tail: 1440
dev.igb.0.queue5.no_desc_avail: 0
dev.igb.0.queue5.tx_packets: 51096272
dev.igb.0.queue5.rxd_head: 2764
dev.igb.0.queue5.rxd_tail: 2762
dev.igb.0.queue5.rx_packets: 33131211
dev.igb.0.queue5.rx_bytes: 18297721719
dev.igb.0.queue5.lro_queued: 0
dev.igb.0.queue5.lro_flushed: 0
dev.igb.0.queue6.interrupt_rate: 1000
dev.igb.0.queue6.txd_head: 612
dev.igb.0.queue6.txd_tail: 612
dev.igb.0.queue6.no_desc_avail: 0
dev.igb.0.queue6.tx_packets: 29516082
dev.igb.0.queue6.rxd_head: 2998
dev.igb.0.queue6.rxd_tail: 2997
dev.igb.0.queue6.rx_packets: 27859894
dev.igb.0.queue6.rx_bytes: 14019910034
dev.igb.0.queue6.lro_queued: 0
dev.igb.0.queue6.lro_flushed: 0
dev.igb.0.queue7.interrupt_rate: 1000
dev.igb.0.queue7.txd_head: 1032
dev.igb.0.queue7.txd_tail: 1032
dev.igb.0.queue7.no_desc_avail: 0
dev.igb.0.queue7.tx_packets: 43895300
dev.igb.0.queue7.rxd_head: 3235
dev.igb.0.queue7.rxd_tail: 3234
dev.igb.0.queue7.rx_packets: 36359331
dev.igb.0.queue7.rx_bytes: 19704509697
dev.igb.0.queue7.lro_queued: 0
dev.igb.0.queue7.lro_flushed: 0
dev.igb.0.mac_stats.excess_coll: 0
dev.igb.0.mac_stats.single_coll: 0
dev.igb.0.mac_stats.multiple_coll: 0
dev.igb.0.mac_stats.late_coll: 0
dev.igb.0.mac_stats.collision_count: 0
dev.igb.0.mac_stats.symbol_errors: 0
dev.igb.0.mac_stats.sequence_errors: 0
dev.igb.0.mac_stats.defer_count: 0
dev.igb.0.mac_stats.missed_packets: 0
dev.igb.0.mac_stats.recv_no_buff: 0
dev.igb.0.mac_stats.recv_undersize: 0
dev.igb.0.mac_stats.recv_fragmented: 0
dev.igb.0.mac_stats.recv_oversize: 0
dev.igb.0.mac_stats.recv_jabber: 0
dev.igb.0.mac_stats.recv_errs: 0
dev.igb.0.mac_stats.crc_errs: 0
dev.igb.0.mac_stats.alignment_errs: 0
dev.igb.0.mac_stats.coll_ext_errs: 0
dev.igb.0.mac_stats.xon_recvd: 0
dev.igb.0.mac_stats.xon_txd: 0
dev.igb.0.mac_stats.xoff_recvd: 0
dev.igb.0.mac_stats.xoff_txd: 0
dev.igb.0.mac_stats.total_pkts_recvd: 293679048
dev.igb.0.mac_stats.good_pkts_recvd: 291424047
dev.igb.0.mac_stats.bcast_pkts_recvd: 2081336
dev.igb.0.mac_stats.mcast_pkts_recvd: 26520
dev.igb.0.mac_stats.rx_frames_64: 0
dev.igb.0.mac_stats.rx_frames_65_127: 179877996
dev.igb.0.mac_stats.rx_frames_128_255: 9035781
dev.igb.0.mac_stats.rx_frames_256_511: 4633737
dev.igb.0.mac_stats.rx_frames_512_1023: 8620689
dev.igb.0.mac_stats.rx_frames_1024_1522: 89255844
dev.igb.0.mac_stats.good_octets_recvd: 154627546923
dev.igb.0.mac_stats.good_octets_txd: 395386062810
dev.igb.0.mac_stats.total_pkts_txd: 361984953
dev.igb.0.mac_stats.good_pkts_txd: 361984953
dev.igb.0.mac_stats.bcast_pkts_txd: 9648
dev.igb.0.mac_stats.mcast_pkts_txd: 342
dev.igb.0.mac_stats.tx_frames_64: 31408955
dev.igb.0.mac_stats.tx_frames_65_127: 52243337
dev.igb.0.mac_stats.tx_frames_128_255: 9840673
dev.igb.0.mac_stats.tx_frames_256_511: 5953245
dev.igb.0.mac_stats.tx_frames_512_1023: 8397409
dev.igb.0.mac_stats.tx_frames_1024_1522: 254141334
dev.igb.0.mac_stats.tso_txd: 0
dev.igb.0.mac_stats.tso_ctx_fail: 0
dev.igb.0.interrupts.asserts: 193999211
dev.igb.0.interrupts.rx_pkt_timer: 291420061
dev.igb.0.interrupts.rx_abs_timer: 0
dev.igb.0.interrupts.tx_pkt_timer: 0
dev.igb.0.interrupts.tx_abs_timer: 291424047
dev.igb.0.interrupts.tx_queue_empty: 361980717
dev.igb.0.interrupts.tx_queue_min_thresh: 0
dev.igb.0.interrupts.rx_desc_min_thresh: 0
dev.igb.0.interrupts.rx_overrun: 0
dev.igb.0.host.breaker_tx_pkt: 0
dev.igb.0.host.host_tx_pkt_discard: 0
dev.igb.0.host.rx_pkt: 3986
dev.igb.0.host.breaker_rx_pkts: 0
dev.igb.0.host.breaker_rx_pkt_drop: 0
dev.igb.0.host.tx_good_pkt: 3895
dev.igb.0.host.breaker_tx_pkt_drop: 0
dev.igb.0.host.rx_good_bytes: 154627546993
dev.igb.0.host.tx_good_bytes: 395385950536
dev.igb.0.host.length_errors: 0
dev.igb.0.host.serdes_violation_pkt: 0
dev.igb.0.host.header_redir_missed: 0