world leader in high performance signal processing
Trace: » tcpip_performance

Linux TCP/IP stack CPU utilization on blackfin

In order to get a rough concept of the Linux 2.6 kernel TCP/IP stack performance on blackfin, we do a simple benchmark by iperf and oprofile.

The kernel source is the latest pre 2011R1 based on kernel 2.6.30. The hardware is BF527 EZKIT v2.2 and bf527 v0.2 connect to a host PC via on chip 100M EMAC. The kernel configuration is the default BF527-EZKIT-V2 with oprofile enabled.

Iperf TCP test

oprofile result when blackfin runs as TCP client

Note we need to run “ipers -s” first on server before running as client on another side.

root:/> iperf -c 10.100.4.174 -t 30
------------------------------------------------------------
Client connecting to 10.100.4.174, TCP port 5001
TCP window size: 16.0 KByte (default)
------------------------------------------------------------
[  5] local 10.100.4.50 port 34152 connected with 10.100.4.174 port 5001
[ ID] Interval       Transfer     Bandwidth
[  5]  0.0-30.0 sec    328 MBytes  91.7 Mbits/sec

CPU: CPU with timer interrupt, speed 525 MHz (estimated)
Profiling through timer interrupt
samples  %        app name                 symbol name
91        1.5968  iperf                    (no symbols)

273       4.7903  vmlinux                  _tcp_transmit_skb
228       4.0007  vmlinux                  _tcp_ack
205       3.5971  vmlinux                  _tcp_sendmsg
124       2.1758  vmlinux                  _tcp_v4_rcv
123       2.1583  vmlinux                  _ip_queue_xmit
102       1.7898  vmlinux                  _tcp_write_xmit
96        1.6845  vmlinux                  _ip_rcv
85        1.4915  vmlinux                  _ip_finish_output
82        1.4388  vmlinux                  _ip_local_deliver
71        1.2458  vmlinux                  ___tcp_select_window
68        1.1932  vmlinux                  _ip_route_input_common
58        1.0177  vmlinux                  _tcp_rcv_established
44        0.7721  vmlinux                  _tcp_event_new_data_sent
43        0.7545  vmlinux                  ___inet_lookup_established
40        0.7019  vmlinux                  _tcp_options_write
39        0.6843  vmlinux                  ___tcp_v4_send_check
39        0.6843  vmlinux                  _bictcp_acked
37        0.6492  vmlinux                  _sock_wfree
27        0.4738  vmlinux                  _tcp_v4_do_rcv
27        0.4738  vmlinux                  _tcp_valid_rtt_meas
25        0.4387  vmlinux                  ___ip_local_out
25        0.4387  vmlinux                  _tcp_set_skb_tso_segs
23        0.4036  vmlinux                  _ip_local_out
23        0.4036  vmlinux                  _ip_output
23        0.4036  vmlinux                  _tcp_established_options
19        0.3334  vmlinux                  _tcp_current_mss
18        0.3158  vmlinux                  _bictcp_cong_avoid
17        0.2983  vmlinux                  _ipv4_dst_check
17        0.2983  vmlinux                  _tcp_v4_send_check
15        0.2632  vmlinux                  _tcp_parse_aligned_timestamp
12        0.2106  vmlinux                  ___tcp_push_pending_frames
10        0.1755  vmlinux                  _ipv4_default_mtu
10        0.1755  vmlinux                  _tcp_init_tso_segs
9         0.1579  vmlinux                  _inet_sendmsg
9         0.1579  vmlinux                  _ip_fast_csum
9         0.1579  vmlinux                  _tcp_is_cwnd_limited
9         0.1579  vmlinux                  _tcp_push_one
8         0.1404  vmlinux                  _tcp_send_mss
4         0.0702  vmlinux                  _release_sock
1         0.0175  vmlinux                  _tcp_validate_incoming

oprofile result when blackfin runs as TCP server

root:/> iperf -s
------------------------------------------------------------
Server listening on TCP port 5001
TCP window size: 85.3 KByte (default)
------------------------------------------------------------
[  6] local 10.100.4.50 port 5001 connected with 10.100.4.174 port 59502
[ ID] Interval       Transfer     Bandwidth
[  6]  0.0-30.0 sec    337 MBytes  94.2 Mbits/sec

CPU: CPU with timer interrupt, speed 525 MHz (estimated)
Profiling through timer interrupt
samples  %        app name                 symbol name
3051     22.1215  iperf                    (no symbols)

349       2.5305  vmlinux                  _tcp_transmit_skb
261       1.8924  vmlinux                  _tcp_v4_rcv
228       1.6531  vmlinux                  _tcp_ack
205       1.4864  vmlinux                  _tcp_sendmsg
198       1.4356  vmlinux                  _ip_rcv
188       1.3631  vmlinux                  _tcp_rcv_established
181       1.3124  vmlinux                  _ip_queue_xmit
154       1.1166  vmlinux                  _ip_route_input_common
135       0.9788  vmlinux                  _ip_local_deliver
133       0.9643  vmlinux                  _ip_finish_output
110       0.7976  vmlinux                  _tcp_recvmsg
105       0.7613  vmlinux                  ___inet_lookup_established
102       0.7396  vmlinux                  _tcp_write_xmit
84        0.6090  vmlinux                  _tcp_event_data_recv
83        0.6018  vmlinux                  _tx_reclaim_skb
81        0.5873  vmlinux                  ___tcp_select_window
60        0.4350  vmlinux                  _tcp_options_write
55        0.3988  vmlinux                  ___tcp_v4_send_check
51        0.3698  vmlinux                  _tcp_v4_do_rcv
47        0.3408  vmlinux                  _ip_output
47        0.3408  vmlinux                  _sock_wfree
44        0.3190  vmlinux                  _tcp_event_new_data_sent
42        0.3045  vmlinux                  _tcp_established_options
39        0.2828  vmlinux                  _bictcp_acked
36        0.2610  vmlinux                  _tcp_parse_aligned_timestamp
35        0.2538  vmlinux                  _udp_recvmsg
34        0.2465  vmlinux                  ___ip_local_out
34        0.2465  vmlinux                  ___sk_dst_check
33        0.2393  vmlinux                  _ip_local_out
30        0.2175  vmlinux                  _sk_filter
30        0.2175  vmlinux                  _sock_recvmsg
29        0.2103  vmlinux                  ___tcp_ack_snd_check
27        0.1958  vmlinux                  _ipv4_dst_check
27        0.1958  vmlinux                  _tcp_rcv_space_adjust
27        0.1958  vmlinux                  _tcp_valid_rtt_meas
25        0.1813  vmlinux                  _tcp_set_skb_tso_segs
23        0.1668  vmlinux                  _sock_rfree
22        0.1595  vmlinux                  _tcp_v4_send_check
19        0.1378  vmlinux                  _ipv4_default_mtu
19        0.1378  vmlinux                  _tcp_cleanup_rbuf
19        0.1378  vmlinux                  _tcp_current_mss
18        0.1305  vmlinux                  _bictcp_cong_avoid
15        0.1088  vmlinux                  _inet_recvmsg
14        0.1015  vmlinux                  _tcp_send_ack
13        0.0943  vmlinux                  _ip_fast_csum
13        0.0943  vmlinux                  _tcp_prequeue_process
12        0.0870  vmlinux                  ___tcp_push_pending_frames
11        0.0798  vmlinux                  _tcp_send_delayed_ack
10        0.0725  vmlinux                  _tcp_init_tso_segs
9         0.0653  vmlinux                  _inet_sendmsg
9         0.0653  vmlinux                  _tcp_is_cwnd_limited
9         0.0653  vmlinux                  _tcp_push_one
8         0.0580  vmlinux                  _tcp_send_mss
6         0.0435  vmlinux                  ___udp4_lib_lookup
6         0.0435  vmlinux                  ___udp4_lib_rcv
2         0.0145  vmlinux                  _ip_queue_rcv_skb
2         0.0145  vmlinux                  _udp_queue_rcv_skb
1         0.0073  vmlinux                  ___udp_queue_rcv_skb
1         0.0073  vmlinux                  _tcp_validate_incoming

Iperf UDP test

oprofile result when blackfin runs as UDP client

root:/> iperf -c 10.100.4.174 -u -b 200M
------------------------------------------------------------
Client connecting to 10.100.4.174, UDP port 5001
Sending 1470 byte datagrams
UDP buffer size:   104 KByte (default)
------------------------------------------------------------
[  5] local 10.100.4.50 port 54202 connected with 10.100.4.174 port 5001
[ ID] Interval       Transfer     Bandwidth
[  5]  0.0-10.0 sec  50.3 MBytes  42.1 Mbits/sec
[  5] Sent 35856 datagrams
[  5] Server Report:
[ ID] Interval       Transfer     Bandwidth       Jitter   Lost/Total Datagrams
[  5]  0.0-10.0 sec  50.3 MBytes  42.2 Mbits/sec  0.361 ms    0/35855 (0%)
[  5]  0.0-10.0 sec  1 datagrams received out-of-order

CPU: CPU with timer interrupt, speed 525 MHz (estimated)
Profiling through timer interrupt
samples  %        app name                 symbol name
2297     26.2755  iperf                    (no symbols)

273       3.1229  vmlinux                  _tcp_transmit_skb
228       2.6081  vmlinux                  _tcp_ack
205       2.3450  vmlinux                  _tcp_sendmsg
124       1.4184  vmlinux                  _tcp_v4_rcv
123       1.4070  vmlinux                  _ip_queue_xmit
102       1.1668  vmlinux                  _tcp_write_xmit
96        1.0981  vmlinux                  _ip_rcv
85        0.9723  vmlinux                  _ip_finish_output
82        0.9380  vmlinux                  _ip_local_deliver
71        0.8122  vmlinux                  ___tcp_select_window
68        0.7779  vmlinux                  _ip_route_input_common
58        0.6635  vmlinux                  _tcp_rcv_established
44        0.5033  vmlinux                  _tcp_event_new_data_sent
43        0.4919  vmlinux                  ___inet_lookup_established
40        0.4576  vmlinux                  _tcp_options_write
39        0.4461  vmlinux                  ___tcp_v4_send_check
39        0.4461  vmlinux                  _bictcp_acked
27        0.3089  vmlinux                  _tcp_v4_do_rcv
27        0.3089  vmlinux                  _tcp_valid_rtt_meas
25        0.2860  vmlinux                  ___ip_local_out
25        0.2860  vmlinux                  _tcp_set_skb_tso_segs
23        0.2631  vmlinux                  _ip_local_out
23        0.2631  vmlinux                  _ip_output
23        0.2631  vmlinux                  _tcp_established_options
19        0.2173  vmlinux                  _tcp_current_mss
18        0.2059  vmlinux                  _bictcp_cong_avoid
17        0.1945  vmlinux                  _ipv4_dst_check
17        0.1945  vmlinux                  _tcp_v4_send_check
15        0.1716  vmlinux                  _tcp_parse_aligned_timestamp
12        0.1373  vmlinux                  ___tcp_push_pending_frames
10        0.1144  vmlinux                  _ipv4_default_mtu
10        0.1144  vmlinux                  _tcp_init_tso_segs
9         0.1030  vmlinux                  _inet_sendmsg
9         0.1030  vmlinux                  _ip_fast_csum
9         0.1030  vmlinux                  _tcp_is_cwnd_limited
9         0.1030  vmlinux                  _tcp_push_one
8         0.0915  vmlinux                  _tcp_send_mss
1         0.0114  vmlinux                  _tcp_validate_incoming

oprofile result when blackfin runs as UDP server

t:/> iperf -s -u
------------------------------------------------------------
Server listening on UDP port 5001
Receiving 1470 byte datagrams
UDP buffer size:   104 KByte (default)
------------------------------------------------------------
[  5] local 10.100.4.50 port 5001 connected with 10.100.4.174 port 60711
[ ID] Interval       Transfer     Bandwidth       Jitter   Lost/Total Datagrams
[  5]  0.0-10.0 sec    114 MBytes  95.7 Mbits/sec  0.435 ms    0/81435 (0%)
[  5]  0.0-10.0 sec  1 datagrams received out-of-order

CPU: CPU with timer interrupt, speed 525 MHz (estimated)
Profiling through timer interrupt
samples  %        app name                 symbol name
2959     29.3522  iperf                    (no symbols)

273       2.7081  vmlinux                  _tcp_transmit_skb
228       2.2617  vmlinux                  _tcp_ack
205       2.0335  vmlinux                  _tcp_sendmsg
124       1.2300  vmlinux                  _tcp_v4_rcv
123       1.2201  vmlinux                  _ip_queue_xmit
104       1.0316  vmlinux                  _ip_rcv
102       1.0118  vmlinux                  _tcp_write_xmit
86        0.8531  vmlinux                  _ip_local_deliver
85        0.8432  vmlinux                  _ip_finish_output
74        0.7341  vmlinux                  _ip_route_input_common
71        0.7043  vmlinux                  ___tcp_select_window
58        0.5753  vmlinux                  _tcp_rcv_established
44        0.4365  vmlinux                  _tcp_event_new_data_sent
43        0.4265  vmlinux                  ___inet_lookup_established
40        0.3968  vmlinux                  _tcp_options_write
39        0.3869  vmlinux                  ___tcp_v4_send_check
39        0.3869  vmlinux                  _bictcp_acked
27        0.2678  vmlinux                  _tcp_v4_do_rcv
27        0.2678  vmlinux                  _tcp_valid_rtt_meas
25        0.2480  vmlinux                  ___ip_local_out
25        0.2480  vmlinux                  _tcp_set_skb_tso_segs
23        0.2282  vmlinux                  _ip_local_out
23        0.2282  vmlinux                  _ip_output
23        0.2282  vmlinux                  _tcp_established_options
19        0.1885  vmlinux                  _tcp_current_mss
18        0.1786  vmlinux                  _bictcp_cong_avoid
17        0.1686  vmlinux                  _ipv4_dst_check
17        0.1686  vmlinux                  _tcp_v4_send_check
15        0.1488  vmlinux                  _tcp_parse_aligned_timestamp
12        0.1190  vmlinux                  ___tcp_push_pending_frames
11        0.1091  vmlinux                  _ip_fast_csum
10        0.0992  vmlinux                  _ipv4_default_mtu
10        0.0992  vmlinux                  _sock_recvmsg
10        0.0992  vmlinux                  _tcp_init_tso_segs
9         0.0893  vmlinux                  _inet_sendmsg
9         0.0893  vmlinux                  _tcp_is_cwnd_limited
9         0.0893  vmlinux                  _tcp_push_one
8         0.0794  vmlinux                  _sock_queue_rcv_skb
8         0.0794  vmlinux                  _tcp_send_mss
6         0.0595  vmlinux                  ___udp4_lib_lookup
6         0.0595  vmlinux                  ___udp4_lib_rcv
2         0.0198  vmlinux                  _ip_queue_rcv_skb
1         0.0099  vmlinux                  ___udp_queue_rcv_skb
1         0.0099  vmlinux                  _tcp_validate_incoming

TCP/IP performance summary

CPU clockSystem clock Test mode TCP/IP stack CPU time Kernel total CPU timeiperf CPU time Bandwidth
525MHz125MHz TCP TX 37%98%1.6% 92Mbps
525MHz125MHz TCP RX 25%78%22% 94Mbps
525MHz125MHz UDP TX 23%47.4%26.3% 42Mbps
525MHz125MHz UDP RX 20%60.7%29.4% 95Mbps