jcs.org
qemu with hax to log dma reads & writes
jcs.org/2018/11/12/vfio
Merge remote-tracking branch 'remotes/ehabkost/tags/python-next-pull-request' into staging
Python queue for 5.0 soft freeze
* Add scripts/simplebench (Vladimir Sementsov-Ogievskiy)
# gpg: Signature made Wed 18 Mar 2020 01:11:49 GMT
# gpg: using RSA key 5A322FD5ABC4D3DBACCFD1AA2807936F984DC5A6
# gpg: issuer "ehabkost@redhat.com"
# gpg: Good signature from "Eduardo Habkost <ehabkost@redhat.com>" [full]
# Primary key fingerprint: 5A32 2FD5 ABC4 D3DB ACCF D1AA 2807 936F 984D C5A6
* remotes/ehabkost/tags/python-next-pull-request:
MAINTAINERS: add simplebench
scripts/simplebench: add example usage of simplebench
scripts/simplebench: add qemu/bench_block_job.py
scripts/simplebench: add simplebench.py
Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
+332
+5
MAINTAINERS
+5
MAINTAINERS
···
2147
2147
F: scripts/*.py
2148
2148
F: tests/*.py
2149
2149
2150
+
Benchmark util
2151
+
M: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
2152
+
S: Maintained
2153
+
F: scripts/simplebench/
2154
+
2150
2155
QAPI
2151
2156
M: Markus Armbruster <armbru@redhat.com>
2152
2157
M: Michael Roth <mdroth@linux.vnet.ibm.com>
+80
scripts/simplebench/bench-example.py
+80
scripts/simplebench/bench-example.py
···
1
+
#!/usr/bin/env python3
2
+
#
3
+
# Benchmark example
4
+
#
5
+
# Copyright (c) 2019 Virtuozzo International GmbH.
6
+
#
7
+
# This program is free software; you can redistribute it and/or modify
8
+
# it under the terms of the GNU General Public License as published by
9
+
# the Free Software Foundation; either version 2 of the License, or
10
+
# (at your option) any later version.
11
+
#
12
+
# This program is distributed in the hope that it will be useful,
13
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+
# GNU General Public License for more details.
16
+
#
17
+
# You should have received a copy of the GNU General Public License
18
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
19
+
#
20
+
21
+
import simplebench
22
+
from bench_block_job import bench_block_copy, drv_file, drv_nbd
23
+
24
+
25
+
def bench_func(env, case):
26
+
""" Handle one "cell" of benchmarking table. """
27
+
return bench_block_copy(env['qemu_binary'], env['cmd'],
28
+
case['source'], case['target'])
29
+
30
+
31
+
# You may set the following five variables to correct values, to turn this
32
+
# example to real benchmark.
33
+
ssd_source = '/path-to-raw-source-image-at-ssd'
34
+
ssd_target = '/path-to-raw-target-image-at-ssd'
35
+
hdd_target = '/path-to-raw-source-image-at-hdd'
36
+
nbd_ip = 'nbd-ip-addr'
37
+
nbd_port = 'nbd-port-number'
38
+
39
+
# Test-cases are "rows" in benchmark resulting table, 'id' is a caption for
40
+
# the row, other fields are handled by bench_func.
41
+
test_cases = [
42
+
{
43
+
'id': 'ssd -> ssd',
44
+
'source': drv_file(ssd_source),
45
+
'target': drv_file(ssd_target)
46
+
},
47
+
{
48
+
'id': 'ssd -> hdd',
49
+
'source': drv_file(ssd_source),
50
+
'target': drv_file(hdd_target)
51
+
},
52
+
{
53
+
'id': 'ssd -> nbd',
54
+
'source': drv_file(ssd_source),
55
+
'target': drv_nbd(nbd_ip, nbd_port)
56
+
},
57
+
]
58
+
59
+
# Test-envs are "columns" in benchmark resulting table, 'id is a caption for
60
+
# the column, other fields are handled by bench_func.
61
+
test_envs = [
62
+
{
63
+
'id': 'backup-1',
64
+
'cmd': 'blockdev-backup',
65
+
'qemu_binary': '/path-to-qemu-binary-1'
66
+
},
67
+
{
68
+
'id': 'backup-2',
69
+
'cmd': 'blockdev-backup',
70
+
'qemu_binary': '/path-to-qemu-binary-2'
71
+
},
72
+
{
73
+
'id': 'mirror',
74
+
'cmd': 'blockdev-mirror',
75
+
'qemu_binary': '/path-to-qemu-binary-1'
76
+
}
77
+
]
78
+
79
+
result = simplebench.bench(bench_func, test_envs, test_cases, count=3)
80
+
print(simplebench.ascii(result))
+119
scripts/simplebench/bench_block_job.py
+119
scripts/simplebench/bench_block_job.py
···
1
+
#!/usr/bin/env python
2
+
#
3
+
# Benchmark block jobs
4
+
#
5
+
# Copyright (c) 2019 Virtuozzo International GmbH.
6
+
#
7
+
# This program is free software; you can redistribute it and/or modify
8
+
# it under the terms of the GNU General Public License as published by
9
+
# the Free Software Foundation; either version 2 of the License, or
10
+
# (at your option) any later version.
11
+
#
12
+
# This program is distributed in the hope that it will be useful,
13
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+
# GNU General Public License for more details.
16
+
#
17
+
# You should have received a copy of the GNU General Public License
18
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
19
+
#
20
+
21
+
22
+
import sys
23
+
import os
24
+
import socket
25
+
import json
26
+
27
+
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
28
+
from qemu.machine import QEMUMachine
29
+
from qemu.qmp import QMPConnectError
30
+
31
+
32
+
def bench_block_job(cmd, cmd_args, qemu_args):
33
+
"""Benchmark block-job
34
+
35
+
cmd -- qmp command to run block-job (like blockdev-backup)
36
+
cmd_args -- dict of qmp command arguments
37
+
qemu_args -- list of Qemu command line arguments, including path to Qemu
38
+
binary
39
+
40
+
Returns {'seconds': int} on success and {'error': str} on failure, dict may
41
+
contain addional 'vm-log' field. Return value is compatible with
42
+
simplebench lib.
43
+
"""
44
+
45
+
vm = QEMUMachine(qemu_args[0], args=qemu_args[1:])
46
+
47
+
try:
48
+
vm.launch()
49
+
except OSError as e:
50
+
return {'error': 'popen failed: ' + str(e)}
51
+
except (QMPConnectError, socket.timeout):
52
+
return {'error': 'qemu failed: ' + str(vm.get_log())}
53
+
54
+
try:
55
+
res = vm.qmp(cmd, **cmd_args)
56
+
if res != {'return': {}}:
57
+
vm.shutdown()
58
+
return {'error': '"{}" command failed: {}'.format(cmd, str(res))}
59
+
60
+
e = vm.event_wait('JOB_STATUS_CHANGE')
61
+
assert e['data']['status'] == 'created'
62
+
start_ms = e['timestamp']['seconds'] * 1000000 + \
63
+
e['timestamp']['microseconds']
64
+
65
+
e = vm.events_wait((('BLOCK_JOB_READY', None),
66
+
('BLOCK_JOB_COMPLETED', None),
67
+
('BLOCK_JOB_FAILED', None)), timeout=True)
68
+
if e['event'] not in ('BLOCK_JOB_READY', 'BLOCK_JOB_COMPLETED'):
69
+
vm.shutdown()
70
+
return {'error': 'block-job failed: ' + str(e),
71
+
'vm-log': vm.get_log()}
72
+
end_ms = e['timestamp']['seconds'] * 1000000 + \
73
+
e['timestamp']['microseconds']
74
+
finally:
75
+
vm.shutdown()
76
+
77
+
return {'seconds': (end_ms - start_ms) / 1000000.0}
78
+
79
+
80
+
# Bench backup or mirror
81
+
def bench_block_copy(qemu_binary, cmd, source, target):
82
+
"""Helper to run bench_block_job() for mirror or backup"""
83
+
assert cmd in ('blockdev-backup', 'blockdev-mirror')
84
+
85
+
source['node-name'] = 'source'
86
+
target['node-name'] = 'target'
87
+
88
+
return bench_block_job(cmd,
89
+
{'job-id': 'job0', 'device': 'source',
90
+
'target': 'target', 'sync': 'full'},
91
+
[qemu_binary,
92
+
'-blockdev', json.dumps(source),
93
+
'-blockdev', json.dumps(target)])
94
+
95
+
96
+
def drv_file(filename):
97
+
return {'driver': 'file', 'filename': filename,
98
+
'cache': {'direct': True}, 'aio': 'native'}
99
+
100
+
101
+
def drv_nbd(host, port):
102
+
return {'driver': 'nbd',
103
+
'server': {'type': 'inet', 'host': host, 'port': port}}
104
+
105
+
106
+
if __name__ == '__main__':
107
+
import sys
108
+
109
+
if len(sys.argv) < 4:
110
+
print('USAGE: {} <qmp block-job command name> '
111
+
'<json string of arguments for the command> '
112
+
'<qemu binary path and arguments>'.format(sys.argv[0]))
113
+
exit(1)
114
+
115
+
res = bench_block_job(sys.argv[1], json.loads(sys.argv[2]), sys.argv[3:])
116
+
if 'seconds' in res:
117
+
print('{:.2f}'.format(res['seconds']))
118
+
else:
119
+
print(res)
+128
scripts/simplebench/simplebench.py
+128
scripts/simplebench/simplebench.py
···
1
+
#!/usr/bin/env python
2
+
#
3
+
# Simple benchmarking framework
4
+
#
5
+
# Copyright (c) 2019 Virtuozzo International GmbH.
6
+
#
7
+
# This program is free software; you can redistribute it and/or modify
8
+
# it under the terms of the GNU General Public License as published by
9
+
# the Free Software Foundation; either version 2 of the License, or
10
+
# (at your option) any later version.
11
+
#
12
+
# This program is distributed in the hope that it will be useful,
13
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+
# GNU General Public License for more details.
16
+
#
17
+
# You should have received a copy of the GNU General Public License
18
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
19
+
#
20
+
21
+
22
+
def bench_one(test_func, test_env, test_case, count=5, initial_run=True):
23
+
"""Benchmark one test-case
24
+
25
+
test_func -- benchmarking function with prototype
26
+
test_func(env, case), which takes test_env and test_case
27
+
arguments and returns {'seconds': int} (which is benchmark
28
+
result) on success and {'error': str} on error. Returned
29
+
dict may contain any other additional fields.
30
+
test_env -- test environment - opaque first argument for test_func
31
+
test_case -- test case - opaque second argument for test_func
32
+
count -- how many times to call test_func, to calculate average
33
+
initial_run -- do initial run of test_func, which don't get into result
34
+
35
+
Returns dict with the following fields:
36
+
'runs': list of test_func results
37
+
'average': average seconds per run (exists only if at least one run
38
+
succeeded)
39
+
'delta': maximum delta between test_func result and the average
40
+
(exists only if at least one run succeeded)
41
+
'n-failed': number of failed runs (exists only if at least one run
42
+
failed)
43
+
"""
44
+
if initial_run:
45
+
print(' #initial run:')
46
+
print(' ', test_func(test_env, test_case))
47
+
48
+
runs = []
49
+
for i in range(count):
50
+
print(' #run {}'.format(i+1))
51
+
res = test_func(test_env, test_case)
52
+
print(' ', res)
53
+
runs.append(res)
54
+
55
+
result = {'runs': runs}
56
+
57
+
successed = [r for r in runs if ('seconds' in r)]
58
+
if successed:
59
+
avg = sum(r['seconds'] for r in successed) / len(successed)
60
+
result['average'] = avg
61
+
result['delta'] = max(abs(r['seconds'] - avg) for r in successed)
62
+
63
+
if len(successed) < count:
64
+
result['n-failed'] = count - len(successed)
65
+
66
+
return result
67
+
68
+
69
+
def ascii_one(result):
70
+
"""Return ASCII representation of bench_one() returned dict."""
71
+
if 'average' in result:
72
+
s = '{:.2f} +- {:.2f}'.format(result['average'], result['delta'])
73
+
if 'n-failed' in result:
74
+
s += '\n({} failed)'.format(result['n-failed'])
75
+
return s
76
+
else:
77
+
return 'FAILED'
78
+
79
+
80
+
def bench(test_func, test_envs, test_cases, *args, **vargs):
81
+
"""Fill benchmark table
82
+
83
+
test_func -- benchmarking function, see bench_one for description
84
+
test_envs -- list of test environments, see bench_one
85
+
test_cases -- list of test cases, see bench_one
86
+
args, vargs -- additional arguments for bench_one
87
+
88
+
Returns dict with the following fields:
89
+
'envs': test_envs
90
+
'cases': test_cases
91
+
'tab': filled 2D array, where cell [i][j] is bench_one result for
92
+
test_cases[i] for test_envs[j] (i.e., rows are test cases and
93
+
columns are test environments)
94
+
"""
95
+
tab = {}
96
+
results = {
97
+
'envs': test_envs,
98
+
'cases': test_cases,
99
+
'tab': tab
100
+
}
101
+
n = 1
102
+
n_tests = len(test_envs) * len(test_cases)
103
+
for env in test_envs:
104
+
for case in test_cases:
105
+
print('Testing {}/{}: {} :: {}'.format(n, n_tests,
106
+
env['id'], case['id']))
107
+
if case['id'] not in tab:
108
+
tab[case['id']] = {}
109
+
tab[case['id']][env['id']] = bench_one(test_func, env, case,
110
+
*args, **vargs)
111
+
n += 1
112
+
113
+
print('Done')
114
+
return results
115
+
116
+
117
+
def ascii(results):
118
+
"""Return ASCII representation of bench() returned dict."""
119
+
from tabulate import tabulate
120
+
121
+
tab = [[""] + [c['id'] for c in results['envs']]]
122
+
for case in results['cases']:
123
+
row = [case['id']]
124
+
for env in results['envs']:
125
+
row.append(ascii_one(results['tab'][case['id']][env['id']]))
126
+
tab.append(row)
127
+
128
+
return tabulate(tab)