jcs.org
qemu with hax to log dma reads & writes
jcs.org/2018/11/12/vfio
Merge remote-tracking branch 'remotes/cleber/tags/python-next-pull-request' into staging
Python queue, 2019-02-22
Python:
* introduce "python" directory with module namespace
* log QEMU launch command line on qemu.QEMUMachine
Acceptance Tests:
* initrd 4GiB+ test
* migration test
* multi vm support in test class
* bump Avocado version and drop ":avocado: enable"
# gpg: Signature made Fri 22 Feb 2019 19:37:07 GMT
# gpg: using RSA key 657E8D33A5F209F3
# gpg: Good signature from "Cleber Rosa <crosa@redhat.com>" [marginal]
# gpg: WARNING: This key is not certified with sufficiently trusted signatures!
# gpg: It is not certain that the signature belongs to the owner.
# Primary key fingerprint: 7ABB 96EB 8B46 B94D 5E0F E9BB 657E 8D33 A5F2 09F3
* remotes/cleber/tags/python-next-pull-request:
Acceptance tests: expect boot to extract 2GiB+ initrd with linux-v4.16
Acceptance tests: use linux-3.6 and set vm memory to 4GiB
tests.acceptance: adds simple migration test
tests.acceptance: adds multi vm capability for acceptance tests
scripts/qemu.py: log QEMU launch command line
Introduce a Python module structure
Acceptance tests: drop usage of ":avocado: enable"
Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
+193
-40
+1
configure
+1
configure
···
7674
7674
LINKS="$LINKS .gdbinit scripts" # scripts needed by relative path in .gdbinit
7675
7675
LINKS="$LINKS tests/acceptance tests/data"
7676
7676
LINKS="$LINKS tests/qemu-iotests/check"
7677
+
LINKS="$LINKS python"
7677
7678
for bios_file in \
7678
7679
$source_path/pc-bios/*.bin \
7679
7680
$source_path/pc-bios/*.lid \
+40
-2
docs/devel/testing.rst
+40
-2
docs/devel/testing.rst
···
600
600
601
601
class Version(Test):
602
602
"""
603
-
:avocado: enable
604
603
:avocado: tags=quick
605
604
"""
606
605
def test_qmp_human_info_version(self):
···
634
633
QEMU command line, launching the QEMUMachine (by using ``self.vm.launch()``)
635
634
is left to the test writer.
636
635
637
-
At test "tear down", ``avocado_qemu.Test`` handles the QEMUMachine
636
+
The base test class has also support for tests with more than one
637
+
QEMUMachine. The way to get machines is through the ``self.get_vm()``
638
+
method which will return a QEMUMachine instance. The ``self.get_vm()``
639
+
method accepts arguments that will be passed to the QEMUMachine creation
640
+
and also an optional `name` attribute so you can identify a specific
641
+
machine and get it more than once through the tests methods. A simple
642
+
and hypothetical example follows:
643
+
644
+
.. code::
645
+
646
+
from avocado_qemu import Test
647
+
648
+
649
+
class MultipleMachines(Test):
650
+
"""
651
+
:avocado: enable
652
+
"""
653
+
def test_multiple_machines(self):
654
+
first_machine = self.get_vm()
655
+
second_machine = self.get_vm()
656
+
self.get_vm(name='third_machine').launch()
657
+
658
+
first_machine.launch()
659
+
second_machine.launch()
660
+
661
+
first_res = first_machine.command(
662
+
'human-monitor-command',
663
+
command_line='info version')
664
+
665
+
second_res = second_machine.command(
666
+
'human-monitor-command',
667
+
command_line='info version')
668
+
669
+
third_res = self.get_vm(name='third_machine').command(
670
+
'human-monitor-command',
671
+
command_line='info version')
672
+
673
+
self.assertEquals(first_res, second_res, third_res)
674
+
675
+
At test "tear down", ``avocado_qemu.Test`` handles all the QEMUMachines
638
676
shutdown.
639
677
640
678
QEMUMachine
+2
scripts/device-crash-test
+2
scripts/device-crash-test
···
25
25
"""
26
26
from __future__ import print_function
27
27
28
+
import os
28
29
import sys
29
30
import glob
30
31
import logging
···
34
35
import argparse
35
36
from itertools import chain
36
37
38
+
sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'python'))
37
39
from qemu import QEMUMachine
38
40
39
41
logger = logging.getLogger('device-crash-test')
+7
-5
scripts/qemu.py
python/qemu/__init__.py
+7
-5
scripts/qemu.py
python/qemu/__init__.py
···
16
16
import logging
17
17
import os
18
18
import subprocess
19
-
import qmp.qmp
20
19
import re
21
20
import shutil
22
21
import socket
23
22
import tempfile
23
+
24
+
from . import qmp
24
25
25
26
26
27
LOG = logging.getLogger(__name__)
···
66
67
failures reported by the QEMU binary itself.
67
68
"""
68
69
69
-
class MonitorResponseError(qmp.qmp.QMPError):
70
+
class MonitorResponseError(qmp.QMPError):
70
71
"""
71
72
Represents erroneous QMP monitor reply
72
73
"""
···
266
267
self._qemu_log_path = os.path.join(self._temp_dir, self._name + ".log")
267
268
self._qemu_log_file = open(self._qemu_log_path, 'wb')
268
269
269
-
self._qmp = qmp.qmp.QEMUMonitorProtocol(self._vm_monitor,
270
-
server=True)
270
+
self._qmp = qmp.QEMUMonitorProtocol(self._vm_monitor,
271
+
server=True)
271
272
272
273
def _post_launch(self):
273
274
self._qmp.accept()
···
319
320
self._pre_launch()
320
321
self._qemu_full_args = (self._wrapper + [self._binary] +
321
322
self._base_args() + self._args)
323
+
LOG.debug('VM launch command: %r', ' '.join(self._qemu_full_args))
322
324
self._popen = subprocess.Popen(self._qemu_full_args,
323
325
stdin=devnull,
324
326
stdout=self._qemu_log_file,
···
383
385
"""
384
386
reply = self.qmp(cmd, conv_keys, **args)
385
387
if reply is None:
386
-
raise qmp.qmp.QMPError("Monitor is closed")
388
+
raise qmp.QMPError("Monitor is closed")
387
389
if "error" in reply:
388
390
raise MonitorResponseError(reply)
389
391
return reply["return"]
scripts/qmp/__init__.py
scripts/qmp/__init__.py
This is a binary file and will not be displayed.
+4
-1
scripts/qmp/qemu-ga-client
+4
-1
scripts/qmp/qemu-ga-client
···
37
37
#
38
38
39
39
from __future__ import print_function
40
+
import os
41
+
import sys
40
42
import base64
41
43
import random
42
44
43
-
import qmp
45
+
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
46
+
from qemu import qmp
44
47
45
48
46
49
class QemuGuestAgent(qmp.QEMUMonitorProtocol):
+3
-1
scripts/qmp/qmp-shell
+3
-1
scripts/qmp/qmp-shell
···
66
66
# sent to QEMU, which is useful for debugging and documentation generation.
67
67
68
68
from __future__ import print_function
69
-
import qmp
70
69
import json
71
70
import ast
72
71
import readline
···
75
74
import errno
76
75
import atexit
77
76
import shlex
77
+
78
+
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
79
+
from qemu import qmp
78
80
79
81
class QMPCompleter(list):
80
82
def complete(self, text, state):
scripts/qmp/qmp.py
python/qemu/qmp.py
scripts/qmp/qmp.py
python/qemu/qmp.py
+3
-2
scripts/qtest.py
python/qemu/qtest.py
+3
-2
scripts/qtest.py
python/qemu/qtest.py
···
13
13
14
14
import socket
15
15
import os
16
-
import qemu
16
+
17
+
from . import QEMUMachine
17
18
18
19
19
20
class QEMUQtestProtocol(object):
···
79
80
self._sock.settimeout(timeout)
80
81
81
82
82
-
class QEMUQtestMachine(qemu.QEMUMachine):
83
+
class QEMUQtestMachine(QEMUMachine):
83
84
'''A QEMU VM'''
84
85
85
86
def __init__(self, binary, args=None, name=None, test_dir="/var/tmp",
+2
scripts/render_block_graph.py
+2
scripts/render_block_graph.py
+23
-7
tests/acceptance/avocado_qemu/__init__.py
+23
-7
tests/acceptance/avocado_qemu/__init__.py
···
10
10
11
11
import os
12
12
import sys
13
+
import uuid
13
14
14
15
import avocado
15
16
16
-
SRC_ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
17
-
SRC_ROOT_DIR = os.path.abspath(os.path.dirname(SRC_ROOT_DIR))
18
-
sys.path.append(os.path.join(SRC_ROOT_DIR, 'scripts'))
17
+
SRC_ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..', '..')
18
+
sys.path.append(os.path.join(SRC_ROOT_DIR, 'python'))
19
19
20
20
from qemu import QEMUMachine
21
21
···
42
42
43
43
class Test(avocado.Test):
44
44
def setUp(self):
45
-
self.vm = None
45
+
self._vms = {}
46
46
self.qemu_bin = self.params.get('qemu_bin',
47
47
default=pick_default_qemu_bin())
48
48
if self.qemu_bin is None:
49
49
self.cancel("No QEMU binary defined or found in the source tree")
50
-
self.vm = QEMUMachine(self.qemu_bin)
50
+
51
+
def _new_vm(self, *args):
52
+
vm = QEMUMachine(self.qemu_bin)
53
+
if args:
54
+
vm.add_args(*args)
55
+
return vm
56
+
57
+
@property
58
+
def vm(self):
59
+
return self.get_vm(name='default')
60
+
61
+
def get_vm(self, *args, name=None):
62
+
if not name:
63
+
name = str(uuid.uuid4())
64
+
if self._vms.get(name) is None:
65
+
self._vms[name] = self._new_vm(*args)
66
+
return self._vms[name]
51
67
52
68
def tearDown(self):
53
-
if self.vm is not None:
54
-
self.vm.shutdown()
69
+
for vm in self._vms.values():
70
+
vm.shutdown()
-1
tests/acceptance/boot_linux_console.py
-1
tests/acceptance/boot_linux_console.py
+44
-8
tests/acceptance/linux_initrd.py
+44
-8
tests/acceptance/linux_initrd.py
···
8
8
# This work is licensed under the terms of the GNU GPL, version 2 or
9
9
# later. See the COPYING file in the top-level directory.
10
10
11
+
import logging
11
12
import tempfile
12
13
from avocado.utils.process import run
13
14
···
18
19
"""
19
20
Checks QEMU evaluates correctly the initrd file passed as -initrd option.
20
21
21
-
:avocado: enable
22
22
:avocado: tags=x86_64
23
23
"""
24
24
25
-
timeout = 60
25
+
timeout = 300
26
26
27
-
def test_with_2gib_file_should_exit_error_msg(self):
27
+
def test_with_2gib_file_should_exit_error_msg_with_linux_v3_6(self):
28
28
"""
29
29
Pretends to boot QEMU with an initrd file with size of 2GiB
30
30
and expect it exits with error message.
31
+
Fedora-18 shipped with linux-3.6 which have not supported xloadflags
32
+
cannot support more than 2GiB initrd.
31
33
"""
32
-
kernel_url = ('https://mirrors.kernel.org/fedora/releases/28/'
33
-
'Everything/x86_64/os/images/pxeboot/vmlinuz')
34
-
kernel_hash = '238e083e114c48200f80d889f7e32eeb2793e02a'
34
+
kernel_url = ('https://archives.fedoraproject.org/pub/archive/fedora/li'
35
+
'nux/releases/18/Fedora/x86_64/os/images/pxeboot/vmlinuz')
36
+
kernel_hash = '41464f68efe42b9991250bed86c7081d2ccdbb21'
35
37
kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash)
36
38
max_size = 2 * (1024 ** 3) - 1
37
39
···
39
41
initrd.seek(max_size)
40
42
initrd.write(b'\0')
41
43
initrd.flush()
42
-
cmd = "%s -kernel %s -initrd %s" % (self.qemu_bin, kernel_path,
43
-
initrd.name)
44
+
cmd = "%s -kernel %s -initrd %s -m 4096" % (
45
+
self.qemu_bin, kernel_path, initrd.name)
44
46
res = run(cmd, ignore_status=True)
45
47
self.assertEqual(res.exit_status, 1)
46
48
expected_msg = r'.*initrd is too large.*max: \d+, need %s.*' % (
47
49
max_size + 1)
48
50
self.assertRegex(res.stderr_text, expected_msg)
51
+
52
+
def test_with_2gib_file_should_work_with_linux_v4_16(self):
53
+
"""
54
+
QEMU has supported up to 4 GiB initrd for recent kernel
55
+
Expect guest can reach 'Unpacking initramfs...'
56
+
"""
57
+
kernel_url = ('https://mirrors.kernel.org/fedora/releases/28/'
58
+
'Everything/x86_64/os/images/pxeboot/vmlinuz')
59
+
kernel_hash = '238e083e114c48200f80d889f7e32eeb2793e02a'
60
+
kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash)
61
+
max_size = 2 * (1024 ** 3) + 1
62
+
63
+
with tempfile.NamedTemporaryFile() as initrd:
64
+
initrd.seek(max_size)
65
+
initrd.write(b'\0')
66
+
initrd.flush()
67
+
68
+
self.vm.set_machine('pc')
69
+
self.vm.set_console()
70
+
kernel_command_line = 'console=ttyS0'
71
+
self.vm.add_args('-kernel', kernel_path,
72
+
'-append', kernel_command_line,
73
+
'-initrd', initrd.name,
74
+
'-m', '5120')
75
+
self.vm.launch()
76
+
console = self.vm.console_socket.makefile()
77
+
console_logger = logging.getLogger('console')
78
+
while True:
79
+
msg = console.readline()
80
+
console_logger.debug(msg.strip())
81
+
if 'Unpacking initramfs...' in msg:
82
+
break
83
+
if 'Kernel panic - not syncing' in msg:
84
+
self.fail("Kernel panic reached")
+53
tests/acceptance/migration.py
+53
tests/acceptance/migration.py
···
1
+
# Migration test
2
+
#
3
+
# Copyright (c) 2019 Red Hat, Inc.
4
+
#
5
+
# Authors:
6
+
# Cleber Rosa <crosa@redhat.com>
7
+
# Caio Carrara <ccarrara@redhat.com>
8
+
#
9
+
# This work is licensed under the terms of the GNU GPL, version 2 or
10
+
# later. See the COPYING file in the top-level directory.
11
+
12
+
13
+
from avocado_qemu import Test
14
+
15
+
from avocado.utils import network
16
+
from avocado.utils import wait
17
+
18
+
19
+
class Migration(Test):
20
+
"""
21
+
:avocado: enable
22
+
"""
23
+
24
+
timeout = 10
25
+
26
+
@staticmethod
27
+
def migration_finished(vm):
28
+
return vm.command('query-migrate')['status'] in ('completed', 'failed')
29
+
30
+
def _get_free_port(self):
31
+
port = network.find_free_port()
32
+
if port is None:
33
+
self.cancel('Failed to find a free port')
34
+
return port
35
+
36
+
37
+
def test_migration_with_tcp_localhost(self):
38
+
source_vm = self.get_vm()
39
+
dest_uri = 'tcp:localhost:%u' % self._get_free_port()
40
+
dest_vm = self.get_vm('-incoming', dest_uri)
41
+
dest_vm.launch()
42
+
source_vm.launch()
43
+
source_vm.qmp('migrate', uri=dest_uri)
44
+
wait.wait_for(
45
+
self.migration_finished,
46
+
timeout=self.timeout,
47
+
step=0.1,
48
+
args=(source_vm,)
49
+
)
50
+
self.assertEqual(dest_vm.command('query-migrate')['status'], 'completed')
51
+
self.assertEqual(source_vm.command('query-migrate')['status'], 'completed')
52
+
self.assertEqual(dest_vm.command('query-status')['status'], 'running')
53
+
self.assertEqual(source_vm.command('query-status')['status'], 'postmigrate')
-1
tests/acceptance/version.py
-1
tests/acceptance/version.py
+1
-2
tests/acceptance/virtio_version.py
+1
-2
tests/acceptance/virtio_version.py
···
11
11
import sys
12
12
import os
13
13
14
-
sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "scripts"))
14
+
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
15
15
from qemu import QEMUMachine
16
16
from avocado_qemu import Test
17
17
···
61
61
same device tree created by `disable-modern` and
62
62
`disable-legacy`.
63
63
64
-
:avocado: enable
65
64
:avocado: tags=x86_64
66
65
"""
67
66
-1
tests/acceptance/vnc.py
-1
tests/acceptance/vnc.py
+4
-3
tests/migration/guestperf/engine.py
+4
-3
tests/migration/guestperf/engine.py
···
24
24
import sys
25
25
import time
26
26
27
-
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'scripts'))
28
-
import qemu
29
-
import qmp.qmp
30
27
from guestperf.progress import Progress, ProgressStats
31
28
from guestperf.report import Report
32
29
from guestperf.timings import TimingRecord, Timings
30
+
31
+
sys.path.append(os.path.join(os.path.dirname(__file__),
32
+
'..', '..', '..', 'python'))
33
+
import qemu
33
34
34
35
35
36
class Engine(object):
+1
-1
tests/qemu-iotests/235
+1
-1
tests/qemu-iotests/235
···
23
23
import iotests
24
24
from iotests import qemu_img_create, qemu_io, file_path, log
25
25
26
-
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'scripts'))
26
+
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
27
27
28
28
from qemu import QEMUMachine
29
29
+1
-1
tests/qemu-iotests/238
+1
-1
tests/qemu-iotests/238
+2
-2
tests/qemu-iotests/iotests.py
+2
-2
tests/qemu-iotests/iotests.py
···
32
32
import io
33
33
from collections import OrderedDict
34
34
35
-
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'scripts'))
36
-
import qtest
35
+
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
36
+
from qemu import qtest
37
37
38
38
39
39
# This will not work if arguments contain spaces but is necessary if we
+1
-1
tests/requirements.txt
+1
-1
tests/requirements.txt
+1
-1
tests/vm/basevm.py
+1
-1
tests/vm/basevm.py
···
17
17
import logging
18
18
import time
19
19
import datetime
20
-
sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "scripts"))
20
+
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
21
21
from qemu import QEMUMachine, kvm_available
22
22
import subprocess
23
23
import hashlib