matrixfurry.com
The open source OpenXR runtime
scripts: pytest-based Android device testing
Part-of: <https://gitlab.freedesktop.org/monado/monado/-/merge_requests/2311>
authored by
Rylie Pavlik
and committed by
Marge Bot
511efda7
f30e5265
+531
+1
doc/changes/misc_features/mr.2311.md
+1
doc/changes/misc_features/mr.2311.md
···
···
1
+
Add: Lifecycle tests that launch and switch OpenXR applications on Android.
+4
tests/android/.gitignore
+4
tests/android/.gitignore
+63
tests/android/README.md
+63
tests/android/README.md
···
···
1
+
# Android device tests
2
+
3
+
<!--
4
+
Copyright 2024, Collabora, Ltd.
5
+
6
+
SPDX-License-Identifier: CC-BY-4.0
7
+
-->
8
+
9
+
This directory contains some tests to run on an Android device over ADB,
10
+
primarily to verify some lifecycle handling right now.
11
+
12
+
It uses [pytest][] as the test runner, which is more readable, maintainable, and
13
+
usable than the earlier bash scripts, and gives the option for e.g. junit-style
14
+
output for CI usage.
15
+
16
+
The actual tests are in the `test_*.py` files, while `conftest.py` configures
17
+
the pytest framework, as well as provides constants and fixtures for use in the
18
+
tests themselves.
19
+
20
+
[pytest]: https://pytest.org
21
+
22
+
## Preconditions
23
+
24
+
- Have `adb` in the path, and only the device under test connected. (or, have
25
+
environment variables set appropriately so that
26
+
`adb shell getprop ro.product.vendor.device` shows you the expected device.)
27
+
- Have Python and pytest installed, either system-wide or in a `venv`. The
28
+
version of pytest in Debian Bookworm is suitable.
29
+
- Have built and installed the "outOfProcessDebug" build of Monado.
30
+
(`./gradlew installOOPD` in root dir)
31
+
- Have set the out-of-process runtime as active in the OpenXR Runtime Broker (if
32
+
applicable).
33
+
- Have installed both Vulkan and OpenGL ES versions of the `hello_xr` sample app
34
+
from Khronos. (Binaries from a release are fine.)
35
+
- If you want the tests to pass, turn on "Draw over other apps" -- but this is
36
+
actually a bug to require this.
37
+
38
+
## Instructions
39
+
40
+
Change your current working directory to this one. (Otherwise the extra
41
+
argument you need to enable the ADB tests will not be recognized by pytest.)
42
+
43
+
```sh
44
+
cd tests/android/
45
+
```
46
+
47
+
Run the following command, or one based on it:
48
+
49
+
```sh
50
+
pytest -v --adb
51
+
```
52
+
53
+
Or, if you want junit-style output, add a variation on these two args:
54
+
`--junit-xml=android-tests.xml -o junit_family=xunit1` for an overall command
55
+
like:
56
+
57
+
```sh
58
+
pytest -v --adb --junit-xml=android-tests.xml -o junit_family=xunit1
59
+
```
60
+
61
+
Another useful options is `-k` which can be followed by a pattern used to
62
+
selecet a subset of tests to run. Handy if you only want to run one or two
63
+
tests.
+273
tests/android/conftest.py
+273
tests/android/conftest.py
···
···
1
+
# Copyright 2024, Collabora, Ltd.
2
+
#
3
+
# SPDX-License-Identifier: BSL-1.0
4
+
#
5
+
# Author: Rylie Pavlik <rylie.pavlik@collabora.com>
6
+
"""
7
+
pytest configuration and helpers for adb/Android.
8
+
9
+
Helper functions to run tests using adb on an Android device.
10
+
11
+
pytest serves as the test runner and collects the results.
12
+
13
+
See README.md in this directory for requirements and running instructions.
14
+
"""
15
+
16
+
import subprocess
17
+
import time
18
+
from pathlib import Path
19
+
from typing import Optional
20
+
21
+
import pytest
22
+
23
+
_NATIVE_ACTIVITY = "android.app.NativeActivity"
24
+
helloxr_vulkan_pkg = "org.khronos.openxr.hello_xr.vulkan"
25
+
helloxr_vulkan_activity = f"{helloxr_vulkan_pkg}/{_NATIVE_ACTIVITY}"
26
+
helloxr_gles_pkg = "org.khronos.openxr.hello_xr.opengles"
27
+
helloxr_gles_activity = f"{helloxr_gles_pkg}/{_NATIVE_ACTIVITY}"
28
+
29
+
RUNTIME = "org.freedesktop.monado.openxr_runtime.out_of_process"
30
+
31
+
_PKGS_TO_STOP = [
32
+
helloxr_gles_pkg,
33
+
helloxr_vulkan_pkg,
34
+
"org.khronos.openxr.cts",
35
+
"org.freedesktop.monado.openxr_runtime.in_process",
36
+
RUNTIME,
37
+
]
38
+
39
+
_REPO_ROOT = Path(__file__).resolve().parent.parent.parent
40
+
41
+
42
+
class AdbTesting:
43
+
"""pytest fixture for testing on an Android device with adb."""
44
+
45
+
def __init__(self, tmp_path: Optional[Path]):
46
+
"""Initialize members."""
47
+
self.tmp_path = tmp_path
48
+
self.logcat_grabbed = False
49
+
self.logcat_echoed = False
50
+
self.log: Optional[str] = None
51
+
52
+
# TODO allow most of the following to be configured
53
+
self._ndk_ver = "26.3.11579264"
54
+
self._build_variant = "outOfProcessDebug"
55
+
self._sdk_root = Path.home() / "Android" / "Sdk"
56
+
self._arch = "arm64-v8a"
57
+
self._ndk_stack = self._sdk_root / "ndk" / self._ndk_ver / "ndk-stack"
58
+
self._caps_build_variant = (
59
+
self._build_variant[0].upper() + self._build_variant[1:]
60
+
)
61
+
self._sym_dir = (
62
+
_REPO_ROOT
63
+
/ "src"
64
+
/ "xrt"
65
+
/ "targets"
66
+
/ "openxr_android"
67
+
/ "build"
68
+
/ "intermediates"
69
+
/ "merged_native_libs"
70
+
/ self._build_variant
71
+
/ f"merge{self._caps_build_variant}NativeLibs"
72
+
/ "out"
73
+
/ "lib"
74
+
/ self._arch
75
+
)
76
+
77
+
def adb_call(self, cmd: list[str], **kwargs):
78
+
"""Call ADB in a subprocess."""
79
+
return subprocess.check_call(cmd, **kwargs)
80
+
81
+
def adb_check_output(self, cmd: list[str], **kwargs):
82
+
"""Call ADB in a subprocess and get the output."""
83
+
return subprocess.check_output(cmd, encoding="utf-8", **kwargs)
84
+
85
+
def get_device_state(self) -> Optional[str]:
86
+
try:
87
+
state = self.adb_check_output(["adb", "get-state"]).strip()
88
+
return state.strip()
89
+
90
+
except subprocess.CalledProcessError:
91
+
return None
92
+
93
+
def send_key(self, keyevent):
94
+
"""Send a key event with `adb shell input keyevent`."""
95
+
time.sleep(1)
96
+
print(f"*** {keyevent}")
97
+
self.adb_call(["adb", "shell", "input", "keyevent", keyevent])
98
+
99
+
def send_tap(self, x, y):
100
+
"""Send touchscreen tap at x, y."""
101
+
# sleep 1
102
+
print(f"*** tap {x} {y}")
103
+
self.adb_call(
104
+
[
105
+
"adb",
106
+
"shell",
107
+
"input",
108
+
"touchscreen",
109
+
"tap",
110
+
str(x),
111
+
str(y),
112
+
]
113
+
)
114
+
115
+
def start_activity(self, activity):
116
+
"""Start an activity and wait with `adb shell am start-activity -W`."""
117
+
time.sleep(1)
118
+
print(f"*** starting {activity} and waiting")
119
+
self.adb_call(["adb", "shell", "am", "start-activity", "-W", activity])
120
+
121
+
def stop_and_clear_all(self):
122
+
"""
123
+
Stop relevant packages and clear their data.
124
+
125
+
This second step removes them from recents.
126
+
"""
127
+
print("*** stopping all relevant packages and clearing their data")
128
+
for pkg in _PKGS_TO_STOP:
129
+
self.adb_call(["adb", "shell", "am", "force-stop", pkg])
130
+
self.adb_call(["adb", "shell", "pm", "clear", pkg])
131
+
132
+
def start_test(
133
+
self,
134
+
):
135
+
"""Ensure a clean starting setup."""
136
+
self.stop_and_clear_all()
137
+
self.adb_call(["adb", "logcat", "-c"])
138
+
139
+
@property
140
+
def logcat_fn(self):
141
+
"""Get computed path of logcat file."""
142
+
assert self.tmp_path
143
+
return self.tmp_path / "logcat.txt"
144
+
145
+
def grab_logcat(self):
146
+
"""Save logcat to the named file and return it as a string."""
147
+
log = self.adb_check_output(["adb", "logcat", "-d"])
148
+
log_fn = self.logcat_fn
149
+
with open(log_fn, "w", encoding="utf-8") as fp:
150
+
fp.write(log)
151
+
print(f"Logcat saved to {log_fn}")
152
+
self.logcat_grabbed = True
153
+
self.log = log
154
+
return log
155
+
156
+
def dump_logcat(self):
157
+
"""Print logcat to stdout."""
158
+
if self.logcat_echoed:
159
+
return
160
+
if not self.logcat_grabbed:
161
+
self.grab_logcat()
162
+
print("*** Logcat:")
163
+
print(self.log)
164
+
self.logcat_echoed = True
165
+
166
+
def check_for_crash(self):
167
+
"""Dump logcat, and look for a crash."""
168
+
__tracebackhide__ = True
169
+
log = self.grab_logcat()
170
+
log_fn = self.logcat_fn
171
+
crash_name = log_fn.with_name("crashes.txt")
172
+
if "backtrace:" in log:
173
+
subprocess.check_call(
174
+
" ".join(
175
+
(
176
+
str(x)
177
+
for x in (
178
+
self._ndk_stack,
179
+
"-sym",
180
+
self._sym_dir,
181
+
"-i",
182
+
log_fn,
183
+
">",
184
+
crash_name,
185
+
)
186
+
)
187
+
),
188
+
shell=True,
189
+
)
190
+
self.dump_logcat()
191
+
192
+
with open(crash_name, "r", encoding="utf-8") as fp:
193
+
print("*** Crash Backtrace(s):")
194
+
print(fp.read())
195
+
print(f"*** Crash backtrace in {crash_name}")
196
+
pytest.fail("Native crash backtrace detected in test")
197
+
198
+
if "FATAL EXCEPTION" in log:
199
+
self.dump_logcat()
200
+
pytest.fail("Java exception detected in test")
201
+
202
+
if "WindowManager: ANR" in log:
203
+
self.dump_logcat()
204
+
pytest.fail("ANR detected in test")
205
+
206
+
self.stop_and_clear_all()
207
+
208
+
209
+
@pytest.fixture(scope="function")
210
+
def adb(tmp_path):
211
+
"""Provide AdbTesting instance and cleanup as pytest fixture."""
212
+
adb_testing = AdbTesting(tmp_path)
213
+
214
+
adb_testing.start_test()
215
+
216
+
# Body of test happens here
217
+
yield adb_testing
218
+
219
+
# Grab logcat in case it wasn't grabbed, and dump to stdout
220
+
adb_testing.dump_logcat()
221
+
222
+
# Cleanup
223
+
adb_testing.stop_and_clear_all()
224
+
225
+
226
+
@pytest.fixture(scope="session", autouse=True)
227
+
def log_device_and_build(record_testsuite_property):
228
+
"""Session fixture, autoused, to record device data."""
229
+
adb = AdbTesting(None)
230
+
231
+
try:
232
+
device = adb.adb_check_output(
233
+
["adb", "shell", "getprop", "ro.product.vendor.device"]
234
+
)
235
+
build = adb.adb_check_output(
236
+
["adb", "shell", "getprop", "ro.product.build.fingerprint"]
237
+
)
238
+
except subprocess.CalledProcessError:
239
+
pytest.skip("adb shell getprop failed - no device connected?")
240
+
241
+
record_testsuite_property("device", device.strip())
242
+
record_testsuite_property("build", build.strip())
243
+
244
+
245
+
def device_is_available():
246
+
"""Return True if an ADB device is available."""
247
+
adb = AdbTesting(None)
248
+
249
+
try:
250
+
state = adb.adb_check_output(["adb", "get-state"]).strip()
251
+
return state == "device"
252
+
253
+
except subprocess.CalledProcessError:
254
+
return False
255
+
256
+
257
+
def pytest_addoption(parser: pytest.Parser):
258
+
"""Pytest addoption function."""
259
+
parser.addoption(
260
+
"--adb",
261
+
action="store_true",
262
+
default=False,
263
+
help="run ADB/Android device tests",
264
+
)
265
+
266
+
267
+
def pytest_configure(config):
268
+
"""Configure function for pytest."""
269
+
config.addinivalue_line(
270
+
"markers",
271
+
"adb: mark test as needing adb and thus only run when an "
272
+
"Android device is available",
273
+
)
+129
tests/android/test_app_switch.py
+129
tests/android/test_app_switch.py
···
···
1
+
#!/usr/bin/env python3
2
+
# Copyright 2024, Collabora, Ltd.
3
+
#
4
+
# SPDX-License-Identifier: BSL-1.0
5
+
#
6
+
# Author: Rylie Pavlik <rylie.pavlik@collabora.com>
7
+
"""
8
+
Tests of app lifecycle behavior (app pause/resume and switch).
9
+
10
+
See README.md in this directory for requirements and running instructions.
11
+
"""
12
+
13
+
import time
14
+
15
+
import pytest
16
+
from conftest import helloxr_gles_activity, helloxr_vulkan_activity, helloxr_gles_pkg
17
+
18
+
# Ignore missing docstrings:
19
+
# flake8: noqa: D103
20
+
21
+
skipif_not_adb = pytest.mark.skipif(
22
+
"not config.getoption('adb')", reason="--adb not passed to pytest"
23
+
)
24
+
25
+
# All the tests in this module require a device and ADB.
26
+
pytestmark = [pytest.mark.adb, skipif_not_adb]
27
+
28
+
29
+
def test_launch_and_back(adb):
30
+
# Launch activity
31
+
adb.start_activity(helloxr_gles_activity)
32
+
time.sleep(2)
33
+
34
+
# Press "Back"
35
+
adb.send_key("KEYCODE_BACK")
36
+
37
+
time.sleep(5)
38
+
39
+
adb.check_for_crash()
40
+
41
+
42
+
def test_home_and_resume(adb):
43
+
44
+
# Launch activity
45
+
adb.start_activity(helloxr_gles_activity)
46
+
time.sleep(2)
47
+
48
+
# Press "Home"
49
+
adb.send_key("KEYCODE_HOME")
50
+
time.sleep(1)
51
+
52
+
# Press "App Switch"
53
+
adb.send_key("KEYCODE_APP_SWITCH")
54
+
time.sleep(1)
55
+
56
+
# Tap "somewhat middle" to re-select recent app
57
+
adb.send_tap(400, 400)
58
+
59
+
time.sleep(5)
60
+
61
+
adb.check_for_crash()
62
+
63
+
64
+
def test_home_and_start(adb):
65
+
66
+
# Launch activity
67
+
adb.start_activity(helloxr_gles_activity)
68
+
time.sleep(2)
69
+
70
+
# Press "Home"
71
+
adb.send_key("KEYCODE_HOME")
72
+
time.sleep(2)
73
+
74
+
# Launch activity again
75
+
adb.start_activity(helloxr_gles_activity)
76
+
77
+
time.sleep(5)
78
+
79
+
adb.check_for_crash()
80
+
81
+
82
+
def test_launch_second(adb):
83
+
84
+
# Launch A
85
+
adb.start_activity(helloxr_gles_activity)
86
+
time.sleep(2)
87
+
88
+
# Launch B
89
+
adb.start_activity(helloxr_vulkan_activity)
90
+
91
+
time.sleep(5)
92
+
93
+
adb.check_for_crash()
94
+
95
+
96
+
def test_home_and_launch_second(adb):
97
+
98
+
# Launch A
99
+
adb.start_activity(helloxr_gles_activity)
100
+
time.sleep(2)
101
+
102
+
# Press "Home"
103
+
adb.send_key("KEYCODE_HOME")
104
+
time.sleep(2)
105
+
106
+
# Launch B
107
+
adb.start_activity(helloxr_vulkan_activity)
108
+
109
+
time.sleep(5)
110
+
111
+
adb.check_for_crash()
112
+
113
+
114
+
def test_launch_a_b_a(adb):
115
+
116
+
# Launch A
117
+
adb.start_activity(helloxr_gles_activity)
118
+
time.sleep(2)
119
+
120
+
# Launch B
121
+
adb.start_activity(helloxr_vulkan_activity)
122
+
time.sleep(2)
123
+
124
+
# Launch A (again)
125
+
adb.start_activity(helloxr_gles_activity)
126
+
127
+
time.sleep(5)
128
+
129
+
adb.check_for_crash()
+61
tests/android/test_single_app_reliability.py
+61
tests/android/test_single_app_reliability.py
···
···
1
+
#!/usr/bin/env python3
2
+
# Copyright 2024, Collabora, Ltd.
3
+
#
4
+
# SPDX-License-Identifier: BSL-1.0
5
+
#
6
+
# Author: Rylie Pavlik <rylie.pavlik@collabora.com>
7
+
"""
8
+
Tests of a single app without inducing extra session state changes.
9
+
10
+
See README.md in this directory for requirements and running instructions.
11
+
"""
12
+
13
+
import time
14
+
15
+
import pytest
16
+
from conftest import helloxr_gles_activity, helloxr_gles_pkg
17
+
18
+
# Ignore missing docstrings:
19
+
# flake8: noqa: D103
20
+
21
+
skipif_not_adb = pytest.mark.skipif(
22
+
"not config.getoption('adb')", reason="--adb not passed to pytest"
23
+
)
24
+
25
+
# All the tests in this module require a device and ADB.
26
+
pytestmark = [pytest.mark.adb, skipif_not_adb]
27
+
28
+
29
+
def test_just_launch(adb):
30
+
# Just launch activity and make sure it starts OK.
31
+
adb.start_activity(helloxr_gles_activity)
32
+
33
+
time.sleep(5)
34
+
35
+
adb.check_for_crash()
36
+
37
+
38
+
def test_launch_and_monkey(adb):
39
+
# Launch activity
40
+
adb.start_activity(helloxr_gles_activity)
41
+
time.sleep(2)
42
+
43
+
# Release the monkey! 1k events.
44
+
adb.adb_call(
45
+
[
46
+
"adb",
47
+
"shell",
48
+
"monkey",
49
+
"-p",
50
+
helloxr_gles_pkg,
51
+
"-v",
52
+
"1000",
53
+
# seed
54
+
"-s",
55
+
"100",
56
+
"--pct-syskeys",
57
+
"0",
58
+
]
59
+
)
60
+
61
+
adb.check_for_crash()