jcs.org
qemu with hax to log dma reads & writes
jcs.org/2018/11/12/vfio
1#!/usr/bin/env python3
2#
3# Docker controlling module
4#
5# Copyright (c) 2016 Red Hat Inc.
6#
7# Authors:
8# Fam Zheng <famz@redhat.com>
9#
10# This work is licensed under the terms of the GNU GPL, version 2
11# or (at your option) any later version. See the COPYING file in
12# the top-level directory.
13
14import os
15import sys
16import subprocess
17import json
18import hashlib
19import atexit
20import uuid
21import argparse
22import enum
23import tempfile
24import re
25import signal
26from tarfile import TarFile, TarInfo
27from io import StringIO
28from shutil import copy, rmtree
29from pwd import getpwuid
30from datetime import datetime, timedelta
31
32
33FILTERED_ENV_NAMES = ['ftp_proxy', 'http_proxy', 'https_proxy']
34
35
36DEVNULL = open(os.devnull, 'wb')
37
38class EngineEnum(enum.IntEnum):
39 AUTO = 1
40 DOCKER = 2
41 PODMAN = 3
42
43 def __str__(self):
44 return self.name.lower()
45
46 def __repr__(self):
47 return str(self)
48
49 @staticmethod
50 def argparse(s):
51 try:
52 return EngineEnum[s.upper()]
53 except KeyError:
54 return s
55
56
57USE_ENGINE = EngineEnum.AUTO
58
59def _bytes_checksum(bytes):
60 """Calculate a digest string unique to the text content"""
61 return hashlib.sha1(bytes).hexdigest()
62
63def _text_checksum(text):
64 """Calculate a digest string unique to the text content"""
65 return _bytes_checksum(text.encode('utf-8'))
66
67def _read_dockerfile(path):
68 return open(path, 'rt', encoding='utf-8').read()
69
70def _file_checksum(filename):
71 return _bytes_checksum(open(filename, 'rb').read())
72
73
74def _guess_engine_command():
75 """ Guess a working engine command or raise exception if not found"""
76 commands = []
77
78 if USE_ENGINE in [EngineEnum.AUTO, EngineEnum.PODMAN]:
79 commands += [["podman"]]
80 if USE_ENGINE in [EngineEnum.AUTO, EngineEnum.DOCKER]:
81 commands += [["docker"], ["sudo", "-n", "docker"]]
82 for cmd in commands:
83 try:
84 # docker version will return the client details in stdout
85 # but still report a status of 1 if it can't contact the daemon
86 if subprocess.call(cmd + ["version"],
87 stdout=DEVNULL, stderr=DEVNULL) == 0:
88 return cmd
89 except OSError:
90 pass
91 commands_txt = "\n".join([" " + " ".join(x) for x in commands])
92 raise Exception("Cannot find working engine command. Tried:\n%s" %
93 commands_txt)
94
95
96def _copy_with_mkdir(src, root_dir, sub_path='.'):
97 """Copy src into root_dir, creating sub_path as needed."""
98 dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path))
99 try:
100 os.makedirs(dest_dir)
101 except OSError:
102 # we can safely ignore already created directories
103 pass
104
105 dest_file = "%s/%s" % (dest_dir, os.path.basename(src))
106 copy(src, dest_file)
107
108
109def _get_so_libs(executable):
110 """Return a list of libraries associated with an executable.
111
112 The paths may be symbolic links which would need to be resolved to
113 ensure the right data is copied."""
114
115 libs = []
116 ldd_re = re.compile(r"(?:\S+ => )?(\S*) \(:?0x[0-9a-f]+\)")
117 try:
118 ldd_output = subprocess.check_output(["ldd", executable]).decode('utf-8')
119 for line in ldd_output.split("\n"):
120 search = ldd_re.search(line)
121 if search:
122 try:
123 libs.append(s.group(1))
124 except IndexError:
125 pass
126 except subprocess.CalledProcessError:
127 print("%s had no associated libraries (static build?)" % (executable))
128
129 return libs
130
131
132def _copy_binary_with_libs(src, bin_dest, dest_dir):
133 """Maybe copy a binary and all its dependent libraries.
134
135 If bin_dest isn't set we only copy the support libraries because
136 we don't need qemu in the docker path to run (due to persistent
137 mapping). Indeed users may get confused if we aren't running what
138 is in the image.
139
140 This does rely on the host file-system being fairly multi-arch
141 aware so the file don't clash with the guests layout.
142 """
143
144 if bin_dest:
145 _copy_with_mkdir(src, dest_dir, os.path.dirname(bin_dest))
146 else:
147 print("only copying support libraries for %s" % (src))
148
149 libs = _get_so_libs(src)
150 if libs:
151 for l in libs:
152 so_path = os.path.dirname(l)
153 real_l = os.path.realpath(l)
154 _copy_with_mkdir(real_l, dest_dir, so_path)
155
156
157def _check_binfmt_misc(executable):
158 """Check binfmt_misc has entry for executable in the right place.
159
160 The details of setting up binfmt_misc are outside the scope of
161 this script but we should at least fail early with a useful
162 message if it won't work.
163
164 Returns the configured binfmt path and a valid flag. For
165 persistent configurations we will still want to copy and dependent
166 libraries.
167 """
168
169 binary = os.path.basename(executable)
170 binfmt_entry = "/proc/sys/fs/binfmt_misc/%s" % (binary)
171
172 if not os.path.exists(binfmt_entry):
173 print ("No binfmt_misc entry for %s" % (binary))
174 return None, False
175
176 with open(binfmt_entry) as x: entry = x.read()
177
178 if re.search("flags:.*F.*\n", entry):
179 print("binfmt_misc for %s uses persistent(F) mapping to host binary" %
180 (binary))
181 return None, True
182
183 m = re.search("interpreter (\S+)\n", entry)
184 interp = m.group(1)
185 if interp and interp != executable:
186 print("binfmt_misc for %s does not point to %s, using %s" %
187 (binary, executable, interp))
188
189 return interp, True
190
191
192def _read_qemu_dockerfile(img_name):
193 # special case for Debian linux-user images
194 if img_name.startswith("debian") and img_name.endswith("user"):
195 img_name = "debian-bootstrap"
196
197 df = os.path.join(os.path.dirname(__file__), "dockerfiles",
198 img_name + ".docker")
199 return _read_dockerfile(df)
200
201
202def _dockerfile_preprocess(df):
203 out = ""
204 for l in df.splitlines():
205 if len(l.strip()) == 0 or l.startswith("#"):
206 continue
207 from_pref = "FROM qemu:"
208 if l.startswith(from_pref):
209 # TODO: Alternatively we could replace this line with "FROM $ID"
210 # where $ID is the image's hex id obtained with
211 # $ docker images $IMAGE --format="{{.Id}}"
212 # but unfortunately that's not supported by RHEL 7.
213 inlining = _read_qemu_dockerfile(l[len(from_pref):])
214 out += _dockerfile_preprocess(inlining)
215 continue
216 out += l + "\n"
217 return out
218
219
220class Docker(object):
221 """ Running Docker commands """
222 def __init__(self):
223 self._command = _guess_engine_command()
224 self._instance = None
225 atexit.register(self._kill_instances)
226 signal.signal(signal.SIGTERM, self._kill_instances)
227 signal.signal(signal.SIGHUP, self._kill_instances)
228
229 def _do(self, cmd, quiet=True, **kwargs):
230 if quiet:
231 kwargs["stdout"] = DEVNULL
232 return subprocess.call(self._command + cmd, **kwargs)
233
234 def _do_check(self, cmd, quiet=True, **kwargs):
235 if quiet:
236 kwargs["stdout"] = DEVNULL
237 return subprocess.check_call(self._command + cmd, **kwargs)
238
239 def _do_kill_instances(self, only_known, only_active=True):
240 cmd = ["ps", "-q"]
241 if not only_active:
242 cmd.append("-a")
243
244 filter = "--filter=label=com.qemu.instance.uuid"
245 if only_known:
246 if self._instance:
247 filter += "=%s" % (self._instance)
248 else:
249 # no point trying to kill, we finished
250 return
251
252 print("filter=%s" % (filter))
253 cmd.append(filter)
254 for i in self._output(cmd).split():
255 self._do(["rm", "-f", i])
256
257 def clean(self):
258 self._do_kill_instances(False, False)
259 return 0
260
261 def _kill_instances(self, *args, **kwargs):
262 return self._do_kill_instances(True)
263
264 def _output(self, cmd, **kwargs):
265 try:
266 return subprocess.check_output(self._command + cmd,
267 stderr=subprocess.STDOUT,
268 encoding='utf-8',
269 **kwargs)
270 except TypeError:
271 # 'encoding' argument was added in 3.6+
272 return subprocess.check_output(self._command + cmd,
273 stderr=subprocess.STDOUT,
274 **kwargs).decode('utf-8')
275
276
277 def inspect_tag(self, tag):
278 try:
279 return self._output(["inspect", tag])
280 except subprocess.CalledProcessError:
281 return None
282
283 def get_image_creation_time(self, info):
284 return json.loads(info)[0]["Created"]
285
286 def get_image_dockerfile_checksum(self, tag):
287 resp = self.inspect_tag(tag)
288 labels = json.loads(resp)[0]["Config"].get("Labels", {})
289 return labels.get("com.qemu.dockerfile-checksum", "")
290
291 def build_image(self, tag, docker_dir, dockerfile,
292 quiet=True, user=False, argv=None, extra_files_cksum=[]):
293 if argv is None:
294 argv = []
295
296 tmp_df = tempfile.NamedTemporaryFile(mode="w+t",
297 encoding='utf-8',
298 dir=docker_dir, suffix=".docker")
299 tmp_df.write(dockerfile)
300
301 if user:
302 uid = os.getuid()
303 uname = getpwuid(uid).pw_name
304 tmp_df.write("\n")
305 tmp_df.write("RUN id %s 2>/dev/null || useradd -u %d -U %s" %
306 (uname, uid, uname))
307
308 tmp_df.write("\n")
309 tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" %
310 _text_checksum(_dockerfile_preprocess(dockerfile)))
311 for f, c in extra_files_cksum:
312 tmp_df.write("LABEL com.qemu.%s-checksum=%s" % (f, c))
313
314 tmp_df.flush()
315
316 self._do_check(["build", "-t", tag, "-f", tmp_df.name] + argv +
317 [docker_dir],
318 quiet=quiet)
319
320 def update_image(self, tag, tarball, quiet=True):
321 "Update a tagged image using "
322
323 self._do_check(["build", "-t", tag, "-"], quiet=quiet, stdin=tarball)
324
325 def image_matches_dockerfile(self, tag, dockerfile):
326 try:
327 checksum = self.get_image_dockerfile_checksum(tag)
328 except Exception:
329 return False
330 return checksum == _text_checksum(_dockerfile_preprocess(dockerfile))
331
332 def run(self, cmd, keep, quiet, as_user=False):
333 label = uuid.uuid4().hex
334 if not keep:
335 self._instance = label
336
337 if as_user:
338 uid = os.getuid()
339 cmd = [ "-u", str(uid) ] + cmd
340 # podman requires a bit more fiddling
341 if self._command[0] == "podman":
342 cmd.insert(0, '--userns=keep-id')
343
344 ret = self._do_check(["run", "--label",
345 "com.qemu.instance.uuid=" + label] + cmd,
346 quiet=quiet)
347 if not keep:
348 self._instance = None
349 return ret
350
351 def command(self, cmd, argv, quiet):
352 return self._do([cmd] + argv, quiet=quiet)
353
354
355class SubCommand(object):
356 """A SubCommand template base class"""
357 name = None # Subcommand name
358
359 def shared_args(self, parser):
360 parser.add_argument("--quiet", action="store_true",
361 help="Run quietly unless an error occurred")
362
363 def args(self, parser):
364 """Setup argument parser"""
365 pass
366
367 def run(self, args, argv):
368 """Run command.
369 args: parsed argument by argument parser.
370 argv: remaining arguments from sys.argv.
371 """
372 pass
373
374
375class RunCommand(SubCommand):
376 """Invoke docker run and take care of cleaning up"""
377 name = "run"
378
379 def args(self, parser):
380 parser.add_argument("--keep", action="store_true",
381 help="Don't remove image when command completes")
382 parser.add_argument("--run-as-current-user", action="store_true",
383 help="Run container using the current user's uid")
384
385 def run(self, args, argv):
386 return Docker().run(argv, args.keep, quiet=args.quiet,
387 as_user=args.run_as_current_user)
388
389
390class BuildCommand(SubCommand):
391 """ Build docker image out of a dockerfile. Arg: <tag> <dockerfile>"""
392 name = "build"
393
394 def args(self, parser):
395 parser.add_argument("--include-executable", "-e",
396 help="""Specify a binary that will be copied to the
397 container together with all its dependent
398 libraries""")
399 parser.add_argument("--extra-files", nargs='*',
400 help="""Specify files that will be copied in the
401 Docker image, fulfilling the ADD directive from the
402 Dockerfile""")
403 parser.add_argument("--add-current-user", "-u", dest="user",
404 action="store_true",
405 help="Add the current user to image's passwd")
406 parser.add_argument("-t", dest="tag",
407 help="Image Tag")
408 parser.add_argument("-f", dest="dockerfile",
409 help="Dockerfile name")
410
411 def run(self, args, argv):
412 dockerfile = _read_dockerfile(args.dockerfile)
413 tag = args.tag
414
415 dkr = Docker()
416 if "--no-cache" not in argv and \
417 dkr.image_matches_dockerfile(tag, dockerfile):
418 if not args.quiet:
419 print("Image is up to date.")
420 else:
421 # Create a docker context directory for the build
422 docker_dir = tempfile.mkdtemp(prefix="docker_build")
423
424 # Validate binfmt_misc will work
425 if args.include_executable:
426 qpath, enabled = _check_binfmt_misc(args.include_executable)
427 if not enabled:
428 return 1
429
430 # Is there a .pre file to run in the build context?
431 docker_pre = os.path.splitext(args.dockerfile)[0]+".pre"
432 if os.path.exists(docker_pre):
433 stdout = DEVNULL if args.quiet else None
434 rc = subprocess.call(os.path.realpath(docker_pre),
435 cwd=docker_dir, stdout=stdout)
436 if rc == 3:
437 print("Skip")
438 return 0
439 elif rc != 0:
440 print("%s exited with code %d" % (docker_pre, rc))
441 return 1
442
443 # Copy any extra files into the Docker context. These can be
444 # included by the use of the ADD directive in the Dockerfile.
445 cksum = []
446 if args.include_executable:
447 # FIXME: there is no checksum of this executable and the linked
448 # libraries, once the image built any change of this executable
449 # or any library won't trigger another build.
450 _copy_binary_with_libs(args.include_executable,
451 qpath, docker_dir)
452
453 for filename in args.extra_files or []:
454 _copy_with_mkdir(filename, docker_dir)
455 cksum += [(filename, _file_checksum(filename))]
456
457 argv += ["--build-arg=" + k.lower() + "=" + v
458 for k, v in os.environ.items()
459 if k.lower() in FILTERED_ENV_NAMES]
460 dkr.build_image(tag, docker_dir, dockerfile,
461 quiet=args.quiet, user=args.user, argv=argv,
462 extra_files_cksum=cksum)
463
464 rmtree(docker_dir)
465
466 return 0
467
468
469class UpdateCommand(SubCommand):
470 """ Update a docker image with new executables. Args: <tag> <executable>"""
471 name = "update"
472
473 def args(self, parser):
474 parser.add_argument("tag",
475 help="Image Tag")
476 parser.add_argument("executable",
477 help="Executable to copy")
478
479 def run(self, args, argv):
480 # Create a temporary tarball with our whole build context and
481 # dockerfile for the update
482 tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz")
483 tmp_tar = TarFile(fileobj=tmp, mode='w')
484
485 # Add the executable to the tarball, using the current
486 # configured binfmt_misc path. If we don't get a path then we
487 # only need the support libraries copied
488 ff, enabled = _check_binfmt_misc(args.executable)
489
490 if not enabled:
491 print("binfmt_misc not enabled, update disabled")
492 return 1
493
494 if ff:
495 tmp_tar.add(args.executable, arcname=ff)
496
497 # Add any associated libraries
498 libs = _get_so_libs(args.executable)
499 if libs:
500 for l in libs:
501 tmp_tar.add(os.path.realpath(l), arcname=l)
502
503 # Create a Docker buildfile
504 df = StringIO()
505 df.write("FROM %s\n" % args.tag)
506 df.write("ADD . /\n")
507 df.seek(0)
508
509 df_tar = TarInfo(name="Dockerfile")
510 df_tar.size = len(df.buf)
511 tmp_tar.addfile(df_tar, fileobj=df)
512
513 tmp_tar.close()
514
515 # reset the file pointers
516 tmp.flush()
517 tmp.seek(0)
518
519 # Run the build with our tarball context
520 dkr = Docker()
521 dkr.update_image(args.tag, tmp, quiet=args.quiet)
522
523 return 0
524
525
526class CleanCommand(SubCommand):
527 """Clean up docker instances"""
528 name = "clean"
529
530 def run(self, args, argv):
531 Docker().clean()
532 return 0
533
534
535class ImagesCommand(SubCommand):
536 """Run "docker images" command"""
537 name = "images"
538
539 def run(self, args, argv):
540 return Docker().command("images", argv, args.quiet)
541
542
543class ProbeCommand(SubCommand):
544 """Probe if we can run docker automatically"""
545 name = "probe"
546
547 def run(self, args, argv):
548 try:
549 docker = Docker()
550 if docker._command[0] == "docker":
551 print("docker")
552 elif docker._command[0] == "sudo":
553 print("sudo docker")
554 elif docker._command[0] == "podman":
555 print("podman")
556 except Exception:
557 print("no")
558
559 return
560
561
562class CcCommand(SubCommand):
563 """Compile sources with cc in images"""
564 name = "cc"
565
566 def args(self, parser):
567 parser.add_argument("--image", "-i", required=True,
568 help="The docker image in which to run cc")
569 parser.add_argument("--cc", default="cc",
570 help="The compiler executable to call")
571 parser.add_argument("--source-path", "-s", nargs="*", dest="paths",
572 help="""Extra paths to (ro) mount into container for
573 reading sources""")
574
575 def run(self, args, argv):
576 if argv and argv[0] == "--":
577 argv = argv[1:]
578 cwd = os.getcwd()
579 cmd = ["--rm", "-w", cwd,
580 "-v", "%s:%s:rw" % (cwd, cwd)]
581 if args.paths:
582 for p in args.paths:
583 cmd += ["-v", "%s:%s:ro,z" % (p, p)]
584 cmd += [args.image, args.cc]
585 cmd += argv
586 return Docker().run(cmd, False, quiet=args.quiet,
587 as_user=True)
588
589
590class CheckCommand(SubCommand):
591 """Check if we need to re-build a docker image out of a dockerfile.
592 Arguments: <tag> <dockerfile>"""
593 name = "check"
594
595 def args(self, parser):
596 parser.add_argument("tag",
597 help="Image Tag")
598 parser.add_argument("dockerfile", default=None,
599 help="Dockerfile name", nargs='?')
600 parser.add_argument("--checktype", choices=["checksum", "age"],
601 default="checksum", help="check type")
602 parser.add_argument("--olderthan", default=60, type=int,
603 help="number of minutes")
604
605 def run(self, args, argv):
606 tag = args.tag
607
608 try:
609 dkr = Docker()
610 except subprocess.CalledProcessError:
611 print("Docker not set up")
612 return 1
613
614 info = dkr.inspect_tag(tag)
615 if info is None:
616 print("Image does not exist")
617 return 1
618
619 if args.checktype == "checksum":
620 if not args.dockerfile:
621 print("Need a dockerfile for tag:%s" % (tag))
622 return 1
623
624 dockerfile = _read_dockerfile(args.dockerfile)
625
626 if dkr.image_matches_dockerfile(tag, dockerfile):
627 if not args.quiet:
628 print("Image is up to date")
629 return 0
630 else:
631 print("Image needs updating")
632 return 1
633 elif args.checktype == "age":
634 timestr = dkr.get_image_creation_time(info).split(".")[0]
635 created = datetime.strptime(timestr, "%Y-%m-%dT%H:%M:%S")
636 past = datetime.now() - timedelta(minutes=args.olderthan)
637 if created < past:
638 print ("Image created @ %s more than %d minutes old" %
639 (timestr, args.olderthan))
640 return 1
641 else:
642 if not args.quiet:
643 print ("Image less than %d minutes old" % (args.olderthan))
644 return 0
645
646
647def main():
648 global USE_ENGINE
649
650 parser = argparse.ArgumentParser(description="A Docker helper",
651 usage="%s <subcommand> ..." %
652 os.path.basename(sys.argv[0]))
653 parser.add_argument("--engine", type=EngineEnum.argparse, choices=list(EngineEnum),
654 help="specify which container engine to use")
655 subparsers = parser.add_subparsers(title="subcommands", help=None)
656 for cls in SubCommand.__subclasses__():
657 cmd = cls()
658 subp = subparsers.add_parser(cmd.name, help=cmd.__doc__)
659 cmd.shared_args(subp)
660 cmd.args(subp)
661 subp.set_defaults(cmdobj=cmd)
662 args, argv = parser.parse_known_args()
663 if args.engine:
664 USE_ENGINE = args.engine
665 return args.cmdobj.run(args, argv)
666
667
668if __name__ == "__main__":
669 sys.exit(main())