jcs.org
qemu with hax to log dma reads & writes
jcs.org/2018/11/12/vfio
1#!/usr/bin/env python3
2#
3# Docker controlling module
4#
5# Copyright (c) 2016 Red Hat Inc.
6#
7# Authors:
8# Fam Zheng <famz@redhat.com>
9#
10# This work is licensed under the terms of the GNU GPL, version 2
11# or (at your option) any later version. See the COPYING file in
12# the top-level directory.
13
14import os
15import sys
16import subprocess
17import json
18import hashlib
19import atexit
20import uuid
21import argparse
22import enum
23import tempfile
24import re
25import signal
26from tarfile import TarFile, TarInfo
27from io import StringIO
28from shutil import copy, rmtree
29from pwd import getpwuid
30from datetime import datetime, timedelta
31
32
33FILTERED_ENV_NAMES = ['ftp_proxy', 'http_proxy', 'https_proxy']
34
35
36DEVNULL = open(os.devnull, 'wb')
37
38class EngineEnum(enum.IntEnum):
39 AUTO = 1
40 DOCKER = 2
41 PODMAN = 3
42
43 def __str__(self):
44 return self.name.lower()
45
46 def __repr__(self):
47 return str(self)
48
49 @staticmethod
50 def argparse(s):
51 try:
52 return EngineEnum[s.upper()]
53 except KeyError:
54 return s
55
56
57USE_ENGINE = EngineEnum.AUTO
58
59def _text_checksum(text):
60 """Calculate a digest string unique to the text content"""
61 return hashlib.sha1(text.encode('utf-8')).hexdigest()
62
63def _read_dockerfile(path):
64 return open(path, 'rt', encoding='utf-8').read()
65
66def _file_checksum(filename):
67 return _text_checksum(_read_dockerfile(filename))
68
69
70def _guess_engine_command():
71 """ Guess a working engine command or raise exception if not found"""
72 commands = []
73
74 if USE_ENGINE in [EngineEnum.AUTO, EngineEnum.PODMAN]:
75 commands += [["podman"]]
76 if USE_ENGINE in [EngineEnum.AUTO, EngineEnum.DOCKER]:
77 commands += [["docker"], ["sudo", "-n", "docker"]]
78 for cmd in commands:
79 try:
80 # docker version will return the client details in stdout
81 # but still report a status of 1 if it can't contact the daemon
82 if subprocess.call(cmd + ["version"],
83 stdout=DEVNULL, stderr=DEVNULL) == 0:
84 return cmd
85 except OSError:
86 pass
87 commands_txt = "\n".join([" " + " ".join(x) for x in commands])
88 raise Exception("Cannot find working engine command. Tried:\n%s" %
89 commands_txt)
90
91
92def _copy_with_mkdir(src, root_dir, sub_path='.'):
93 """Copy src into root_dir, creating sub_path as needed."""
94 dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path))
95 try:
96 os.makedirs(dest_dir)
97 except OSError:
98 # we can safely ignore already created directories
99 pass
100
101 dest_file = "%s/%s" % (dest_dir, os.path.basename(src))
102 copy(src, dest_file)
103
104
105def _get_so_libs(executable):
106 """Return a list of libraries associated with an executable.
107
108 The paths may be symbolic links which would need to be resolved to
109 ensure the right data is copied."""
110
111 libs = []
112 ldd_re = re.compile(r"(?:\S+ => )?(\S*) \(:?0x[0-9a-f]+\)")
113 try:
114 ldd_output = subprocess.check_output(["ldd", executable]).decode('utf-8')
115 for line in ldd_output.split("\n"):
116 search = ldd_re.search(line)
117 if search:
118 try:
119 libs.append(s.group(1))
120 except IndexError:
121 pass
122 except subprocess.CalledProcessError:
123 print("%s had no associated libraries (static build?)" % (executable))
124
125 return libs
126
127
128def _copy_binary_with_libs(src, bin_dest, dest_dir):
129 """Maybe copy a binary and all its dependent libraries.
130
131 If bin_dest isn't set we only copy the support libraries because
132 we don't need qemu in the docker path to run (due to persistent
133 mapping). Indeed users may get confused if we aren't running what
134 is in the image.
135
136 This does rely on the host file-system being fairly multi-arch
137 aware so the file don't clash with the guests layout.
138 """
139
140 if bin_dest:
141 _copy_with_mkdir(src, dest_dir, os.path.dirname(bin_dest))
142 else:
143 print("only copying support libraries for %s" % (src))
144
145 libs = _get_so_libs(src)
146 if libs:
147 for l in libs:
148 so_path = os.path.dirname(l)
149 real_l = os.path.realpath(l)
150 _copy_with_mkdir(real_l, dest_dir, so_path)
151
152
153def _check_binfmt_misc(executable):
154 """Check binfmt_misc has entry for executable in the right place.
155
156 The details of setting up binfmt_misc are outside the scope of
157 this script but we should at least fail early with a useful
158 message if it won't work.
159
160 Returns the configured binfmt path and a valid flag. For
161 persistent configurations we will still want to copy and dependent
162 libraries.
163 """
164
165 binary = os.path.basename(executable)
166 binfmt_entry = "/proc/sys/fs/binfmt_misc/%s" % (binary)
167
168 if not os.path.exists(binfmt_entry):
169 print ("No binfmt_misc entry for %s" % (binary))
170 return None, False
171
172 with open(binfmt_entry) as x: entry = x.read()
173
174 if re.search("flags:.*F.*\n", entry):
175 print("binfmt_misc for %s uses persistent(F) mapping to host binary" %
176 (binary))
177 return None, True
178
179 m = re.search("interpreter (\S+)\n", entry)
180 interp = m.group(1)
181 if interp and interp != executable:
182 print("binfmt_misc for %s does not point to %s, using %s" %
183 (binary, executable, interp))
184
185 return interp, True
186
187
188def _read_qemu_dockerfile(img_name):
189 # special case for Debian linux-user images
190 if img_name.startswith("debian") and img_name.endswith("user"):
191 img_name = "debian-bootstrap"
192
193 df = os.path.join(os.path.dirname(__file__), "dockerfiles",
194 img_name + ".docker")
195 return _read_dockerfile(df)
196
197
198def _dockerfile_preprocess(df):
199 out = ""
200 for l in df.splitlines():
201 if len(l.strip()) == 0 or l.startswith("#"):
202 continue
203 from_pref = "FROM qemu:"
204 if l.startswith(from_pref):
205 # TODO: Alternatively we could replace this line with "FROM $ID"
206 # where $ID is the image's hex id obtained with
207 # $ docker images $IMAGE --format="{{.Id}}"
208 # but unfortunately that's not supported by RHEL 7.
209 inlining = _read_qemu_dockerfile(l[len(from_pref):])
210 out += _dockerfile_preprocess(inlining)
211 continue
212 out += l + "\n"
213 return out
214
215
216class Docker(object):
217 """ Running Docker commands """
218 def __init__(self):
219 self._command = _guess_engine_command()
220 self._instance = None
221 atexit.register(self._kill_instances)
222 signal.signal(signal.SIGTERM, self._kill_instances)
223 signal.signal(signal.SIGHUP, self._kill_instances)
224
225 def _do(self, cmd, quiet=True, **kwargs):
226 if quiet:
227 kwargs["stdout"] = DEVNULL
228 return subprocess.call(self._command + cmd, **kwargs)
229
230 def _do_check(self, cmd, quiet=True, **kwargs):
231 if quiet:
232 kwargs["stdout"] = DEVNULL
233 return subprocess.check_call(self._command + cmd, **kwargs)
234
235 def _do_kill_instances(self, only_known, only_active=True):
236 cmd = ["ps", "-q"]
237 if not only_active:
238 cmd.append("-a")
239
240 filter = "--filter=label=com.qemu.instance.uuid"
241 if only_known:
242 if self._instance:
243 filter += "=%s" % (self._instance)
244 else:
245 # no point trying to kill, we finished
246 return
247
248 print("filter=%s" % (filter))
249 cmd.append(filter)
250 for i in self._output(cmd).split():
251 self._do(["rm", "-f", i])
252
253 def clean(self):
254 self._do_kill_instances(False, False)
255 return 0
256
257 def _kill_instances(self, *args, **kwargs):
258 return self._do_kill_instances(True)
259
260 def _output(self, cmd, **kwargs):
261 if sys.version_info[1] >= 6:
262 return subprocess.check_output(self._command + cmd,
263 stderr=subprocess.STDOUT,
264 encoding='utf-8',
265 **kwargs)
266 else:
267 return subprocess.check_output(self._command + cmd,
268 stderr=subprocess.STDOUT,
269 **kwargs).decode('utf-8')
270
271
272 def inspect_tag(self, tag):
273 try:
274 return self._output(["inspect", tag])
275 except subprocess.CalledProcessError:
276 return None
277
278 def get_image_creation_time(self, info):
279 return json.loads(info)[0]["Created"]
280
281 def get_image_dockerfile_checksum(self, tag):
282 resp = self.inspect_tag(tag)
283 labels = json.loads(resp)[0]["Config"].get("Labels", {})
284 return labels.get("com.qemu.dockerfile-checksum", "")
285
286 def build_image(self, tag, docker_dir, dockerfile,
287 quiet=True, user=False, argv=None, extra_files_cksum=[]):
288 if argv is None:
289 argv = []
290
291 tmp_df = tempfile.NamedTemporaryFile(mode="w+t",
292 encoding='utf-8',
293 dir=docker_dir, suffix=".docker")
294 tmp_df.write(dockerfile)
295
296 if user:
297 uid = os.getuid()
298 uname = getpwuid(uid).pw_name
299 tmp_df.write("\n")
300 tmp_df.write("RUN id %s 2>/dev/null || useradd -u %d -U %s" %
301 (uname, uid, uname))
302
303 tmp_df.write("\n")
304 tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" %
305 _text_checksum(_dockerfile_preprocess(dockerfile)))
306 for f, c in extra_files_cksum:
307 tmp_df.write("LABEL com.qemu.%s-checksum=%s" % (f, c))
308
309 tmp_df.flush()
310
311 self._do_check(["build", "-t", tag, "-f", tmp_df.name] + argv +
312 [docker_dir],
313 quiet=quiet)
314
315 def update_image(self, tag, tarball, quiet=True):
316 "Update a tagged image using "
317
318 self._do_check(["build", "-t", tag, "-"], quiet=quiet, stdin=tarball)
319
320 def image_matches_dockerfile(self, tag, dockerfile):
321 try:
322 checksum = self.get_image_dockerfile_checksum(tag)
323 except Exception:
324 return False
325 return checksum == _text_checksum(_dockerfile_preprocess(dockerfile))
326
327 def run(self, cmd, keep, quiet, as_user=False):
328 label = uuid.uuid4().hex
329 if not keep:
330 self._instance = label
331
332 if as_user:
333 uid = os.getuid()
334 cmd = [ "-u", str(uid) ] + cmd
335 # podman requires a bit more fiddling
336 if self._command[0] == "podman":
337 cmd.insert(0, '--userns=keep-id')
338
339 ret = self._do_check(["run", "--label",
340 "com.qemu.instance.uuid=" + label] + cmd,
341 quiet=quiet)
342 if not keep:
343 self._instance = None
344 return ret
345
346 def command(self, cmd, argv, quiet):
347 return self._do([cmd] + argv, quiet=quiet)
348
349
350class SubCommand(object):
351 """A SubCommand template base class"""
352 name = None # Subcommand name
353
354 def shared_args(self, parser):
355 parser.add_argument("--quiet", action="store_true",
356 help="Run quietly unless an error occurred")
357
358 def args(self, parser):
359 """Setup argument parser"""
360 pass
361
362 def run(self, args, argv):
363 """Run command.
364 args: parsed argument by argument parser.
365 argv: remaining arguments from sys.argv.
366 """
367 pass
368
369
370class RunCommand(SubCommand):
371 """Invoke docker run and take care of cleaning up"""
372 name = "run"
373
374 def args(self, parser):
375 parser.add_argument("--keep", action="store_true",
376 help="Don't remove image when command completes")
377 parser.add_argument("--run-as-current-user", action="store_true",
378 help="Run container using the current user's uid")
379
380 def run(self, args, argv):
381 return Docker().run(argv, args.keep, quiet=args.quiet,
382 as_user=args.run_as_current_user)
383
384
385class BuildCommand(SubCommand):
386 """ Build docker image out of a dockerfile. Arg: <tag> <dockerfile>"""
387 name = "build"
388
389 def args(self, parser):
390 parser.add_argument("--include-executable", "-e",
391 help="""Specify a binary that will be copied to the
392 container together with all its dependent
393 libraries""")
394 parser.add_argument("--extra-files", "-f", nargs='*',
395 help="""Specify files that will be copied in the
396 Docker image, fulfilling the ADD directive from the
397 Dockerfile""")
398 parser.add_argument("--add-current-user", "-u", dest="user",
399 action="store_true",
400 help="Add the current user to image's passwd")
401 parser.add_argument("tag",
402 help="Image Tag")
403 parser.add_argument("dockerfile",
404 help="Dockerfile name")
405
406 def run(self, args, argv):
407 dockerfile = _read_dockerfile(args.dockerfile)
408 tag = args.tag
409
410 dkr = Docker()
411 if "--no-cache" not in argv and \
412 dkr.image_matches_dockerfile(tag, dockerfile):
413 if not args.quiet:
414 print("Image is up to date.")
415 else:
416 # Create a docker context directory for the build
417 docker_dir = tempfile.mkdtemp(prefix="docker_build")
418
419 # Validate binfmt_misc will work
420 if args.include_executable:
421 qpath, enabled = _check_binfmt_misc(args.include_executable)
422 if not enabled:
423 return 1
424
425 # Is there a .pre file to run in the build context?
426 docker_pre = os.path.splitext(args.dockerfile)[0]+".pre"
427 if os.path.exists(docker_pre):
428 stdout = DEVNULL if args.quiet else None
429 rc = subprocess.call(os.path.realpath(docker_pre),
430 cwd=docker_dir, stdout=stdout)
431 if rc == 3:
432 print("Skip")
433 return 0
434 elif rc != 0:
435 print("%s exited with code %d" % (docker_pre, rc))
436 return 1
437
438 # Copy any extra files into the Docker context. These can be
439 # included by the use of the ADD directive in the Dockerfile.
440 cksum = []
441 if args.include_executable:
442 # FIXME: there is no checksum of this executable and the linked
443 # libraries, once the image built any change of this executable
444 # or any library won't trigger another build.
445 _copy_binary_with_libs(args.include_executable,
446 qpath, docker_dir)
447
448 for filename in args.extra_files or []:
449 _copy_with_mkdir(filename, docker_dir)
450 cksum += [(filename, _file_checksum(filename))]
451
452 argv += ["--build-arg=" + k.lower() + "=" + v
453 for k, v in os.environ.items()
454 if k.lower() in FILTERED_ENV_NAMES]
455 dkr.build_image(tag, docker_dir, dockerfile,
456 quiet=args.quiet, user=args.user, argv=argv,
457 extra_files_cksum=cksum)
458
459 rmtree(docker_dir)
460
461 return 0
462
463
464class UpdateCommand(SubCommand):
465 """ Update a docker image with new executables. Args: <tag> <executable>"""
466 name = "update"
467
468 def args(self, parser):
469 parser.add_argument("tag",
470 help="Image Tag")
471 parser.add_argument("executable",
472 help="Executable to copy")
473
474 def run(self, args, argv):
475 # Create a temporary tarball with our whole build context and
476 # dockerfile for the update
477 tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz")
478 tmp_tar = TarFile(fileobj=tmp, mode='w')
479
480 # Add the executable to the tarball, using the current
481 # configured binfmt_misc path. If we don't get a path then we
482 # only need the support libraries copied
483 ff, enabled = _check_binfmt_misc(args.executable)
484
485 if not enabled:
486 print("binfmt_misc not enabled, update disabled")
487 return 1
488
489 if ff:
490 tmp_tar.add(args.executable, arcname=ff)
491
492 # Add any associated libraries
493 libs = _get_so_libs(args.executable)
494 if libs:
495 for l in libs:
496 tmp_tar.add(os.path.realpath(l), arcname=l)
497
498 # Create a Docker buildfile
499 df = StringIO()
500 df.write("FROM %s\n" % args.tag)
501 df.write("ADD . /\n")
502 df.seek(0)
503
504 df_tar = TarInfo(name="Dockerfile")
505 df_tar.size = len(df.buf)
506 tmp_tar.addfile(df_tar, fileobj=df)
507
508 tmp_tar.close()
509
510 # reset the file pointers
511 tmp.flush()
512 tmp.seek(0)
513
514 # Run the build with our tarball context
515 dkr = Docker()
516 dkr.update_image(args.tag, tmp, quiet=args.quiet)
517
518 return 0
519
520
521class CleanCommand(SubCommand):
522 """Clean up docker instances"""
523 name = "clean"
524
525 def run(self, args, argv):
526 Docker().clean()
527 return 0
528
529
530class ImagesCommand(SubCommand):
531 """Run "docker images" command"""
532 name = "images"
533
534 def run(self, args, argv):
535 return Docker().command("images", argv, args.quiet)
536
537
538class ProbeCommand(SubCommand):
539 """Probe if we can run docker automatically"""
540 name = "probe"
541
542 def run(self, args, argv):
543 try:
544 docker = Docker()
545 if docker._command[0] == "docker":
546 print("docker")
547 elif docker._command[0] == "sudo":
548 print("sudo docker")
549 elif docker._command[0] == "podman":
550 print("podman")
551 except Exception:
552 print("no")
553
554 return
555
556
557class CcCommand(SubCommand):
558 """Compile sources with cc in images"""
559 name = "cc"
560
561 def args(self, parser):
562 parser.add_argument("--image", "-i", required=True,
563 help="The docker image in which to run cc")
564 parser.add_argument("--cc", default="cc",
565 help="The compiler executable to call")
566 parser.add_argument("--source-path", "-s", nargs="*", dest="paths",
567 help="""Extra paths to (ro) mount into container for
568 reading sources""")
569
570 def run(self, args, argv):
571 if argv and argv[0] == "--":
572 argv = argv[1:]
573 cwd = os.getcwd()
574 cmd = ["--rm", "-w", cwd,
575 "-v", "%s:%s:rw" % (cwd, cwd)]
576 if args.paths:
577 for p in args.paths:
578 cmd += ["-v", "%s:%s:ro,z" % (p, p)]
579 cmd += [args.image, args.cc]
580 cmd += argv
581 return Docker().run(cmd, False, quiet=args.quiet,
582 as_user=True)
583
584
585class CheckCommand(SubCommand):
586 """Check if we need to re-build a docker image out of a dockerfile.
587 Arguments: <tag> <dockerfile>"""
588 name = "check"
589
590 def args(self, parser):
591 parser.add_argument("tag",
592 help="Image Tag")
593 parser.add_argument("dockerfile", default=None,
594 help="Dockerfile name", nargs='?')
595 parser.add_argument("--checktype", choices=["checksum", "age"],
596 default="checksum", help="check type")
597 parser.add_argument("--olderthan", default=60, type=int,
598 help="number of minutes")
599
600 def run(self, args, argv):
601 tag = args.tag
602
603 try:
604 dkr = Docker()
605 except subprocess.CalledProcessError:
606 print("Docker not set up")
607 return 1
608
609 info = dkr.inspect_tag(tag)
610 if info is None:
611 print("Image does not exist")
612 return 1
613
614 if args.checktype == "checksum":
615 if not args.dockerfile:
616 print("Need a dockerfile for tag:%s" % (tag))
617 return 1
618
619 dockerfile = _read_dockerfile(args.dockerfile)
620
621 if dkr.image_matches_dockerfile(tag, dockerfile):
622 if not args.quiet:
623 print("Image is up to date")
624 return 0
625 else:
626 print("Image needs updating")
627 return 1
628 elif args.checktype == "age":
629 timestr = dkr.get_image_creation_time(info).split(".")[0]
630 created = datetime.strptime(timestr, "%Y-%m-%dT%H:%M:%S")
631 past = datetime.now() - timedelta(minutes=args.olderthan)
632 if created < past:
633 print ("Image created @ %s more than %d minutes old" %
634 (timestr, args.olderthan))
635 return 1
636 else:
637 if not args.quiet:
638 print ("Image less than %d minutes old" % (args.olderthan))
639 return 0
640
641
642def main():
643 global USE_ENGINE
644
645 parser = argparse.ArgumentParser(description="A Docker helper",
646 usage="%s <subcommand> ..." %
647 os.path.basename(sys.argv[0]))
648 parser.add_argument("--engine", type=EngineEnum.argparse, choices=list(EngineEnum),
649 help="specify which container engine to use")
650 subparsers = parser.add_subparsers(title="subcommands", help=None)
651 for cls in SubCommand.__subclasses__():
652 cmd = cls()
653 subp = subparsers.add_parser(cmd.name, help=cmd.__doc__)
654 cmd.shared_args(subp)
655 cmd.args(subp)
656 subp.set_defaults(cmdobj=cmd)
657 args, argv = parser.parse_known_args()
658 if args.engine:
659 USE_ENGINE = args.engine
660 return args.cmdobj.run(args, argv)
661
662
663if __name__ == "__main__":
664 sys.exit(main())