scsi: add multipath support to qemu-pr-helper

+3

Makefile

··· 373 373 fsdev/virtfs-proxy-helper$(EXESUF): LIBS += -lcap 374 374 375 375 scsi/qemu-pr-helper$(EXESUF): scsi/qemu-pr-helper.o scsi/utils.o $(crypto-obj-y) $(io-obj-y) $(qom-obj-y) $(COMMON_LDADDS) 376 + ifdef CONFIG_MPATH 377 + scsi/qemu-pr-helper$(EXESUF): LIBS += -ludev -lmultipath -lmpathpersist 378 + endif 376 379 377 380 qemu-img-cmds.h: $(SRC_PATH)/qemu-img-cmds.hx $(SRC_PATH)/scripts/hxtool 378 381 $(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -h < $< > $@,"GEN","$@")

+46

configure

··· 290 290 sdl="" 291 291 sdlabi="" 292 292 virtfs="" 293 + mpath="" 293 294 vnc="yes" 294 295 sparse="no" 295 296 vde="" ··· 936 937 ;; 937 938 --enable-virtfs) virtfs="yes" 938 939 ;; 940 + --disable-mpath) mpath="no" 941 + ;; 942 + --enable-mpath) mpath="yes" 943 + ;; 939 944 --disable-vnc) vnc="no" 940 945 ;; 941 946 --enable-vnc) vnc="yes" ··· 1479 1484 vnc-png PNG compression for VNC server 1480 1485 cocoa Cocoa UI (Mac OS X only) 1481 1486 virtfs VirtFS 1487 + mpath Multipath persistent reservation passthrough 1482 1488 xen xen backend driver support 1483 1489 xen-pci-passthrough 1484 1490 brlapi BrlAPI (Braile) ··· 3300 3306 fi 3301 3307 3302 3308 ########################################## 3309 + # libmpathpersist probe 3310 + 3311 + if test "$mpath" != "no" ; then 3312 + cat > $TMPC <<EOF 3313 + #include <libudev.h> 3314 + #include <mpath_persist.h> 3315 + unsigned mpath_mx_alloc_len = 1024; 3316 + int logsink; 3317 + int main(void) { 3318 + struct udev *udev = udev_new(); 3319 + mpath_lib_init(udev); 3320 + return 0; 3321 + } 3322 + EOF 3323 + if compile_prog "" "-ludev -lmultipath -lmpathpersist" ; then 3324 + mpathpersist=yes 3325 + else 3326 + mpathpersist=no 3327 + fi 3328 + else 3329 + mpathpersist=no 3330 + fi 3331 + 3332 + ########################################## 3303 3333 # libcap probe 3304 3334 3305 3335 if test "$cap" != "no" ; then ··· 5044 5074 fi 5045 5075 virtfs=no 5046 5076 fi 5077 + if test "$mpath" != no && test "$mpathpersist" = yes ; then 5078 + mpath=yes 5079 + else 5080 + if test "$mpath" = yes; then 5081 + error_exit "Multipath requires libmpathpersist devel" 5082 + fi 5083 + mpath=no 5084 + fi 5047 5085 tools="$tools scsi/qemu-pr-helper\$(EXESUF)" 5048 5086 else 5049 5087 if test "$virtfs" = yes; then 5050 5088 error_exit "VirtFS is supported only on Linux" 5051 5089 fi 5052 5090 virtfs=no 5091 + if test "$mpath" = yes; then 5092 + error_exit "Multipath is supported only on Linux" 5093 + fi 5094 + mpath=no 5053 5095 fi 5054 5096 fi 5055 5097 ··· 5295 5337 echo "Block whitelist (rw) $block_drv_rw_whitelist" 5296 5338 echo "Block whitelist (ro) $block_drv_ro_whitelist" 5297 5339 echo "VirtFS support $virtfs" 5340 + echo "Multipath support $mpath" 5298 5341 echo "VNC support $vnc" 5299 5342 if test "$vnc" = "yes" ; then 5300 5343 echo "VNC SASL support $vnc_sasl" ··· 5737 5780 fi 5738 5781 if test "$virtfs" = "yes" ; then 5739 5782 echo "CONFIG_VIRTFS=y" >> $config_host_mak 5783 + fi 5784 + if test "$mpath" = "yes" ; then 5785 + echo "CONFIG_MPATH=y" >> $config_host_mak 5740 5786 fi 5741 5787 if test "$vhost_scsi" = "yes" ; then 5742 5788 echo "CONFIG_VHOST_SCSI=y" >> $config_host_mak

+27

docs/pr-manager.rst

··· 60 60 61 61 -d, --daemon run in the background 62 62 -q, --quiet decrease verbosity 63 + -v, --verbose increase verbosity 63 64 -f, --pidfile=path PID file when running as a daemon 64 65 -k, --socket=path path to the socket 65 66 -T, --trace=trace-opts tracing options ··· 82 83 83 84 -u, --user=user user to drop privileges to 84 85 -g, --group=group group to drop privileges to 86 + 87 + --------------------------------------------- 88 + Multipath devices and persistent reservations 89 + --------------------------------------------- 90 + 91 + Proper support of persistent reservation for multipath devices requires 92 + communication with the multipath daemon, so that the reservation is 93 + registered and applied when a path is newly discovered or becomes online 94 + again. :command:`qemu-pr-helper` can do this if the ``libmpathpersist`` 95 + library was available on the system at build time. 96 + 97 + As of August 2017, a reservation key must be specified in ``multipath.conf`` 98 + for ``multipathd`` to check for persistent reservation for newly 99 + discovered paths or reinstated paths. The attribute can be added 100 + to the ``defaults`` section or the ``multipaths`` section; for example:: 101 + 102 + multipaths { 103 + multipath { 104 + wwid XXXXXXXXXXXXXXXX 105 + alias yellow 106 + reservation_key 0x123abc 107 + } 108 + } 109 + 110 + Linking :program:`qemu-pr-helper` to ``libmpathpersist`` does not impede 111 + its usage on regular SCSI devices.

+4

include/scsi/utils.h

··· 72 72 extern const struct SCSISense sense_code_I_T_NEXUS_LOSS; 73 73 /* Command aborted, Logical Unit failure */ 74 74 extern const struct SCSISense sense_code_LUN_FAILURE; 75 + /* Command aborted, LUN Communication failure */ 76 + extern const struct SCSISense sense_code_LUN_COMM_FAILURE; 75 77 /* Command aborted, Overlapped Commands Attempted */ 76 78 extern const struct SCSISense sense_code_OVERLAPPED_COMMANDS; 77 79 /* LUN not ready, Capacity data has changed */ 78 80 extern const struct SCSISense sense_code_CAPACITY_CHANGED; 81 + /* Unit attention, SCSI bus reset */ 82 + extern const struct SCSISense sense_code_SCSI_BUS_RESET; 79 83 /* LUN not ready, Medium not present */ 80 84 extern const struct SCSISense sense_code_UNIT_ATTENTION_NO_MEDIUM; 81 85 /* Unit attention, Power on, reset or bus device reset occurred */

+343 -3

scsi/qemu-pr-helper.c

··· 30 30 #include <pwd.h> 31 31 #include <grp.h> 32 32 33 + #ifdef CONFIG_MPATH 34 + #include <libudev.h> 35 + #include <mpath_cmd.h> 36 + #include <mpath_persist.h> 37 + #endif 38 + 33 39 #include "qapi/error.h" 34 40 #include "qemu-common.h" 35 41 #include "qemu/cutils.h" ··· 60 66 static QIOChannelSocket *server_ioc; 61 67 static int server_watch; 62 68 static int num_active_sockets = 1; 69 + static int noisy; 63 70 static int verbose; 64 71 65 72 #ifdef CONFIG_LIBCAP ··· 204 211 return r; 205 212 } 206 213 214 + /* Device mapper interface */ 215 + 216 + #ifdef CONFIG_MPATH 217 + #define CONTROL_PATH "/dev/mapper/control" 218 + 219 + typedef struct DMData { 220 + struct dm_ioctl dm; 221 + uint8_t data[1024]; 222 + } DMData; 223 + 224 + static int control_fd; 225 + 226 + static void *dm_ioctl(int ioc, struct dm_ioctl *dm) 227 + { 228 + static DMData d; 229 + memcpy(&d.dm, dm, sizeof(d.dm)); 230 + QEMU_BUILD_BUG_ON(sizeof(d.data) < sizeof(struct dm_target_spec)); 231 + 232 + d.dm.version[0] = DM_VERSION_MAJOR; 233 + d.dm.version[1] = 0; 234 + d.dm.version[2] = 0; 235 + d.dm.data_size = 1024; 236 + d.dm.data_start = offsetof(DMData, data); 237 + if (ioctl(control_fd, ioc, &d) < 0) { 238 + return NULL; 239 + } 240 + memcpy(dm, &d.dm, sizeof(d.dm)); 241 + return &d.data; 242 + } 243 + 244 + static void *dm_dev_ioctl(int fd, int ioc, struct dm_ioctl *dm) 245 + { 246 + struct stat st; 247 + int r; 248 + 249 + r = fstat(fd, &st); 250 + if (r < 0) { 251 + perror("fstat"); 252 + exit(1); 253 + } 254 + 255 + dm->dev = st.st_rdev; 256 + return dm_ioctl(ioc, dm); 257 + } 258 + 259 + static void dm_init(void) 260 + { 261 + control_fd = open(CONTROL_PATH, O_RDWR); 262 + if (control_fd < 0) { 263 + perror("Cannot open " CONTROL_PATH); 264 + exit(1); 265 + } 266 + struct dm_ioctl dm = { 0 }; 267 + if (!dm_ioctl(DM_VERSION, &dm)) { 268 + perror("ioctl"); 269 + exit(1); 270 + } 271 + if (dm.version[0] != DM_VERSION_MAJOR) { 272 + fprintf(stderr, "Unsupported device mapper interface"); 273 + exit(1); 274 + } 275 + } 276 + 277 + /* Variables required by libmultipath and libmpathpersist. */ 278 + QEMU_BUILD_BUG_ON(PR_HELPER_DATA_SIZE > MPATH_MAX_PARAM_LEN); 279 + unsigned mpath_mx_alloc_len = PR_HELPER_DATA_SIZE; 280 + int logsink; 281 + 282 + static void multipath_pr_init(void) 283 + { 284 + static struct udev *udev; 285 + 286 + udev = udev_new(); 287 + mpath_lib_init(udev); 288 + } 289 + 290 + static int is_mpath(int fd) 291 + { 292 + struct dm_ioctl dm = { .flags = DM_NOFLUSH_FLAG }; 293 + struct dm_target_spec *tgt; 294 + 295 + tgt = dm_dev_ioctl(fd, DM_TABLE_STATUS, &dm); 296 + if (!tgt) { 297 + if (errno == ENXIO) { 298 + return 0; 299 + } 300 + perror("ioctl"); 301 + exit(EXIT_FAILURE); 302 + } 303 + return !strncmp(tgt->target_type, "multipath", DM_MAX_TYPE_NAME); 304 + } 305 + 306 + static int mpath_reconstruct_sense(int fd, int r, uint8_t *sense) 307 + { 308 + switch (r) { 309 + case MPATH_PR_SUCCESS: 310 + return GOOD; 311 + case MPATH_PR_SENSE_NOT_READY: 312 + case MPATH_PR_SENSE_MEDIUM_ERROR: 313 + case MPATH_PR_SENSE_HARDWARE_ERROR: 314 + case MPATH_PR_SENSE_ABORTED_COMMAND: 315 + { 316 + /* libmpathpersist ate the exact sense. Try to find it by 317 + * issuing TEST UNIT READY. 318 + */ 319 + uint8_t cdb[6] = { TEST_UNIT_READY }; 320 + int sz = 0; 321 + return do_sgio(fd, cdb, sense, NULL, &sz, SG_DXFER_NONE); 322 + } 323 + 324 + case MPATH_PR_SENSE_UNIT_ATTENTION: 325 + /* Congratulations libmpathpersist, you ruined the Unit Attention... 326 + * Return a heavyweight one. 327 + */ 328 + scsi_build_sense(sense, SENSE_CODE(SCSI_BUS_RESET)); 329 + return CHECK_CONDITION; 330 + case MPATH_PR_SENSE_INVALID_OP: 331 + /* Only one valid sense. */ 332 + scsi_build_sense(sense, SENSE_CODE(INVALID_OPCODE)); 333 + return CHECK_CONDITION; 334 + case MPATH_PR_ILLEGAL_REQ: 335 + /* Guess. */ 336 + scsi_build_sense(sense, SENSE_CODE(INVALID_PARAM)); 337 + return CHECK_CONDITION; 338 + case MPATH_PR_NO_SENSE: 339 + scsi_build_sense(sense, SENSE_CODE(NO_SENSE)); 340 + return CHECK_CONDITION; 341 + 342 + case MPATH_PR_RESERV_CONFLICT: 343 + return RESERVATION_CONFLICT; 344 + 345 + case MPATH_PR_OTHER: 346 + default: 347 + scsi_build_sense(sense, SENSE_CODE(LUN_COMM_FAILURE)); 348 + return CHECK_CONDITION; 349 + } 350 + } 351 + 352 + static int multipath_pr_in(int fd, const uint8_t *cdb, uint8_t *sense, 353 + uint8_t *data, int sz) 354 + { 355 + int rq_servact = cdb[1]; 356 + struct prin_resp resp; 357 + size_t written; 358 + int r; 359 + 360 + switch (rq_servact) { 361 + case MPATH_PRIN_RKEY_SA: 362 + case MPATH_PRIN_RRES_SA: 363 + case MPATH_PRIN_RCAP_SA: 364 + break; 365 + case MPATH_PRIN_RFSTAT_SA: 366 + /* Nobody implements it anyway, so bail out. */ 367 + default: 368 + /* Cannot parse any other output. */ 369 + scsi_build_sense(sense, SENSE_CODE(INVALID_FIELD)); 370 + return CHECK_CONDITION; 371 + } 372 + 373 + r = mpath_persistent_reserve_in(fd, rq_servact, &resp, noisy, verbose); 374 + if (r == MPATH_PR_SUCCESS) { 375 + switch (rq_servact) { 376 + case MPATH_PRIN_RKEY_SA: 377 + case MPATH_PRIN_RRES_SA: { 378 + struct prin_readdescr *out = &resp.prin_descriptor.prin_readkeys; 379 + assert(sz >= 8); 380 + written = MIN(out->additional_length + 8, sz); 381 + stl_be_p(&data[0], out->prgeneration); 382 + stl_be_p(&data[4], out->additional_length); 383 + memcpy(&data[8], out->key_list, written - 8); 384 + break; 385 + } 386 + case MPATH_PRIN_RCAP_SA: { 387 + struct prin_capdescr *out = &resp.prin_descriptor.prin_readcap; 388 + assert(sz >= 6); 389 + written = 6; 390 + stw_be_p(&data[0], out->length); 391 + data[2] = out->flags[0]; 392 + data[3] = out->flags[1]; 393 + stw_be_p(&data[4], out->pr_type_mask); 394 + break; 395 + } 396 + default: 397 + scsi_build_sense(sense, SENSE_CODE(INVALID_OPCODE)); 398 + return CHECK_CONDITION; 399 + } 400 + assert(written <= sz); 401 + memset(data + written, 0, sz - written); 402 + } 403 + 404 + return mpath_reconstruct_sense(fd, r, sense); 405 + } 406 + 407 + static int multipath_pr_out(int fd, const uint8_t *cdb, uint8_t *sense, 408 + const uint8_t *param, int sz) 409 + { 410 + int rq_servact = cdb[1]; 411 + int rq_scope = cdb[2] >> 4; 412 + int rq_type = cdb[2] & 0xf; 413 + struct prout_param_descriptor paramp; 414 + char transportids[PR_HELPER_DATA_SIZE]; 415 + int r; 416 + 417 + switch (rq_servact) { 418 + case MPATH_PROUT_REG_SA: 419 + case MPATH_PROUT_RES_SA: 420 + case MPATH_PROUT_REL_SA: 421 + case MPATH_PROUT_CLEAR_SA: 422 + case MPATH_PROUT_PREE_SA: 423 + case MPATH_PROUT_PREE_AB_SA: 424 + case MPATH_PROUT_REG_IGN_SA: 425 + break; 426 + case MPATH_PROUT_REG_MOV_SA: 427 + /* Not supported by struct prout_param_descriptor. */ 428 + default: 429 + /* Cannot parse any other input. */ 430 + scsi_build_sense(sense, SENSE_CODE(INVALID_FIELD)); 431 + return CHECK_CONDITION; 432 + } 433 + 434 + /* Convert input data, especially transport IDs, to the structs 435 + * used by libmpathpersist (which, of course, will immediately 436 + * do the opposite). 437 + */ 438 + memset(&paramp, 0, sizeof(paramp)); 439 + memcpy(&paramp.key, &param[0], 8); 440 + memcpy(&paramp.sa_key, &param[8], 8); 441 + paramp.sa_flags = param[10]; 442 + if (sz > PR_OUT_FIXED_PARAM_SIZE) { 443 + size_t transportid_len; 444 + int i, j; 445 + if (sz < PR_OUT_FIXED_PARAM_SIZE + 4) { 446 + scsi_build_sense(sense, SENSE_CODE(INVALID_PARAM_LEN)); 447 + return CHECK_CONDITION; 448 + } 449 + transportid_len = ldl_be_p(&param[24]) + PR_OUT_FIXED_PARAM_SIZE + 4; 450 + if (transportid_len > sz) { 451 + scsi_build_sense(sense, SENSE_CODE(INVALID_PARAM)); 452 + return CHECK_CONDITION; 453 + } 454 + for (i = PR_OUT_FIXED_PARAM_SIZE + 4, j = 0; i < transportid_len; ) { 455 + struct transportid *id = (struct transportid *) &transportids[j]; 456 + int len; 457 + 458 + id->format_code = param[i] & 0xc0; 459 + id->protocol_id = param[i] & 0x0f; 460 + switch (param[i] & 0xcf) { 461 + case 0: 462 + /* FC transport. */ 463 + if (i + 24 > transportid_len) { 464 + goto illegal_req; 465 + } 466 + memcpy(id->n_port_name, &param[i + 8], 8); 467 + j += offsetof(struct transportid, n_port_name[8]); 468 + i += 24; 469 + break; 470 + case 3: 471 + case 0x43: 472 + /* iSCSI transport. */ 473 + len = lduw_be_p(&param[i + 2]); 474 + if (len > 252 || (len & 3) || i + len + 4 > transportid_len) { 475 + /* For format code 00, the standard says the maximum is 223 476 + * plus the NUL terminator. For format code 01 there is no 477 + * maximum length, but libmpathpersist ignores the first 478 + * byte of id->iscsi_name so our maximum is 252. 479 + */ 480 + goto illegal_req; 481 + } 482 + if (memchr(&param[i + 4], 0, len) == NULL) { 483 + goto illegal_req; 484 + } 485 + memcpy(id->iscsi_name, &param[i + 2], len + 2); 486 + j += offsetof(struct transportid, iscsi_name[len + 2]); 487 + i += len + 4; 488 + break; 489 + case 6: 490 + /* SAS transport. */ 491 + if (i + 24 > transportid_len) { 492 + goto illegal_req; 493 + } 494 + memcpy(id->sas_address, &param[i + 4], 8); 495 + j += offsetof(struct transportid, sas_address[8]); 496 + i += 24; 497 + break; 498 + default: 499 + illegal_req: 500 + scsi_build_sense(sense, SENSE_CODE(INVALID_PARAM)); 501 + return CHECK_CONDITION; 502 + } 503 + 504 + paramp.trnptid_list[paramp.num_transportid++] = id; 505 + } 506 + } 507 + 508 + r = mpath_persistent_reserve_out(fd, rq_servact, rq_scope, rq_type, 509 + &paramp, noisy, verbose); 510 + return mpath_reconstruct_sense(fd, r, sense); 511 + } 512 + #endif 513 + 207 514 static int do_pr_in(int fd, const uint8_t *cdb, uint8_t *sense, 208 515 uint8_t *data, int *resp_sz) 209 516 { 517 + #ifdef CONFIG_MPATH 518 + if (is_mpath(fd)) { 519 + /* multipath_pr_in fills the whole input buffer. */ 520 + return multipath_pr_in(fd, cdb, sense, data, *resp_sz); 521 + } 522 + #endif 523 + 210 524 return do_sgio(fd, cdb, sense, data, resp_sz, 211 525 SG_DXFER_FROM_DEV); 212 526 } ··· 214 528 static int do_pr_out(int fd, const uint8_t *cdb, uint8_t *sense, 215 529 const uint8_t *param, int sz) 216 530 { 217 - int resp_sz = sz; 531 + int resp_sz; 532 + #ifdef CONFIG_MPATH 533 + if (is_mpath(fd)) { 534 + return multipath_pr_out(fd, cdb, sense, param, sz); 535 + } 536 + #endif 537 + 538 + resp_sz = sz; 218 539 return do_sgio(fd, cdb, sense, (uint8_t *)param, &resp_sz, 219 540 SG_DXFER_TO_DEV); 220 541 } ··· 525 846 return -1; 526 847 } 527 848 849 + #ifdef CONFIG_MPATH 850 + /* For /dev/mapper/control ioctls */ 851 + if (capng_update(CAPNG_ADD, CAPNG_EFFECTIVE | CAPNG_PERMITTED, 852 + CAP_SYS_ADMIN) < 0) { 853 + return -1; 854 + } 855 + #endif 856 + 528 857 /* Change user/group id, retaining the capabilities. Because file descriptors 529 858 * are passed via SCM_RIGHTS, we don't need supplementary groups (and in 530 859 * fact the helper can run as "nobody"). ··· 541 870 542 871 int main(int argc, char **argv) 543 872 { 544 - const char *sopt = "hVk:fdT:u:g:q"; 873 + const char *sopt = "hVk:fdT:u:g:vq"; 545 874 struct option lopt[] = { 546 875 { "help", no_argument, NULL, 'h' }, 547 876 { "version", no_argument, NULL, 'V' }, ··· 551 880 { "trace", required_argument, NULL, 'T' }, 552 881 { "user", required_argument, NULL, 'u' }, 553 882 { "group", required_argument, NULL, 'g' }, 883 + { "verbose", no_argument, NULL, 'v' }, 554 884 { "quiet", no_argument, NULL, 'q' }, 555 885 { NULL, 0, NULL, 0 } 556 886 }; 557 887 int opt_ind = 0; 888 + int loglevel = 1; 558 889 int quiet = 0; 559 890 int ch; 560 891 Error *local_err = NULL; ··· 631 962 case 'q': 632 963 quiet = 1; 633 964 break; 965 + case 'v': 966 + ++loglevel; 967 + break; 634 968 case 'T': 635 969 g_free(trace_file); 636 970 trace_file = trace_opt_parse(optarg); ··· 650 984 } 651 985 652 986 /* set verbosity */ 653 - verbose = !quiet; 987 + noisy = !quiet && (loglevel >= 3); 988 + verbose = quiet ? 0 : MIN(loglevel, 3); 654 989 655 990 if (!trace_init_backends()) { 656 991 exit(EXIT_FAILURE); 657 992 } 658 993 trace_init_file(trace_file); 659 994 qemu_set_log(LOG_TRACE); 995 + 996 + #ifdef CONFIG_MPATH 997 + dm_init(); 998 + multipath_pr_init(); 999 + #endif 660 1000 661 1001 socket_activation = check_socket_activation(); 662 1002 if (socket_activation == 0) {

+10

scsi/utils.c

··· 206 206 .key = ABORTED_COMMAND, .asc = 0x4e, .ascq = 0x00 207 207 }; 208 208 209 + /* Command aborted, LUN Communication Failure */ 210 + const struct SCSISense sense_code_LUN_COMM_FAILURE = { 211 + .key = ABORTED_COMMAND, .asc = 0x08, .ascq = 0x00 212 + }; 213 + 209 214 /* Unit attention, Capacity data has changed */ 210 215 const struct SCSISense sense_code_CAPACITY_CHANGED = { 211 216 .key = UNIT_ATTENTION, .asc = 0x2a, .ascq = 0x09 ··· 214 219 /* Unit attention, Power on, reset or bus device reset occurred */ 215 220 const struct SCSISense sense_code_RESET = { 216 221 .key = UNIT_ATTENTION, .asc = 0x29, .ascq = 0x00 222 + }; 223 + 224 + /* Unit attention, SCSI bus reset */ 225 + const struct SCSISense sense_code_SCSI_BUS_RESET = { 226 + .key = UNIT_ATTENTION, .asc = 0x29, .ascq = 0x02 217 227 }; 218 228 219 229 /* Unit attention, No medium */