From 2c53bae01f819ea3788cdf278c5113d6d5819b7c Mon Sep 17 00:00:00 2001 From: Zhongze Liu Date: Sat, 19 Aug 2017 01:31:31 +0800 Subject: [PATCH 1/7] libxc: add xc_domain_remove_from_physmap to wrap XENMEM_remove_from_physmap This is for the proposal "Allow setting up shared memory areas between VMs from xl config file". See: https://lists.xen.org/archives/html/xen-devel/2017-08/msg03242.html Then plan is to use XENMEM_add_to_physmap_batch to map the shared pages from one domU to another and use XENMEM_remove_from_physmap to cancel the sharing. A wrapper to XENMEM_add_to_physmap_batch was added in the following commit: commit 20e725e9364cff4a29945f66986ecd88cca8743d Now add the wrapper to XENMEM_remove_from_physmap. Signed-off-by: Zhongze Liu Reviewed-by: Stefano Stabellini Acked-by: Wei Liu Cc: Ian Jackson Cc: Wei Liu Cc: Stefano Stabellini Cc: Julien Grall Cc: xen-devel@lists.xen.org V4: * change domid_t to uint32_t --- tools/libxc/include/xenctrl.h | 4 ++++ tools/libxc/xc_domain.c | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/tools/libxc/include/xenctrl.h b/tools/libxc/include/xenctrl.h index 666db0b9193e..3c10bce63db8 100644 --- a/tools/libxc/include/xenctrl.h +++ b/tools/libxc/include/xenctrl.h @@ -1415,6 +1415,10 @@ int xc_domain_add_to_physmap_batch(xc_interface *xch, xen_pfn_t *gfpns, int *errs); +int xc_domain_remove_from_physmap(xc_interface *xch, + uint32_t domid, + xen_pfn_t gpfn); + int xc_domain_populate_physmap(xc_interface *xch, uint32_t domid, unsigned long nr_extents, diff --git a/tools/libxc/xc_domain.c b/tools/libxc/xc_domain.c index 3ccd27f10142..3b4b22428da5 100644 --- a/tools/libxc/xc_domain.c +++ b/tools/libxc/xc_domain.c @@ -1104,6 +1104,17 @@ int xc_domain_add_to_physmap_batch(xc_interface *xch, return rc; } +int xc_domain_remove_from_physmap(xc_interface *xch, + uint32_t domid, + xen_pfn_t gpfn) +{ + struct xen_remove_from_physmap xrfp = { + .domid = domid, + .gpfn = gpfn, + }; + return do_memory_op(xch, XENMEM_remove_from_physmap, &xrfp, sizeof(xrfp)); +} + int xc_domain_claim_pages(xc_interface *xch, uint32_t domid, unsigned long nr_pages) From 630047a5d53935a44db7dcc27047ff450a9fd1c7 Mon Sep 17 00:00:00 2001 From: Zhongze Liu Date: Wed, 23 Aug 2017 00:19:46 +0800 Subject: [PATCH 2/7] xen: xsm: flask: introduce XENMAPSPACE_gmfn_share for memory sharing The existing XENMAPSPACE_gmfn_foreign subop of XENMEM_add_to_physmap forbids a Dom0 to map memory pages from one DomU to another, which restricts some useful yet not dangerous use cases -- such as sharing pages among DomU's so that they can do shm-based communication. This patch introduces XENMAPSPACE_gmfn_share to address this inconvenience, which is mostly the same as XENMAPSPACE_gmfn_foreign but has its own xsm check. Specifically, the patch: * Introduces a new av permission MMU__SHARE_MEM to denote if two domains can share memory by using the new subop; * Introduces xsm_map_gmfn_share() to check if (current) has proper permission over (t) AND MMU__SHARE_MEM is allowed between (d) and (t); * Modify the default xen.te to allow MMU__SHARE_MEM for normal domains that allow grant mapping/event channels. The new subop is marked unsupported for x86 because calling p2m_add_foregin on two DomU's is currently not supported on x86. This is for the proposal "Allow setting up shared memory areas between VMs from xl config file" (see [1]). [1] https://lists.xen.org/archives/html/xen-devel/2017-08/msg03242.html Signed-off-by: Zhongze Liu Cc: Daniel De Graaf Cc: Ian Jackson Cc: Wei Liu Cc: Stefano Stabellini Cc: Julien Grall Cc: xen-devel@lists.xen.org V3: * Change several if statements to the GCC '... = a ?: b' extention. * lookup the current domain in the hooks instead of passing it as an arg V4: * eliminate the duplicated (c) over (d) check * Added a new subop instead of modifying the old subop --- tools/flask/policy/modules/xen.if | 2 ++ xen/arch/arm/mm.c | 8 +++++++- xen/arch/x86/mm.c | 4 ++++ xen/include/public/memory.h | 4 ++++ xen/include/xsm/dummy.h | 6 ++++++ xen/include/xsm/xsm.h | 6 ++++++ xen/xsm/dummy.c | 1 + xen/xsm/flask/hooks.c | 7 +++++++ xen/xsm/flask/policy/access_vectors | 5 +++++ 9 files changed, 42 insertions(+), 1 deletion(-) diff --git a/tools/flask/policy/modules/xen.if b/tools/flask/policy/modules/xen.if index 55437496f6a0..3ffd1c623966 100644 --- a/tools/flask/policy/modules/xen.if +++ b/tools/flask/policy/modules/xen.if @@ -127,6 +127,8 @@ define(`domain_comms', ` domain_event_comms($1, $2) allow $1 $2:grant { map_read map_write copy unmap }; allow $2 $1:grant { map_read map_write copy unmap }; + allow $1 $2:mmu share_mem; + allow $2 $1:mmu share_mem; ') # domain_self_comms(domain) diff --git a/xen/arch/arm/mm.c b/xen/arch/arm/mm.c index 3c328e2df501..8e385d62a881 100644 --- a/xen/arch/arm/mm.c +++ b/xen/arch/arm/mm.c @@ -1251,6 +1251,7 @@ int xenmem_add_to_physmap_one( break; case XENMAPSPACE_gmfn_foreign: + case XENMAPSPACE_gmfn_share: { struct domain *od; p2m_type_t p2mt; @@ -1265,7 +1266,12 @@ int xenmem_add_to_physmap_one( return -EINVAL; } - rc = xsm_map_gmfn_foreign(XSM_TARGET, d, od); + if ( space == XENMAPSPACE_gmfn_foreign ) { + rc = xsm_map_gmfn_foreign(XSM_TARGET, d, od); + } else { + rc = xsm_map_gmfn_share(XSM_TARGET, d, od); + } + if ( rc ) { rcu_unlock_domain(od); diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c index 1ff4a9d629b9..180053490463 100644 --- a/xen/arch/x86/mm.c +++ b/xen/arch/x86/mm.c @@ -4083,6 +4083,10 @@ int xenmem_add_to_physmap_one( } case XENMAPSPACE_gmfn_foreign: return p2m_add_foreign(d, idx, gfn_x(gpfn), extra.foreign_domid); + case XENMAPSPACE_gmfn_share: + gdprintk(XENLOG_WARNING, + "XENMAPSPACE_gmfn_share is currently not supported on x86\n"); + break; default: break; } diff --git a/xen/include/public/memory.h b/xen/include/public/memory.h index 29386df98b01..ce707886602f 100644 --- a/xen/include/public/memory.h +++ b/xen/include/public/memory.h @@ -227,6 +227,10 @@ DEFINE_XEN_GUEST_HANDLE(xen_machphys_mapping_t); Stage-2 using the Normal Memory Inner/Outer Write-Back Cacheable memory attribute. */ +#define XENMAPSPACE_gmfn_share 6 /* Same as *_gmfn_foreign, but this is + for a privileged dom to share pages + between two doms. */ + /* ` } */ /* diff --git a/xen/include/xsm/dummy.h b/xen/include/xsm/dummy.h index b2cd56cdc5ff..9754d5a0d0a8 100644 --- a/xen/include/xsm/dummy.h +++ b/xen/include/xsm/dummy.h @@ -519,6 +519,12 @@ static XSM_INLINE int xsm_map_gmfn_foreign(XSM_DEFAULT_ARG struct domain *d, str return xsm_default_action(action, d, t); } +static XSM_INLINE int xsm_map_gmfn_share(XSM_DEFAULT_ARG struct domain *d, struct domain *t) +{ + XSM_ASSERT_ACTION(XSM_TARGET); + return xsm_default_action(action, current->domain, t); +} + static XSM_INLINE int xsm_hvm_param(XSM_DEFAULT_ARG struct domain *d, unsigned long op) { XSM_ASSERT_ACTION(XSM_TARGET); diff --git a/xen/include/xsm/xsm.h b/xen/include/xsm/xsm.h index 7f7feffc68ac..49594b52fb86 100644 --- a/xen/include/xsm/xsm.h +++ b/xen/include/xsm/xsm.h @@ -86,6 +86,7 @@ struct xsm_operations { int (*add_to_physmap) (struct domain *d1, struct domain *d2); int (*remove_from_physmap) (struct domain *d1, struct domain *d2); int (*map_gmfn_foreign) (struct domain *d, struct domain *t); + int (*map_gmfn_share) (struct domain *d, struct domain *t); int (*claim_pages) (struct domain *d); int (*console_io) (struct domain *d, int cmd); @@ -375,6 +376,11 @@ static inline int xsm_map_gmfn_foreign (xsm_default_t def, struct domain *d, str return xsm_ops->map_gmfn_foreign(d, t); } +static inline int xsm_map_gmfn_share (xsm_default_t def, struct domain *d, struct domain *t) +{ + return xsm_ops->map_gmfn_share(d, t); +} + static inline int xsm_claim_pages(xsm_default_t def, struct domain *d) { return xsm_ops->claim_pages(d); diff --git a/xen/xsm/dummy.c b/xen/xsm/dummy.c index 479b103614b2..3017dfc9a651 100644 --- a/xen/xsm/dummy.c +++ b/xen/xsm/dummy.c @@ -124,6 +124,7 @@ void __init xsm_fixup_ops (struct xsm_operations *ops) set_to_dummy_if_null(ops, add_to_physmap); set_to_dummy_if_null(ops, remove_from_physmap); set_to_dummy_if_null(ops, map_gmfn_foreign); + set_to_dummy_if_null(ops, map_gmfn_share); set_to_dummy_if_null(ops, vm_event_control); diff --git a/xen/xsm/flask/hooks.c b/xen/xsm/flask/hooks.c index f01b4cfaaa2b..74f9b120618b 100644 --- a/xen/xsm/flask/hooks.c +++ b/xen/xsm/flask/hooks.c @@ -1202,6 +1202,12 @@ static int flask_map_gmfn_foreign(struct domain *d, struct domain *t) return domain_has_perm(d, t, SECCLASS_MMU, MMU__MAP_READ | MMU__MAP_WRITE); } +static int flask_map_gmfn_share(struct domain *d, struct domain *t) +{ + return current_has_perm(t, SECCLASS_MMU, MMU__MAP_READ | MMU__MAP_WRITE) ?: + domain_has_perm(d, t, SECCLASS_MMU, MMU__SHARE_MEM); +} + static int flask_hvm_param(struct domain *d, unsigned long op) { u32 perm; @@ -1821,6 +1827,7 @@ static struct xsm_operations flask_ops = { .add_to_physmap = flask_add_to_physmap, .remove_from_physmap = flask_remove_from_physmap, .map_gmfn_foreign = flask_map_gmfn_foreign, + .map_gmfn_share = flask_map_gmfn_share, #if defined(CONFIG_HAS_PASSTHROUGH) && defined(CONFIG_HAS_PCI) .get_device_group = flask_get_device_group, diff --git a/xen/xsm/flask/policy/access_vectors b/xen/xsm/flask/policy/access_vectors index 3a2d863b8f39..1962a85d5b6c 100644 --- a/xen/xsm/flask/policy/access_vectors +++ b/xen/xsm/flask/policy/access_vectors @@ -387,6 +387,11 @@ class mmu # Allow a privileged domain to install a map of a page it does not own. Used # for stub domain device models with the PV framebuffer. target_hack +# Checked when using XENMEM_add_to_physmap with XENMAPSPACE_gmfn_share +# to share memory between two domains: +# source = domain whose memory is being shared +# target = client domain + share_mem } # control of the paging_domctl split by subop From 549c921e53cb62dcc9c6533cdaa1ae8bc1a60f75 Mon Sep 17 00:00:00 2001 From: Zhongze Liu Date: Fri, 4 Aug 2017 00:01:17 +0800 Subject: [PATCH 3/7] libxl: introduce a new structure to represent static shared memory regions Add a new structure to the IDL familiy to represent static shared memory regions as proposed in the proposal "Allow setting up shared memory areas between VMs from xl config file" (see [1]). And deleted some trailing white spaces. [1] https://lists.xen.org/archives/html/xen-devel/2017-08/msg03242.html Signed-off-by: Zhongze Liu Reviewed-by: Stefano Stabellini Acked-by: Wei Liu Cc: Wei Liu Cc: Ian Jackson Cc: Stefano Stabellini Cc: Julien Grall Cc: xen-devel@lists.xen.org --- tools/libxl/libxl.h | 4 ++++ tools/libxl/libxl_types.idl | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h index 5e9aed739d7a..5fbbe266e59a 100644 --- a/tools/libxl/libxl.h +++ b/tools/libxl/libxl.h @@ -2320,6 +2320,10 @@ int libxl_fd_set_nonblock(libxl_ctx *ctx, int fd, int nonblock); int libxl_qemu_monitor_command(libxl_ctx *ctx, uint32_t domid, const char *command_line, char **output); +/* Constants for libxl_static_shm */ +#define LIBXL_SSHM_RANGE_UNKNOWN UINT64_MAX +#define LIBXL_SSHM_ID_MAXLEN 128 + #include #endif /* LIBXL_H */ diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl index a23932434163..78cd2696b0eb 100644 --- a/tools/libxl/libxl_types.idl +++ b/tools/libxl/libxl_types.idl @@ -559,10 +559,10 @@ libxl_domain_build_info = Struct("domain_build_info",[ ("keymap", string), ("sdl", libxl_sdl_info), ("spice", libxl_spice_info), - + ("gfx_passthru", libxl_defbool), ("gfx_passthru_kind", libxl_gfx_passthru_kind), - + ("serial", string), ("boot", string), ("usb", libxl_defbool), @@ -812,6 +812,33 @@ libxl_device_vdispl = Struct("device_vdispl", [ ("connectors", Array(libxl_connector_param, "num_connectors")) ]) +libxl_sshm_cachepolicy = Enumeration("sshm_cachepolicy", [ + (-1, "UNKNOWN"), + (0, "ARM_NORMAL"), # ARM policies should be < 32 + (32, "X86_NORMAL"), # X86 policies should be >= 32 + ], init_val = "LIBXL_SSHM_CHCHE_POLICY_UNKNOWN") + +libxl_sshm_prot = Enumeration("sshm_prot", [ + (-1, "UNKNOWN"), + (3, "RW"), + ], init_val = "LIBXL_SSHM_PROT_UNKNOWN") + +libxl_sshm_role = Enumeration("sshm_role", [ + (-1, "UNKNOWN"), + (0, "MASTER"), + (1, "SLAVE"), + ], init_val = "LIBXL_SSHM_ROLE_UNKNOWN") + +libxl_static_shm = Struct("static_shm", [ + ("id", string), + ("offset", uint64, {'init_val': 'LIBXL_SSHM_RANGE_UNKNOWN'}), + ("begin", uint64, {'init_val': 'LIBXL_SSHM_RANGE_UNKNOWN'}), + ("end", uint64, {'init_val': 'LIBXL_SSHM_RANGE_UNKNOWN'}), + ("prot", libxl_sshm_prot, {'init_val': 'LIBXL_SSHM_PROT_UNKNOWN'}), + ("cache_policy", libxl_sshm_cachepolicy, {'init_val': 'LIBXL_SSHM_CACHEPOLICY_UNKNOWN'}), + ("role", libxl_sshm_role, {'init_val': 'LIBXL_SSHM_ROLE_UNKNOWN'}), +]) + libxl_domain_config = Struct("domain_config", [ ("c_info", libxl_domain_create_info), ("b_info", libxl_domain_build_info), @@ -831,6 +858,7 @@ libxl_domain_config = Struct("domain_config", [ ("channels", Array(libxl_device_channel, "num_channels")), ("usbctrls", Array(libxl_device_usbctrl, "num_usbctrls")), ("usbdevs", Array(libxl_device_usbdev, "num_usbdevs")), + ("sshms", Array(libxl_static_shm, "num_sshms")), ("on_poweroff", libxl_action_on_shutdown), ("on_reboot", libxl_action_on_shutdown), From 242413a4c86203e407ee43288a9ceb37fb54cd69 Mon Sep 17 00:00:00 2001 From: Zhongze Liu Date: Sat, 19 Aug 2017 18:29:44 +0800 Subject: [PATCH 4/7] libxl: support mapping static shared memory areas during domain creation Add libxl__sshm_add to map shared pages from one DomU to another, The mapping process involves the follwing steps: * Set defaults and check for further errors in the static_shm configs: overlapping areas, invalid ranges, duplicated master domain, no master domain etc. * Write infomation of static shared memory areas into the appropriate xenstore paths. * Use xc_domain_add_to_physmap_batch to do the page sharing. * Set the refcount of the shared region accordingly Temporarily mark this as unsupported on x86 because calling p2m_add_foreign on two domU's is currently not allowd on x86 (see the comments in x86/mm/p2m.c:p2m_add_foreign for more details). This is for the proposal "Allow setting up shared memory areas between VMs from xl config file" (see [1]). [1] https://lists.xen.org/archives/html/xen-devel/2017-08/msg03242.html Signed-off-by: Zhongze Liu Cc: Wei Liu Cc: Ian Jackson Cc: Stefano Stabellini Cc: Julien Grall Cc: xen-devel@lists.xen.org V3: * unmap the successfully mapped pages whenever rc != 0 V4: * use XENMAPSPACE_gmfn_share instead of *_gmfn_foreign --- tools/libxl/Makefile | 2 +- tools/libxl/libxl_arch.h | 6 + tools/libxl/libxl_arm.c | 15 ++ tools/libxl/libxl_create.c | 27 +++ tools/libxl/libxl_internal.h | 13 ++ tools/libxl/libxl_sshm.c | 395 +++++++++++++++++++++++++++++++++++ tools/libxl/libxl_x86.c | 18 ++ 7 files changed, 475 insertions(+), 1 deletion(-) create mode 100644 tools/libxl/libxl_sshm.c diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile index 5a861f72cbb8..91bc70cda2fb 100644 --- a/tools/libxl/Makefile +++ b/tools/libxl/Makefile @@ -139,7 +139,7 @@ LIBXL_OBJS = flexarray.o libxl.o libxl_create.o libxl_dm.o libxl_pci.o \ libxl_dom_suspend.o libxl_dom_save.o libxl_usb.o \ libxl_vtpm.o libxl_nic.o libxl_disk.o libxl_console.o \ libxl_cpupool.o libxl_mem.o libxl_sched.o libxl_tmem.o \ - libxl_9pfs.o libxl_domain.o libxl_vdispl.o \ + libxl_9pfs.o libxl_domain.o libxl_vdispl.o libxl_sshm.o \ $(LIBXL_OBJS-y) LIBXL_OBJS += libxl_genid.o LIBXL_OBJS += _libxl_types.o libxl_flask.o _libxl_types_internal.o diff --git a/tools/libxl/libxl_arch.h b/tools/libxl/libxl_arch.h index 784ec7f60914..11433fe4660e 100644 --- a/tools/libxl/libxl_arch.h +++ b/tools/libxl/libxl_arch.h @@ -78,6 +78,12 @@ int libxl__arch_extra_memory(libxl__gc *gc, const libxl_domain_build_info *info, uint64_t *out); +_hidden +bool libxl__arch_domain_support_sshm(const libxl_domain_build_info *b_info); + +_hidden +int libxl__arch_domain_sshm_cachepolicy_setdefault(libxl_static_shm *sshm); + #if defined(__i386__) || defined(__x86_64__) #define LAPIC_BASE_ADDRESS 0xfee00000 diff --git a/tools/libxl/libxl_arm.c b/tools/libxl/libxl_arm.c index de1840bece45..d687dad0bdcf 100644 --- a/tools/libxl/libxl_arm.c +++ b/tools/libxl/libxl_arm.c @@ -1154,6 +1154,21 @@ void libxl__arch_domain_build_info_acpi_setdefault( libxl_defbool_setdefault(&b_info->acpi, false); } +bool libxl__arch_domain_support_sshm(const libxl_domain_build_info *b_info) +{ + return true; +} + +int libxl__arch_domain_sshm_cachepolicy_setdefault(libxl_static_shm *sshm) +{ + if (sshm->cache_policy == LIBXL_SSHM_CACHEPOLICY_UNKNOWN) + sshm->cache_policy = LIBXL_SSHM_CACHEPOLICY_ARM_NORMAL; + if (sshm->cache_policy >= LIBXL_SSHM_CACHEPOLICY_X86_NORMAL) + return ERROR_INVAL; + + return 0; +} + /* * Local variables: * mode: C diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c index f15fb215c24b..fecc873b7a35 100644 --- a/tools/libxl/libxl_create.c +++ b/tools/libxl/libxl_create.c @@ -514,6 +514,14 @@ int libxl__domain_build(libxl__gc *gc, ret = ERROR_INVAL; goto out; } + + /* the p2m has been setup, we could map the static shared memory now. */ + ret = libxl__sshm_add(gc, domid, d_config->sshms, d_config->num_sshms); + if (ret != 0) { + LOG(ERROR, "failed to map static shared memory"); + goto out; + } + ret = libxl__build_post(gc, domid, info, state, vments, localents); out: return ret; @@ -939,6 +947,25 @@ static void initiate_domain_create(libxl__egc *egc, goto error_out; } + if (d_config->num_sshms != 0 && + !libxl__arch_domain_support_sshm(&d_config->b_info)) { + LOGD(ERROR, domid, "static_shm is not supported by this domain type."); + ret = ERROR_INVAL; + goto error_out; + } + + for (i = 0; i < d_config->num_sshms; ++i) { + ret = libxl__sshm_setdefault(gc, domid, &d_config->sshms[i]); + if (ret) { + LOGD(ERROR, domid, "Unable to set defaults for static shm"); + goto error_out; + } + } + + ret = libxl__sshm_check_overlap(gc, domid, + d_config->sshms, d_config->num_sshms); + if (ret) goto error_out; + ret = libxl__domain_make(gc, d_config, &domid, &state->config); if (ret) { LOGD(ERROR, domid, "cannot make domain: %d", ret); diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h index 45e6df6c82d6..fc2819162553 100644 --- a/tools/libxl/libxl_internal.h +++ b/tools/libxl/libxl_internal.h @@ -4350,6 +4350,19 @@ static inline bool libxl__string_is_default(char **s) } #endif +/* + * Set up static shared ram pages for HVM domains to communicate + * + * This function should only be called after the memory map is constructed + * and before any further memory access. */ +_hidden int libxl__sshm_add(libxl__gc *gc, uint32_t domid, + libxl_static_shm *sshm, int len); + +_hidden int libxl__sshm_check_overlap(libxl__gc *gc, uint32_t domid, + libxl_static_shm *sshms, int len); +_hidden int libxl__sshm_setdefault(libxl__gc *gc, uint32_t domid, + libxl_static_shm *sshm); + /* * Local variables: * mode: C diff --git a/tools/libxl/libxl_sshm.c b/tools/libxl/libxl_sshm.c new file mode 100644 index 000000000000..8d3ca06ad399 --- /dev/null +++ b/tools/libxl/libxl_sshm.c @@ -0,0 +1,395 @@ +#include "libxl_osdeps.h" +#include "libxl_internal.h" +#include "libxl_arch.h" + +#define SSHM_PATH(id) GCSPRINTF("/local/static_shm/%s", id) + +#define SSHM_ERROR(domid, sshmid, f, ...) \ + LOGD(ERROR, domid, "static_shm id = %s: " f, sshmid, ##__VA_ARGS__) + + +/* Set default values for libxl_static_shm */ +int libxl__sshm_setdefault(libxl__gc *gc, uint32_t domid, + libxl_static_shm *sshm) +{ + int rc; + + if (sshm->role == LIBXL_SSHM_ROLE_UNKNOWN) + sshm->role = LIBXL_SSHM_ROLE_SLAVE; + if (sshm->prot == LIBXL_SSHM_PROT_UNKNOWN) + sshm->prot = LIBXL_SSHM_PROT_RW; + + /* role-specific checks */ + if (sshm->role == LIBXL_SSHM_ROLE_SLAVE) { + if (sshm->offset == LIBXL_SSHM_RANGE_UNKNOWN) + sshm->offset = 0; + if (sshm->cache_policy != LIBXL_SSHM_CACHEPOLICY_UNKNOWN) { + SSHM_ERROR(domid, sshm->id, + "cache_policy is only applicable to master domains"); + rc = ERROR_INVAL; + goto out; + } + } else { + if (sshm->offset != LIBXL_SSHM_RANGE_UNKNOWN) { + SSHM_ERROR(domid, sshm->id, + "offset is only applicable to slave domains"); + rc = ERROR_INVAL; + goto out; + } + + rc = libxl__arch_domain_sshm_cachepolicy_setdefault(sshm); + if (rc) { + SSHM_ERROR(domid, sshm->id, + "cache policy not supported on this platform"); + goto out; + } + } + + rc = 0; +out: + return rc; +} + +/* Compare function for sorting sshm ranges by sshm->begin */ +static int sshm_range_cmp(const void *a, const void *b) +{ + libxl_static_shm *const *sshma = a, *const *sshmb = b; + return (*sshma)->begin > (*sshmb)->begin ? 1 : -1; +} + +/* check if the sshm slave configs in @sshm overlap */ +int libxl__sshm_check_overlap(libxl__gc *gc, uint32_t domid, + libxl_static_shm *sshms, int len) +{ + + const libxl_static_shm **slave_sshms = NULL; + int num_slaves; + int i; + + if (!len) return 0; + + slave_sshms = libxl__calloc(gc, len, sizeof(slave_sshms[0])); + num_slaves = 0; + for (i = 0; i < len; ++i) { + if (sshms[i].role == LIBXL_SSHM_ROLE_SLAVE) + slave_sshms[num_slaves++] = sshms + i; + } + qsort(slave_sshms, num_slaves, sizeof(slave_sshms[0]), sshm_range_cmp); + + for (i = 0; i < num_slaves - 1; ++i) { + if (slave_sshms[i+1]->begin < slave_sshms[i]->end) { + SSHM_ERROR(domid, slave_sshms[i+1]->id, "slave ranges overlap."); + return ERROR_INVAL; + } + } + + return 0; +} + +/* libxl__sshm_do_map -- map pages into slave's physmap + * + * This functions maps + * mater gfn: [@msshm->begin + @sshm->offset, @msshm->end + @sshm->offset) + * into + * slave gfn: [@sshm->begin, @sshm->end) + * + * The gfns of the pages that are successfully mapped will be stored + * in @mapped, and the number of the gfns will be stored in @nmapped. + * + * The caller have to guarentee that sshm->begin < sshm->end */ +static int libxl__sshm_do_map(libxl__gc *gc, uint32_t mid, uint32_t sid, + libxl_static_shm *sshm, libxl_static_shm *msshm, + xen_pfn_t *mapped, unsigned int *nmapped) +{ + int rc; + int i; + unsigned int num_mpages, num_spages, num_success, offset; + int *errs; + xen_ulong_t *idxs; + xen_pfn_t *gpfns; + + num_mpages = (msshm->end - msshm->begin) >> XC_PAGE_SHIFT; + num_spages = (sshm->end - sshm->begin) >> XC_PAGE_SHIFT; + offset = sshm->offset >> XC_PAGE_SHIFT; + + /* Check range. Test offset < mpages first to avoid overflow */ + if ((offset >= num_mpages) || (num_mpages - offset < num_spages)) { + SSHM_ERROR(sid, sshm->id, "exceeds master's address space."); + rc = ERROR_INVAL; + goto out; + } + + /* fill out the pfn's and do the mapping */ + errs = libxl__calloc(gc, num_spages, sizeof(int)); + idxs = libxl__calloc(gc, num_spages, sizeof(xen_ulong_t)); + gpfns = libxl__calloc(gc, num_spages, sizeof(xen_pfn_t)); + for (i = 0; i < num_spages; i++) { + idxs[i] = (msshm->begin >> XC_PAGE_SHIFT) + offset + i; + gpfns[i]= (sshm->begin >> XC_PAGE_SHIFT) + i; + } + rc = xc_domain_add_to_physmap_batch(CTX->xch, + sid, mid, + XENMAPSPACE_gmfn_share, + num_spages, + idxs, gpfns, errs); + + num_success = 0; + for (i = 0; i < num_spages; i++) { + if (errs[i]) { + SSHM_ERROR(sid, sshm->id, + "can't map at address 0x%"PRIx64".", + gpfns[i] << XC_PAGE_SHIFT); + rc = ERROR_FAIL; + } else { + mapped[num_success++] = gpfns[i]; + } + } + *nmapped = num_success; + if (rc) goto out; + + rc = 0; +out: + return rc; +} + +static int libxl__sshm_incref(libxl__gc *gc, xs_transaction_t xt, + const char *sshm_path) +{ + int rc, count; + const char *count_path, *count_string; + + count_path = GCSPRINTF("%s/users", sshm_path); + rc = libxl__xs_read_checked(gc, xt, count_path, &count_string); + if (rc) goto out; + count = atoi(count_string); + + count_string = GCSPRINTF("%d", count+1); + rc = libxl__xs_write_checked(gc, xt, count_path, count_string); + if (rc) goto out; + + rc = 0; +out: + return rc; +} + +static int libxl__sshm_add_slave(libxl__gc *gc, uint32_t domid, + libxl_static_shm *sshm) +{ + int rc, i; + const char *sshm_path, *slave_path; + const char *dom_path, *dom_sshm_path, *dom_role_path; + const char *xs_value; + char *ents[9]; + libxl_static_shm master_sshm; + uint32_t master_domid; + xen_pfn_t *mapped; + unsigned int nmapped = 0; + xs_transaction_t xt = XBT_NULL; + bool isretry; + + sshm_path = SSHM_PATH(sshm->id); + slave_path = GCSPRINTF("%s/slaves/%"PRIu32, sshm_path, domid); + dom_path = libxl__xs_get_dompath(gc, domid); + /* the domain should be in xenstore by now */ + assert(dom_path); + dom_sshm_path = GCSPRINTF("%s/static_shm/%s", dom_path, sshm->id); + dom_role_path = GCSPRINTF("%s/role", dom_sshm_path); + + /* prepare the slave xenstore entries */ + ents[0] = "begin"; + ents[1] = GCSPRINTF("0x%"PRIx64, sshm->begin); + ents[2] = "end"; + ents[3] = GCSPRINTF("0x%"PRIx64, sshm->end); + ents[4] = "offset"; + ents[5] = GCSPRINTF("0x%"PRIx64, sshm->offset); + ents[6] = "prot"; + ents[7] = libxl__strdup(gc, libxl_sshm_prot_to_string(sshm->prot)); + ents[8] = NULL; + + mapped = libxl__calloc(gc, (sshm->end - sshm->begin) >> XC_PAGE_SHIFT, + sizeof(xen_pfn_t)); + + isretry = false; + for (;;) { + rc = libxl__xs_transaction_start(gc, &xt); + if (rc) goto out; + + if (!libxl__xs_read(gc, xt, sshm_path)) { + SSHM_ERROR(domid, sshm->id, "no master found."); + rc = ERROR_FAIL; + goto out; + } + + /* every ID can appear in each domain at most once */ + if (libxl__xs_read(gc, xt, dom_sshm_path)) { + SSHM_ERROR(domid, sshm->id, + "domain tried to map the same ID twice."); + rc = ERROR_FAIL; + goto out; + } + + /* look at the master info and see if we could do the mapping */ + rc = libxl__xs_read_checked(gc, xt, + GCSPRINTF("%s/prot", sshm_path), + &xs_value); + if (rc) goto out; + libxl_sshm_prot_from_string(xs_value, &master_sshm.prot); + + rc = libxl__xs_read_checked(gc, xt, + GCSPRINTF("%s/begin", sshm_path), + &xs_value); + if (rc) goto out; + master_sshm.begin = strtoull(xs_value, NULL, 16); + + rc = libxl__xs_read_checked(gc, xt, + GCSPRINTF("%s/end", sshm_path), + &xs_value); + if (rc) goto out; + master_sshm.end = strtoull(xs_value, NULL, 16); + + rc = libxl__xs_read_checked(gc, xt, + GCSPRINTF("%s/master", sshm_path), + &xs_value); + if (rc) goto out; + master_domid = strtoull(xs_value, NULL, 16); + + if (sshm->prot == LIBXL_SSHM_PROT_UNKNOWN) + sshm->prot = master_sshm.prot; + + /* check if the slave is asking too much permission */ + if (master_sshm.prot < sshm->prot) { + SSHM_ERROR(domid, sshm->id, "slave is asking too much permission."); + rc = ERROR_INVAL; + goto out; + } + + /* all checks passed, do the job */ + if (!isretry) { + rc = libxl__sshm_do_map(gc, master_domid, domid, + sshm, &master_sshm, + mapped, &nmapped); + if (rc) goto out; + } + + /* write the result to xenstore and commit */ + rc = libxl__xs_write_checked(gc, xt, dom_role_path, "slave"); + if (rc) goto out; + rc = libxl__xs_writev(gc, xt, slave_path, ents); + if (rc) goto out; + rc = libxl__sshm_incref(gc, xt, sshm_path); + if (rc) goto out; + + rc = libxl__xs_transaction_commit(gc, &xt); + if (!rc) break; + if (rc < 0) goto out; + isretry = true; + } + + rc = 0; +out: + if (rc) { + /* role back successfully mapped pages */ + SSHM_ERROR(domid, sshm->id, "failed to map some pages, cancelling."); + for (i = 0; i < nmapped; i++) { + xc_domain_remove_from_physmap(CTX->xch, domid, mapped[i]); + } + } + + libxl__xs_transaction_abort(gc, &xt); + + return rc; +} + +static int libxl__sshm_add_master(libxl__gc *gc, uint32_t domid, + libxl_static_shm *sshm) +{ + int rc; + const char *sshm_path, *dom_path, *dom_role_path; + char *ents[13]; + struct xs_permissions noperm; + xs_transaction_t xt = XBT_NULL; + + sshm_path = SSHM_PATH(sshm->id); + dom_path = libxl__xs_get_dompath(gc, domid); + /* the domain should be in xenstore by now */ + assert(dom_path); + dom_role_path = GCSPRINTF("%s/static_shm/%s/role", dom_path, sshm->id); + + /* prepare the xenstore entries */ + ents[0] = "master"; + ents[1] = GCSPRINTF("%"PRIu32, domid); + ents[2] = "begin"; + ents[3] = GCSPRINTF("0x%"PRIx64, sshm->begin); + ents[4] = "end"; + ents[5] = GCSPRINTF("0x%"PRIx64, sshm->end); + ents[6] = "prot"; + ents[7] = libxl__strdup(gc, libxl_sshm_prot_to_string(sshm->prot)); + ents[8] = "cache_policy"; + ents[9] = libxl__strdup(gc, + libxl_sshm_cachepolicy_to_string(sshm->cache_policy)); + ents[10] = "users"; + ents[11] = "1"; + ents[12] = NULL; + + /* could only be accessed by Dom0 */ + noperm.id = 0; + noperm.perms = XS_PERM_NONE; + + for (;;) { + rc = libxl__xs_transaction_start(gc, &xt); + if (rc) goto out; + + if (!libxl__xs_read(gc, xt, sshm_path)) { + /* every ID can appear in each domain at most once */ + if (libxl__xs_read(gc, xt, dom_role_path)) { + SSHM_ERROR(domid, sshm->id, + "domain tried to map the same ID twice."); + rc = ERROR_FAIL; + goto out; + } + rc = libxl__xs_write_checked(gc, xt, dom_role_path, "master"); + if (rc) goto out;; + + libxl__xs_mknod(gc, xt, sshm_path, &noperm, 1); + libxl__xs_writev(gc, xt, sshm_path, ents); + } else { + SSHM_ERROR(domid, sshm->id, "can only have one master."); + rc = ERROR_FAIL; + goto out; + } + + rc = libxl__xs_transaction_commit(gc, &xt); + if (!rc) break; + if (rc < 0) goto out; + } + + rc = 0; +out: + libxl__xs_transaction_abort(gc, &xt); + return rc; +} + +int libxl__sshm_add(libxl__gc *gc, uint32_t domid, + libxl_static_shm *sshms, int len) +{ + int rc, i; + + for (i = 0; i < len; ++i) { + if (sshms[i].role == LIBXL_SSHM_ROLE_SLAVE) { + rc = libxl__sshm_add_slave(gc, domid, sshms+i); + } else { + rc = libxl__sshm_add_master(gc, domid, sshms+i); + } + if (rc) return rc; + } + + return 0; +} + +/* + * Local variables: + * mode: C + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + */ diff --git a/tools/libxl/libxl_x86.c b/tools/libxl/libxl_x86.c index 5f91fe4f92d8..450fb3d3546c 100644 --- a/tools/libxl/libxl_x86.c +++ b/tools/libxl/libxl_x86.c @@ -596,6 +596,24 @@ void libxl__arch_domain_build_info_acpi_setdefault( libxl_defbool_setdefault(&b_info->acpi, true); } +bool libxl__arch_domain_support_sshm(const libxl_domain_build_info *b_info) +{ + /* FIXME: Mark this as unsupported for calling p2m_add_foreign on two + * DomU's is currently not allowd on x86, see the comments in + * x86/mm/p2m.c: p2m_add_foreign */ + return false; +} + +int libxl__arch_domain_sshm_cachepolicy_setdefault(libxl_static_shm *sshm) +{ + if (sshm->cache_policy == LIBXL_SSHM_CACHEPOLICY_UNKNOWN) + sshm->cache_policy = LIBXL_SSHM_CACHEPOLICY_X86_NORMAL; + if (sshm->cache_policy < LIBXL_SSHM_CACHEPOLICY_X86_NORMAL) + return ERROR_INVAL; + + return 0; +} + /* * Local variables: * mode: C From ea6b1be74274797ecfe06d27c2f0cf0ce611368e Mon Sep 17 00:00:00 2001 From: Zhongze Liu Date: Sat, 19 Aug 2017 18:55:06 +0800 Subject: [PATCH 5/7] libxl: support unmapping static shared memory areas during domain destruction Add libxl__sshm_del to unmap static shared memory areas mapped by libxl__sshm_add during domain creation. The unmapping process is: * For a master: decrease the refcount of the sshm region, if the refcount reaches 0, cleanup the whole sshm path. * For a slave: unmap the shared pages, and cleanup related xs entries. decrease the refcount of the sshm region, if the refcount reaches 0, cleanup the whole sshm path. This is for the proposal "Allow setting up shared memory areas between VMs from xl config file" (see [1]). [1] https://lists.xen.org/archives/html/xen-devel/2017-08/msg03242.html Signed-off-by: Zhongze Liu Cc: Ian Jackson Cc: Wei Liu Cc: Stefano Stabellini Cc: Julien Grall Cc: xen-devel@lists.xen.org --- tools/libxl/libxl_domain.c | 5 ++ tools/libxl/libxl_internal.h | 2 + tools/libxl/libxl_sshm.c | 105 +++++++++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+) diff --git a/tools/libxl/libxl_domain.c b/tools/libxl/libxl_domain.c index 814f8128a1cc..ce9038717331 100644 --- a/tools/libxl/libxl_domain.c +++ b/tools/libxl/libxl_domain.c @@ -1026,6 +1026,11 @@ void libxl__destroy_domid(libxl__egc *egc, libxl__destroy_domid_state *dis) goto out; } + rc = libxl__sshm_del(gc, domid); + if (rc) { + LOGD(ERROR, domid, "Deleting static shm failed."); + } + if (libxl__device_pci_destroy_all(gc, domid) < 0) LOGD(ERROR, domid, "Pci shutdown failed"); rc = xc_domain_pause(ctx->xch, domid); diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h index fc2819162553..dc5937e71208 100644 --- a/tools/libxl/libxl_internal.h +++ b/tools/libxl/libxl_internal.h @@ -4358,6 +4358,8 @@ static inline bool libxl__string_is_default(char **s) _hidden int libxl__sshm_add(libxl__gc *gc, uint32_t domid, libxl_static_shm *sshm, int len); +_hidden int libxl__sshm_del(libxl__gc *gc, uint32_t domid); + _hidden int libxl__sshm_check_overlap(libxl__gc *gc, uint32_t domid, libxl_static_shm *sshms, int len); _hidden int libxl__sshm_setdefault(libxl__gc *gc, uint32_t domid, diff --git a/tools/libxl/libxl_sshm.c b/tools/libxl/libxl_sshm.c index 8d3ca06ad399..c74441da6662 100644 --- a/tools/libxl/libxl_sshm.c +++ b/tools/libxl/libxl_sshm.c @@ -86,6 +86,111 @@ int libxl__sshm_check_overlap(libxl__gc *gc, uint32_t domid, return 0; } +/* Decrease the refcount of an sshm. When refcount reaches 0, + * clean up the whole sshm path */ +static void libxl__sshm_decref(libxl__gc *gc, xs_transaction_t xt, + const char *sshm_path) +{ + int count; + const char *count_path, *count_string; + + count_path = GCSPRINTF("%s/users", sshm_path); + if (libxl__xs_read_checked(gc, xt, count_path, &count_string)) + return; + count = atoi(count_string); + + if (--count == 0) { + libxl__xs_path_cleanup(gc, xt, sshm_path); + return; + } + + count_string = GCSPRINTF("%d", count); + libxl__xs_write_checked(gc, xt, count_path, count_string); + + return; +} + +static void libxl__sshm_do_unmap(libxl__gc *gc, uint32_t domid, const char *id, + uint64_t begin, uint64_t end) +{ + begin >>= XC_PAGE_SHIFT; + end >>= XC_PAGE_SHIFT; + for (; begin < end; ++begin) { + if (xc_domain_remove_from_physmap(CTX->xch, domid, begin)) { + SSHM_ERROR(domid, id, + "unable to unmap shared page at 0x%"PRIx64".", + begin); + } + } +} + +static void libxl__sshm_del_slave(libxl__gc *gc, xs_transaction_t xt, + uint32_t domid, const char *id, bool isretry) +{ + const char *slave_path, *begin_str, *end_str; + uint64_t begin, end; + + slave_path = GCSPRINTF("%s/slaves/%"PRIu32, SSHM_PATH(id), domid); + + begin_str = libxl__xs_read(gc, xt, GCSPRINTF("%s/begin", slave_path)); + end_str = libxl__xs_read(gc, xt, GCSPRINTF("%s/end", slave_path)); + begin = strtoull(begin_str, NULL, 16); + end = strtoull(end_str, NULL, 16); + + /* Avoid calling do_unmap many times in case of xs transaction retry */ + if (!isretry) + libxl__sshm_do_unmap(gc, domid, id, begin, end); + + libxl__xs_path_cleanup(gc, xt, slave_path); +} + +/* Delete static_shm entries in the xensotre. */ +int libxl__sshm_del(libxl__gc *gc, uint32_t domid) +{ + int rc, i; + bool isretry; + xs_transaction_t xt = XBT_NULL; + const char *dom_path, *dom_sshm_path, *role; + char **sshm_ents; + unsigned int sshm_num; + + dom_path = libxl__xs_get_dompath(gc, domid); + dom_sshm_path = GCSPRINTF("%s/static_shm", dom_path); + + isretry = false; + for (;;) { + rc = libxl__xs_transaction_start(gc, &xt); + if (rc) goto out; + + if (libxl__xs_read(gc, xt, dom_sshm_path)) { + sshm_ents = libxl__xs_directory(gc, xt, dom_sshm_path, &sshm_num); + if (!sshm_ents) continue; + + for (i = 0; i < sshm_num; ++i) { + role = libxl__xs_read(gc, xt, + GCSPRINTF("%s/%s/role", + dom_sshm_path, + sshm_ents[i])); + assert(role); + if (!strncmp(role, "slave", 5)) + libxl__sshm_del_slave(gc, xt, domid, sshm_ents[i], isretry); + + libxl__sshm_decref(gc, xt, SSHM_PATH(sshm_ents[i])); + } + } + + rc = libxl__xs_transaction_commit(gc, &xt); + if (!rc) break; + if (rc < 0) goto out; + isretry = true; + } + + rc = 0; +out: + libxl__xs_transaction_abort(gc, &xt); + return rc; +} + /* libxl__sshm_do_map -- map pages into slave's physmap * * This functions maps From 019a3c0edee2b349884f56bcea8f761881e9c394 Mon Sep 17 00:00:00 2001 From: Zhongze Liu Date: Fri, 4 Aug 2017 00:36:19 +0800 Subject: [PATCH 6/7] libxl:xl: add parsing code to parse "libxl_static_sshm" from xl config files Add the parsing utils for the newly introduced libxl_static_sshm struct to the libxl/libxlu_* family. And add realated parsing code in xl to parse the struct from xl config files. This is for the proposal "Allow setting up shared memory areas between VMs from xl config file" (see [1]). [1] https://lists.xen.org/archives/html/xen-devel/2017-08/msg03242.html Signed-off-by: Zhongze Liu Cc: Wei Liu Cc: Ian Jackson Cc: Stefano Stabellini Cc: Julien Grall Cc: xen-devel@lists.xen.org --- tools/libxl/Makefile | 2 +- tools/libxl/libxlu_sshm.c | 210 ++++++++++++++++++++++++++++++++++++++ tools/libxl/libxlutil.h | 6 ++ tools/xl/xl_parse.c | 24 ++++- 4 files changed, 240 insertions(+), 2 deletions(-) create mode 100644 tools/libxl/libxlu_sshm.c diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile index 91bc70cda2fb..25e6f0bd7fe5 100644 --- a/tools/libxl/Makefile +++ b/tools/libxl/Makefile @@ -176,7 +176,7 @@ AUTOINCS= libxlu_cfg_y.h libxlu_cfg_l.h _libxl_list.h _paths.h \ AUTOSRCS= libxlu_cfg_y.c libxlu_cfg_l.c AUTOSRCS += _libxl_save_msgs_callout.c _libxl_save_msgs_helper.c LIBXLU_OBJS = libxlu_cfg_y.o libxlu_cfg_l.o libxlu_cfg.o \ - libxlu_disk_l.o libxlu_disk.o libxlu_vif.o libxlu_pci.o + libxlu_disk_l.o libxlu_disk.o libxlu_vif.o libxlu_pci.o libxlu_sshm.o $(LIBXLU_OBJS): CFLAGS += $(CFLAGS_libxenctrl) # For xentoollog.h $(TEST_PROG_OBJS) _libxl.api-for-check: CFLAGS += $(CFLAGS_libxentoollog) $(CFLAGS_libxentoolcore) diff --git a/tools/libxl/libxlu_sshm.c b/tools/libxl/libxlu_sshm.c new file mode 100644 index 000000000000..5276ff939589 --- /dev/null +++ b/tools/libxl/libxlu_sshm.c @@ -0,0 +1,210 @@ +#include "libxl_osdeps.h" /* must come before any other headers */ +#include "libxlu_internal.h" +#include "xenctrl.h" + +#include + +#define PARAM_RE(EXPR) "^\\s*" EXPR "\\s*(,|$)" +#define WORD_RE "([_a-zA-Z0-9]+)" +#define EQU_RE PARAM_RE(WORD_RE "\\s*=\\s*" WORD_RE) + +#define RET_INVAL(msg, curr_str) do { \ + xlu__sshm_err(cfg, msg, curr_str); \ + rc = EINVAL; \ + goto out; \ + } while(0) + +/* set a member in libxl_static_shm and report an error if it's respecified, + * @curr_str indicates the head of the remaining string. */ +#define SET_VAL(var, name, type, value, curr_str) do { \ + if ((var) != LIBXL_SSHM_##type##_UNKNOWN && (var) != value) { \ + RET_INVAL("\"" name "\" respecified", curr_str); \ + } \ + (var) = value; \ + } while(0) + + +static void xlu__sshm_err(XLU_Config *cfg, const char *msg, + const char *curr_str) { + fprintf(cfg->report, + "%s: config parsing error in shared_memory: %s at '%s'\n", + cfg->config_source, msg, curr_str); +} + +static int parse_prot(XLU_Config *cfg, char *str, libxl_sshm_prot *prot) +{ + int rc; + libxl_sshm_prot new_prot; + + if (!strcmp(str, "rw")) { + new_prot = LIBXL_SSHM_PROT_RW; + } else { + RET_INVAL("invalid permission flags", str); + } + + SET_VAL(*prot, "permission flags", PROT, new_prot, str); + + rc = 0; + + out: + return rc; +} + +static int parse_cachepolicy(XLU_Config *cfg, char *str, + libxl_sshm_cachepolicy *policy) +{ + int rc; + libxl_sshm_cachepolicy new_policy; + + if (!strcmp(str, "ARM_normal")) { + new_policy = LIBXL_SSHM_CACHEPOLICY_ARM_NORMAL; + } else if (!strcmp(str, "x86_normal")) { + new_policy = LIBXL_SSHM_CACHEPOLICY_X86_NORMAL; + } else { + RET_INVAL("invalid cache policy", str); + } + + SET_VAL(*policy, "cache policy", CACHEPOLICY, new_policy, str); + rc = 0; + + out: + return rc; +} + +/* handle key = value pairs */ +static int handle_equ(XLU_Config *cfg, char *key, char *val, + libxl_static_shm *sshm) +{ + int rc; + + if (!strcmp(key, "id")) { + if (strlen(val) > LIBXL_SSHM_ID_MAXLEN) { RET_INVAL("id too long", val); } + if (sshm->id && !strcmp(sshm->id, val)) { + RET_INVAL("id respecified", val); + } + + sshm->id = strdup(val); + if (!sshm->id) { + fprintf(stderr, "sshm parser out of memory\n"); + rc = ENOMEM; + goto out; + } + } else if (!strcmp(key, "role")) { + libxl_sshm_role new_role; + + if (!strcmp("master", val)) { + new_role = LIBXL_SSHM_ROLE_MASTER; + } else if (!strcmp("slave", val)) { + new_role = LIBXL_SSHM_ROLE_SLAVE; + } else { + RET_INVAL("invalid role", val); + } + + SET_VAL(sshm->role, "role", ROLE, new_role, val); + } else if (!strcmp(key, "begin") || + !strcmp(key, "end") || + !strcmp(key, "offset")) { + char *endptr; + int base = 10; + uint64_t new_addr; + + /* Could be in hex form. Note that we don't need to check the length here, + * for val[] is NULL-terminated */ + if (val[0] == '0' && val[1] == 'x') { base = 16; } + new_addr = strtoull(val, &endptr, base); + if (errno == ERANGE || *endptr) + RET_INVAL("invalid begin/end/offset", val); + if (new_addr & ~XC_PAGE_MASK) + RET_INVAL("begin/end/offset is not a multiple of 4K", val); + + /* begin or end */ + if (key[0] == 'b') { + SET_VAL(sshm->begin, "beginning address", RANGE, new_addr, val); + } else if(key[0] == 'e'){ + SET_VAL(sshm->end, "ending address", RANGE, new_addr, val); + } else { + SET_VAL(sshm->offset, "offset", RANGE, new_addr, val); + } + } else if (!strcmp(key, "prot")) { + rc = parse_prot(cfg, val, &sshm->prot); + if (rc) { goto out; } + } else if (!strcmp(key, "cache_policy")) { + rc = parse_cachepolicy(cfg, val, &sshm->cache_policy); + if (rc) { goto out; } + } else { + RET_INVAL("invalid option", key); + } + + rc = 0; + + out: + return rc; +} + +int xlu_sshm_parse(XLU_Config *cfg, const char *spec, + libxl_static_shm *sshm) +{ + int rc; + regex_t equ_rec; + char *buf2 = NULL, *ptr = NULL; + regmatch_t pmatch[3]; + + rc = regcomp(&equ_rec, EQU_RE, REG_EXTENDED); + if (rc) { + fprintf(stderr, "sshm parser failed to initialize\n"); + goto out; + } + + buf2 = ptr = strdup(spec); + if (!buf2) { + fprintf(stderr, "sshm parser out of memory\n"); + rc = ENOMEM; + goto out; + } + + /* main parsing loop */ + while (true) { + if (!*ptr) { break; } + if (regexec(&equ_rec, ptr, 3, pmatch, 0)) + RET_INVAL("unrecognized token", ptr); + + ptr[pmatch[1].rm_eo] = '\0'; + ptr[pmatch[2].rm_eo] = '\0'; + rc = handle_equ(cfg, ptr + pmatch[1].rm_so, + ptr + pmatch[2].rm_so, sshm); + if (rc) { goto out; } + + ptr += pmatch[0].rm_eo; + } + + if (*ptr) { RET_INVAL("invalid syntax", ptr); } + + /* do some early checks */ + if (!sshm->id) { + RET_INVAL("id not specified", spec); + } + if (sshm->begin == LIBXL_SSHM_RANGE_UNKNOWN) { + RET_INVAL("begin address not specified", spec); + } + if (sshm->end == LIBXL_SSHM_RANGE_UNKNOWN) { + RET_INVAL("end address not specified", spec); + } + if (sshm->begin > sshm->end) { + RET_INVAL("begin address larger that end address", spec); + } + + rc = 0; + + out: + if (buf2) { free(buf2); } + regfree(&equ_rec); + return rc; +} + +/* + * Local variables: + * mode: C + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + */ diff --git a/tools/libxl/libxlutil.h b/tools/libxl/libxlutil.h index e81b644c01f1..ee39cb5bdcac 100644 --- a/tools/libxl/libxlutil.h +++ b/tools/libxl/libxlutil.h @@ -118,6 +118,12 @@ int xlu_rdm_parse(XLU_Config *cfg, libxl_rdm_reserve *rdm, const char *str); int xlu_vif_parse_rate(XLU_Config *cfg, const char *rate, libxl_device_nic *nic); +/* + * static shared memory specification parsing + */ +int xlu_sshm_parse(XLU_Config *cfg, const char *spec, + libxl_static_shm *sshm); + #endif /* LIBXLUTIL_H */ /* diff --git a/tools/xl/xl_parse.c b/tools/xl/xl_parse.c index 9a692d5ae644..bf425f0ff3cc 100644 --- a/tools/xl/xl_parse.c +++ b/tools/xl/xl_parse.c @@ -860,7 +860,7 @@ void parse_config_data(const char *config_source, long l, vcpus = 0; XLU_Config *config; XLU_ConfigList *cpus, *vbds, *nics, *pcis, *cvfbs, *cpuids, *vtpms, - *usbctrls, *usbdevs, *p9devs, *vdispls; + *usbctrls, *usbdevs, *p9devs, *vdispls, *sshms; XLU_ConfigList *channels, *ioports, *irqs, *iomem, *viridian, *dtdevs, *mca_caps; int num_ioports, num_irqs, num_iomem, num_cpus, num_viridian, num_mca_caps; @@ -1555,6 +1555,28 @@ void parse_config_data(const char *config_source, } } + if (!xlu_cfg_get_list (config, "static_shm", &sshms, 0, 0)) { + d_config->num_sshms = 0; + d_config->sshms = NULL; + while ((buf = xlu_cfg_get_listitem (sshms, d_config->num_sshms)) != NULL) { + libxl_static_shm *sshm; + char *buf2 = strdup(buf); + int ret; + + sshm = ARRAY_EXTEND_INIT_NODEVID(d_config->sshms, + d_config->num_sshms, + libxl_static_shm_init); + ret = xlu_sshm_parse(config, buf2, sshm); + if (ret) { + fprintf(stderr, + "xl: Invalid argument for static_shm: %s", buf2); + exit(EXIT_FAILURE); + } + + free(buf2); + } + } + if (!xlu_cfg_get_list(config, "p9", &p9devs, 0, 0)) { libxl_device_p9 *p9; char *security_model = NULL; From cd3822a2f9a1610a89e23afbc5f32fbb298dd97e Mon Sep 17 00:00:00 2001 From: Zhongze Liu Date: Thu, 19 Oct 2017 09:04:59 +0800 Subject: [PATCH 7/7] docs: documentation about static shared memory regions Add docs to document the motivation, usage, use cases and other relevant information about the static shared memory feature. This is for the proposal "Allow setting up shared memory areas between VMs from xl config file". See: https://lists.xen.org/archives/html/xen-devel/2017-08/msg03242.html Signed-off-by: Zhongze Liu Cc: Ian Jackson Cc: Wei Liu Cc: Stefano Stabellini Cc: Julien Grall Cc: xen-devel@lists.xen.org --- docs/man/xl-static-shm-configuration.pod.5 | 257 +++++++++++++++++++++ docs/man/xl.cfg.pod.5.in | 8 + docs/misc/xenstore-paths.markdown | 47 ++++ 3 files changed, 312 insertions(+) create mode 100644 docs/man/xl-static-shm-configuration.pod.5 diff --git a/docs/man/xl-static-shm-configuration.pod.5 b/docs/man/xl-static-shm-configuration.pod.5 new file mode 100644 index 000000000000..d68ed0ebf718 --- /dev/null +++ b/docs/man/xl-static-shm-configuration.pod.5 @@ -0,0 +1,257 @@ +=head1 NAME + +xl-static-shm-configuration - XL Static Shared Memeory Configuration Syntax + + +(B: This is currently only available to ARM guests.) + +=head1 DESCRIPTION + +The static_shm option allows users to statically setup shared memory regions +among a group of VMs, enabling guests without grant table support to do +shm-based communication. + +Every shared region is: + +=over 4 + +* Uniquely identified by a string that is no longer than 128 characters, which +is called an B in this document. + +* Backed by exactely one domain, which is called a B domain, and all +the other domains who are also sharing this region are called Bs. + +=back + +=head1 SYNTAX + +This document specifies syntax of the static shared memory configuration in +the xl config file. It has the following form: + + static_shm = [ "SSHM_SPEC", "SSHM_SPEC", ... ] + +where each C is in this form: + + [=,]* + +Valid examples of C are: + + id=ID1, begin=0x100000, end=0x200000, role=master, cache_policy=x86_normal + id=ID1, offset = 0, begin=0x500000, end=0x600000, role=slave, prot=rw + id=ID2, begin=0x300000, end=0x400000, role=master + id=ID2, offset = 0x10000, begin=0x690000, end=0x800000, role=slave + id=ID2, offset = 0x10000, begin=0x690000, end=0x800000, role=slave + +These might be specified in the domain config file like this: + + static_shm = ["id=ID2, offset = 0x10000, begin=0x690000, end=0x800000, +role=slave"] + + +More formally, the string is a series of comma-separated keyword/value +pairs. Each parameter may be specified at most once. Default values apply if +the parameter is not specified. + +=head1 Parameters + +=over 4 + +=item B + +=over 4 + +=item Description + +The unique identifier of the shared memory region. + +Every identifier could appear only once in each xl config file. + +=item Supported values + +A string that contains alphanumerics and "_"s, and is no longer than 128 +characters. + +=item Default value + +None, this parameter is mandatory. + +=back + +=item B/B + +=over 4 + +=item Description + +The boundaries of the shared memory area. + +=item Supported values + +Same with B. + +=item Default Value + +None, this parameter is mandatory. + +=back + +=item B + +=over 4 + +=item Description + +Can only appear when B = slave. If set, the address mapping will not +start from the beginning the backing memory region, but from the middle +(B bytes away from the beginning) of it. See the graph below: + +With B = 0, the mapping will look like: + + backing memory region: ######################################### + | | + | | + | | + V V + slave's shared region: ######################### + +With B > 0: + + backing memory region: ######################################### + |<-- offset -->|| | + | | + | | + V V + slave's memory region: ######################### + +=item Supported values + +Decimals or hexadecimals with a prefix "0x", and should be the multiple of the +hypervisor page granularity (currently 4K on both ARM and x86). + +=item Default value + +0x0 + +=back + +=item B + +=over 4 + +=item Description + +The backing area would be taken from one domain, which we will mark +as the "master domain", and this domain should be created prior to any +other slave domains that depend on it. + +This arugment specifies the role of this domain. + +=item Supported values + +master, slave + +=item Default value + +slave + +=back + +=item B + +=over 4 + +=item Description + +When B = master, this means the largest set of stage-2 permission flags +that can be granted to the slave domains. When B = slave, this means the +stage-2 permission flags of the shared memory area. + +=item Supported values + +Currently only 'rw' is supported. + +=item Default value + +rw + +=back + +=item B + +=over 4 + +=item Description + +The stage-2 cacheability/shareability attributes of the shared memory area. +This can only appear when B = master. + +=item Supported values + +Currently, only the following policy is supported: + +=over 4 + +=item B + +Only applicable to ARM guests. This would mean Inner and Outer Write-Back +Cacheable, and Inner Shareable. + +=back + +=item Default value + +ARM_normal + +=back + +=back + +=head1 TYPICAL USAGE + +A typical procedure of setting up a shared mem region among several VMs is: + +=over 4 + +1. Add a static_shm option to the master domain's xl config file, assign an +B to it and mark it's B as master, and set up the boundaries, prot +flag, and B appropriately. + +2. Add a static_shm option to every slave domain's xl config file, set +their B to the same value as the master's, and set up the B, +boundaries and prot flag appropriately. + +3. Create the master domain. + +4. Create the slaves. + +=back + +Remember that the master domain must be created before any slave domains could +be created, for the slaves depend on the memory pages backed by their master. + +=head1 Example + +Suppose that we have 3 domains: vm1~vm3. And we want to setup two shared +regions, say, ID1 and ID2, among the three domains, with the following address +mapping: + + ID1: (vm1 : 0x100000~0x200000) <=====> (vm2 : 0x500000~0x600000) + ID2: (vm1 : 0x310000~0x400000) <=====> (vm3 : 0x690000~0x800000) + +According to the syntax defined above, the xl config files of the three domains +should contains the following content: + +In xl config file of vm1: + static_shm = [ "id=ID1, begin=0x100000, end=0x200000, role=master, +cache_policy=x86_normal, prot=rw", +"id=ID2, begin=0x300000, end=0x400000, role=master" ] + +In xl config file of vm2: + static_shm = [ "id=ID1, offset=0, begin=0x500000, end=0x600000, +role=slave, prot=rw" ] + +In xl config file of vm3: + static_shm = [ "id=ID2, offset=0x10000, begin=0x690000, +end=0x800000, role=slave" ] + +After that, just create vm1 first, and then create vm2 and vm3 in any order. diff --git a/docs/man/xl.cfg.pod.5.in b/docs/man/xl.cfg.pod.5.in index b7b91d86274f..21540ba15a18 100644 --- a/docs/man/xl.cfg.pod.5.in +++ b/docs/man/xl.cfg.pod.5.in @@ -277,6 +277,14 @@ memory=8096 will report significantly less memory available for use than a system with maxmem=8096 memory=8096 due to the memory overhead of having to track the unused pages. +=item B + +Specifies the static shared memory regions of this guest. Static shared +memory regions enables guests to communicate with each other through +one or more shared memory regions, even without grant table support. +Currently, this only works on ARM guests. +See L for more details. + =back =head3 Guest Virtual NUMA Configuration diff --git a/docs/misc/xenstore-paths.markdown b/docs/misc/xenstore-paths.markdown index 7be2592c7479..e3fa8e29ea60 100644 --- a/docs/misc/xenstore-paths.markdown +++ b/docs/misc/xenstore-paths.markdown @@ -174,6 +174,14 @@ than this amount of RAM. The size of the video RAM this domain is configured with. +#### ~/static_shm/[_a-zA-Z0-9]+/role = ("master"|"slave") [] + +(Note: Currently, this will only appear on ARM guests.) + +The role of this domain in the static shared memory region whose id matches +the `[_a-zA-Z0-9]+` part in the path. (Described in the manpage +**xl-static-shm-configuration(5)**). + #### ~/device/suspend/event-channel = ""|EVTCHN [w] The domain's suspend event channel. The toolstack will create this @@ -610,6 +618,45 @@ for the toolstack to obtain e.g. the domain id of a xenstore domain. Domain Id of the xenstore domain in case xenstore is provided via a domain instead of a daemon in dom0. +#### /local/static_shm/[_a-zA-Z0-9]+/* [] + +(Note: Currently, this will only appear on ARM guests.) + +The following paths contain backing memory parameters of a static shared memory +whose id matches the `[_a-zA-Z0-9]+` part in the path. Their formats and +meanings are the same as those in an xl config file, described in the manpage +**xl-static-shm-configuration(5)**. + +* begin/end: the boundary of the backing memory region. +* prot: the largest set of stage-2 permission flags that can be granted to + the slave domains. +* cache_policy: the stage-2 cacheability/shareability attributes of the backing + memory region. + +The following paths contain run-time information about the static shared memory +region. + +* master: the domid of the backing domain. +* slaves: information about the slaves that are sharing the region, see + ** /local/static_shm/[_a-zA-Z0-9]+/slaves/$DOMID/* ** below. +* users: An integer. This is the reference count of the backing memory region, + including the master domain itself. When this value reachies 0, the backing + memory region will be freed. + +#### /local/staitc_shm/[_a-zA-Z0-9]+/slaves/$DOMID/* [] + +(Note: Currently, this will only appear on ARM guests.) + +The following paths contain static shared memory region parameters of a slave +domain. Their formats and meanings are the same as those in xl config files, +described in the manpage **xl-static-shm-configuration(5)**. + +* begin/end: the boundary of the shared memory region. +* prot: the stage-2 permission flags of the shared memory area. +* offset: when mapping the backing memory region to the slave's memory space, + the mapping will start from offset bytes after the beginning of the backing + memory region. + [BLKIF]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,io,blkif.h.html [FBIF]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,io,fbif.h.html [HVMPARAMS]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,hvm,params.h.html