diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index 658721d39f16b6d91c4378c685a7f46ea3700e91..193d0ee8caa1f51a137080016478b937c77dbd7a 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -104,6 +104,7 @@ bool kvm_readonly_mem_allowed; bool kvm_vm_attributes_allowed; bool kvm_msi_use_devid; bool kvm_csv3_allowed; +bool kvm_csv3_npt_ex_allowed; bool kvm_has_guest_debug; static int kvm_sstep_flags; static bool kvm_immediate_exit; diff --git a/accel/stubs/kvm-stub.c b/accel/stubs/kvm-stub.c index 3a057aceace2db0efd2afa4d3c1c39847b1337b7..41aa846f956266e8ec0079c82821307a832ca7e8 100644 --- a/accel/stubs/kvm-stub.c +++ b/accel/stubs/kvm-stub.c @@ -25,6 +25,7 @@ bool kvm_allowed; bool kvm_readonly_mem_allowed; bool kvm_msi_use_devid; bool kvm_csv3_allowed; +bool kvm_csv3_npt_ex_allowed; bool virtcca_cvm_allowed; diff --git a/backends/hostmem.c b/backends/hostmem.c index 5cef25a13d0b570e657335b0598f36a697379eab..dc66864f2fa6688d9cb60a89e9901022aa697280 100644 --- a/backends/hostmem.c +++ b/backends/hostmem.c @@ -20,6 +20,8 @@ #include "qom/object_interfaces.h" #include "qemu/mmap-alloc.h" #include "qemu/madvise.h" +#include "qemu/error-report.h" +#include "sysemu/kvm.h" #ifdef CONFIG_NUMA #include @@ -305,6 +307,14 @@ MemoryRegion *host_memory_backend_get_memory(HostMemoryBackend *backend) void host_memory_backend_set_mapped(HostMemoryBackend *backend, bool mapped) { + if (mapped && kvm_csv3_npt_ex_enabled() && + host_memory_backend_mr_inited(backend) && + host_memory_backend_pagesize(backend) != (1UL << 30)) { + error_report("memory backend '%s' is not backed by 1G huge pages, " + "which is required by CSV3 NPT extension", + object_get_canonical_path_component(OBJECT(backend))); + exit(EXIT_FAILURE); + } backend->is_mapped = mapped; } @@ -336,6 +346,12 @@ host_memory_backend_memory_complete(UserCreatable *uc, Error **errp) return; } + if (kvm_csv3_npt_ex_enabled() && + host_memory_backend_pagesize(backend) != (1UL << 30)) { + error_setg(errp, "CSV3 NPT extension requires 1G huge page memory backend"); + return; + } + ptr = memory_region_get_ram_ptr(&backend->mr); sz = memory_region_size(&backend->mr); diff --git a/hw/vfio/container.c b/hw/vfio/container.c index 97f99612bec10cc8e0050428ceb4ed7776d1979f..3c3aae9e575c988062e773faed5f4fc41fab1945 100644 --- a/hw/vfio/container.c +++ b/hw/vfio/container.c @@ -640,13 +640,13 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as, container->listener = vfio_memory_listener; - if (kvm_csv3_enabled()) { + if (kvm_csv3_enabled() && !kvm_csv3_npt_ex_enabled()) { container->csv3_ram_listener = csv3_vfio_ram_listener; container->csv3_mmio_listener = csv3_vfio_mmio_listener; shared_memory_listener_register(&container->csv3_ram_listener, container->space->as); memory_listener_register(&container->csv3_mmio_listener, - container->space->as); + container->space->as); } else { memory_listener_register(&container->listener, container->space->as); } @@ -665,7 +665,7 @@ listener_release_exit: QLIST_REMOVE(group, container_next); QLIST_REMOVE(container, next); vfio_kvm_device_del_group(group); - if (kvm_csv3_enabled()) { + if (kvm_csv3_enabled() && !kvm_csv3_npt_ex_enabled()) { shared_memory_listener_unregister(); memory_listener_unregister(&container->csv3_mmio_listener); } else { @@ -704,8 +704,9 @@ static void vfio_disconnect_container(VFIOGroup *group) * group. */ if (QLIST_EMPTY(&container->group_list)) { - if (kvm_csv3_enabled()) { + if (kvm_csv3_enabled() && !kvm_csv3_npt_ex_enabled()) { shared_memory_listener_unregister(); + memory_listener_unregister(&container->csv3_mmio_listener); } else { memory_listener_unregister(&container->listener); } diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c index 2c9ac794680ea9b65eba6cc22e70cf141e90aa73..1a60f50ced6b166b1a93ff7c755829c8f4915d59 100644 --- a/hw/virtio/vhost.c +++ b/hw/virtio/vhost.c @@ -27,6 +27,7 @@ #include "migration/blocker.h" #include "migration/qemu-file-types.h" #include "sysemu/dma.h" +#include "sysemu/kvm.h" #include "trace.h" /* enabled until disconnected backend stabilizes */ @@ -1540,7 +1541,11 @@ int vhost_dev_init(struct vhost_dev *hdev, void *opaque, hdev->log_size = 0; hdev->log_enabled = false; hdev->started = false; - memory_listener_register(&hdev->memory_listener, &address_space_memory); + if (kvm_csv3_npt_ex_enabled()) + memory_listener_register(&hdev->memory_listener, + &address_space_csv3_shared_memory); + else + memory_listener_register(&hdev->memory_listener, &address_space_memory); QLIST_INSERT_HEAD(&vhost_devices, hdev, entry); /* diff --git a/include/exec/address-spaces.h b/include/exec/address-spaces.h index 0d0aa61d68966a56e65e35cc6f38687b7e16816f..6887d1dce2ffdcfbed8ba017c513c75e42c86010 100644 --- a/include/exec/address-spaces.h +++ b/include/exec/address-spaces.h @@ -33,6 +33,7 @@ MemoryRegion *get_system_io(void); extern AddressSpace address_space_memory; extern AddressSpace address_space_io; +extern AddressSpace address_space_csv3_shared_memory; #endif diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h index 44f1fe29c1cb580296474b290dafe6bdf6ce197c..ec36cf382e852b58cd6db5c565eecb79ca8471ee 100644 --- a/include/exec/cpu-common.h +++ b/include/exec/cpu-common.h @@ -28,6 +28,7 @@ typedef uint64_t vaddr; void cpu_exec_init_all(void); void cpu_exec_step_atomic(CPUState *cpu); +void csv3_shared_memory_address_space_init(void); /* Using intptr_t ensures that qemu_*_page_mask is sign-extended even * when intptr_t is 32-bit and we are aligning a long long. diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h index b384ab3267dcf547c283220d0357a47d9f3f523a..db2d30acca51360886e46e4a2dcae9093a898836 100644 --- a/include/sysemu/kvm.h +++ b/include/sysemu/kvm.h @@ -45,6 +45,7 @@ extern bool kvm_gsi_direct_mapping; extern bool kvm_readonly_mem_allowed; extern bool kvm_msi_use_devid; extern bool kvm_csv3_allowed; +extern bool kvm_csv3_npt_ex_allowed; #define kvm_enabled() (kvm_allowed) #define virtcca_cvm_enabled() (virtcca_cvm_allowed) @@ -162,6 +163,13 @@ extern bool kvm_csv3_allowed; #define kvm_csv3_should_set_priv_mem() \ (kvm_hygon_coco_ext_inuse & KVM_CAP_HYGON_COCO_EXT_CSV3_SET_PRIV_MEM) +/** + * kvm_csv3_npt_ex_enabled: + * Returns: true if CSV3 npt extension is used for the VM. + */ +#define kvm_csv3_npt_ex_enabled() \ + (kvm_csv3_enabled() && kvm_csv3_npt_ex_allowed) + #else #define kvm_enabled() (0) @@ -180,6 +188,7 @@ extern bool kvm_csv3_allowed; #define kvm_msi_devid_required() (false) #define kvm_csv3_enabled() (false) #define kvm_csv3_should_set_priv_mem() (false) +#define kvm_csv3_npt_ex_enabled() (false) #endif /* CONFIG_KVM_IS_POSSIBLE */ diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h index d9e5bdadc20e7b14556e2d8dee9e054a127679d2..d41b3911834b05d15702f0f55657d4145eafa7c1 100644 --- a/linux-headers/linux/kvm.h +++ b/linux-headers/linux/kvm.h @@ -181,6 +181,8 @@ struct kvm_xen_exit { #define KVM_EXIT_LOONGARCH_IOCSR 38 #define KVM_EXIT_MEMORY_FAULT 39 +#define KVM_EXIT_CSV3_SECURE_CALL 200 + /* For KVM_EXIT_INTERNAL_ERROR */ /* Emulate instruction failed. */ #define KVM_INTERNAL_ERROR_EMULATION 1 @@ -432,6 +434,20 @@ struct kvm_run { __u64 gpa; __u64 size; } memory_fault; + /* KVM_EXIT_CSV3_SECURE_CALL */ + struct { +#define KVM_CSV3_SECURE_CALL_PG_ENC 1 +#define KVM_CSV3_SECURE_CALL_PG_DEC 2 + __u32 type; + union { + struct { + __u64 gpa; + __u64 size; + __u32 smr_size; + __u32 enc; + } enc_dec_info; + }; + } secure_call; /* Fix the size of the union. */ char padding[256]; }; @@ -929,6 +945,19 @@ struct kvm_enable_cap { #define KVM_CAP_HYGON_COCO_EXT_CSV3_LFINISH_EX (1 << 3) /* support userspace to request management of CSV3 shared pages */ #define KVM_CAP_HYGON_COCO_EXT_CSV3_SP_MGR (1 << 4) +/* support update NPT by CSV3 NPT_EX */ +#define KVM_CAP_HYGON_COCO_EXT_CSV3_NPT_EX (1 << 5) +#define KVM_CAP_HYGON_COCO_EXT_VALID_MASK \ + (KVM_CAP_HYGON_COCO_EXT_CSV3_SET_PRIV_MEM | \ + KVM_CAP_HYGON_COCO_EXT_CSV3_MULT_LUP_DATA | \ + KVM_CAP_HYGON_COCO_EXT_CSV3_INJ_SECRET | \ + KVM_CAP_HYGON_COCO_EXT_CSV3_LFINISH_EX | \ + KVM_CAP_HYGON_COCO_EXT_CSV3_SP_MGR | \ + KVM_CAP_HYGON_COCO_EXT_CSV3_NPT_EX) +#define KVM_CAP_EXIT_CSV3_SECURE_CALL 502 +#define KVM_CSV3_SECURE_CALL_PG_ENC_DEC 0 +#define KVM_CSV3_SECURE_CALL_PG_ENC_DEC_MASK (1UL << KVM_CSV3_SECURE_CALL_PG_ENC_DEC) +#define KVM_EXIT_CSV3_SECURE_CALL_VALID_MASK KVM_CSV3_SECURE_CALL_PG_ENC_DEC_MASK #define KVM_EXIT_HYPERCALL_VALID_MASK (1 << KVM_HC_MAP_GPA_RANGE) diff --git a/system/physmem.c b/system/physmem.c index 96255caf10ce469250682323f98b36ffd175de2d..07b887ed064168b1ee45fd00cefc8dcebabceb28 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -89,9 +89,11 @@ RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) }; static MemoryRegion *system_memory; static MemoryRegion *system_io; +static MemoryRegion *csv3_root_shared_memory; AddressSpace address_space_io; AddressSpace address_space_memory; +AddressSpace address_space_csv3_shared_memory; static MemoryRegion io_mem_unassigned; @@ -2631,6 +2633,16 @@ static void memory_map_init(void) address_space_init(&address_space_io, system_io, "I/O"); } +void csv3_shared_memory_address_space_init(void) +{ + csv3_root_shared_memory = g_malloc(sizeof(*csv3_root_shared_memory)); + + memory_region_init(csv3_root_shared_memory, NULL, + "csv3_root_shared_memory", UINT64_MAX); + address_space_init(&address_space_csv3_shared_memory, csv3_root_shared_memory, + "CSV3-SHARED-MEMORY-AS"); +} + MemoryRegion *get_system_memory(void) { return system_memory; diff --git a/system/vl.c b/system/vl.c index 4f9ecf1cf9bb74e770ab3ca27cc6914af09c8a5c..48f84c623fbfc5b9cf4b10d70b6286664316c240 100644 --- a/system/vl.c +++ b/system/vl.c @@ -3773,6 +3773,9 @@ void qemu_init(int argc, char **argv) configure_accelerators(argv[0]); phase_advance(PHASE_ACCEL_CREATED); + if (kvm_csv3_npt_ex_enabled()) + csv3_shared_memory_address_space_init(); + /* * Beware, QOM objects created before this point miss global and * compat properties. diff --git a/target/i386/csv-sysemu-stub.c b/target/i386/csv-sysemu-stub.c index 00722c9ba07c9eb83571f2ff44ee54be295b0dae..07842f388df5f594d0984be10f0a5c3052b528b4 100644 --- a/target/i386/csv-sysemu-stub.c +++ b/target/i386/csv-sysemu-stub.c @@ -31,6 +31,12 @@ int csv3_launch_encrypt_vmcb(void) } void csv3_launch_finish_ex(char *host_data) +{ + g_assert_not_reached(); +} + +void csv3_shared_memory_init(void) +{ g_assert_not_reached(); } @@ -54,6 +60,18 @@ void csv3_shared_region_get(uint64_t gpa, uint32_t num_pages) } +void csv3_secure_call_convert_private_to_shared(unsigned long gpa, unsigned long size, + unsigned int smr_size) +{ + g_assert_not_reached(); +} + +void csv3_secure_call_convert_shared_to_private(unsigned long gpa, unsigned long size, + unsigned int smr_size) +{ + g_assert_not_reached(); +} + int csv3_set_guest_private_memory(Error **errp) { g_assert_not_reached(); diff --git a/target/i386/csv.c b/target/i386/csv.c index cdff19f515630e801b558b5d9754845b5b0faa4a..faefc4c274c236da9e554f5f8948a46f3802dcb1 100644 --- a/target/i386/csv.c +++ b/target/i386/csv.c @@ -14,14 +14,21 @@ #include "qemu/osdep.h" #include "qemu/base64.h" #include "qemu/error-report.h" +#include "qemu/memfd.h" +#include "qemu/range.h" #include "qapi/error.h" #include "sysemu/kvm.h" +#include "sysemu/runstate.h" #include "exec/address-spaces.h" #include "exec/ramblock.h" +#include "exec/memory.h" +#include "exec/ram_addr.h" #include "migration/blocker.h" #include "migration/qemu-file.h" #include "migration/misc.h" +#include "migration/vmstate.h" #include "monitor/monitor.h" +#include "hw/i386/x86.h" #include #include @@ -41,6 +48,9 @@ bool csv_kvm_cpu_reset_inhibit; uint32_t kvm_hygon_coco_ext; uint32_t kvm_hygon_coco_ext_inuse; +static MemoryRegion *csv3_shared_memory; +static int csv3_shared_memfd = -1; + struct ConfidentialGuestMemoryEncryptionOps csv3_memory_encryption_ops = { .save_setup = sev_save_setup, .save_outgoing_page = NULL, @@ -362,6 +372,16 @@ void csv3_shared_region_release(uint64_t gpa, uint32_t num_pages) memory_region_unref(mrs.mr); block = qemu_ram_block_from_host(hva, false, &offset); + + /* When a CSV3 VM is using KVM_CAP_EXIT_CSV3_SECURE_CALL, we should + * not attempt to unpin memory backed by 1G huge pages in the userspace + * VMM. + */ + if (kvm_csv3_npt_ex_enabled() && qemu_ram_pagesize(block) == (1UL << 30)) { + gpa_start += length; + continue; + } + if (!block || offset < ((uint64_t)hva & (block->mr->align - 1))) { gpa_start += 1 << TARGET_PAGE_BITS; continue; @@ -456,6 +476,16 @@ void csv3_shared_region_get(uint64_t gpa, uint32_t num_pages) memory_region_unref(mrs.mr); block = qemu_ram_block_from_host(hva, false, &offset); + + /* When a CSV3 VM is using KVM_CAP_EXIT_CSV3_SECURE_CALL, we should + * not attempt to pin memory backed by 1G huge pages in the userspace + * VMM. + */ + if (kvm_csv3_npt_ex_enabled() && qemu_ram_pagesize(block) == (1UL << 30)) { + gpa_start += length; + continue; + } + if (!block || offset < ((uint64_t)hva & (block->mr->align - 1))) { gpa_start += 1 << TARGET_PAGE_BITS; continue; @@ -546,6 +576,310 @@ end: return; } +void csv3_shared_memory_init(void) +{ + MachineState *machine = MACHINE(qdev_get_machine()); + X86MachineState *x86ms = X86_MACHINE(machine); + MemoryRegion *csv3_ram_below_4g, *csv3_ram_above_4g; + + if (csv3_shared_memfd != -1) + return; + + csv3_shared_memfd = qemu_memfd_create("csv3_shared_memfd", + machine->ram_size, + false, + 0, + 0, + NULL); + g_assert(csv3_shared_memfd != -1); + + csv3_ram_below_4g = g_malloc(sizeof(*csv3_ram_below_4g)); + csv3_ram_above_4g = g_malloc(sizeof(*csv3_ram_above_4g)); + + csv3_shared_memory = g_malloc(sizeof(*csv3_shared_memory)); + + memory_region_init_ram_from_fd(csv3_shared_memory, + NULL, + "csv3_shared_memory", + machine->ram_size, + RAM_SHARED | RAM_NORESERVE, + csv3_shared_memfd, + 0, + NULL); + + memory_region_set_unmergeable(csv3_shared_memory, 1); + vmstate_register_ram_global(csv3_shared_memory); + memory_region_init_alias(csv3_ram_below_4g, NULL, "csv3_ram_below-4g", + csv3_shared_memory, 0, x86ms->below_4g_mem_size); + memory_region_add_subregion(address_space_csv3_shared_memory.root, 0, + csv3_ram_below_4g); + memory_region_init_alias(csv3_ram_above_4g, NULL, "csv3_ram_above-4g", + csv3_shared_memory, x86ms->below_4g_mem_size, + x86ms->above_4g_mem_size); + memory_region_add_subregion(address_space_csv3_shared_memory.root, + x86ms->above_4g_mem_start, + csv3_ram_above_4g); +} + +/* + * Install a shared-memory alias for an SMR-unaligned GPA range. + * + * For a GPA range that is NOT aligned to the CSV3 hardware SMR, the + * guest's actual memory backend is csv3_shared_memfd (already mapped + * into address_space_csv3_shared_memory by csv3_shared_memory_init()). + * + * Here we alias that csv3_shared_memfd-backed range into the system + * address space at the same GPA with higher priority, so that guest + * accesses to this GPA range hit the csv3_shared_memfd backend + * instead of machine->ram. + */ +static void +__csv3_sc_handle_smr_unaligned_shared_region(unsigned long gpa, + unsigned long size) +{ + MemoryRegion *alias_mr = g_new(MemoryRegion, 1); + char *alias_name = g_strdup_printf("csv3_unalgn_alias_0x%lx", gpa); + bool need_lock = !runstate_check(RUN_STATE_INMIGRATE); + + if (need_lock) + bql_lock(); + + memory_region_init_alias(alias_mr, NULL, alias_name, + address_space_csv3_shared_memory.root, + gpa, size); + memory_region_add_subregion_overlap(get_system_memory(), gpa, alias_mr, 1); + + if (need_lock) + bql_unlock(); + + g_free(alias_name); +} + +/* + * Split an SMR-unaligned shared GPA range into 2MB-aligned chunks + * and install a csv3_shared_memfd alias for each chunk. + * + * After this function returns, every byte in [gpa, gpa + size) that + * is not SMR-aligned is served by csv3_shared_memfd through the + * aliases installed in the system address space. + */ +static void +csv3_sc_handle_smr_unaligned_shared_region(unsigned long gpa, unsigned long size) +{ + unsigned long head_size, hugepage_aligned_size, tail_size; + unsigned long hugepage_size = (1UL << 21); + + head_size = QEMU_ALIGN_UP(gpa, hugepage_size) - gpa; + if (head_size && size) { + __csv3_sc_handle_smr_unaligned_shared_region(gpa, MIN(head_size, size)); + if (head_size >= size) + return; + } + + hugepage_aligned_size = QEMU_ALIGN_DOWN(gpa + size, hugepage_size) - + QEMU_ALIGN_UP(gpa, hugepage_size); + if (hugepage_aligned_size) { + if (hugepage_aligned_size >= hugepage_size) { + __csv3_sc_handle_smr_unaligned_shared_region( + QEMU_ALIGN_UP(gpa, hugepage_size), + hugepage_aligned_size); + } + } + + tail_size = (gpa + size) - QEMU_ALIGN_DOWN(gpa + size, hugepage_size); + if (tail_size) { + __csv3_sc_handle_smr_unaligned_shared_region( + QEMU_ALIGN_DOWN(gpa + size, hugepage_size), + tail_size); + } +} + +/* + * Expose an SMR-aligned shared GPA range in the CSV3 shared address + * space. + * + * For a GPA range that is aligned to the CSV3 hardware SMR, the + * guest's actual memory backend is machine->ram (i.e. the system + * memory region). + * + * Here we alias that machine->ram-backed range into + * address_space_csv3_shared_memory at the same GPA, so the shared + * address space sees the SMR-aligned portion as shared while the + * guest still accesses it through machine->ram in the system + * address space. No alias is added to system_memory because + * machine->ram already serves it there. + */ +static void +csv3_sc_handle_smr_aligned_shared_region(unsigned long gpa, unsigned long size) +{ + MemoryRegion *alias_mr = g_new(MemoryRegion, 1); + char *alias_name = g_strdup_printf("csv3_algn_alias_0x%lx", gpa); + bool need_lock = !runstate_check(RUN_STATE_INMIGRATE); + + if (need_lock) + bql_lock(); + + memory_region_init_alias(alias_mr, NULL, alias_name, + get_system_memory(), gpa, size); + memory_region_set_unmergeable(alias_mr, 1); + memory_region_add_subregion_overlap(address_space_csv3_shared_memory.root, + gpa, alias_mr, 1); + + if (need_lock) + bql_unlock(); + + g_free(alias_name); +} + +/* + * Handle a guest secure call that converts a [gpa, gpa + size) range + * from private to shared. + * + * Invariant: + * - If a GPA range is aligned to the CSV3 hardware SMR, the guest's + * actual memory backend for that range is machine->ram. + * - If a GPA range is NOT aligned to the CSV3 hardware SMR, the + * guest's actual memory backend for that range is + * csv3_shared_memfd. + * + * Accordingly the range is split into head (unaligned), middle + * (SMR-aligned) and tail (unaligned) parts. The unaligned parts are + * served from csv3_shared_memfd via aliases installed in + * system_memory; the aligned part continues to be served from + * machine->ram in system_memory, and is additionally aliased into + * address_space_csv3_shared_memory so that the shared address space + * tracks it as shared. + */ +void csv3_secure_call_convert_private_to_shared(unsigned long gpa, unsigned long size, + unsigned int smr_size) +{ + MachineState *machine = MACHINE(qdev_get_machine()); + X86MachineState *x86ms = X86_MACHINE(machine); + unsigned long head_size, smr_aligned_size, tail_size; + + if (!size || !smr_size) { + error_report("%s: size or smr_size cannot be 0", __func__); + return; + } + + /* The enc/dec range should not intersect the memory hole. */ + g_assert((gpa < x86ms->below_4g_mem_size && + (gpa + size) <= x86ms->below_4g_mem_size) || + gpa >= x86ms->above_4g_mem_start); + + trace_kvm_csv3_secure_call_dec(gpa, size, smr_size); + + /* SMR-unaligned shared memory lives within the csv3_root_shared_memory. */ + head_size = QEMU_ALIGN_UP(gpa, smr_size) - gpa; + if (head_size) { + csv3_sc_handle_smr_unaligned_shared_region(gpa, MIN(head_size, size)); + if (head_size >= size) { + goto end; + } + } + + smr_aligned_size = QEMU_ALIGN_DOWN(gpa + size, smr_size) + - QEMU_ALIGN_UP(gpa, smr_size); + if (smr_aligned_size) + csv3_sc_handle_smr_aligned_shared_region(QEMU_ALIGN_UP(gpa, smr_size), + smr_aligned_size); + + tail_size = (gpa + size) - QEMU_ALIGN_DOWN(gpa + size, smr_size); + if (tail_size) + csv3_sc_handle_smr_unaligned_shared_region( + QEMU_ALIGN_DOWN(gpa + size, smr_size), tail_size); + +end: + /* Pin the SMR-unaligned unencrypted memory regions after applying + * the MemoryRegion overlay. + */ + csv3_shared_region_get(gpa, + (size + TARGET_PAGE_SIZE - 1) >> TARGET_PAGE_BITS); +} + +/* + * Handle a guest secure call that converts a [gpa, gpa + size) range + * from shared back to private. + * + * This is the inverse of csv3_secure_call_convert_private_to_shared(): + * it removes the per-call aliases that were installed in either + * system_memory (for SMR-unaligned, csv3_shared_memfd-backed sub- + * ranges) or address_space_csv3_shared_memory (for SMR-aligned, + * machine->ram-backed sub-ranges), so the guest goes back to + * accessing the private machine->ram backend through the system + * address space alone. + */ +void csv3_secure_call_convert_shared_to_private(unsigned long gpa, unsigned long size, + unsigned int smr_size) +{ + MachineState *machine = MACHINE(qdev_get_machine()); + X86MachineState *x86ms = X86_MACHINE(machine); + MemoryRegion *system = get_system_memory(); + MemoryRegion *shared = address_space_csv3_shared_memory.root; + MemoryRegion *child, *next; + bool need_lock; + + if (!size || !smr_size) { + error_report("%s: size or smr_size cannot be 0", __func__); + return; + } + + /* The enc/dec range should not intersect the memory hole. */ + g_assert((gpa < x86ms->below_4g_mem_size && + (gpa + size) <= x86ms->below_4g_mem_size) || + gpa >= x86ms->above_4g_mem_start); + + /* Unpin the SMR-unaligned unencrypted memory regions before + * revoking the MemoryRegion overlay. + */ + csv3_shared_region_release(gpa, + (size + TARGET_PAGE_SIZE - 1) >> TARGET_PAGE_BITS); + + trace_kvm_csv3_secure_call_enc(gpa, size, smr_size); + + need_lock = !runstate_check(RUN_STATE_INMIGRATE); + + /* + * Remove every alias previously installed by + * __csv3_sc_handle_smr_unaligned_shared_region() that falls within + * [gpa, gpa + size). Match by the well-known name prefix to avoid + * touching unrelated subregions (e.g. pc.ram, vga.vram) that may + * happen to be covered by this GPA range. + */ + QTAILQ_FOREACH_SAFE(child, &system->subregions, subregions_link, next) { + if (!child->name || + !g_str_has_prefix(child->name, "csv3_unalgn_alias_")) + continue; + if (gpa <= child->addr && + (gpa + size) >= (child->addr + int128_get64(child->size))) { + if (need_lock) + bql_lock(); + memory_region_del_subregion(system, child); + if (need_lock) + bql_unlock(); + } + } + + /* + * Remove every alias previously installed by + * csv3_sc_handle_smr_aligned_shared_region() in the CSV3 shared + * address space within [gpa, gpa + size). + */ + QTAILQ_FOREACH_SAFE(child, &shared->subregions, subregions_link, next) { + if (!child->name || + !g_str_has_prefix(child->name, "csv3_algn_alias_")) + continue; + if (gpa <= child->addr && + (gpa + size) >= (child->addr + int128_get64(child->size))) { + if (need_lock) + bql_lock(); + memory_region_del_subregion(shared, child); + if (need_lock) + bql_unlock(); + } + } +} + static inline hwaddr csv3_hva_to_gfn(uint8_t *ptr) { ram_addr_t offset = RAM_ADDR_INVALID; diff --git a/target/i386/csv.h b/target/i386/csv.h index 8b10dc27a5d85e3eb566383057804d6b474cb547..ee9ccdef67943796c20569e89328520909283574 100644 --- a/target/i386/csv.h +++ b/target/i386/csv.h @@ -120,6 +120,7 @@ extern struct ConfidentialGuestMemoryEncryptionOps csv3_memory_encryption_ops; extern int csv3_init(uint32_t policy, int fd, void *state, struct sev_ops *ops); extern int csv3_launch_encrypt_vmcb(void); void csv3_launch_finish_ex(char *host_data); +void csv3_shared_memory_init(void); int csv3_load_data(uint64_t gpa, uint8_t *ptr, uint64_t len, Error **errp); @@ -127,6 +128,10 @@ int csv3_shared_region_dma_map(uint64_t start, uint64_t end); void csv3_shared_region_dma_unmap(uint64_t start, uint64_t end); void csv3_shared_region_release(uint64_t gpa, uint32_t num_pages); void csv3_shared_region_get(uint64_t gpa, uint32_t num_pages); +void csv3_secure_call_convert_private_to_shared(unsigned long gpa, unsigned long size, + unsigned int smr_size); +void csv3_secure_call_convert_shared_to_private(unsigned long gpa, unsigned long size, + unsigned int smr_size); int csv3_load_incoming_page(QEMUFile *f, uint8_t *ptr); int csv3_load_incoming_context(QEMUFile *f); int csv3_queue_outgoing_page(uint8_t *ptr, uint32_t sz, uint64_t addr); diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c index 29d105412157aadb7a689d2233711fc277848590..66e49d54cecefeca7bb68730ead1d9bddbf90bd0 100644 --- a/target/i386/kvm/kvm.c +++ b/target/i386/kvm/kvm.c @@ -2727,15 +2727,36 @@ int kvm_arch_init(MachineState *ms, KVMState *s) /* check and enable Hygon coco extensions */ kvm_hygon_coco_ext = (uint32_t)kvm_vm_check_extension(s, KVM_CAP_HYGON_COCO_EXT); + kvm_hygon_coco_ext &= KVM_CAP_HYGON_COCO_EXT_VALID_MASK; if (kvm_hygon_coco_ext) { + if (kvm_hygon_coco_ext & KVM_CAP_HYGON_COCO_EXT_CSV3_NPT_EX) { + int has_csv3_secure_call = 0; + + /* check and enable Hygon secure call*/ + has_csv3_secure_call = kvm_vm_check_extension(s, + KVM_CAP_EXIT_CSV3_SECURE_CALL); + if (has_csv3_secure_call) { + ret = kvm_vm_enable_cap(s, KVM_CAP_EXIT_CSV3_SECURE_CALL, 0, + KVM_EXIT_CSV3_SECURE_CALL_VALID_MASK); + if (ret < 0) { + error_report("kvm: Failed to enable CSV3_SECURE_CALL cap: %s", + strerror(-ret)); + kvm_hygon_coco_ext &= ~KVM_CAP_HYGON_COCO_EXT_CSV3_NPT_EX; + } + } + } + ret = kvm_vm_enable_cap(s, KVM_CAP_HYGON_COCO_EXT, 0, (uint64_t)kvm_hygon_coco_ext); if (ret == -EINVAL) { error_report("kvm: Failed to enable KVM_CAP_HYGON_COCO_EXT cap: %s", strerror(-ret)); kvm_hygon_coco_ext_inuse = 0; + kvm_csv3_npt_ex_allowed = false; } else { kvm_hygon_coco_ext_inuse = (uint32_t)ret; + kvm_csv3_npt_ex_allowed = kvm_hygon_coco_ext_inuse & + KVM_CAP_HYGON_COCO_EXT_CSV3_NPT_EX; } } } @@ -5183,17 +5204,42 @@ static int kvm_handle_exit_hypercall(X86CPU *cpu, struct kvm_run *run) if (enc) { sev_remove_shared_regions_list(gfn_start, gfn_end); - csv3_shared_region_dma_unmap(gpa, gfn_end << TARGET_PAGE_BITS); - csv3_shared_region_release(gpa, npages); + if (kvm_csv3_enabled() && !kvm_csv3_npt_ex_enabled()) { + csv3_shared_region_dma_unmap(gpa, gfn_end << TARGET_PAGE_BITS); + csv3_shared_region_release(gpa, npages); + } } else { sev_add_shared_regions_list(gfn_start, gfn_end); - csv3_shared_region_dma_map(gpa, gfn_end << TARGET_PAGE_BITS); - csv3_shared_region_get(gpa, npages); + if (kvm_csv3_enabled() && !kvm_csv3_npt_ex_enabled()) { + csv3_shared_region_dma_map(gpa, gfn_end << TARGET_PAGE_BITS); + csv3_shared_region_get(gpa, npages); + } } } return 0; } +static int kvm_handle_exit_csv3_secure_call(X86CPU *cpu, struct kvm_run *run) +{ + if (!kvm_csv3_npt_ex_enabled()) + return 0; + + if (run->secure_call.type == KVM_CSV3_SECURE_CALL_PG_ENC || + run->secure_call.type == KVM_CSV3_SECURE_CALL_PG_DEC) { + unsigned long gpa = run->secure_call.enc_dec_info.gpa; + unsigned long size = run->secure_call.enc_dec_info.size; + unsigned int smr_size = run->secure_call.enc_dec_info.smr_size; + unsigned int enc = run->secure_call.enc_dec_info.enc; + + if (!enc) { + csv3_secure_call_convert_private_to_shared(gpa, size, smr_size); + } else { + csv3_secure_call_convert_shared_to_private(gpa, size, smr_size); + } + } + return 0; +} + int kvm_arch_insert_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp) { static const uint8_t int3 = 0xcc; @@ -5668,6 +5714,9 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run) ret = kvm_handle_hypercall(run); } break; + case KVM_EXIT_CSV3_SECURE_CALL: + ret = kvm_handle_exit_csv3_secure_call(cpu, run); + break; default: fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason); ret = -1; diff --git a/target/i386/sev.c b/target/i386/sev.c index b8447f29659b68883a630bb2123c64f08ec9b515..c3553c2ea53457b710847745188dbfb1343a64d7 100644 --- a/target/i386/sev.c +++ b/target/i386/sev.c @@ -1272,12 +1272,16 @@ sev_launch_finish(SevCommonState *sev_common) sev_guest = (SevGuestState *)obj; } - if (csv3_enabled() && - sev_guest && - sev_guest->host_data && - (kvm_hygon_coco_ext_inuse & KVM_CAP_HYGON_COCO_EXT_CSV3_LFINISH_EX)) { - csv3_launch_finish_ex(sev_guest->host_data); - goto common_finish; + if (csv3_enabled()) { + if (kvm_hygon_coco_ext_inuse & KVM_CAP_HYGON_COCO_EXT_CSV3_NPT_EX) + csv3_shared_memory_init(); + + if (sev_guest && + sev_guest->host_data && + (kvm_hygon_coco_ext_inuse & KVM_CAP_HYGON_COCO_EXT_CSV3_LFINISH_EX)) { + csv3_launch_finish_ex(sev_guest->host_data); + goto common_finish; + } } trace_kvm_sev_launch_finish(); @@ -1842,6 +1846,8 @@ static int sev_common_kvm_init(ConfidentialGuestSupport *cgs, Error **errp) g_free(user_id); g_free(init_cmd_buf); } + } else { + ret = sev_ioctl(sev_common->sev_fd, cmd, NULL, &fw_error); } } else { switch (x86_klass->kvm_type(X86_CONFIDENTIAL_GUEST(sev_common))) { diff --git a/target/i386/trace-events b/target/i386/trace-events index 873811e8d56b05bb3f42cc2de3d9db9cfff1b40f..b4bb693d2f4e569e830a07e56e28b6c885b2703f 100644 --- a/target/i386/trace-events +++ b/target/i386/trace-events @@ -37,3 +37,5 @@ kvm_csv3_madvise_release(uint64_t hva, uint32_t num_pages) "hva 0x%" PRIx64 " nu kvm_csv3_pin_pages_count(uint64_t pin_count) "pinned count 0x%" PRIx64 kvm_csv3_shared_region_get(uint64_t gpa, uint32_t num_pages) "gpa 0x%" PRIx64 " num_pages 0x%x" kvm_csv3_pinned(uint64_t gpa, uint32_t num_pages) "gpa 0x%" PRIx64 " num_pages 0x%x" +kvm_csv3_secure_call_dec(unsigned long gpa, unsigned long size, unsigned int smr_size) "gpa 0x%" PRIx64 " size 0x%" PRIx64 " smr_size 0x%x" +kvm_csv3_secure_call_enc(unsigned long gpa, unsigned long size, unsigned int smr_size) "gpa 0x%" PRIx64 " size 0x%" PRIx64 " smr_size 0x%x"