diff --git a/arch/x86/include/asm/csv.h b/arch/x86/include/asm/csv.h index 027b595c72b18a1850006a1f066a2fe155c04ff7..52ebdf1f5d7c512bfd639035f661a65871fa6404 100644 --- a/arch/x86/include/asm/csv.h +++ b/arch/x86/include/asm/csv.h @@ -55,6 +55,11 @@ enum csv_smr_source get_csv_smr_source(void); uint32_t csv_get_smr_entry_shift(void); +void __init early_csv_guest_mem_init(void); +phys_addr_t csv3_alloc_mem_block(void); +void csv3_free_mem_block(phys_addr_t phys_addr); +size_t csv3_get_mem_block_size(void); + #else /* !CONFIG_HYGON_CSV */ #define csv_smr NULL @@ -114,6 +119,10 @@ static inline void csv_memory_enc_dec(u64 vaddr, u64 pages, bool enc) { } static inline int csv3_issue_request_report(phys_addr_t paddr, size_t size) { return -EIO; } static inline int csv3_issue_request_rtmr(void *req_buffer, size_t buffer_size) { return -ENODEV; } +static inline void __init early_csv_guest_mem_init(void) { } +static inline phys_addr_t csv3_alloc_mem_block(void) { return 0; } +static inline void csv3_free_mem_block(phys_addr_t phys_addr) { } +static inline size_t csv3_get_mem_block_size(void) { return 0; } #endif /* CONFIG_HYGON_CSV */ #endif /* __ASSEMBLY__ */ diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index 8d3c8d4eb77836d25833ceeb63300fbccf9961e0..3adc91b1b37604060e7811ebbef92f662324a2b2 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -252,34 +252,6 @@ static u64 __init get_ramdisk_size(void) return ramdisk_size; } -#define MAX_MAP_CHUNK (NR_FIX_BTMAPS << PAGE_SHIFT) - -static void __init copy_early_initrd(void *dest, phys_addr_t src, - unsigned long size) -{ - unsigned long slop, clen; - char *p; - - while (size) { - slop = offset_in_page(src); - clen = size; - if (clen > MAX_MAP_CHUNK - slop) - clen = MAX_MAP_CHUNK - slop; - /* - * _ENC flag should be preserved so that when SME is enabled initrd - * can be mapped as encrypted, as it had been encrypted earlier. - * This flag won't impact on other platforms like TDX/SEV enabled. - */ - p = early_memremap_prot(src & PAGE_MASK, clen + slop, - pgprot_val(FIXMAP_PAGE_NORMAL)); - memcpy(dest, p + slop, clen); - early_memunmap(p, clen + slop); - dest += clen; - src += clen; - size -= clen; - } -} - static void __init relocate_initrd(void) { /* Assume only end is not page aligned */ @@ -299,7 +271,7 @@ static void __init relocate_initrd(void) printk(KERN_INFO "Allocated new RAMDISK: [mem %#010llx-%#010llx]\n", relocated_ramdisk, relocated_ramdisk + ramdisk_size - 1); - copy_early_initrd((void *)initrd_start, ramdisk_image, ramdisk_size); + copy_from_early_mem((void *)initrd_start, ramdisk_image, ramdisk_size); printk(KERN_INFO "Move RAMDISK from [mem %#010llx-%#010llx] to" " [mem %#010llx-%#010llx]\n", @@ -1011,27 +983,6 @@ void __init setup_arch(char **cmdline_p) parse_early_param(); -#ifdef CONFIG_IEE_SIP - /* - * Perform a one-time check for IEE_SIP prerequisites. This must be done - * early in setup_arch() before any code might rely on these features. - * At this point, CPU features (from early_cpu_init) and kernel command - * line (from parse_early_param) are both available. - */ - if (haoc_enabled) { - bool smep_ok = cpu_feature_enabled(X86_FEATURE_SMEP); - bool smap_ok = cpu_feature_enabled(X86_FEATURE_SMAP); - if (smep_ok && smap_ok) { - pr_info("IEE_SIP: Feature is active. CPU supports SMEP and SMAP.\n"); - } else { - // Fail-fast: The user wants the feature, but the hardware - // doesn't support it. This is a fatal configuration error. - panic("IEE_SIP: FATAL: Feature enabled via 'haoc=on' but hardware is missing support (SMEP:%d, SMAP:%d).\n", - smep_ok, smap_ok); - } - } -#endif - if (efi_enabled(EFI_BOOT)) efi_memblock_x86_reserve_range(); @@ -1166,7 +1117,7 @@ void __init setup_arch(char **cmdline_p) * Needs to run after memblock setup because it needs the physical * memory size. */ - mem_encrypt_setup_arch(); + sev_setup_arch(); cc_random_init(); efi_fake_memmap(); @@ -1270,7 +1221,16 @@ void __init setup_arch(char **cmdline_p) initmem_init(); - /* Try to reserve contiguous memory to support CSV3 */ + /* CSV guest memory specific initialization */ + early_csv_guest_mem_init(); + + /* + * Try to reserve contiguous memory to support CSV3. The + * contiguous memory will be reserved iff running on bare metal + * and support CSV3 feature. + * Note: If CONFIG_HYGON_CSV is not set, early_csv_reserve_mem + * will do nothing. + */ early_csv_reserve_mem(); dma_contiguous_reserve(max_pfn_mapped << PAGE_SHIFT); diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile index 699cd989f6af363c85df101e718f2430e8697c06..c9d9bee63d7db81cad119ab8c63222a488124eee 100644 --- a/arch/x86/mm/Makefile +++ b/arch/x86/mm/Makefile @@ -69,3 +69,4 @@ obj-$(CONFIG_AMD_MEM_ENCRYPT) += mem_encrypt_identity.o obj-$(CONFIG_AMD_MEM_ENCRYPT) += mem_encrypt_boot.o obj-$(CONFIG_HYGON_CSV) += mem_encrypt_hygon.o +obj-$(CONFIG_HYGON_CSV) += csv_guest_mem.o diff --git a/arch/x86/mm/csv_guest_mem.c b/arch/x86/mm/csv_guest_mem.c new file mode 100644 index 0000000000000000000000000000000000000000..200c26f56d1e8e72fbd0a19e7776e572f880992f --- /dev/null +++ b/arch/x86/mm/csv_guest_mem.c @@ -0,0 +1,322 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Hygon China Secure Virtualization (CSV) + * + * Copyright (C) Hygon Info Technologies Ltd. + * + * Description: The file is used to reserve memblock for CSV3 guest. + * + * Author: Wencheng Yang + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CSV3_MEM_BLOCK_SIZE (SZ_1G) +#define CSV3_MEM_BLOCK_ALIGN (SZ_1G) + +/** + * The struct is used to record the memory block information. + * If the block is the header of the large block, the head is true. + * The size is the size of the large block, each sub block size is + * mem_block->block_size. + */ +struct block { + phys_addr_t start; + unsigned long size; // the size of the large block + bool used; + bool head; // head of the block +}; + +struct mem_block { + unsigned long block_align; + unsigned long block_size; + unsigned long nr_blocks; + unsigned long nr_free; + bool decrypted; + bool decrypt_failed; + struct block blocks[]; +}; + +static struct mem_block *csv3_mem_block; + +static phys_addr_t size_cmdline __initdata = -1; + +static DEFINE_MUTEX(lock); + +/** + * Config string format: csv3_guest_rsv_mb=size + * For example: csv3_guest_rsv_mb=1G + * csv3_guest_rsv_mb=2048M + * + * size: the size of memory to reserve, it should be multiple + * of CSV3_MEM_BLOCK_SIZE, i.e. 1G. + */ +static int __init early_csv3_reserve_mem_block(char *p) +{ + if (!p) + return -EINVAL; + + size_cmdline = memparse(p, &p); + + return 0; +} + +early_param("csv3_guest_rsv_mb", early_csv3_reserve_mem_block); + +/** + * csv3_reserve_mem_block() - reserve memblock for csv3 guest. + */ +static void __init csv3_reserve_mem_block(void) +{ + unsigned long csv3_mem_size; + unsigned long size, reserved_size, meta_size; + unsigned long block_cnt, reserved_cnt, try_cnt; + size_t align = CSV3_MEM_BLOCK_ALIGN; + size_t block_size = CSV3_MEM_BLOCK_SIZE; + phys_addr_t base; + int i; + + if (!csv3_active()) + return; + + if (size_cmdline == -1 || size_cmdline == 0) + return; + + csv3_mem_size = size_cmdline; + if (size_cmdline % align) { + csv3_mem_size = ALIGN(size_cmdline, block_size); + pr_warn("csv3_guest_rsv_mb: size must be multiple of %u MB, align up to %lu MB\n", + CSV3_MEM_BLOCK_ALIGN / SZ_1M, csv3_mem_size / SZ_1M); + } + + block_cnt = csv3_mem_size / block_size; + meta_size = sizeof(struct mem_block) + block_cnt * sizeof(struct block); + csv3_mem_block = memblock_alloc_node(meta_size, SMP_CACHE_BYTES, + NUMA_NO_NODE); + if (!csv3_mem_block) { + pr_err("csv3_guest_rsv_mb: fail to allocate size 0x%lx from memblock\n", + meta_size); + return; + } + + memset(csv3_mem_block, 0, meta_size); + csv3_mem_block->block_size = block_size; + csv3_mem_block->block_align = align; + + try_cnt = 0; + reserved_size = 0; + reserved_cnt = 0; + while (reserved_size < csv3_mem_size) { + size = csv3_mem_size - reserved_size - try_cnt * SZ_1G; + base = memblock_phys_alloc_try_nid(size, + csv3_mem_block->block_align, + NUMA_NO_NODE); + if (!base) { + pr_warn("csv3_guest_rsv_mb: reserve memblock failed\n"); + // can't alloc any more, no retry. + if (size <= CSV3_MEM_BLOCK_SIZE) + goto fail_free; + + try_cnt++; + continue; + } else { + /* success, reset try_cnt */ + try_cnt = 0; + } + + pr_debug("csv3_guest_rsv_mb: reserve memblock[%ld] 0x%lx MB memory, pa 0x%llx\n", + reserved_cnt, size / SZ_1M, base); + + block_cnt = size / csv3_mem_block->block_size; + csv3_mem_block->blocks[reserved_cnt].head = true; + csv3_mem_block->blocks[reserved_cnt].size = size; + for (i = 0; i < block_cnt; i++) { + csv3_mem_block->blocks[reserved_cnt].start = base; + csv3_mem_block->blocks[reserved_cnt].used = false; + csv3_mem_block->nr_blocks++; + csv3_mem_block->nr_free++; + base += csv3_mem_block->block_size; + reserved_cnt++; + } + + reserved_size += size; + } + + pr_info("csv3_guest_rsv_mb: reserved %lu MB memory from memblock\n", + reserved_size / SZ_1M); + + return; + +fail_free: + /* find the head block, free the block */ + for (i = 0; i < reserved_cnt; i++) { + if (!csv3_mem_block->blocks[i].head) + continue; + base = csv3_mem_block->blocks[i].start; + size = csv3_mem_block->blocks[i].size; + memblock_phys_free(base, size); + } + + memblock_phys_free(__pa(csv3_mem_block), meta_size); + csv3_mem_block = NULL; +} + +static bool decrypt_mem_block(void) +{ + unsigned long size; + unsigned long virt; + phys_addr_t phys_addr; + int i, j; + + for (i = 0; i < csv3_mem_block->nr_blocks; i++) { + if (!csv3_mem_block->blocks[i].head) + continue; + phys_addr = csv3_mem_block->blocks[i].start; + size = csv3_mem_block->blocks[i].size; + virt = (unsigned long)phys_to_virt(phys_addr); + if (set_memory_decrypted(virt, size >> PAGE_SHIFT)) { + pr_err("csv3_guest_rsv_mb: decrypt pa 0x%llx size 0x%lx va 0x%llx failed\n", + phys_addr, size, (u64)virt); + break; + } + } + + if (i == csv3_mem_block->nr_blocks) + return true; + + // decrypt failed, restore to encrypted state + for (j = 0; j < i; j++) { + if (!csv3_mem_block->blocks[j].head) + continue; + + phys_addr = csv3_mem_block->blocks[j].start; + size = csv3_mem_block->blocks[j].size; + virt = (unsigned long)phys_to_virt(phys_addr); + set_memory_encrypted(virt, size >> PAGE_SHIFT); + } + + return false; +} + +phys_addr_t csv3_alloc_mem_block(void) +{ + phys_addr_t phys_addr = 0; + int i; + + if (!csv3_active()) + return 0; + + if (!csv3_mem_block) + return 0; + + mutex_lock(&lock); + + if (unlikely(csv3_mem_block->decrypt_failed)) { + mutex_unlock(&lock); + return 0; + } + + if (unlikely(!csv3_mem_block->decrypted)) { + if (!decrypt_mem_block()) { + csv3_mem_block->decrypt_failed = true; + mutex_unlock(&lock); + return 0; + } + + csv3_mem_block->decrypted = true; + } + + if (csv3_mem_block->nr_free == 0) + goto out; + + // iterate csv3_mem_block->blocks to find a free block. + for (i = 0; i < csv3_mem_block->nr_blocks; i++) { + if (csv3_mem_block->blocks[i].used) + continue; + + csv3_mem_block->blocks[i].used = true; + csv3_mem_block->nr_free--; + break; + } + + if (i == csv3_mem_block->nr_blocks) + goto out; + else + phys_addr = csv3_mem_block->blocks[i].start; +out: + mutex_unlock(&lock); + + pr_debug("csv3_guest_rsv_mb: allocate 0x%lx MB memory, pa 0x%llx\n", + phys_addr ? csv3_mem_block->block_size / SZ_1M : 0, phys_addr); + + return phys_addr; +} +EXPORT_SYMBOL_GPL(csv3_alloc_mem_block); + +void csv3_free_mem_block(phys_addr_t phys_addr) +{ + int i; + bool free_done = false; + + if (!csv3_active()) + return; + + if (!csv3_mem_block || !phys_addr) + return; + + mutex_lock(&lock); + + if (csv3_mem_block->nr_free >= csv3_mem_block->nr_blocks) { + mutex_unlock(&lock); + pr_err("csv3_guest_rsv_mb: all blocks are free, please check\n"); + return; + } + + // find the block and mark it as free. + for (i = 0; i < csv3_mem_block->nr_blocks; i++) { + if (!csv3_mem_block->blocks[i].used || + csv3_mem_block->blocks[i].start != phys_addr) + continue; + + csv3_mem_block->blocks[i].used = false; + + csv3_mem_block->nr_free++; + free_done = true; + break; + } + + mutex_unlock(&lock); + + if (free_done) { + pr_debug("csv3_guest_rsv_mb: free 0x%lx MB memory, pa 0x%llx\n", + csv3_mem_block->block_size / SZ_1M, phys_addr); + } else { + pr_err("csv3_guest_rsv_mb: fail to free 0x%lx MB memory, pa 0x%llx\n", + csv3_mem_block->block_size / SZ_1M, phys_addr); + } +} +EXPORT_SYMBOL_GPL(csv3_free_mem_block); + +size_t csv3_get_mem_block_size(void) +{ + if (!csv3_mem_block) + return 0; + + return csv3_mem_block->block_size; +} +EXPORT_SYMBOL_GPL(csv3_get_mem_block_size); + +void __init early_csv_guest_mem_init(void) +{ + csv3_reserve_mem_block(); +} +