Squash commits for public release

kernel/include/mem/bits/mmu.h

@@ -0,0 +1,42 @@
+#ifndef _KERNEL_MEM_BITS_MMU_H
+#define _KERNEL_MEM_BITS_MMU_H
+
+#include <libkern/types.h>
+
+enum MMU_FLAGS {
+    MMU_FLAG_PERM_WRITE = (1 << 0),
+    MMU_FLAG_PERM_READ = (1 << 1),
+    MMU_FLAG_PERM_EXEC = (1 << 2),
+    MMU_FLAG_UNCACHED = (1 << 3),
+    MMU_FLAG_NONPRIV = (1 << 4),
+    MMU_FLAG_INVALID = (1 << 5),
+    MMU_FLAG_COW = (1 << 6), // TODO: Remove this flag.
+    MMU_FLAG_HUGE_PAGE = (1 << 7),
+    MMU_FLAG_DEVICE = MMU_FLAG_PERM_READ | MMU_FLAG_PERM_WRITE | MMU_FLAG_UNCACHED,
+};
+typedef uint32_t mmu_flags_t;
+
+enum MMU_PF_INFO_FLAGS {
+    MMU_PF_INFO_ON_NOT_PRESENT = (1 << 0),
+    MMU_PF_INFO_ON_WRITE = (1 << 1),
+    MMU_PF_INFO_ON_NONPRIV_ACCESS = (1 << 2),
+    MMU_PF_INFO_SECURITY_VIOLATION = (1 << 3),
+};
+typedef uint32_t mmu_pf_info_flags_t;
+
+enum PTABLE_LEVELS {
+    PTABLE_LV0 = 0,
+    PTABLE_LV1 = 1,
+    PTABLE_LV2 = 2,
+    PTABLE_LV3 = 3,
+};
+typedef enum PTABLE_LEVELS ptable_lv_t;
+
+enum PTABLE_ENTITY_STATES {
+    PTABLE_ENTITY_PRESENT,
+    PTABLE_ENTITY_INVALID,
+    PTABLE_ENTITY_ALLOC, // For arm32 pspace, other targets never return this state.
+};
+typedef uint32_t ptable_state_t;
+
+#endif // _KERNEL_MEM_BITS_MMU_H

kernel/include/mem/bits/swap.h

@@ -0,0 +1,10 @@
+#ifndef _KERNEL_MEM_BITS_SWAP_H
+#define _KERNEL_MEM_BITS_SWAP_H
+
+enum SWAP_TYPE {
+    SWAP_TO_DEV,
+    SWAP_DROP,
+    SWAP_NOT_ALLOWED,
+};
+
+#endif // _KERNEL_MEM_BITS_SWAP_H

kernel/include/mem/bits/vm.h

@@ -0,0 +1,68 @@
+#ifndef _KERNEL_MEM_BITS_VM_H
+#define _KERNEL_MEM_BITS_VM_H
+
+#include <libkern/types.h>
+#include <mem/bits/mmu.h>
+#include <platform/generic/vmm/consts.h>
+#include <platform/generic/vmm/mmu.h>
+
+#ifndef VMM_LV2_ENTITY_COUNT
+#define VMM_LV2_ENTITY_COUNT (1)
+#endif
+
+#ifndef VMM_LV3_ENTITY_COUNT
+#define VMM_LV3_ENTITY_COUNT (1)
+#endif
+
+#ifndef PTABLE_LV2_VADDR_OFFSET
+#define PTABLE_LV2_VADDR_OFFSET (32)
+#endif
+
+#ifndef PTABLE_LV3_VADDR_OFFSET
+#define PTABLE_LV3_VADDR_OFFSET (32)
+#endif
+
+#define PTABLE_ENTITY_COUNT(lv) (ptable_entity_count_at_level[lv])
+#define PTABLE_SIZE(lv) (ptable_size_at_level[lv])
+#define IS_INDIVIDUAL_PER_DIR(index) (index < PTABLE_TOP_KERNEL_OFFSET || (index == VMM_OFFSET_IN_DIRECTORY(pspace_zone.start)))
+
+typedef struct {
+    ptable_entity_t entities[1];
+} ptable_t;
+
+static const size_t ptable_entity_count_at_level[] = {
+    [PTABLE_LV0] = VMM_LV0_ENTITY_COUNT,
+    [PTABLE_LV1] = VMM_LV1_ENTITY_COUNT,
+    [PTABLE_LV2] = VMM_LV2_ENTITY_COUNT,
+    [PTABLE_LV3] = VMM_LV3_ENTITY_COUNT,
+};
+
+static const size_t ptable_size_at_level[] = {
+    [PTABLE_LV0] = VMM_LV0_ENTITY_COUNT * sizeof(ptable_entity_t),
+    [PTABLE_LV1] = VMM_LV1_ENTITY_COUNT * sizeof(ptable_entity_t),
+    [PTABLE_LV2] = VMM_LV2_ENTITY_COUNT * sizeof(ptable_entity_t),
+    [PTABLE_LV3] = VMM_LV3_ENTITY_COUNT * sizeof(ptable_entity_t),
+};
+
+static const size_t ptable_entity_vaddr_offset_at_level[] = {
+    [PTABLE_LV0] = PTABLE_LV0_VADDR_OFFSET,
+    [PTABLE_LV1] = PTABLE_LV1_VADDR_OFFSET,
+    [PTABLE_LV2] = PTABLE_LV2_VADDR_OFFSET,
+    [PTABLE_LV3] = PTABLE_LV3_VADDR_OFFSET,
+};
+
+#define VM_VADDR_OFFSET_AT_LEVEL(vaddr, lv) ((vaddr >> ptable_entity_vaddr_offset_at_level[lv]) % ptable_entity_count_at_level[lv])
+
+static inline ptable_lv_t lower_level(ptable_lv_t lv)
+{
+    ASSERT(lv != PTABLE_LV0);
+    return lv - 1;
+}
+
+static inline ptable_lv_t upper_level(ptable_lv_t lv)
+{
+    ASSERT(lv != PTABLE_LV_TOP);
+    return lv + 1;
+}
+
+#endif // _KERNEL_MEM_BITS_VM_H

kernel/include/mem/bits/zone.h

@@ -0,0 +1,18 @@
+#ifndef _KERNEL_MEM_BITS_ZONE_H
+#define _KERNEL_MEM_BITS_ZONE_H
+
+#include <mem/bits/mmu.h>
+
+enum ZONE_TYPES {
+    ZONE_TYPE_NULL = 0x0,
+    ZONE_TYPE_CODE = 0x1,
+    ZONE_TYPE_DATA = 0x2,
+    ZONE_TYPE_STACK = 0x4,
+    ZONE_TYPE_BSS = 0x8,
+    ZONE_TYPE_DEVICE = 0x10,
+    ZONE_TYPE_MAPPED = 0x20,
+    ZONE_TYPE_MAPPED_FILE_PRIVATLY = 0x40,
+    ZONE_TYPE_MAPPED_FILE_SHAREDLY = 0x80,
+};
+
+#endif // _KERNEL_MEM_BITS_ZONE_H

kernel/include/mem/boot.h

@@ -0,0 +1,45 @@
+#ifndef _KERNEL_MEM_BOOT_H
+#define _KERNEL_MEM_BOOT_H
+
+#include <libkern/types.h>
+
+enum MEMORY_LAYOUT_FLAGS {
+    MEMORY_LAYOUT_FLAG_TERMINATE = (1 << 0),
+};
+
+struct memory_layout {
+    uint64_t base;
+    uint64_t size;
+    uint32_t flags;
+};
+typedef struct memory_layout memory_layout_t;
+
+struct memory_boot_desc {
+    uint64_t ram_base;
+    uint64_t ram_size;
+    memory_layout_t* reserved_areas;
+};
+typedef struct memory_boot_desc memory_boot_desc_t;
+
+struct fb_boot_desc {
+    uintptr_t vaddr;
+    uintptr_t paddr;
+    size_t width;
+    size_t height;
+    size_t pixels_per_row;
+};
+typedef struct fb_boot_desc fb_boot_desc_t;
+
+struct boot_args {
+    size_t paddr;
+    size_t vaddr;
+    size_t kernel_data_size;
+    memory_boot_desc_t mem_boot_desc;
+    fb_boot_desc_t fb_boot_desc;
+    void* devtree;
+    char cmd_args[32];
+    char init_process[32];
+};
+typedef struct boot_args boot_args_t;
+
+#endif // _KERNEL_MEM_BOOT_H

kernel/include/mem/kmalloc.h

@@ -0,0 +1,18 @@
+#ifndef _KERNEL_MEM_KMALLOC_H
+#define _KERNEL_MEM_KMALLOC_H
+
+#include <libkern/types.h>
+#include <mem/vmm.h>
+
+#define KMALLOC_SPACE_SIZE (4 * MB)
+#define KMALLOC_BLOCK_SIZE 32
+
+void kmalloc_init();
+void* kmalloc(size_t size);
+void* kmalloc_aligned(size_t size, size_t alignment);
+void* kmalloc_page_aligned();
+void kfree(void* ptr);
+void kfree_aligned(void* ptr);
+void* krealloc(void* ptr, size_t size);
+
+#endif // _KERNEL_MEM_KMALLOC_H

kernel/include/mem/kmemzone.h

@@ -0,0 +1,35 @@
+#ifndef _KERNEL_MEM_KMEMZONE_H
+#define _KERNEL_MEM_KMEMZONE_H
+
+#include <libkern/types.h>
+#include <mem/bits/zone.h>
+
+extern char __text_start[];
+extern char __text_end[];
+extern char __rodata_start[];
+extern char __rodata_end[];
+extern char __data_start[];
+extern char __data_end[];
+extern char __bss_start[];
+extern char __bss_end[];
+extern char __stack_start[];
+extern char __stack_end[];
+extern char __end[];
+
+struct __kmemzone {
+    union {
+        uintptr_t start;
+        uint8_t* ptr;
+    };
+    size_t len;
+};
+typedef struct __kmemzone kmemzone_t;
+
+void kmemzone_init();
+void kmemzone_init_stage2();
+
+kmemzone_t kmemzone_new(size_t size);
+kmemzone_t kmemzone_new_aligned(size_t size, size_t alignment);
+int kmemzone_free(kmemzone_t zone);
+
+#endif // _KERNEL_MEM_KMEMZONE_H

kernel/include/mem/kswapd.h

@@ -0,0 +1,6 @@
+#ifndef _KERNEL_MEM_KSWAPD_H
+#define _KERNEL_MEM_KSWAPD_H
+
+void kswapd();
+
+#endif // _KERNEL_MEM_KSWAPD_H

kernel/include/mem/memzone.h

@@ -0,0 +1,34 @@
+#ifndef _KERNEL_MEM_MEMZONE_H
+#define _KERNEL_MEM_MEMZONE_H
+
+#include <algo/dynamic_array.h>
+#include <fs/vfs.h>
+#include <libkern/types.h>
+#include <mem/bits/zone.h>
+
+struct vm_ops;
+struct memzone {
+    uintptr_t vaddr;
+    size_t len;
+    mmu_flags_t mmu_flags;
+    uint32_t type;
+    file_t* file;
+    off_t file_offset;
+    size_t file_size;
+    struct vm_ops* ops;
+};
+typedef struct memzone memzone_t;
+
+struct vm_address_space;
+memzone_t* memzone_new(struct vm_address_space* vm_aspace, size_t start, size_t len);
+memzone_t* memzone_new_random(struct vm_address_space* vm_aspace, size_t len);
+memzone_t* memzone_new_random_backward(struct vm_address_space* vm_aspace, size_t len);
+memzone_t* memzone_find(struct vm_address_space* vm_aspace, size_t addr);
+memzone_t* memzone_find_no_proc(dynamic_array_t* zones, size_t addr);
+memzone_t* memzone_split(struct vm_address_space* vm_aspace, memzone_t* zone, uintptr_t addr);
+int memzone_free_no_proc(dynamic_array_t*, memzone_t*);
+int memzone_free(struct vm_address_space* vm_aspace, memzone_t*);
+
+int memzone_copy(struct vm_address_space* to_vm_aspace, struct vm_address_space* from_vm_aspace);
+
+#endif // _KERNEL_MEM_MEMZONE_H

kernel/include/mem/pmm.h

@@ -0,0 +1,40 @@
+#ifndef _KERNEL_MEM_PMM_H
+#define _KERNEL_MEM_PMM_H
+
+#include <algo/bitmap.h>
+#include <libkern/types.h>
+#include <mem/boot.h>
+#include <platform/generic/pmm/settings.h>
+
+struct pmm_state {
+    size_t kernel_va_base;
+    size_t kernel_data_size; // Kernel + MAT size.
+
+    bitmap_t mat;
+    boot_args_t* boot_args;
+
+    size_t ram_size;
+    size_t ram_offset;
+
+    size_t max_blocks;
+    size_t used_blocks;
+};
+typedef struct pmm_state pmm_state_t;
+
+void pmm_setup(boot_args_t* boot_args);
+
+void* pmm_alloc(size_t size);
+void* pmm_alloc_aligned(size_t size, size_t alignment);
+int pmm_free(void* ptr, size_t size);
+
+size_t pmm_get_ram_size();
+size_t pmm_get_max_blocks();
+size_t pmm_get_used_blocks();
+size_t pmm_get_free_blocks();
+size_t pmm_get_block_size();
+size_t pmm_get_ram_in_kb();
+size_t pmm_get_free_space_in_kb();
+const pmm_state_t* pmm_get_state();
+const boot_args_t* boot_args();
+
+#endif // _KERNEL_MEM_PMM_H

kernel/include/mem/swapfile.h

@@ -0,0 +1,15 @@
+#ifndef _KERNEL_MEM_SWAPFILE_H
+#define _KERNEL_MEM_SWAPFILE_H
+
+#include <libkern/libkern.h>
+#include <mem/bits/mmu.h>
+#include <mem/bits/swap.h>
+#include <platform/generic/vmm/consts.h>
+
+int swapfile_init();
+int swapfile_new_ref(int id);
+int swapfile_rem_ref(int id);
+int swapfile_load(uintptr_t vaddr, int id);
+int swapfile_store(uintptr_t vaddr);
+
+#endif // _KERNEL_MEM_SWAPFILE_H

kernel/include/mem/vm_address_space.h

@@ -0,0 +1,20 @@
+#ifndef _KERNEL_MEM_VM_ADDRESS_SPACE_H
+#define _KERNEL_MEM_VM_ADDRESS_SPACE_H
+
+#include <algo/dynamic_array.h>
+#include <libkern/libkern.h>
+#include <libkern/lock.h>
+#include <mem/bits/vm.h>
+
+struct vm_address_space {
+    ptable_t* pdir;
+    dynamic_array_t zones;
+    int count;
+    spinlock_t lock;
+};
+typedef struct vm_address_space vm_address_space_t;
+
+vm_address_space_t* vm_address_space_alloc();
+int vm_address_space_free(vm_address_space_t* old);
+
+#endif // _KERNEL_MEM_VM_ADDRESS_SPACE_H

kernel/include/mem/vm_alloc.h

@@ -0,0 +1,21 @@
+#ifndef _KERNEL_MEM_VM_ALLOC_H
+#define _KERNEL_MEM_VM_ALLOC_H
+
+#include <libkern/libkern.h>
+#include <mem/bits/vm.h>
+#include <mem/kmemzone.h>
+
+uintptr_t vm_alloc_pdir_paddr();
+uintptr_t vm_alloc_ptable_paddr(ptable_lv_t lv);
+uintptr_t vm_alloc_ptables_to_cover_page();
+void vm_free_ptables_to_cover_page(uintptr_t addr);
+uintptr_t vm_alloc_page_paddr();
+void vm_free_page_paddr(uintptr_t addr);
+
+int vm_alloc_mapped_zone(size_t size, size_t alignment, kmemzone_t* zone, mmu_flags_t flags);
+int vm_free_mapped_zone(kmemzone_t zone);
+
+ptable_t* vm_alloc_ptable_lv_top();
+void vm_free_ptable_lv_top(ptable_t* pdir);
+
+#endif // _KERNEL_MEM_VM_ALLOC_H

kernel/include/mem/vm_pspace.h

@@ -0,0 +1,27 @@
+#ifndef _KERNEL_MEM_VM_PSPACE_H
+#define _KERNEL_MEM_VM_PSPACE_H
+
+#include <libkern/libkern.h>
+#include <mem/bits/vm.h>
+#include <mem/boot.h>
+
+struct vm_address_space;
+
+ptable_t* vm_pspace_get_nth_active_ptable(size_t n, ptable_lv_t lv);
+ptable_t* vm_pspace_get_vaddr_of_active_ptable(uintptr_t vaddr, ptable_lv_t lv);
+
+void vm_pspace_init(boot_args_t* args);
+void vm_pspace_gen(ptable_t* pdir);
+void vm_pspace_free(ptable_t* pdir);
+int vm_pspace_on_ptable_mapped(uintptr_t vaddr, uintptr_t ptable_paddr, ptable_lv_t lv);
+
+ptable_t* vm_get_table(uintptr_t vaddr, ptable_lv_t lv);
+ptable_entity_t* vm_get_entity(uintptr_t vaddr, ptable_lv_t lv);
+int vm_pspace_free_address_space_locked(struct vm_address_space* vm_aspace);
+
+static inline ptable_entity_t* vm_lookup(ptable_t* table, ptable_lv_t lv, uintptr_t vaddr)
+{
+    return &table->entities[VM_VADDR_OFFSET_AT_LEVEL(vaddr, lv)];
+}
+
+#endif // _KERNEL_MEM_VM_PSPACE_H

kernel/include/mem/vmm.h

@@ -0,0 +1,73 @@
+#ifndef _KERNEL_MEM_VMM_H
+#define _KERNEL_MEM_VMM_H
+
+#include <libkern/libkern.h>
+#include <mem/bits/mmu.h>
+#include <mem/bits/swap.h>
+#include <mem/bits/vm.h>
+#include <mem/boot.h>
+#include <mem/pmm.h>
+#include <mem/vm_address_space.h>
+#include <platform/generic/vmm/consts.h>
+#include <platform/generic/vmm/mmu.h>
+
+#define IS_KERNEL_VADDR(vaddr) (vaddr >= KERNEL_BASE)
+#define IS_USER_VADDR(vaddr) (vaddr <= USER_HIGH)
+
+#define VMM_INVALID_PADDR ((uintptr_t)(-1))
+#define VMM_INVALID_VADDR ((uintptr_t)(-1))
+
+struct memzone;
+struct vm_ops {
+    int (*load_page_content)(struct memzone* zone, uintptr_t vaddr);
+    int (*swap_page_mode)(struct memzone* zone, uintptr_t vaddr);
+    int (*restore_swapped_page)(struct memzone* zone, uintptr_t vaddr);
+};
+typedef struct vm_ops vm_ops_t;
+
+/**
+ * PUBLIC FUNCTIONS
+ */
+
+struct dynamic_array;
+
+int vmm_setup(boot_args_t* boot_args);
+int vmm_setup_secondary_cpu();
+
+int vmm_free_address_space(vm_address_space_t* vm_aspace);
+
+int vmm_alloc_page(uintptr_t vaddr, mmu_flags_t mmu_flags);
+int vmm_tune_page(uintptr_t vaddr, mmu_flags_t mmu_flags);
+int vmm_tune_pages(uintptr_t vaddr, size_t length, mmu_flags_t mmu_flags);
+
+int vmm_map_page(uintptr_t vaddr, uintptr_t paddr, mmu_flags_t mmu_flags);
+int vmm_map_pages(uintptr_t vaddr, uintptr_t paddr, size_t n_pages, mmu_flags_t mmu_flags);
+int vmm_unmap_page(uintptr_t vaddr);
+int vmm_unmap_pages(uintptr_t vaddr, size_t n_pages);
+int vmm_swap_page(ptable_entity_t* page_desc, struct memzone* zone, uintptr_t vaddr);
+
+int vmm_map_page_locked(uintptr_t vaddr, uintptr_t paddr, mmu_flags_t mmu_flags);
+int vmm_map_pages_locked(uintptr_t vaddr, uintptr_t paddr, size_t n_pages, mmu_flags_t mmu_flags);
+int vmm_unmap_page_locked(uintptr_t vaddr);
+int vmm_unmap_pages_locked(uintptr_t vaddr, size_t n_pages);
+
+vm_address_space_t* vmm_new_address_space();
+vm_address_space_t* vmm_new_forked_address_space();
+
+bool vmm_is_copy_on_write(uintptr_t vaddr);
+
+void vmm_ensure_writing_to_active_address_space(uintptr_t dest_vaddr, size_t length);
+void vmm_ensure_reading_from_active_address_space(uintptr_t dest_vaddr, size_t length);
+void vmm_copy_to_address_space(vm_address_space_t* vm_aspace, void* src, uintptr_t dest_vaddr, size_t length);
+
+vm_address_space_t* vmm_get_active_address_space();
+vm_address_space_t* vmm_get_kernel_address_space();
+
+int vmm_switch_address_space_locked(vm_address_space_t* vm_aspace);
+int vmm_switch_address_space(vm_address_space_t* vm_aspace);
+
+int vmm_page_fault_handler(arch_pf_info_t info, uintptr_t vaddr);
+
+uintptr_t vmm_convert_kernel_vaddr_to_paddr(uintptr_t vaddr);
+
+#endif // _KERNEL_MEM_VMM_H