Machine snapshots: serialize RAM, harts and devices

include/rvvm/rvvm.h

@@ -184,11 +184,8 @@ typedef struct {
     /** Called on machine reset */
     void (*reset)(rvvm_mmio_dev_t* dev);
 
-    /*
-     * TODO
-     * void (*suspend)(rvvm_mmio_dev_t* dev, rvvm_state_t* state);
-     * void (*resume)(rvvm_mmio_dev_t* dev, rvvm_state_t* state);
-     */
+    /** Called to serialize/deserialize device state on a paused machine */
+    void (*suspend)(rvvm_mmio_dev_t* dev, rvvm_snapshot_t* snap, bool resume);
 
 } rvvm_mmio_type_t;
 

include/rvvm/rvvm_snapshot.h

@@ -119,6 +119,25 @@ RVVM_PUBLIC bool rvvm_snapshot_host(rvvm_snapshot_t* snap, void* data, size_t si
  */
 #define rvvm_snapshot_field(snap, field) rvvm_snapshot_host(snap, &(field), sizeof(field))
 
+/**
+ * Serialize or deserialize whole machine state to/from a snapshot
+ *
+ * The machine must be paused (See rvvm_pause_machine()), and is left paused
+ *
+ * Writes or reads RAM, hart state and every device state in a fixed section order
+ *
+ * A machine is only resumable from a snapshot of an identically constructed machine:
+ * same RAM size, hart count and device set, as built by the same machine description
+ *
+ * Block devices backing the machine storage are not part of a snapshot and must be
+ * restored to their matching state by the caller
+ *
+ * \param machine Machine handle
+ * \param snap    Snapshot handle, opened for reading or writing
+ * \return        Success (No IO error or corruption)
+ */
+RVVM_PUBLIC bool rvvm_machine_snapshot(rvvm_machine_t* machine, rvvm_snapshot_t* snap);
+
 /** @}*/
 
 RVVM_EXTERN_C_END

src/core/rvvm_pci.c

@@ -979,10 +979,66 @@ static void pci_ecam_cleanup(rvvm_reg_dev_t* ecam)
     free(bus);
 }
 
+static void pci_func_suspend(rvvm_snapshot_t* snap, rvvm_pci_func_t* func)
+{
+    rvvm_snapshot_field(snap, func->command);
+    rvvm_snapshot_field(snap, func->status);
+    rvvm_snapshot_field(snap, func->irq_line);
+    rvvm_snapshot_field(snap, func->bridge_io);
+    rvvm_snapshot_field(snap, func->bridge_mem);
+    rvvm_snapshot_field(snap, func->pm_csr);
+
+    rvvm_snapshot_field(snap, func->msi_ctl);
+    rvvm_snapshot_field(snap, func->msi_addr_low);
+    rvvm_snapshot_field(snap, func->msi_addr_high);
+    rvvm_snapshot_field(snap, func->msi_data);
+    rvvm_snapshot_field(snap, func->msi_mask);
+    rvvm_snapshot_field(snap, func->msi_pending);
+
+    rvvm_snapshot_field(snap, func->msix_ctl);
+    rvvm_snapshot_field(snap, func->msix_bar);
+    for (size_t i = 0; i < PCI_MSIX_BAR_SIZE; ++i) {
+        rvvm_snapshot_field(snap, func->msix[i]);
+    }
+
+    // Where the guest mapped each BAR, which the regions are placed at on resume
+    for (size_t bar_id = 0; bar_id < PCI_FUNC_BARS; ++bar_id) {
+        rvvm_reg_desc_t desc = ZERO_INIT;
+        uint64_t        addr = 0;
+        bool            bar  = rvvm_region_get_desc(func->bar[bar_id], &desc);
+        if (bar) {
+            addr = desc.addr;
+        }
+        rvvm_snapshot_field(snap, addr);
+        if (bar && !rvvm_snapshot_writing(snap) && desc.addr != addr) {
+            desc.addr = addr;
+            rvvm_region_set_desc(func->bar[bar_id], &desc);
+        }
+    }
+}
+
+static void pci_ecam_suspend(rvvm_reg_dev_t* ecam, rvvm_snapshot_t* snap, bool resume)
+{
+    if (snap) {
+        rvvm_pci_bus_t* bus = rvvm_region_data(ecam);
+        rvvm_snapshot_section(snap, "pci-ecam");
+        vector_foreach (bus->dev, i) {
+            rvvm_pci_dev_t* dev = vector_at(bus->dev, i);
+            for (size_t fn_id = 0; fn_id < PCI_DEV_FUNCS; ++fn_id) {
+                if (dev->func[fn_id]) {
+                    pci_func_suspend(snap, dev->func[fn_id]);
+                }
+            }
+        }
+    }
+    UNUSED(resume);
+}
+
 static const rvvm_reg_type_t pci_ecam_type = {
     .name     = "pci-ecam",
     .read     = pci_ecam_read,
     .write    = pci_ecam_write,
+    .suspend  = pci_ecam_suspend,
     .cleanup  = pci_ecam_cleanup,
     .min_size = 4,
     .max_size = 4,

src/core/rvvm_region.c

@@ -72,6 +72,14 @@ static void rvvm_region_legacy_reset(rvvm_mmio_dev_t* mmio)
     }
 }
 
+static void rvvm_region_legacy_suspend(rvvm_mmio_dev_t* mmio, rvvm_snapshot_t* snap, bool resume)
+{
+    rvvm_reg_dev_t* dev = (rvvm_reg_dev_t*)mmio->data;
+    if (dev->desc.type && dev->desc.type->suspend) {
+        dev->desc.type->suspend(dev, snap, resume);
+    }
+}
+
 RVVM_PUBLIC rvvm_reg_dev_t* rvvm_region_init(rvvm_machine_t* machine, const rvvm_reg_desc_t* desc)
 {
     rvvm_reg_dev_t* dev = safe_new_obj(rvvm_reg_dev_t);
@@ -107,6 +115,7 @@ RVVM_PUBLIC rvvm_reg_dev_t* rvvm_region_init(rvvm_machine_t* machine, const rvvm
         mmio_type->remove     = rvvm_region_legacy_remove;
         mmio_type->update     = rvvm_region_legacy_update;
         mmio_type->reset      = rvvm_region_legacy_reset;
+        mmio_type->suspend    = rvvm_region_legacy_suspend;
         rvvm_mmio_dev_t* mmio = rvvm_attach_mmio(machine, &mmio_desc);
         if (mmio) {
             dev->mmio = mmio;

src/core/rvvm_snapshot.c

@@ -10,11 +10,21 @@ file, You can obtain one at https://mozilla.org/MPL/2.0/.
 #include <rvvm/rvvm_blk.h>
 #include <rvvm/rvvm_snapshot.h>
 
+#include <cpu/riscv_cpu.h>
+#include <cpu/riscv_hart.h>
+#include <cpu/riscv_mmu.h>
+#include <util/rvtimer.h>
+#include <util/vector.h>
+
 #include "mem_ops.h"
+#include "rvvm.h"
 #include "utils.h"
 
 #define SNAP_MAGIC "\x7Frvvm-snapshot0\xFF"
 
+// Bumped whenever the machine, hart or device state layout changes
+#define RVVM_SNAPSHOT_VERSION 1
+
 PUSH_OPTIMIZATION_SIZE
 
 struct rvvm_snapshot {
@@ -162,4 +172,158 @@ RVVM_PUBLIC bool rvvm_snapshot_host(rvvm_snapshot_t* snap, void* data, size_t si
     return false;
 }
 
+static bool rvvm_snapshot_hart(rvvm_snapshot_t* snap, rvvm_hart_t* vm)
+{
+    bool ok = true;
+    for (size_t i = 0; i < RISCV_REGS_MAX; ++i) {
+        ok &= rvvm_snapshot_field(snap, vm->registers[i]);
+    }
+
+#if defined(USE_FPU)
+    for (size_t i = 0; i < RISCV_FPU_REGS_MAX; ++i) {
+        ok &= rvvm_snapshot_host(snap, &vm->fpu_registers[i], sizeof(vm->fpu_registers[i]));
+    }
+#endif
+
+#if defined(USE_RVV)
+    ok &= rvvm_snapshot_data(snap, vm->rvv_state, sizeof(vm->rvv_state));
+#endif
+
+    ok &= rvvm_snapshot_field(snap, vm->root_page_table);
+    ok &= rvvm_snapshot_field(snap, vm->mmu_mode);
+    ok &= rvvm_snapshot_field(snap, vm->priv_mode);
+    ok &= rvvm_snapshot_field(snap, vm->trap);
+    ok &= rvvm_snapshot_field(snap, vm->trap_pc);
+    ok &= rvvm_snapshot_field(snap, vm->lrsc);
+    ok &= rvvm_snapshot_field(snap, vm->lrsc_addr);
+    ok &= rvvm_snapshot_field(snap, vm->lrsc_cas);
+
+    ok &= rvvm_snapshot_field(snap, vm->csr.status);
+    ok &= rvvm_snapshot_field(snap, vm->csr.ie);
+    ok &= rvvm_snapshot_field(snap, vm->csr.ip);
+    ok &= rvvm_snapshot_field(snap, vm->csr.isa);
+    for (size_t i = 0; i < RISCV_PRIVS_MAX; ++i) {
+        ok &= rvvm_snapshot_field(snap, vm->csr.edeleg[i]);
+        ok &= rvvm_snapshot_field(snap, vm->csr.ideleg[i]);
+        ok &= rvvm_snapshot_field(snap, vm->csr.tvec[i]);
+        ok &= rvvm_snapshot_field(snap, vm->csr.scratch[i]);
+        ok &= rvvm_snapshot_field(snap, vm->csr.epc[i]);
+        ok &= rvvm_snapshot_field(snap, vm->csr.cause[i]);
+        ok &= rvvm_snapshot_field(snap, vm->csr.tval[i]);
+        ok &= rvvm_snapshot_field(snap, vm->csr.iselect[i]);
+        ok &= rvvm_snapshot_field(snap, vm->csr.counteren[i]);
+        ok &= rvvm_snapshot_field(snap, vm->csr.envcfg[i]);
+    }
+    ok &= rvvm_snapshot_field(snap, vm->csr.mseccfg);
+    ok &= rvvm_snapshot_field(snap, vm->csr.fcsr);
+    ok &= rvvm_snapshot_field(snap, vm->csr.vcsr);
+    ok &= rvvm_snapshot_field(snap, vm->csr.vtype);
+    ok &= rvvm_snapshot_field(snap, vm->csr.hartid);
+
+    // AIA register files are allocated in pairs for M/S modes
+    uint8_t aia = !!vm->aia;
+    ok &= rvvm_snapshot_field(snap, aia);
+    if (aia) {
+        if (!vm->aia) {
+            riscv_hart_aia_init(vm);
+        }
+        for (size_t i = 0; i < 2; ++i) {
+            ok &= rvvm_snapshot_field(snap, vm->aia[i].eidelivery);
+            ok &= rvvm_snapshot_field(snap, vm->aia[i].eithreshold);
+            for (size_t j = 0; j < RVVM_AIA_ARR_LEN; ++j) {
+                ok &= rvvm_snapshot_field(snap, vm->aia[i].eip[j]);
+                ok &= rvvm_snapshot_field(snap, vm->aia[i].eie[j]);
+            }
+        }
+    }
+
+    uint64_t mtimecmp = rvtimecmp_get(&vm->mtimecmp);
+    uint64_t stimecmp = rvtimecmp_get(&vm->stimecmp);
+    ok &= rvvm_snapshot_field(snap, mtimecmp);
+    ok &= rvvm_snapshot_field(snap, stimecmp);
+    ok &= rvvm_snapshot_field(snap, vm->pending_irqs);
+
+    if (!rvvm_snapshot_writing(snap)) {
+        rvtimecmp_set(&vm->mtimecmp, mtimecmp);
+        rvtimecmp_set(&vm->stimecmp, stimecmp);
+
+        // Address translation and compiled code are caches over guest memory,
+        // which the loaded state has no relation to
+        riscv_tlb_flush(vm);
+#if defined(USE_JIT)
+        if (vm->jit_enabled) {
+            riscv_jit_flush_cache(vm);
+        }
+#endif
+        riscv_hart_check_timer(vm);
+    }
+    return ok;
+}
+
+RVVM_PUBLIC bool rvvm_machine_snapshot(rvvm_machine_t* machine, rvvm_snapshot_t* snap)
+{
+    if (!machine || !snap) {
+        return false;
+    }
+    if (atomic_load_uint32(&machine->running)) {
+        rvvm_error("Snapshot of a running machine, pause it first");
+        return false;
+    }
+
+    bool     resume     = !rvvm_snapshot_writing(snap);
+    uint64_t mem_size   = machine->mem.size;
+    uint64_t hart_count = vector_size(machine->harts);
+    uint64_t freq       = machine->timer.freq;
+    uint64_t time       = rvtimer_get(&machine->timer);
+    uint8_t  rv64       = machine->rv64;
+
+    uint32_t version = RVVM_SNAPSHOT_VERSION;
+
+    bool ok = rvvm_snapshot_section(snap, "machine");
+    ok &= rvvm_snapshot_field(snap, version);
+    ok &= rvvm_snapshot_field(snap, mem_size);
+    ok &= rvvm_snapshot_field(snap, hart_count);
+    ok &= rvvm_snapshot_field(snap, rv64);
+    ok &= rvvm_snapshot_field(snap, freq);
+    ok &= rvvm_snapshot_field(snap, time);
+    ok &= rvvm_snapshot_field(snap, machine->power_state);
+    if (!ok) {
+        return false;
+    }
+
+    if (resume) {
+        if (version != RVVM_SNAPSHOT_VERSION) {
+            rvvm_error("Snapshot version %u is not supported", version);
+            return false;
+        }
+
+        // A snapshot only fits the machine it was taken from
+        if (mem_size != machine->mem.size || hart_count != vector_size(machine->harts) || rv64 != machine->rv64) {
+            rvvm_error("Snapshot does not match this machine");
+            return false;
+        }
+
+        // The timebase is set up by a machine reset, which resuming skips over
+        rvtimer_init(&machine->timer, freq);
+        rvtimer_rebase(&machine->timer, time);
+    }
+
+    ok &= rvvm_snapshot_section(snap, "ram");
+    ok &= rvvm_snapshot_data(snap, machine->mem.data, machine->mem.size);
+
+    vector_foreach (machine->harts, i) {
+        ok &= rvvm_snapshot_section(snap, "hart");
+        ok &= rvvm_snapshot_hart(snap, vector_at(machine->harts, i));
+    }
+
+    vector_foreach (machine->mmio_devs, i) {
+        rvvm_mmio_dev_t* dev = vector_at(machine->mmio_devs, i);
+        if (dev->type && dev->type->suspend) {
+            dev->type->suspend(dev, snap, resume);
+        }
+    }
+
+    return ok;
+}
+
 POP_OPTIMIZATION_SIZE

src/devices/i2c-oc.c

@@ -7,6 +7,8 @@ License, v. 2.0. If a copy of the MPL was not distributed with this
 file, You can obtain one at https://mozilla.org/MPL/2.0/.
 */
 
+#include <rvvm/rvvm_snapshot.h>
+
 #include "i2c-oc.h"
 #include "fdtlib.h"
 #include "mem_ops.h"
@@ -188,9 +190,25 @@ static void i2c_oc_remove(rvvm_mmio_dev_t* dev)
     free(bus);
 }
 
+static void i2c_oc_suspend(rvvm_mmio_dev_t* dev, rvvm_snapshot_t* snap, bool resume)
+{
+    if (snap) {
+        i2c_bus_t* bus = dev->data;
+        rvvm_snapshot_section(snap, "i2c_opencores");
+        rvvm_snapshot_field(snap, bus->sel_addr);
+        rvvm_snapshot_field(snap, bus->clock);
+        rvvm_snapshot_field(snap, bus->control);
+        rvvm_snapshot_field(snap, bus->status);
+        rvvm_snapshot_field(snap, bus->tx_byte);
+        rvvm_snapshot_field(snap, bus->rx_byte);
+    }
+    UNUSED(resume);
+}
+
 static rvvm_mmio_type_t i2c_oc_dev_type = {
-    .name   = "i2c_opencores",
-    .remove = i2c_oc_remove,
+    .name    = "i2c_opencores",
+    .remove  = i2c_oc_remove,
+    .suspend = i2c_oc_suspend,
 };
 
 PUBLIC i2c_bus_t* i2c_oc_init(rvvm_machine_t* machine, rvvm_addr_t addr, rvvm_intc_t* intc, rvvm_irq_t irq)

src/devices/nvme.c

@@ -1138,10 +1138,46 @@ static void nvme_cleanup(rvvm_reg_dev_t* dev)
     free(nvme);
 }
 
+static void nvme_queue_suspend(rvvm_snapshot_t* snap, nvme_queue_t* queue)
+{
+    rvvm_snapshot_field(snap, queue->addr_l);
+    rvvm_snapshot_field(snap, queue->addr_h);
+    rvvm_snapshot_field(snap, queue->size);
+    rvvm_snapshot_field(snap, queue->head);
+    rvvm_snapshot_field(snap, queue->tail);
+    rvvm_snapshot_field(snap, queue->data);
+}
+
+static void nvme_suspend(rvvm_reg_dev_t* dev, rvvm_snapshot_t* snap, bool resume)
+{
+    nvme_dev_t* nvme = rvvm_region_data(dev);
+
+    // Commands are processed by worker threads, which must be done before
+    // the queues are read out or overwritten
+    while (atomic_load_uint32(&nvme->threads)) {
+        rvvm_sched_yield();
+    }
+
+    if (snap) {
+        rvvm_snapshot_section(snap, "nvme");
+        for (size_t qid = 0; qid < STATIC_ARRAY_SIZE(nvme->sq); ++qid) {
+            nvme_queue_suspend(snap, &nvme->sq[qid]);
+        }
+        for (size_t qid = 0; qid < STATIC_ARRAY_SIZE(nvme->cq); ++qid) {
+            nvme_queue_suspend(snap, &nvme->cq[qid]);
+        }
+        rvvm_snapshot_field(snap, nvme->conf);
+        rvvm_snapshot_field(snap, nvme->irq_mask);
+        rvvm_snapshot_field(snap, nvme->temp_thresh);
+    }
+    UNUSED(resume);
+}
+
 static rvvm_reg_type_t nvme_type = {
     .name     = "nvme",
     .read     = nvme_pci_read,
     .write    = nvme_pci_write,
+    .suspend  = nvme_suspend,
     .cleanup  = nvme_cleanup,
     .min_size = 4,
     .max_size = 4,