Add 'hotswap/' from commit '5487901d10f5a7ed200b2fbfe0a5b5cd730ba3a9'

git-subtree-dir: hotswap
git-subtree-mainline: 614695d55b
git-subtree-split: 5487901d10

hotswap/.gitignore

@@ -0,0 +1,5 @@
+/.idea
+/.envrc
+
+/zig-out/
+/.zig-cache/

hotswap/build.zig

@@ -0,0 +1,47 @@
+const std = @import("std");
+
+pub fn installModule(b: *std.Build, plug: *std.Build.Step.Compile) void {
+    std.debug.assert(plug.isDynamicLibrary());
+    const install = b.addInstallArtifact(plug, .{
+        .dest_dir = .{ .override = .{ .custom = "mods" } },
+    });
+    b.getInstallStep().dependOn(&install.step);
+}
+
+pub fn build(b: *std.Build) void {
+    const target = b.standardTargetOptions(.{});
+    const optimize = b.standardOptimizeOption(.{});
+
+    const exedriver = b.addExecutable(.{
+        .name = "driver",
+        .root_source_file = b.path("src/main.zig"),
+        .target = target,
+        .optimize = optimize,
+        .link_libc = true,
+    });
+    b.installArtifact(exedriver);
+
+    const libfoo = b.addSharedLibrary(.{
+        .name = "foo",
+        .root_source_file = b.path("src/foo.zig"),
+        .target = target,
+        .optimize = optimize,
+        .link_libc = true,
+    });
+    installModule(b, libfoo);
+
+    const libbar = b.addSharedLibrary(.{
+        .name = "bar",
+        .root_source_file = b.path("src/bar.zig"),
+        .target = target,
+        .optimize = optimize,
+        .link_libc = true,
+    });
+    installModule(b, libbar);
+
+    const run = b.addRunArtifact(exedriver);
+    run.step.dependOn(b.getInstallStep());
+    run.setEnvironmentVariable("MODS_DIR", b.getInstallPath(.{ .custom = "mods" }, ""));
+
+    b.step("run", "").dependOn(&run.step);
+}

hotswap/build.zig.zon

@@ -0,0 +1,86 @@
+.{
+    // This is the default name used by packages depending on this one. For
+    // example, when a user runs `zig fetch --save <url>`, this field is used
+    // as the key in the `dependencies` table. Although the user can choose a
+    // different name, most users will stick with this provided value.
+    //
+    // It is redundant to include "zig" in this name because it is already
+    // within the Zig package namespace.
+    .name = .zig_hotswap,
+
+    // This is a [Semantic Version](https://semver.org/).
+    // In a future version of Zig it will be used for package deduplication.
+    .version = "0.0.0",
+
+    // Together with name, this represents a globally unique package
+    // identifier. This field is generated by the Zig toolchain when the
+    // package is first created, and then *never changes*. This allows
+    // unambiguous detection of one package being an updated version of
+    // another.
+    //
+    // When forking a Zig project, this id should be regenerated (delete the
+    // field and run `zig build`) if the upstream project is still maintained.
+    // Otherwise, the fork is *hostile*, attempting to take control over the
+    // original project's identity. Thus it is recommended to leave the comment
+    // on the following line intact, so that it shows up in code reviews that
+    // modify the field.
+    .fingerprint = 0xa7823a599d65a583, // Changing this has security and trust implications.
+
+    // Tracks the earliest Zig version that the package considers to be a
+    // supported use case.
+    .minimum_zig_version = "0.14.0",
+
+    // This field is optional.
+    // Each dependency must either provide a `url` and `hash`, or a `path`.
+    // `zig build --fetch` can be used to fetch all dependencies of a package, recursively.
+    // Once all dependencies are fetched, `zig build` no longer requires
+    // internet connectivity.
+    .dependencies = .{
+        // See `zig fetch --save <url>` for a command-line interface for adding dependencies.
+        //.example = .{
+        //    // When updating this field to a new URL, be sure to delete the corresponding
+        //    // `hash`, otherwise you are communicating that you expect to find the old hash at
+        //    // the new URL. If the contents of a URL change this will result in a hash mismatch
+        //    // which will prevent zig from using it.
+        //    .url = "https://example.com/foo.tar.gz",
+        //
+        //    // This is computed from the file contents of the directory of files that is
+        //    // obtained after fetching `url` and applying the inclusion rules given by
+        //    // `paths`.
+        //    //
+        //    // This field is the source of truth; packages do not come from a `url`; they
+        //    // come from a `hash`. `url` is just one of many possible mirrors for how to
+        //    // obtain a package matching this `hash`.
+        //    //
+        //    // Uses the [multihash](https://multiformats.io/multihash/) format.
+        //    .hash = "...",
+        //
+        //    // When this is provided, the package is found in a directory relative to the
+        //    // build root. In this case the package's hash is irrelevant and therefore not
+        //    // computed. This field and `url` are mutually exclusive.
+        //    .path = "foo",
+        //
+        //    // When this is set to `true`, a package is declared to be lazily
+        //    // fetched. This makes the dependency only get fetched if it is
+        //    // actually used.
+        //    .lazy = false,
+        //},
+    },
+
+    // Specifies the set of files and directories that are included in this package.
+    // Only files and directories listed here are included in the `hash` that
+    // is computed for this package. Only files listed here will remain on disk
+    // when using the zig package manager. As a rule of thumb, one should list
+    // files required for compilation plus any license(s).
+    // Paths are relative to the build root. Use the empty string (`""`) to refer to
+    // the build root itself.
+    // A directory listed here means that all files within, recursively, are included.
+    .paths = .{
+        "build.zig",
+        "build.zig.zon",
+        "src",
+        // For example...
+        //"LICENSE",
+        //"README.md",
+    },
+}

hotswap/src/bar.zig

@@ -0,0 +1,25 @@
+const std = @import("std");
+const core = @import("core.zig");
+
+const Self = @This();
+
+export const MODULE = core.module(Self);
+
+pub fn startup(alloc: std.mem.Allocator) !*Self {
+    std.log.debug("!! startup {s} !!", .{@typeName(@This())});
+    return try alloc.create(Self);
+}
+
+pub fn unload(_: *Self, _: std.mem.Allocator) ![]u8 {
+    std.log.debug("!! unload {s} !!", .{@typeName(@This())});
+    return &.{};
+}
+
+pub fn reload(_: *Self, _: std.mem.Allocator, _: []u8) !void {
+    std.log.debug("!! reload {s} !!", .{@typeName(@This())});
+}
+
+pub fn shutdown(self: *Self, alloc: std.mem.Allocator) void {
+    std.log.debug("!! shutdown {s} !!", .{@typeName(@This())});
+    alloc.destroy(self);
+}

hotswap/src/core.zig

@@ -0,0 +1,95 @@
+const std = @import("std");
+
+// Notes:
+//
+// "init" is an overloaded term. Try not to use it. Instead, have hooks:
+// - startup / shutdown. executed once each per program lifecycle.
+// - load / unload. executed once per module load cycle.
+//
+// So the flow is:
+// - startup. prepares long-lived state.
+// - load. deserializes and prepares short-lived state.
+// - unload. serializes and releases short-lived state.
+// - shutdown. releases long-lived state.
+//
+// So when the hotplug system reloads the module, it performs an unload/load cycle.
+// Normal operation without any reloads includes all four stages.
+//
+// Some considerations:
+// - a module should be able to indicate specific failure:
+//   - startup (preparing state)
+//   - load (deserialization or preparing state)
+//   - unload (serialization)
+// - a module should be able to indicate it cannot be reloaded.
+// - serialization data might come from disk, or from a reload, or be missing.
+//
+// On reloading and rollback:
+// - when loading a module, copy the dynlib to a versioned temporary file, and load that temporary file.
+// - procedure for reloading a module:
+//   - current old version unload (and serialize)
+//   - attempt new version load (and deserialize)
+//   - if success: release old version and delete lib
+//   - if failure:
+//      - release new version and delete lib.
+//      - attempt old version load (and deserialize) (catch fatal error)
+// - Configure signal handlers to always clean up the temporary files on errors like segfault?
+//   - temp files needs to gracefully handle multiple instances of the app running at once with shared hotloading.
+//
+// Things should be done in a thread-safe manner. Signal to a module that it should unload gracefully, and the
+// module signal when it can safely be unloaded.
+
+pub fn Slot(Function: type) type {
+    const Arguments = std.meta.ArgsTuple(Function);
+    const Return = @typeInfo(Function).@"fn".return_type.?;
+
+    return extern struct {
+        ptr: *const fn (*const Arguments, *Return) callconv(.c) void,
+
+        pub fn wrap(impl: Function) @This() {
+            return .{
+                .ptr = &struct {
+                    pub fn cimpl(args: *const Arguments, ret: *Return) callconv(.c) void {
+                        ret.* = @call(.auto, impl, args.*);
+                    }
+                }.cimpl,
+            };
+        }
+
+        pub fn invoke(self: @This(), args: Arguments) Return {
+            var ret: Return = undefined;
+            self.ptr(&args, &ret);
+            return ret;
+        }
+    };
+}
+
+pub fn Module(State: type) type {
+    return extern struct {
+        pub const Error = std.mem.Allocator.Error;
+
+        pub const StartupError = Error;
+        pub const UnloadError = Error;
+        pub const ReloadError = Error;
+
+        const Startup = Slot(fn (std.mem.Allocator) StartupError!*State);
+        const Unload = Slot(fn (*State, std.mem.Allocator) UnloadError![]u8);
+        const Reload = Slot(fn (*State, std.mem.Allocator, []u8) ReloadError!void);
+        const Shutdown = Slot(fn (*State, std.mem.Allocator) void);
+
+        startup: Startup,
+        unload: Unload,
+        reload: Reload,
+        shutdown: Shutdown,
+    };
+}
+
+pub fn module(State: type) Module(State) {
+    return .{
+        .startup = Module(State).Startup.wrap(State.startup),
+        .unload = Module(State).Unload.wrap(State.unload),
+        .reload = Module(State).Reload.wrap(State.reload),
+        .shutdown = Module(State).Shutdown.wrap(State.shutdown),
+    };
+}
+
+pub const AnyModule = Module(anyopaque);

hotswap/src/foo.zig

@@ -0,0 +1,25 @@
+const std = @import("std");
+const core = @import("core.zig");
+
+const Self = @This();
+
+export const MODULE = core.module(Self);
+
+pub fn startup(alloc: std.mem.Allocator) !*Self {
+    std.log.debug("!! startup {s} !!", .{@typeName(@This())});
+    return try alloc.create(Self);
+}
+
+pub fn unload(_: *Self, _: std.mem.Allocator) ![]u8 {
+    std.log.debug("!! unload {s} !!", .{@typeName(@This())});
+    return &.{};
+}
+
+pub fn reload(_: *Self, _: std.mem.Allocator, _: []u8) !void {
+    std.log.debug("!! reload {s} !!", .{@typeName(@This())});
+}
+
+pub fn shutdown(self: *Self, alloc: std.mem.Allocator) void {
+    std.log.debug("!! shutdown {s} !!", .{@typeName(@This())});
+    alloc.destroy(self);
+}

hotswap/src/main.zig

@@ -0,0 +1,222 @@
+const std = @import("std");
+const linux = std.os.linux;
+const core = @import("core.zig");
+
+const ModLib = struct {
+    path: []const u8,
+    lib: std.DynLib,
+    mod: *core.Module(anyopaque),
+
+    fn open(realpath: []const u8, version: u32, alloc: std.mem.Allocator) !ModLib {
+        const basename = std.fs.path.basename(realpath);
+        const name = try std.fmt.allocPrint(
+            alloc,
+            ".{s}.{d}",
+            .{ basename, version },
+        );
+        defer alloc.free(name);
+
+        const path = if (std.fs.path.dirname(realpath)) |dirname|
+            try std.fs.path.join(alloc, &.{ dirname, name })
+        else
+            try alloc.dupe(u8, name);
+        errdefer alloc.free(path);
+
+        try std.fs.copyFileAbsolute(realpath, path, .{});
+
+        var lib = try std.DynLib.open(path);
+        errdefer lib.close();
+
+        const mod = lib.lookup(*core.Module(anyopaque), "MODULE") orelse
+            return error.MissingModuleDefinition;
+
+        return .{
+            .path = path,
+            .lib = lib,
+            .mod = mod,
+        };
+    }
+
+    fn close(self: *ModLib, alloc: std.mem.Allocator) void {
+        self.lib.close();
+        //         std.fs.deleteFileAbsolute(self.path) catch std.log.warn("Failed to delete {s}", .{self.path});
+        alloc.free(self.path);
+    }
+};
+
+const DynamicModule = struct {
+    gpa: std.heap.GeneralPurposeAllocator(.{}),
+    realpath: []const u8,
+    curr: ModLib,
+    state: *anyopaque,
+    next: u32,
+
+    pub fn init(path: []const u8, alloc: std.mem.Allocator) !DynamicModule {
+        const realpath = try std.fs.cwd().realpathAlloc(alloc, path);
+        errdefer alloc.free(realpath);
+
+        var gpa = std.heap.GeneralPurposeAllocator(.{}).init;
+        errdefer if (gpa.deinit() == .leak) {
+            std.log.warn("Module {s} leaked.", .{realpath});
+        };
+
+        var self: DynamicModule = .{
+            .gpa = gpa,
+            .realpath = realpath,
+            .curr = undefined,
+            .state = undefined,
+            .next = 0,
+        };
+
+        self.curr = try ModLib.open(realpath, self.next, alloc);
+        errdefer self.curr.close(alloc);
+
+        self.state = try self.curr.mod.startup.invoke(.{self.gpa.allocator()});
+        errdefer self.curr.mod.shutdown.invoke(.{ self.state, self.gpa.allocator() });
+
+        // todo deserialize from disk.
+        try self.curr.mod.reload.invoke(.{ self.state, self.gpa.allocator(), &.{} });
+
+        self.next += 1;
+
+        return self;
+    }
+
+    pub fn reload(self: *@This(), alloc: std.mem.Allocator) !void {
+        var next = try ModLib.open(self.realpath, self.next, alloc);
+        errdefer next.close(alloc);
+
+        const data = try self.curr.mod.unload.invoke(.{ self.state, self.gpa.allocator() });
+        errdefer self.curr.mod.reload.invoke(.{ self.state, self.gpa.allocator(), data }) catch
+            std.debug.panic("Failed to rollback to {s}", .{self.curr.path});
+
+        try next.mod.reload.invoke(.{ self.state, self.gpa.allocator(), data });
+
+        self.curr.close(alloc);
+        self.next += 1;
+        self.curr = next;
+    }
+
+    pub fn deinit(self: *DynamicModule, alloc: std.mem.Allocator) void {
+        const data = self.curr.mod.unload.invoke(.{ self.state, self.gpa.allocator() }) catch
+            std.debug.panic("Failed to unload {s}", .{self.curr.path});
+
+        _ = data; // todo serialize to disk.
+
+        self.curr.mod.shutdown.invoke(.{ self.state, self.gpa.allocator() });
+
+        self.curr.close(alloc);
+
+        if (self.gpa.deinit() == .leak) {
+            std.log.warn("Module {s} leaked.", .{self.realpath});
+        }
+
+        alloc.free(self.realpath);
+    }
+};
+
+pub fn main() !void {
+    var gpa = std.heap.GeneralPurposeAllocator(.{}).init;
+    defer _ = gpa.deinit();
+    const alloc = gpa.allocator();
+
+    const self = try std.fs.selfExeDirPathAlloc(alloc);
+    defer alloc.free(self);
+    std.log.debug("running from '{s}'", .{self});
+
+    var env = try std.process.getEnvMap(alloc);
+    defer env.deinit();
+
+    const mods_path = if (env.get("MODS_DIR")) |envvar| blk: {
+        break :blk try alloc.dupe(u8, envvar);
+    } else blk: {
+        const dir = try std.fs.selfExeDirPathAlloc(alloc);
+        defer alloc.free(dir);
+        break :blk try std.fs.path.resolve(alloc, &.{ dir, "..", "mods" });
+    };
+    defer alloc.free(mods_path);
+
+    var mods: std.StringHashMapUnmanaged(DynamicModule) = .{};
+    defer {
+        var it = mods.valueIterator();
+        while (it.next()) |mod| {
+            mod.deinit(alloc);
+        }
+        mods.deinit(alloc);
+    }
+
+    std.log.debug("loading mods in '{s}'", .{mods_path});
+    {
+        var dir = try std.fs.openDirAbsolute(mods_path, .{ .iterate = true });
+        defer dir.close();
+
+        var it = dir.iterate();
+        while (try it.next()) |entry| {
+            const mod_path = try std.fs.path.resolve(alloc, &.{ mods_path, entry.name });
+            defer alloc.free(mod_path);
+
+            if (std.mem.startsWith(u8, std.fs.path.basename(mod_path), "."))
+                continue;
+
+            var mod = try DynamicModule.init(mod_path, alloc);
+            errdefer mod.deinit(alloc);
+
+            try mods.putNoClobber(alloc, entry.name, mod);
+        }
+    }
+
+    const fd: i32 = blk: {
+        const ret: isize = @bitCast(linux.inotify_init1(linux.IN.NONBLOCK));
+        if (ret < 0) return error.inotify_init_error;
+        break :blk @intCast(ret);
+    };
+    defer _ = linux.close(fd);
+
+    const wd: i32 = blk: {
+        const pathz = try alloc.dupeZ(u8, mods_path);
+        defer alloc.free(pathz);
+        const ret: isize = @bitCast(linux.inotify_add_watch(
+            fd,
+            pathz,
+            linux.IN.MOVED_TO,
+        ));
+        if (ret < 0) return error.inotify_add_watch_error;
+        break :blk @intCast(ret);
+    };
+    defer _ = linux.inotify_rm_watch(fd, wd);
+
+    var fds = [_]linux.pollfd{
+        .{ .fd = fd, .events = linux.POLL.IN, .revents = undefined },
+    };
+
+    const eventbuf = try alloc.alloc(u8, 5 * (@sizeOf(linux.inotify_event) + linux.NAME_MAX));
+    defer alloc.free(eventbuf);
+
+    for (0..60 * 10) |_| {
+        const n = linux.poll(&fds, fds.len, std.time.ms_per_s);
+
+        if (n > 0) {
+            const c = linux.read(fd, eventbuf.ptr, eventbuf.len);
+
+            var i: usize = 0;
+            while (i < c) {
+                const event: *linux.inotify_event = @alignCast(@ptrCast(eventbuf.ptr + i));
+                const name = event.getName().?; // Impossible as IN.MOVE_TO includes a name.
+
+                if (mods.getPtr(name)) |mod| {
+                    if (mod.reload(alloc)) |_| {
+                        std.log.info("Reloaded {s}.", .{name});
+                    } else |_| {
+                        std.log.err("Failed to reload {s}. Rolled back.", .{mod.realpath});
+                    }
+                } else {
+                    // Ignore unloaded module.
+                }
+
+                i += @sizeOf(linux.inotify_event) + event.len;
+            }
+        }
+    }
+
+    std.log.debug("-" ** 80, .{});
+}