Dingen

build.zig

@@ -6,7 +6,7 @@ pub fn build(b: *Builder) void {
     generator.setBuildMode(b.standardReleaseOptions());
 
     var test_step = b.step("test", "Run all the tests");
-    test_step.dependOn(&b.addTest("generator/xml.zig").step);
+    test_step.dependOn(&b.addTest("generator/main.zig").step);
 
     const run_cmd = generator.run();
 

generator/main.zig

@@ -3,3 +3,8 @@ const std = @import("std");
 pub fn main() void {
     std.debug.warn("Test\n");
 }
+
+test "main" {
+    _ = @import("xml.zig");
+    _ = @import("utf8.zig");
+}

generator/utf8.zig

@@ -0,0 +1,109 @@
+const std = @import("std");
+const unicode = std.unicode;
+const testing = std.testing;
+
+fn Utf8Iterator(comptime ReadError: type) type {
+    return struct {
+        const Self = @This();
+        pub const Stream = std.io.InStream(ReadError);
+
+        in: *Stream,
+
+        pub fn init(in: *Stream) Self {
+            return Self{
+                .in = in
+            };
+        }
+
+        pub fn next(self: *Self) !?u32 {
+            var cp: [4]u8 = undefined;
+            if ((try self.in.readFull(cp[0 .. 1])) != 1) {
+                return null;
+            }
+
+            const cp_len = try unicode.utf8ByteSequenceLength(cp[0]);
+            try self.in.readNoEof(cp[1 .. cp_len]);
+            return try unicode.utf8Decode(cp[0 .. cp_len]);
+        }
+    };
+}
+
+pub fn PeekUtf8Iterator(comptime buffer_size: usize, comptime ReadError: type) type {
+    return struct {
+        const Self = @This();
+        pub const Stream = Utf8Iterator(ReadError).Stream;
+
+        it: Utf8Iterator(ReadError),
+        buf: [buffer_size]u32,
+        head: usize,
+        size: usize,
+
+        pub fn init(in: *Utf8Iterator(ReadError).Stream) Self {
+            return Self{
+                .it = Utf8Iterator(ReadError).init(in),
+                .buf = undefined,
+                .head = 0,
+                .size = 0
+            };
+        }
+
+        pub fn next(self: *Self) !?u32 {
+            if (self.size > 0) {
+                const cp = self.buf[self.head];
+                self.head = (self.head + 1) % buffer_size;
+                self.size -= 1;
+                return cp;
+            }
+
+            return try self.it.next();
+        }
+
+        pub fn peek(self: *Self, offset: usize) !?u32 {
+            std.debug.assert(offset < buffer_size);
+            while (self.size <= offset) {
+                const cp = (try self.it.next()) orelse return null;
+                self.buf[(self.head + self.size) % buffer_size] = cp;
+                self.size += 1;
+            }
+
+            return self.buf[(self.head + offset) % buffer_size];
+        }
+
+        pub fn peekNoEof(self: *Self, offset: usize) !?u32 {
+            return (try self.peek(offset)) orelse return error.EndOfStream;
+        }
+
+        pub fn discard(self: *Self, amount: usize) void {
+            std.debug.assert(amount < self.size);
+
+            var i: usize = 0;
+            while (i < amount) : (i += 1) {
+                _ = self.next();
+            }
+        }
+    };
+}
+
+test "PeekUtf8Iterator" {
+    var slice_in = std.io.SliceInStream.init("abcd");
+    var it = PeekUtf8Iterator(4, std.io.SliceInStream.Error).init(&slice_in.stream);
+    testing.expect((try it.peek(0)).? == 'a');
+    testing.expect((try it.peek(1)).? == 'b');
+    testing.expect((try it.peek(2)).? == 'c');
+    testing.expect((try it.peek(3)).? == 'd');
+
+    testing.expect((try it.next()).? == 'a');
+    testing.expect((try it.peek(0)).? == 'b');
+    testing.expect((try it.peek(1)).? == 'c');
+    testing.expect((try it.peek(2)).? == 'd');
+
+    testing.expect((try it.next()).? == 'b');
+    testing.expect((try it.peek(0)).? == 'c');
+    testing.expect((try it.peek(1)).? == 'd');
+    testing.expect((try it.peek(2)) == null);
+
+    testing.expect((try it.next()).? == 'c');
+    testing.expect((try it.peek(0)).? == 'd');
+    testing.expect((try it.peek(1)) == null);
+    testing.expectError(error.EndOfStream, it.peekNoEof(1));
+}

generator/xml.zig

@@ -1,316 +1,74 @@
 const std = @import("std");
-const unicode = std.unicode;
+const Allocator = std.mem.Allocator;
-const testing = std.testing;
+const ArenaAllocator = std.heap.ArenaAllocator;
 
-const Entity = struct {
+pub const TagType = enum {
-    substitute: []const u8,
+    tag,
-    char: u8
+    proc_instr
 };
 
-const entities = [_]Entity{
+pub const Node = struct {
-    Entity{.substitute = "amp;", .char = '&'},
+    pub name: []const u8,
-    Entity{.substitute = "lt;", .char = '<'},
+    pub type: TagType,
-    Entity{.substitute = "gt;", .char = '>'},
+    pub attributes: []Attribute,
-    Entity{.substitute = "quot;", .char = '"'},
+    pub children: []Element
-    Entity{.substitute = "apos;", .char = '\''},
 };
 
-fn XmlUtf8Iterator(comptime ReadError: type) type {
+pub const Attribute = struct {
-    return struct {
+    pub key: []const u8,
-        const Self = @This();
+    pub value: []const u8
-        pub const Stream = std.io.InStream(ReadError);
-
-        in: *Stream,
-
-        pub fn init(in: *Stream) Self {
-            return Self{
-                .in = in
-            };
-        }
-
-        pub fn next(self: *Self) !?u32 {
-            const cp = (try self.nextCodepoint()) orelse return null;
-            return if (cp == '&') try self.nextEntity() else cp;
-        }
-
-        fn nextEntity(self: *Self) !u32 {
-            var entity = [_]u8{0} ** 5;
-
-            for (entity) |*c| {
-                const cp = (try self.nextCodepoint()) orelse return error.InvalidEntity;
-                c.* = std.math.cast(u8, cp) catch return error.InvalidEntity;
-
-                if (cp == ';') {
-                    break;
-                }
-            } else {
-                return error.InvalidEntity;
-            }
-
-            for (entities) |*e| {
-                if (std.mem.startsWith(u8, entity, e.substitute)) {
-                    return e.char;
-                }
-            }
-
-            return error.InvalidEntity;
-        }
-
-        fn nextCodepoint(self: *Self) !?u32 {
-            var cp: [4]u8 = undefined;
-            if ((try self.in.readFull(cp[0 .. 1])) != 1) {
-                return null;
-            }
-
-            const cp_len = try unicode.utf8ByteSequenceLength(cp[0]);
-            try self.in.readNoEof(cp[1 .. cp_len]);
-            return try unicode.utf8Decode(cp[0 .. cp_len]);
-        }
-    };
-}
-
-fn testXmlUtf8Iterator(text: []const u8, expected: []const u8) !void {
-    var slice_in = std.io.SliceInStream.init(text);
-    var it = XmlUtf8Iterator(std.io.SliceInStream.Error).init(&slice_in.stream);
-
-    var i: usize = 0;
-    defer testing.expect(i == expected.len);
-
-    while (try it.next()) |cp| {
-        testing.expect(cp == expected[i]);
-        i += 1;
-    }
-}
-
-test "XmlUtf8Iterator" {
-    try testXmlUtf8Iterator("simpleabc", "simpleabc");
-    try testXmlUtf8Iterator("a&lt;b&amp;c&gt;d&quot;e&apos;f&lt;", "a<b&c>d\"e'f<");
-    testing.expectError(error.InvalidEntity, testXmlUtf8Iterator("python&", "python"));
-    testing.expectError(error.InvalidEntity, testXmlUtf8Iterator("python&&", "python"));
-    testing.expectError(error.InvalidEntity, testXmlUtf8Iterator("python&test;", "python"));
-    testing.expectError(error.InvalidEntity, testXmlUtf8Iterator("python&boa", "python"));
-}
-
-pub fn PeekXmlUtf8Iterator(comptime buffer_size: usize, comptime ReadError: type) type {
-    return struct {
-        const Self = @This();
-        pub const Stream = XmlUtf8Iterator(ReadError).Stream;
-
-        it: XmlUtf8Iterator(ReadError),
-        buf: [buffer_size]u32,
-        head: usize,
-        size: usize,
-
-        pub fn init(in: *XmlUtf8Iterator(ReadError).Stream) Self {
-            return Self{
-                .it = XmlUtf8Iterator(ReadError).init(in),
-                .buf = undefined,
-                .head = 0,
-                .size = 0
-            };
-        }
-
-        pub fn next(self: *Self) !?u32 {
-            if (self.size > 0) {
-                const cp = self.buf[self.head];
-                self.head = (self.head + 1) % buffer_size;
-                self.size -= 1;
-                return cp;
-            }
-
-            return try self.it.next();
-        }
-
-        pub fn peek(self: *Self, offset: usize) !?u32 {
-            std.debug.assert(offset < buffer_size);
-            while (self.size <= offset) {
-                self.buf[(self.head + self.size) % buffer_size] = (try self.it.next()) orelse return null;
-                self.size += 1;
-            }
-
-            return self.buf[(self.head + offset) % buffer_size];
-        }
-
-        pub fn peekCheckStr(self: *Self, str: []const u8) !bool {
-            for (str) |c, i| {
-                if (try self.peek(i) != c) {
-                    return false;
-                }
-            }
-
-            return true;
-        }
-    };
-}
-
-test "PeekXmlUtf8Iterator" {
-    var slice_in = std.io.SliceInStream.init("abcd");
-    var it = PeekXmlUtf8Iterator(4, std.io.SliceInStream.Error).init(&slice_in.stream);
-    testing.expect((try it.peek(0)).? == 'a');
-    testing.expect((try it.peek(1)).? == 'b');
-    testing.expect((try it.peek(2)).? == 'c');
-    testing.expect((try it.peek(3)).? == 'd');
-
-    testing.expect((try it.next()).? == 'a');
-    testing.expect((try it.peek(0)).? == 'b');
-    testing.expect((try it.peek(1)).? == 'c');
-    testing.expect((try it.peek(2)).? == 'd');
-
-    testing.expect((try it.next()).? == 'b');
-    testing.expect((try it.peek(0)).? == 'c');
-    testing.expect((try it.peek(1)).? == 'd');
-    testing.expect((try it.peek(2)) == null);
-
-    testing.expect((try it.next()).? == 'c');
-    testing.expect((try it.peek(0)).? == 'd');
-    testing.expect((try it.peek(1)) == null);
-}
-
-pub const Event = enum {
-    open,
-    open_pi,
-    open_comment,
-    close,
-    close_slash,
-    close_pi,
-    close_comment,
-    attrib_key,
-    attrib_value,
-    content
 };
 
-pub fn Parser(ReadError: type) type {
+pub const Element = union(enum) {
-    return struct {
+    pub text: []const u8,
-        const Self = @This();
+    pub node: *Node
-        const Iterator = PeekXmlUtf8Iterator(4, ReadError);
+};
 
-        in: Iterator,
+pub const Document = struct {
-        state: Event,
+    arena: ArenaAllocator,
+
+    pub xml_decl: ?*Node,
+    pub root: *Node,
+
+    pub fn deinit(self: *Document) void {
+        self.arena.deinit();
+    }
+};
+
+const Parser = struct {
+    alloc: *Allocator,
+    source: []const u8,
+    offset: usize,
+
+    fn element(self: *Self) !Element {
 
-        pub fn init(in: *Iterator.Stream) Self {
-            return Self{
-                .in = Iterator.init(in),
-                .state = .content
-            };
     }
 
-        pub fn nextEvent(self: *Self) !?Event {
+    fn text(self: *Self) ![]const u8 {
-            const state = (try self.parseNextEvent()) orelse return null;
+        const start = self.offset;
-            return state;
+        const end = if (std.mem.indexOfPos(self.source, self.offset, "<")) |offset| offset else self.source.len;
+        self.offset = end;
+        return self.source[start .. end]; // TODO: Decode
     }
 
-        fn parseNextEvent(self: *Self) !?Event {
+    fn node(self: *Self) !*Node {
-            while (true) {
+        std.debug.assert(try self.peekNoEof(0) == '<');
-                switch ((try self.in.peek(0)) orelse return null) {
-                    ' ', '\t', '\n', '\r' => {
-                        self.discard(1);
-                        continue;
-                    },
-                    '<' => {
-                        switch (try self.peekNoEof(1)) {
-                            '?' => {
-                                self.discard(2);
-                                return .open_pi;
-                            },
-                            '!' => {
-                                self.discard(2);
-                                try self.matchStr("--");
-                                return .open_comment;
-                            },
-                            else => {
-                                self.discard(1);
-                                return .open;
-                            }
-                        }
-                    },
-                    '>' => {
-                        self.discard(1);
-                        return .close;
-                    },
-                    '?' => {
-                        self.discard(1);
-                        try self.matchChar('>');
-                        return .close_pi;
-                    },
-                    '/' => {
-                        self.discard(1);
-                        try self.matchChar('>');
-                        return .close_slash;
-                    },
-                    '-' => {
-                        self.discard(1);
-                        try self.matchStr("-!>");
-                        return .close_comment;
-                    },
-                    else => return .content
-                }
-            }
     }
 
-        pub fn next(self: *Self) !?u32 {
+    fn peekNoEof(self: *const Self, offset: usize) !u8 {
-            switch (self.state) {
+        if (offset + self.offset >= self.source.len) {
-                .open => {
+            return error.EndOfStream;
-                    switch (try self.peekNoEof(0)) {
-                        '>' => return null,
-                        '/' => {
-                            if ((try self.peekNoEof(1)) == '>') {
-                                return null;
-                            }
-                        },
-                        else => |cp| {
-                            self.discard(1);
-                            return cp;
-                        }
-                    }
-                },
-                .open_pi => {
-                    switch (try self.peekNoEof(0)) {
-                        '?' => {
-                            if ((try self.peekNoEof(1)) == '>') {
-                                return null;
-                            }
-                        },
-                        else => |cp| {
-                            self.discard(1);
-                            return cp;
-                        }
-                    }
-                },
-                .open_comment => {
-                    if (try self.in.peekCheckStr("--!>")) {
-                        return null;
         }
 
-                    const cp = try self.peekNoEof(0);
+        return self.source[offset + self.offset];
-                    self.discard(1);
-                    return cp;
-                }
-            }
     }
+};
 
-        fn matchStr(self: *Self, seq: []const u8) !void {
+pub fn parse(alloc: *Allocator, source: []const u8) !Document {
-            for (seq) |c| {
+    var arena = ArenaAllocator.init(alloc);
-                try self.matchChar(c);
+    var parser = Parser {
-            }
+        .alloc = alloc,
-        }
+        .source = source,
-
+        .offset = 0
-        fn matchChar(self: *Self, expected: u32) !void {
-            if ((try self.peekNoEof(0)) != expected) {
-                return error.Syntax;
-            }
-
-            self.discard(1);
-        }
-
-        fn peekNoEof(self: *Self, offset: usize) !u32 {
-            return (try self.in.peek(offset)) orelse error.EndOfStream;
-        }
-
-        fn discard(self: *Self, num: usize) void {
-            var i: usize = 0;
-            while (i < num) : (i += 1) {
-                _ = self.in.next();
-            }
-        }
     };
 }