Get tests running and passing for tokenization

build

@@ -20,6 +20,7 @@ compile() {
 }
 
 test() {
+    zig build test
     cd "$project_root"/old-rust-tests || exit 1
     cargo test
 }

src/interpret.zig

@@ -95,7 +95,10 @@ pub const DtMachine = struct {
             .bool => |b| try self.push(DtVal{ .bool = b }),
             .int => |i| try self.push(DtVal{ .int = i }),
             .float => |f| try self.push(DtVal{ .float = f }),
-            .string => |s| try self.push(DtVal{ .string = s }),
+            .string => |s| {
+                const unescaped = try string.unescape(self.alloc, s);
+                try self.push(DtVal{ .string = unescaped });
+            },
             .deferred_term => |cmd| try self.push(DtVal{ .deferred_command = cmd }),
             .err => |e| try self.push(.{ .err = e }),
             .none => {},

src/main.zig

@@ -115,3 +115,7 @@ fn readShebangFile(allocator: Allocator) !?[]const u8 {
 
     return null;
 }
+
+test {
+    std.testing.refAllDecls(@This());
+}

src/string.zig

@@ -27,11 +27,14 @@ pub fn escape(allocator: Allocator, unescaped: String) !String {
 }
 
 pub fn unescape(allocator: Allocator, unescaped: String) !String {
-    var result = try allocator.dupe(u8, unescaped);
+    var buf: []u8 = undefined;
 
     for (escapes) |esc| {
-        result = try std.mem.replaceOwned(u8, allocator, result, esc.to, esc.from);
+        buf = std.mem.replaceOwned(u8, allocator, unescaped, esc.to, esc.from) catch |e| {
+            allocator.free(buf);
+            return e;
+        };
     }
 
-    return result;
+    return buf;
 }

src/tokens.zig

@@ -51,8 +51,7 @@ pub const TokenIterator = struct {
                     }
                 }
                 self.index = end + 1;
-                const unescaped = try string.unescape(self.allocator, self.buf[strStart..end]);
-                return .{ .string = unescaped };
+                return .{ .string = self.buf[strStart..end] };
             },
             '~' => { // Parse an error
                 const lastChar = std.mem.indexOfPos(u8, self.buf, start, "~") orelse self.buf.len;
@@ -142,6 +141,7 @@ pub const Token = union(enum) {
             .string => |s| std.debug.assert(std.mem.eql(u8, other.string, s)),
             .term => |t| std.debug.assert(std.mem.eql(u8, other.term, t)),
             .deferred_term => |t| std.debug.assert(std.mem.eql(u8, other.deferred_term, t)),
+            .err => |e| std.debug.assert(std.mem.eql(u8, other.err, e)),
             .none => std.debug.assert(other == Token.none),
         }
     }
@@ -160,15 +160,12 @@ test "parse hello.dt" {
     try expected.append(Token{ .term = "def" });
 
     const helloFile = @embedFile("test/hello.dt");
-    const tokens = try Token.parse(std.testing.allocator, helloFile);
-    defer tokens.deinit();
+    var tokens = Token.parse(std.testing.allocator, helloFile);
 
-    std.debug.assert(tokens.items.len == 6);
     var i: u8 = 0;
-    while (i < 6) {
-        std.debug.print("Expected: {any}, Actual: {any} ... ", .{ expected.items[i], tokens.items[i] });
-        expected.items[i].assertEql(tokens.items[i]);
-        std.debug.print("PASS\n", .{});
-        i += 1;
+    while (try tokens.next()) |token| : (i += 1) {
+        std.log.info("Expected: {any}, Actual: {any} ... ", .{ expected.items[i], token });
+        expected.items[i].assertEql(token);
+        std.log.info("PASS\n", .{});
     }
 }