From c19b774bd8f4e28715cd42920df04f270fd2c458 Mon Sep 17 00:00:00 2001
From: booniepepper <justin@so.dang.cool>
Date: Sat, 8 Jul 2023 01:01:43 -0700
Subject: [PATCH] Get tests running and passing for tokenization

---
 build             |  1 +
 src/interpret.zig |  5 ++++-
 src/main.zig      |  4 ++++
 src/string.zig    |  9 ++++++---
 src/tokens.zig    | 17 +++++++----------
 5 files changed, 22 insertions(+), 14 deletions(-)

diff --git a/build b/build
index 5443088..603be25 100755
--- a/build
+++ b/build
@@ -20,6 +20,7 @@ compile() {
 }
 
 test() {
+    zig build test
     cd "$project_root"/old-rust-tests || exit 1
     cargo test
 }
diff --git a/src/interpret.zig b/src/interpret.zig
index efd3ad2..3fa9610 100644
--- a/src/interpret.zig
+++ b/src/interpret.zig
@@ -95,7 +95,10 @@ pub const DtMachine = struct {
             .bool => |b| try self.push(DtVal{ .bool = b }),
             .int => |i| try self.push(DtVal{ .int = i }),
             .float => |f| try self.push(DtVal{ .float = f }),
-            .string => |s| try self.push(DtVal{ .string = s }),
+            .string => |s| {
+                const unescaped = try string.unescape(self.alloc, s);
+                try self.push(DtVal{ .string = unescaped });
+            },
             .deferred_term => |cmd| try self.push(DtVal{ .deferred_command = cmd }),
             .err => |e| try self.push(.{ .err = e }),
             .none => {},
diff --git a/src/main.zig b/src/main.zig
index 1edcd4e..ded0714 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -115,3 +115,7 @@ fn readShebangFile(allocator: Allocator) !?[]const u8 {
 
     return null;
 }
+
+test {
+    std.testing.refAllDecls(@This());
+}
diff --git a/src/string.zig b/src/string.zig
index aa10f0c..ffeefb4 100644
--- a/src/string.zig
+++ b/src/string.zig
@@ -27,11 +27,14 @@ pub fn escape(allocator: Allocator, unescaped: String) !String {
 }
 
 pub fn unescape(allocator: Allocator, unescaped: String) !String {
-    var result = try allocator.dupe(u8, unescaped);
+    var buf: []u8 = undefined;
 
     for (escapes) |esc| {
-        result = try std.mem.replaceOwned(u8, allocator, result, esc.to, esc.from);
+        buf = std.mem.replaceOwned(u8, allocator, unescaped, esc.to, esc.from) catch |e| {
+            allocator.free(buf);
+            return e;
+        };
     }
 
-    return result;
+    return buf;
 }
diff --git a/src/tokens.zig b/src/tokens.zig
index fd8f028..b391884 100644
--- a/src/tokens.zig
+++ b/src/tokens.zig
@@ -51,8 +51,7 @@ pub const TokenIterator = struct {
                     }
                 }
                 self.index = end + 1;
-                const unescaped = try string.unescape(self.allocator, self.buf[strStart..end]);
-                return .{ .string = unescaped };
+                return .{ .string = self.buf[strStart..end] };
             },
             '~' => { // Parse an error
                 const lastChar = std.mem.indexOfPos(u8, self.buf, start, "~") orelse self.buf.len;
@@ -142,6 +141,7 @@ pub const Token = union(enum) {
             .string => |s| std.debug.assert(std.mem.eql(u8, other.string, s)),
             .term => |t| std.debug.assert(std.mem.eql(u8, other.term, t)),
             .deferred_term => |t| std.debug.assert(std.mem.eql(u8, other.deferred_term, t)),
+            .err => |e| std.debug.assert(std.mem.eql(u8, other.err, e)),
             .none => std.debug.assert(other == Token.none),
         }
     }
@@ -160,15 +160,12 @@ test "parse hello.dt" {
     try expected.append(Token{ .term = "def" });
 
     const helloFile = @embedFile("test/hello.dt");
-    const tokens = try Token.parse(std.testing.allocator, helloFile);
-    defer tokens.deinit();
+    var tokens = Token.parse(std.testing.allocator, helloFile);
 
-    std.debug.assert(tokens.items.len == 6);
     var i: u8 = 0;
-    while (i < 6) {
-        std.debug.print("Expected: {any}, Actual: {any} ... ", .{ expected.items[i], tokens.items[i] });
-        expected.items[i].assertEql(tokens.items[i]);
-        std.debug.print("PASS\n", .{});
-        i += 1;
+    while (try tokens.next()) |token| : (i += 1) {
+        std.log.info("Expected: {any}, Actual: {any} ... ", .{ expected.items[i], token });
+        expected.items[i].assertEql(token);
+        std.log.info("PASS\n", .{});
     }
 }