From 9d386221c9d6265f8ab85b42fcb93b4a0cafbb54 Mon Sep 17 00:00:00 2001
From: sweiglbosker <stefan@s00.xyz>
Date: Tue, 24 Feb 2026 13:04:50 -0500
Subject: [willow][tools]: unit testing for tokenizer (#14)

---
 willow/tools/willowc/lib/tokenizer.cpp | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

(limited to 'willow/tools/willowc/lib/tokenizer.cpp')

diff --git a/willow/tools/willowc/lib/tokenizer.cpp b/willow/tools/willowc/lib/tokenizer.cpp
index 7ad28a6..f7dd04f 100644
--- a/willow/tools/willowc/lib/tokenizer.cpp
+++ b/willow/tools/willowc/lib/tokenizer.cpp
@@ -1,3 +1,4 @@
+#include <iostream>
 #include <tokenizer.hpp>
 
 namespace willowc {
@@ -84,6 +85,15 @@ Token Tokenizer::scan() {
     case '=':
       skip();
       return TokenKind::Equals;
+    case '*':
+      skip();
+      return TokenKind::Star;
+    case '<':
+      skip();
+      return TokenKind::LTriangle;
+    case '>':
+      skip();
+      return TokenKind::RTriangle;
     case '-': {
       if (peek(1) == '>') {
         skip(2);
@@ -133,12 +143,21 @@ Token Tokenizer::scan() {
   }();
 
   Token t{start, offset, k};
+
   if (t.kind == TokenKind::Invalid) {
-    if (t.start == t.end)
+    if (t.start == t.end) {
       t.end++;
+      skip();
+    }
     recover();
   }
 
+  if (t.kind == TokenKind::Inst) {
+    auto lexeme = buf.substr(t.start, t.end - t.start);
+    if (lexeme == "func")
+      t.kind = TokenKind::FuncKW;
+  }
+
   return t;
 }
 
@@ -214,6 +233,8 @@ void Tokenizer::recover() {
       [[fallthrough]];
     case '=':
       [[fallthrough]];
+    case '^':
+      [[fallthrough]];
     case '\0':
       return true;
     default:
-- 
cgit v1.2.3