From 9d386221c9d6265f8ab85b42fcb93b4a0cafbb54 Mon Sep 17 00:00:00 2001
From: sweiglbosker <stefan@s00.xyz>
Date: Tue, 24 Feb 2026 13:04:50 -0500
Subject: [willow][tools]: unit testing for tokenizer (#14)

---
 willow/tools/willowc/include/tokenizer.hpp | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

(limited to 'willow/tools/willowc/include')

diff --git a/willow/tools/willowc/include/tokenizer.hpp b/willow/tools/willowc/include/tokenizer.hpp
index 47577ab..d3970f6 100644
--- a/willow/tools/willowc/include/tokenizer.hpp
+++ b/willow/tools/willowc/include/tokenizer.hpp
@@ -22,6 +22,9 @@ enum class TokenKind {
   LCurly,
   RCurly,
   Equals,
+  Star,
+  LTriangle,
+  RTriangle,
   RArrow,
   Comment,
 
@@ -66,7 +69,7 @@ class Tokenizer {
   bool scan_hex();
   bool scan_constant();
 public:
-  explicit Tokenizer(std::string_view buf, std::size_t offset = 0)
+  explicit constexpr Tokenizer(std::string_view buf, std::size_t offset = 0)
       : buf{buf}, offset{offset} {}
 
   Token scan();
@@ -103,6 +106,12 @@ constexpr std::string_view TokenKindName(TokenKind t) {
     return "RCurly";
   case TokenKind::Equals:
     return "Equals";
+  case TokenKind::Star:
+    return "Star";
+  case TokenKind::LTriangle:
+    return "LTriangle";
+  case TokenKind::RTriangle:
+    return "RTriangle";
   case TokenKind::RArrow:
     return "RArrow";
   case TokenKind::Comment:
-- 
cgit v1.2.3