[willow][tools]: unit testing for tokenizer

1 files changed, 10 insertions, 1 deletions

diff --git a/willow/tools/willowc/include/tokenizer.hpp b/willow/tools/willowc/include/tokenizer.hpp
index 47577ab..d3970f6 100644
--- a/willow/tools/willowc/include/tokenizer.hpp
+++ b/willow/tools/willowc/include/tokenizer.hpp
@@ -22,6 +22,9 @@ enum class TokenKind {
   LCurly,
   RCurly,
   Equals,
+  Star,
+  LTriangle,
+  RTriangle,
   RArrow,
   Comment,
 
@@ -66,7 +69,7 @@ class Tokenizer {
   bool scan_hex();
   bool scan_constant();
 public:
-  explicit Tokenizer(std::string_view buf, std::size_t offset = 0)
+  explicit constexpr Tokenizer(std::string_view buf, std::size_t offset = 0)
       : buf{buf}, offset{offset} {}
 
   Token scan();
@@ -103,6 +106,12 @@ constexpr std::string_view TokenKindName(TokenKind t) {
     return "RCurly";
   case TokenKind::Equals:
     return "Equals";
+  case TokenKind::Star:
+    return "Star";
+  case TokenKind::LTriangle:
+    return "LTriangle";
+  case TokenKind::RTriangle:
+    return "RTriangle";
   case TokenKind::RArrow:
     return "RArrow";
   case TokenKind::Comment: