#ifndef WILLOWC_INCLUDE_TOKENIZER_HPP #define WILLOWC_INCLUDE_TOKENIZER_HPP #include namespace willowc { enum class TokenKind { Function, Variable, Constant, Type, Label, Inst, Comma, Semicolon, LParen, RParen, LCurly, RCurly, Equals, RArrow, Comment, FuncKW, Eof, Invalid, }; struct Token { std::size_t start, end; TokenKind kind; }; class Tokenizer { std::string_view buf; std::size_t offset; void skip(std::size_t idx = 1) { offset += idx; } char eat(std::size_t num = 1) { if (offset >= buf.length()) return '\0'; char c = buf[offset]; offset += num; return c; } char peek(std::size_t idx = 0) { if (offset + idx >= buf.length()) return '\0'; return buf[offset + idx]; } bool scan_id(bool accept_digits); bool scan_dec(); bool scan_hex(); bool scan_constant(); public: explicit Tokenizer(std::string_view buf, std::size_t offset = 0) : buf{buf}, offset{offset} {} Token scan(); void seek(uint64_t offset); }; } // namespace willowc #endif // WILLOWC_INCLUDE_TOKENIZER_HPP