Generic tokenizer written in Nim language, powered by Nim's Macros 👑
nimble install toktok
std/lexbase
/ Zero Regular Expressionenum
, lexbase
tables
, lexbase
MIT
[!NOTE] This is a generic Lexer, based on std/
streams
,lexbase
andmacros
. It is meant to be used by higher level parsers for writing any kind of tools or programs.[!NOTE] Compile with
-d:toktokdebug
to inspect the generated code.
# Register your custom handlers
handlers:
proc handleImport(lex: var Lexer, kind: TokenKind) =
# tokenize `import x, y, z`
lex.kind = kind
# Register your tokens
registerTokens defaultSettings:
`const` = "const"
`echo` = "hello"
asgn = '=': # `=` is tkAsgn
eq = '=' # `==` is tkEQ
excl = '!':
ne = '='
at = '@':
import = tokenizer(handleImport, "import")
# Tokenizing...
var
tok = lexer.init(sample)
prev: TokenTuple
curr: TokenTuple = tok.getToken
next: TokenTuple = tok.getToken
proc walk(tok: var Lexer) =
prev = curr
curr = next
next = tok.getToken
while likely(curr.kind != tkEOF):
if tok.hasError: break
echo curr # use `getToken` consumer to get token by token
walk tok
MIT
license. Made by Humans from OpenPeeps.
Copyright © 2023 OpenPeeps & Contributors — All rights reserved.