//go:generate stringer -type=ItemType package wikilink import ( "fmt" "strings" // "unicode" "unicode/utf8" "go.uber.org/zap" ) const ( ItemError ItemType = iota ItemEOF ItemIdent ItemOpenLink ItemCloseLink ItemHeading ItemBlockRef ItemAlias ItemText ) const ( EOF rune = 0 ) const ( OpenLink = "[[" CloseLink = "]]" Alias = "|" Heading = "#" BlockRef = "#^" ) func Lex(name, input string) *Lexer { l := &Lexer{ L: zap.NewExample().Sugar().Named("lexer"), name: name, input: input, state: lexText, items: make(chan Item, 2), } go l.run() return l } func (l *Lexer) NextItem() Item { for { select { case item := <-l.items: return item default: if l.state == nil { return Item{ Typ: ItemError, Val: "state is nil, should not be", } } l.state = l.state(l) } } } func (l *Lexer) ignore() { l.start = l.pos } func (l *Lexer) backup() { l.pos -= l.width } type Lexer struct { L *zap.SugaredLogger name, input string start, pos, width int state stateFn items chan Item } func (l *Lexer) peek() rune { r := l.next() l.backup() return r } func (l *Lexer) accept(valid string) bool { if strings.ContainsRune(valid, l.next()) { return true } l.backup() return false } func (l *Lexer) acceptRun(valid string) { for strings.ContainsRune(valid, l.next()) { } l.backup() } func (l *Lexer) emit(t ItemType) { i := Item{t, l.input[l.start:l.pos]} L := l.L.With( zap.Int("pos", l.pos), zap.Int("width", l.width), ).Named("emit") L.Debugw("emitting item", zap.String("item", i.String()), ) l.items <- i l.start = l.pos } func (l *Lexer) errorf(format string, args ...interface{}) stateFn { L := l.L.Named("errorf") errorItem := Item{ ItemError, fmt.Sprintf(format, args...), } L.Debugw("emitting errorItem", zap.String("error", errorItem.String()), ) l.items <- errorItem return nil } func (l *Lexer) next() rune { var r rune if l.pos >= len(l.input) { l.width = 0 return EOF } r, l.width = utf8.DecodeRuneInString(l.input[l.pos:]) l.pos += l.width return r } func (l *Lexer) run() { for state := lexText; state != nil; { state = state(l) } close(l.items) } type stateFn func(*Lexer) stateFn type ItemType int type Item struct { Typ ItemType Val string } func (i Item) String() string { switch i.Typ { case ItemEOF: return "EOF" case ItemError: return i.Val } if len(i.Val) > 10 { // return fmt.Sprintf("%s:%.10q...", i.Typ, i.Val) return fmt.Sprintf("%s:%q", i.Typ, i.Val) } return fmt.Sprintf("%s:%q", i.Typ, i.Val) }