diff --git a/.gitignore b/.gitignore index f60da40..e771611 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ node_modules/* package*.json dist/* reports/* +lexer.log +parser.log diff --git a/lexer.go b/lexer.go index 7242f2c..a97bd4b 100644 --- a/lexer.go +++ b/lexer.go @@ -3,12 +3,15 @@ package wikilink import ( "fmt" + // "os" "strings" "sync" + // "unicode" "unicode/utf8" "go.uber.org/zap" + "go.uber.org/zap/zapcore" ) const ( @@ -36,8 +39,28 @@ const ( ) func Lex(name, input string) *Lexer { + encoderCfg := zap.NewProductionEncoderConfig() + encoderCfg.EncodeTime = zapcore.ISO8601TimeEncoder + + config := zap.Config{ + Level: zap.NewAtomicLevelAt(zap.InfoLevel), + EncoderConfig: encoderCfg, + OutputPaths: []string{ + "./lexer.log", + "stdout", + }, + Encoding: "console", + ErrorOutputPaths: []string{ + "stderr", + }, + InitialFields: map[string]interface{}{ + "lexer": name, + // "pid": os.Getpid(), + }, + } + l := &Lexer{ - L: zap.NewExample().Sugar().Named("lexer"), + L: zap.Must(config.Build()).Named("lexer"), name: name, input: input, state: lexText, @@ -79,7 +102,7 @@ func (l *Lexer) backup() { } type Lexer struct { - L *zap.SugaredLogger + L *zap.Logger name, input string start, pos, width int state stateFn @@ -127,7 +150,7 @@ func (l *Lexer) emit(t LexemeType) { L := l.L.Named("emit").With( zap.String("item", i.String()), ) - L.Debug("emitting lexeme") + L.Info("emitting lexeme") l.Items = append(l.Items, i) l.SetStart(l.GetPos()) /* original concurrent implementation @@ -139,7 +162,7 @@ func (l *Lexer) emit(t LexemeType) { zap.Int("width", l.GetWidth()), ).Named("emit") - L.Debugw("emitting item", + L.Debug("emitting item", zap.String("item", i.String()), ) l.items <- i @@ -152,7 +175,7 @@ func (l *Lexer) errorf(format string, args ...interface{}) stateFn { LexError, fmt.Sprintf(format, args...), } - L.Debugw("emitting errorItem", + L.Debug("emitting errorItem", zap.String("error", errorItem.String()), ) @@ -169,7 +192,7 @@ func (l *Lexer) next() rune { return EOF } r, width := utf8.DecodeRuneInString(l.input[l.GetPos():]) - L.Debugw("found rune", + L.Debug("found rune", zap.String("rune", string(r)), zap.Int("width", width), ) @@ -195,7 +218,7 @@ func (l *Lexer) run() { func (l *Lexer) GetPos() int { defer l.posMutex.Unlock() l.posMutex.Lock() - l.L.Named("GetPos").Debugw("getting current position", + l.L.Named("GetPos").Debug("getting current position", zap.Int("old", l.pos), ) return l.pos @@ -204,7 +227,7 @@ func (l *Lexer) GetPos() int { func (l *Lexer) SetPos(pos int) { defer l.posMutex.Unlock() l.posMutex.Lock() - l.L.Named("SetPos").Debugw("setting new position", + l.L.Named("SetPos").Debug("setting new position", zap.Int("new", pos), zap.Int("old", l.pos), ) @@ -214,7 +237,7 @@ func (l *Lexer) SetPos(pos int) { func (l *Lexer) GetWidth() int { defer l.widthMutex.Unlock() l.widthMutex.Lock() - l.L.Named("GetWidth").Debugw("setting new width", + l.L.Named("GetWidth").Debug("setting new width", zap.Int("old", l.width), ) return l.width @@ -223,7 +246,7 @@ func (l *Lexer) GetWidth() int { func (l *Lexer) SetWidth(width int) { defer l.widthMutex.Unlock() l.widthMutex.Lock() - l.L.Named("SetWidth").Debugw("setting new width", + l.L.Named("SetWidth").Debug("setting new width", zap.Int("new", width), zap.Int("old", l.width), ) @@ -233,7 +256,7 @@ func (l *Lexer) SetWidth(width int) { func (l *Lexer) GetStart() int { defer l.startMutex.Unlock() l.startMutex.Lock() - l.L.Named("GetStart").Debugw("getting old start", + l.L.Named("GetStart").Debug("getting old start", zap.Int("old", l.start), ) return l.start @@ -242,7 +265,7 @@ func (l *Lexer) GetStart() int { func (l *Lexer) SetStart(start int) { defer l.startMutex.Unlock() l.startMutex.Lock() - l.L.Named("SetStart").Debugw("setting new start", + l.L.Named("SetStart").Debug("setting new start", zap.Int("new", start), zap.Int("old", l.start), ) diff --git a/lexer_test.go b/lexer_test.go index e28a6ea..0debba1 100644 --- a/lexer_test.go +++ b/lexer_test.go @@ -6,8 +6,249 @@ import ( "code.ndumas.com/ndumas/wikilink-parser" ) -func Test_ObsidianWikilinks(t *testing.T) { - // t.Parallel() +func Test_ObsidianWikilinks_LinksIntext(t *testing.T) { + tcs := []struct { + name string + in string + expected []wikilink.Lexeme + }{ + { + name: "wikilink", + in: "this is a [[wikilink]]", + expected: []wikilink.Lexeme{ + {Typ: wikilink.LexText, Val: "this is a "}, + {Typ: wikilink.LexOpenLink, Val: "[["}, + {Typ: wikilink.LexIdent, Val: "wikilink"}, + {Typ: wikilink.LexCloseLink, Val: "]]"}, + {Typ: wikilink.LexText, Val: ""}, + }, + }, + { + name: "wikilink|display name", + in: "this is a [[wikilink|display name]]", + expected: []wikilink.Lexeme{ + {Typ: wikilink.LexText, Val: "this is a "}, + {Typ: wikilink.LexOpenLink, Val: "[["}, + {Typ: wikilink.LexIdent, Val: "wikilink"}, + {Typ: wikilink.LexAlias, Val: "|"}, + {Typ: wikilink.LexIdent, Val: "display name"}, + {Typ: wikilink.LexCloseLink, Val: "]]"}, + {Typ: wikilink.LexText, Val: ""}, + }, + }, + { + name: "wikilink|display name|second pipe", + in: "[[wikilink|display name|second pipe]]", + expected: []wikilink.Lexeme{ + {Typ: wikilink.LexOpenLink, Val: "[["}, + {Typ: wikilink.LexIdent, Val: "wikilink"}, + {Typ: wikilink.LexAlias, Val: "|"}, + {Typ: wikilink.LexIdent, Val: "display name"}, + {Typ: wikilink.LexAlias, Val: "|"}, + {Typ: wikilink.LexIdent, Val: "second pipe"}, + {Typ: wikilink.LexCloseLink, Val: "]]"}, + {Typ: wikilink.LexText, Val: ""}, + }, + }, + { + name: "wikilink with numeric alias|420|second pipe", + in: "[[wikilink|420|second pipe]]", + expected: []wikilink.Lexeme{ + {Typ: wikilink.LexOpenLink, Val: "[["}, + {Typ: wikilink.LexIdent, Val: "wikilink"}, + {Typ: wikilink.LexAlias, Val: "|"}, + {Typ: wikilink.LexIdent, Val: "420"}, + {Typ: wikilink.LexAlias, Val: "|"}, + {Typ: wikilink.LexIdent, Val: "second pipe"}, + {Typ: wikilink.LexCloseLink, Val: "]]"}, + {Typ: wikilink.LexText, Val: ""}, + }, + }, + { + name: "wikilink with spaces in filename", + in: "[[wikilink spaces]]", + expected: []wikilink.Lexeme{ + {Typ: wikilink.LexOpenLink, Val: "[["}, + {Typ: wikilink.LexIdent, Val: "wikilink spaces"}, + {Typ: wikilink.LexCloseLink, Val: "]]"}, + {Typ: wikilink.LexText, Val: ""}, + }, + }, + { + name: "#heading", + in: "[[#heading]]", + expected: []wikilink.Lexeme{ + {Typ: wikilink.LexOpenLink, Val: "[["}, + {Typ: wikilink.LexIdent, Val: ""}, + {Typ: wikilink.LexHeading, Val: "#"}, + {Typ: wikilink.LexIdent, Val: "heading"}, + {Typ: wikilink.LexCloseLink, Val: "]]"}, + {Typ: wikilink.LexText, Val: ""}, + }, + }, + { + name: "wikilink#heading", + in: "[[wikilink#heading]]", + expected: []wikilink.Lexeme{ + {Typ: wikilink.LexOpenLink, Val: "[["}, + {Typ: wikilink.LexIdent, Val: "wikilink"}, + {Typ: wikilink.LexHeading, Val: "#"}, + {Typ: wikilink.LexIdent, Val: "heading"}, + {Typ: wikilink.LexCloseLink, Val: "]]"}, + {Typ: wikilink.LexText, Val: ""}, + }, + }, + { + name: "wikilink#heading|display name", + in: "[[wikilink#heading|display name]]", + expected: []wikilink.Lexeme{ + {Typ: wikilink.LexOpenLink, Val: "[["}, + {Typ: wikilink.LexIdent, Val: "wikilink"}, + {Typ: wikilink.LexHeading, Val: "#"}, + {Typ: wikilink.LexIdent, Val: "heading"}, + {Typ: wikilink.LexAlias, Val: "|"}, + {Typ: wikilink.LexIdent, Val: "display name"}, + {Typ: wikilink.LexCloseLink, Val: "]]"}, + {Typ: wikilink.LexText, Val: ""}, + }, + }, + { + name: "wikilink#heading|display name|second pipe", + in: "[[wikilink#heading|display name|second pipe]]", + expected: []wikilink.Lexeme{ + {Typ: wikilink.LexOpenLink, Val: "[["}, + {Typ: wikilink.LexIdent, Val: "wikilink"}, + {Typ: wikilink.LexHeading, Val: "#"}, + {Typ: wikilink.LexIdent, Val: "heading"}, + {Typ: wikilink.LexAlias, Val: "|"}, + {Typ: wikilink.LexIdent, Val: "display name"}, + {Typ: wikilink.LexAlias, Val: "|"}, + {Typ: wikilink.LexIdent, Val: "second pipe"}, + {Typ: wikilink.LexCloseLink, Val: "]]"}, + {Typ: wikilink.LexText, Val: ""}, + }, + }, + { + name: "wikilink with numeric aliases#heading|420|display name", + in: "[[wikilink#heading|420|second pipe]]", + expected: []wikilink.Lexeme{ + {Typ: wikilink.LexOpenLink, Val: "[["}, + {Typ: wikilink.LexIdent, Val: "wikilink"}, + {Typ: wikilink.LexHeading, Val: "#"}, + {Typ: wikilink.LexIdent, Val: "heading"}, + {Typ: wikilink.LexAlias, Val: "|"}, + {Typ: wikilink.LexIdent, Val: "420"}, + {Typ: wikilink.LexAlias, Val: "|"}, + {Typ: wikilink.LexIdent, Val: "second pipe"}, + {Typ: wikilink.LexCloseLink, Val: "]]"}, + {Typ: wikilink.LexText, Val: ""}, + }, + }, + { + name: "#^blockRef", + in: "[[#^blockRef]]", + expected: []wikilink.Lexeme{ + {Typ: wikilink.LexOpenLink, Val: "[["}, + {Typ: wikilink.LexIdent, Val: ""}, + {Typ: wikilink.LexBlockRef, Val: "#^"}, + {Typ: wikilink.LexIdent, Val: "blockRef"}, + {Typ: wikilink.LexCloseLink, Val: "]]"}, + {Typ: wikilink.LexText, Val: ""}, + }, + }, + { + name: "wikilink#^blockRef", + in: "[[wikilink#^blockRef]]", + expected: []wikilink.Lexeme{ + {Typ: wikilink.LexOpenLink, Val: "[["}, + {Typ: wikilink.LexIdent, Val: "wikilink"}, + {Typ: wikilink.LexBlockRef, Val: "#^"}, + {Typ: wikilink.LexIdent, Val: "blockRef"}, + {Typ: wikilink.LexCloseLink, Val: "]]"}, + {Typ: wikilink.LexText, Val: ""}, + }, + }, + { + name: "wikilink#^blockRef|display name", + in: "[[wikilink#^blockRef|display name]]", + expected: []wikilink.Lexeme{ + {Typ: wikilink.LexOpenLink, Val: "[["}, + {Typ: wikilink.LexIdent, Val: "wikilink"}, + {Typ: wikilink.LexBlockRef, Val: "#^"}, + {Typ: wikilink.LexIdent, Val: "blockRef"}, + {Typ: wikilink.LexAlias, Val: "|"}, + {Typ: wikilink.LexIdent, Val: "display name"}, + {Typ: wikilink.LexCloseLink, Val: "]]"}, + {Typ: wikilink.LexText, Val: ""}, + }, + }, + { + name: "wikilink#^blockRef|display name|second pipe", + in: "[[wikilink#^blockRef|display name|second pipe]]", + expected: []wikilink.Lexeme{ + {Typ: wikilink.LexOpenLink, Val: "[["}, + {Typ: wikilink.LexIdent, Val: "wikilink"}, + {Typ: wikilink.LexBlockRef, Val: "#^"}, + {Typ: wikilink.LexIdent, Val: "blockRef"}, + {Typ: wikilink.LexAlias, Val: "|"}, + {Typ: wikilink.LexIdent, Val: "display name"}, + {Typ: wikilink.LexAlias, Val: "|"}, + {Typ: wikilink.LexIdent, Val: "second pipe"}, + {Typ: wikilink.LexCloseLink, Val: "]]"}, + {Typ: wikilink.LexText, Val: ""}, + }, + }, + { + name: "wikilink with numeric aliases#^blockRef|420|second pipe", + in: "[[wikilink#^blockRef|420|second pipe]]", + expected: []wikilink.Lexeme{ + {Typ: wikilink.LexOpenLink, Val: "[["}, + {Typ: wikilink.LexIdent, Val: "wikilink"}, + {Typ: wikilink.LexBlockRef, Val: "#^"}, + {Typ: wikilink.LexIdent, Val: "blockRef"}, + {Typ: wikilink.LexAlias, Val: "|"}, + {Typ: wikilink.LexIdent, Val: "420"}, + {Typ: wikilink.LexAlias, Val: "|"}, + {Typ: wikilink.LexIdent, Val: "second pipe"}, + {Typ: wikilink.LexCloseLink, Val: "]]"}, + {Typ: wikilink.LexText, Val: ""}, + }, + }, + } + for _, tc := range tcs { + tc := tc + t.Run(tc.name, func(t *testing.T) { + // t.Parallel() + l := wikilink.Lex("testLexer", tc.in) + defer l.L.Sync() + if len(tc.expected) != len(l.Items) { + t.Logf("expected %d tokens, got %d\n", len(tc.expected), len(l.Items)) + t.Fail() + + return + } + for i, e := range tc.expected { + n := l.Items[i] + if e.Typ != n.Typ { + t.Logf("expected Type %s, received %s", e.Typ.String(), n.Typ.String()) + t.Fail() + + return + } + + if e.Val != n.Val { + t.Logf("expected Value %q, received %q", e.Val, n.Val) + t.Fail() + + return + } + } + }) + } +} + +func Test_ObsidianWikilinks_Basic(t *testing.T) { + t.Parallel() tcs := []struct { name string in string @@ -218,7 +459,7 @@ func Test_ObsidianWikilinks(t *testing.T) { for _, tc := range tcs { tc := tc t.Run(tc.name, func(t *testing.T) { - t.Parallel() + // t.Parallel() l := wikilink.Lex("testLexer", tc.in) defer l.L.Sync() if len(tc.expected) != len(l.Items) {