ndumas
/
wikilinks-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Expand test cases, refine logging configuration

Browse Source 
Wiki links don't occur in a vacuum, I need to be able to parse entire
blocks of markdown and extract the wikilinks properly.

Now that I've gotten the lexer working I mostly don't need to see every
single change to the position/start/width values, but the
instrumentation will be useful for future debugging.
main
Nick Dumas 1 year ago
parent ebeea16b0d
commit 0b377a0500
3 changed files with 281 additions and 15 deletions
Show Stats Download Patch File Download Diff File

2
.gitignore vendored
Unescape Escape View File

@ -2,3 +2,5 @@ node_modules/*
package*.json
dist/*
reports/*
lexer.log
parser.log

47
lexer.go
Unescape Escape View File

@ -3,12 +3,15 @@ package wikilink
import (
"fmt"
// "os"
"strings"
"sync"
// "unicode"
"unicode/utf8"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)
const (
@ -36,8 +39,28 @@ const (
)
func Lex(name, input string) *Lexer {
encoderCfg := zap.NewProductionEncoderConfig()
encoderCfg.EncodeTime = zapcore.ISO8601TimeEncoder
config := zap.Config{
Level: zap.NewAtomicLevelAt(zap.InfoLevel),
EncoderConfig: encoderCfg,
OutputPaths: []string{
"./lexer.log",
"stdout",
},
Encoding: "console",
ErrorOutputPaths: []string{
"stderr",
},
InitialFields: map[string]interface{}{
"lexer": name,
// "pid": os.Getpid(),
},
}
l := &Lexer{
L: zap.NewExample().Sugar().Named("lexer"),
L: zap.Must(config.Build()).Named("lexer"),
name: name,
input: input,
state: lexText,
@ -79,7 +102,7 @@ func (l *Lexer) backup() {
}
type Lexer struct {
L *zap.SugaredLogger
L *zap.Logger
name, input string
start, pos, width int
state stateFn
@ -127,7 +150,7 @@ func (l *Lexer) emit(t LexemeType) {
L := l.L.Named("emit").With(
zap.String("item", i.String()),
)
L.Debug("emitting lexeme")
L.Info("emitting lexeme")
l.Items = append(l.Items, i)
l.SetStart(l.GetPos())
/* original concurrent implementation
@ -139,7 +162,7 @@ func (l *Lexer) emit(t LexemeType) {
zap.Int("width", l.GetWidth()),
).Named("emit")
L.Debugw("emitting item",
L.Debug("emitting item",
zap.String("item", i.String()),
)
l.items <- i
@ -152,7 +175,7 @@ func (l *Lexer) errorf(format string, args ...interface{}) stateFn {
LexError,
fmt.Sprintf(format, args...),
}
L.Debugw("emitting errorItem",
L.Debug("emitting errorItem",
zap.String("error", errorItem.String()),
)
@ -169,7 +192,7 @@ func (l *Lexer) next() rune {
return EOF
}
r, width := utf8.DecodeRuneInString(l.input[l.GetPos():])
L.Debugw("found rune",
L.Debug("found rune",
zap.String("rune", string(r)),
zap.Int("width", width),
)
@ -195,7 +218,7 @@ func (l *Lexer) run() {
func (l *Lexer) GetPos() int {
defer l.posMutex.Unlock()
l.posMutex.Lock()
l.L.Named("GetPos").Debugw("getting current position",
l.L.Named("GetPos").Debug("getting current position",
zap.Int("old", l.pos),
)
return l.pos
@ -204,7 +227,7 @@ func (l *Lexer) GetPos() int {
func (l *Lexer) SetPos(pos int) {
defer l.posMutex.Unlock()
l.posMutex.Lock()
l.L.Named("SetPos").Debugw("setting new position",
l.L.Named("SetPos").Debug("setting new position",
zap.Int("new", pos),
zap.Int("old", l.pos),
)
@ -214,7 +237,7 @@ func (l *Lexer) SetPos(pos int) {
func (l *Lexer) GetWidth() int {
defer l.widthMutex.Unlock()
l.widthMutex.Lock()
l.L.Named("GetWidth").Debugw("setting new width",
l.L.Named("GetWidth").Debug("setting new width",
zap.Int("old", l.width),
)
return l.width
@ -223,7 +246,7 @@ func (l *Lexer) GetWidth() int {
func (l *Lexer) SetWidth(width int) {
defer l.widthMutex.Unlock()
l.widthMutex.Lock()
l.L.Named("SetWidth").Debugw("setting new width",
l.L.Named("SetWidth").Debug("setting new width",
zap.Int("new", width),
zap.Int("old", l.width),
)
@ -233,7 +256,7 @@ func (l *Lexer) SetWidth(width int) {
func (l *Lexer) GetStart() int {
defer l.startMutex.Unlock()
l.startMutex.Lock()
l.L.Named("GetStart").Debugw("getting old start",
l.L.Named("GetStart").Debug("getting old start",
zap.Int("old", l.start),
)
return l.start
@ -242,7 +265,7 @@ func (l *Lexer) GetStart() int {
func (l *Lexer) SetStart(start int) {
defer l.startMutex.Unlock()
l.startMutex.Lock()
l.L.Named("SetStart").Debugw("setting new start",
l.L.Named("SetStart").Debug("setting new start",
zap.Int("new", start),
zap.Int("old", l.start),
)

245
lexer_test.go
Unescape Escape View File

@ -6,8 +6,249 @@ import (
"code.ndumas.com/ndumas/wikilink-parser"
)
func Test_ObsidianWikilinks(t *testing.T) {
func Test_ObsidianWikilinks_LinksIntext(t *testing.T) {
tcs := []struct {
name string
in string
expected []wikilink.Lexeme
}{
{
name: "wikilink",
in: "this is a [[wikilink]]",
expected: []wikilink.Lexeme{
{Typ: wikilink.LexText, Val: "this is a "},
{Typ: wikilink.LexOpenLink, Val: "[["},
{Typ: wikilink.LexIdent, Val: "wikilink"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
},
},
{
name: "wikilink|display name",
in: "this is a [[wikilink|display name]]",
expected: []wikilink.Lexeme{
{Typ: wikilink.LexText, Val: "this is a "},
{Typ: wikilink.LexOpenLink, Val: "[["},
{Typ: wikilink.LexIdent, Val: "wikilink"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "display name"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
},
},
{
name: "wikilink|display name|second pipe",
in: "[[wikilink|display name|second pipe]]",
expected: []wikilink.Lexeme{
{Typ: wikilink.LexOpenLink, Val: "[["},
{Typ: wikilink.LexIdent, Val: "wikilink"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "display name"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "second pipe"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
},
},
{
name: "wikilink with numeric alias|420|second pipe",
in: "[[wikilink|420|second pipe]]",
expected: []wikilink.Lexeme{
{Typ: wikilink.LexOpenLink, Val: "[["},
{Typ: wikilink.LexIdent, Val: "wikilink"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "420"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "second pipe"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
},
},
{
name: "wikilink with spaces in filename",
in: "[[wikilink spaces]]",
expected: []wikilink.Lexeme{
{Typ: wikilink.LexOpenLink, Val: "[["},
{Typ: wikilink.LexIdent, Val: "wikilink spaces"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
},
},
{
name: "#heading",
in: "[[#heading]]",
expected: []wikilink.Lexeme{
{Typ: wikilink.LexOpenLink, Val: "[["},
{Typ: wikilink.LexIdent, Val: ""},
{Typ: wikilink.LexHeading, Val: "#"},
{Typ: wikilink.LexIdent, Val: "heading"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
},
},
{
name: "wikilink#heading",
in: "[[wikilink#heading]]",
expected: []wikilink.Lexeme{
{Typ: wikilink.LexOpenLink, Val: "[["},
{Typ: wikilink.LexIdent, Val: "wikilink"},
{Typ: wikilink.LexHeading, Val: "#"},
{Typ: wikilink.LexIdent, Val: "heading"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
},
},
{
name: "wikilink#heading|display name",
in: "[[wikilink#heading|display name]]",
expected: []wikilink.Lexeme{
{Typ: wikilink.LexOpenLink, Val: "[["},
{Typ: wikilink.LexIdent, Val: "wikilink"},
{Typ: wikilink.LexHeading, Val: "#"},
{Typ: wikilink.LexIdent, Val: "heading"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "display name"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
},
},
{
name: "wikilink#heading|display name|second pipe",
in: "[[wikilink#heading|display name|second pipe]]",
expected: []wikilink.Lexeme{
{Typ: wikilink.LexOpenLink, Val: "[["},
{Typ: wikilink.LexIdent, Val: "wikilink"},
{Typ: wikilink.LexHeading, Val: "#"},
{Typ: wikilink.LexIdent, Val: "heading"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "display name"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "second pipe"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
},
},
{
name: "wikilink with numeric aliases#heading|420|display name",
in: "[[wikilink#heading|420|second pipe]]",
expected: []wikilink.Lexeme{
{Typ: wikilink.LexOpenLink, Val: "[["},
{Typ: wikilink.LexIdent, Val: "wikilink"},
{Typ: wikilink.LexHeading, Val: "#"},
{Typ: wikilink.LexIdent, Val: "heading"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "420"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "second pipe"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
},
},
{
name: "#^blockRef",
in: "[[#^blockRef]]",
expected: []wikilink.Lexeme{
{Typ: wikilink.LexOpenLink, Val: "[["},
{Typ: wikilink.LexIdent, Val: ""},
{Typ: wikilink.LexBlockRef, Val: "#^"},
{Typ: wikilink.LexIdent, Val: "blockRef"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
},
},
{
name: "wikilink#^blockRef",
in: "[[wikilink#^blockRef]]",
expected: []wikilink.Lexeme{
{Typ: wikilink.LexOpenLink, Val: "[["},
{Typ: wikilink.LexIdent, Val: "wikilink"},
{Typ: wikilink.LexBlockRef, Val: "#^"},
{Typ: wikilink.LexIdent, Val: "blockRef"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
},
},
{
name: "wikilink#^blockRef|display name",
in: "[[wikilink#^blockRef|display name]]",
expected: []wikilink.Lexeme{
{Typ: wikilink.LexOpenLink, Val: "[["},
{Typ: wikilink.LexIdent, Val: "wikilink"},
{Typ: wikilink.LexBlockRef, Val: "#^"},
{Typ: wikilink.LexIdent, Val: "blockRef"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "display name"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
},
},
{
name: "wikilink#^blockRef|display name|second pipe",
in: "[[wikilink#^blockRef|display name|second pipe]]",
expected: []wikilink.Lexeme{
{Typ: wikilink.LexOpenLink, Val: "[["},
{Typ: wikilink.LexIdent, Val: "wikilink"},
{Typ: wikilink.LexBlockRef, Val: "#^"},
{Typ: wikilink.LexIdent, Val: "blockRef"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "display name"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "second pipe"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
},
},
{
name: "wikilink with numeric aliases#^blockRef|420|second pipe",
in: "[[wikilink#^blockRef|420|second pipe]]",
expected: []wikilink.Lexeme{
{Typ: wikilink.LexOpenLink, Val: "[["},
{Typ: wikilink.LexIdent, Val: "wikilink"},
{Typ: wikilink.LexBlockRef, Val: "#^"},
{Typ: wikilink.LexIdent, Val: "blockRef"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "420"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "second pipe"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
},
},
}
for _, tc := range tcs {
tc := tc
t.Run(tc.name, func(t *testing.T) {
// t.Parallel()
l := wikilink.Lex("testLexer", tc.in)
defer l.L.Sync()
if len(tc.expected) != len(l.Items) {
t.Logf("expected %d tokens, got %d\n", len(tc.expected), len(l.Items))
t.Fail()
return
}
for i, e := range tc.expected {
n := l.Items[i]
if e.Typ != n.Typ {
t.Logf("expected Type %s, received %s", e.Typ.String(), n.Typ.String())
t.Fail()
return
}
if e.Val != n.Val {
t.Logf("expected Value %q, received %q", e.Val, n.Val)
t.Fail()
return
}
}
})
}
}
func Test_ObsidianWikilinks_Basic(t *testing.T) {
t.Parallel()
tcs := []struct {
name string
in string
@ -218,7 +459,7 @@ func Test_ObsidianWikilinks(t *testing.T) {
for _, tc := range tcs {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
// t.Parallel()
l := wikilink.Lex("testLexer", tc.in)
defer l.L.Sync()
if len(tc.expected) != len(l.Items) {

Write Preview
Loading…
Cancel
Save