Expand test cases, refine logging configuration

Wiki links don't occur in a vacuum, I need to be able to parse entire
blocks of markdown and extract the wikilinks properly.

Now that I've gotten the lexer working I mostly don't need to see every
single change to the position/start/width values, but the
instrumentation will be useful for future debugging.

.gitignore

@@ -2,3 +2,5 @@ node_modules/*
 package*.json
 dist/*
 reports/*
+lexer.log
+parser.log

lexer.go

@@ -3,12 +3,15 @@ package wikilink
 
 import (
 	"fmt"
+	// "os"
 	"strings"
 	"sync"
+
 	// "unicode"
 	"unicode/utf8"
 
 	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
 )
 
 const (
@@ -36,8 +39,28 @@ const (
 )
 
 func Lex(name, input string) *Lexer {
+	encoderCfg := zap.NewProductionEncoderConfig()
+	encoderCfg.EncodeTime = zapcore.ISO8601TimeEncoder
+
+	config := zap.Config{
+		Level:         zap.NewAtomicLevelAt(zap.InfoLevel),
+		EncoderConfig: encoderCfg,
+		OutputPaths: []string{
+			"./lexer.log",
+			"stdout",
+		},
+		Encoding: "console",
+		ErrorOutputPaths: []string{
+			"stderr",
+		},
+		InitialFields: map[string]interface{}{
+			"lexer": name,
+			// "pid": os.Getpid(),
+		},
+	}
+
 	l := &Lexer{
-		L:     zap.NewExample().Sugar().Named("lexer"),
+		L:     zap.Must(config.Build()).Named("lexer"),
 		name:  name,
 		input: input,
 		state: lexText,
@@ -79,7 +102,7 @@ func (l *Lexer) backup() {
 }
 
 type Lexer struct {
-	L                 *zap.SugaredLogger
+	L                 *zap.Logger
 	name, input       string
 	start, pos, width int
 	state             stateFn
@@ -127,7 +150,7 @@ func (l *Lexer) emit(t LexemeType) {
 	L := l.L.Named("emit").With(
 		zap.String("item", i.String()),
 	)
-	L.Debug("emitting lexeme")
+	L.Info("emitting lexeme")
 	l.Items = append(l.Items, i)
 	l.SetStart(l.GetPos())
 	/* original concurrent implementation
@@ -139,7 +162,7 @@ func (l *Lexer) emit(t LexemeType) {
 		zap.Int("width", l.GetWidth()),
 	).Named("emit")
 
-	L.Debugw("emitting item",
+	L.Debug("emitting item",
 		zap.String("item", i.String()),
 	)
 	l.items <- i
@@ -152,7 +175,7 @@ func (l *Lexer) errorf(format string, args ...interface{}) stateFn {
 		LexError,
 		fmt.Sprintf(format, args...),
 	}
-	L.Debugw("emitting errorItem",
+	L.Debug("emitting errorItem",
 		zap.String("error", errorItem.String()),
 	)
 
@@ -169,7 +192,7 @@ func (l *Lexer) next() rune {
 		return EOF
 	}
 	r, width := utf8.DecodeRuneInString(l.input[l.GetPos():])
-	L.Debugw("found rune",
+	L.Debug("found rune",
 		zap.String("rune", string(r)),
 		zap.Int("width", width),
 	)
@@ -195,7 +218,7 @@ func (l *Lexer) run() {
 func (l *Lexer) GetPos() int {
 	defer l.posMutex.Unlock()
 	l.posMutex.Lock()
-	l.L.Named("GetPos").Debugw("getting current position",
+	l.L.Named("GetPos").Debug("getting current position",
 		zap.Int("old", l.pos),
 	)
 	return l.pos
@@ -204,7 +227,7 @@ func (l *Lexer) GetPos() int {
 func (l *Lexer) SetPos(pos int) {
 	defer l.posMutex.Unlock()
 	l.posMutex.Lock()
-	l.L.Named("SetPos").Debugw("setting new position",
+	l.L.Named("SetPos").Debug("setting new position",
 		zap.Int("new", pos),
 		zap.Int("old", l.pos),
 	)
@@ -214,7 +237,7 @@ func (l *Lexer) SetPos(pos int) {
 func (l *Lexer) GetWidth() int {
 	defer l.widthMutex.Unlock()
 	l.widthMutex.Lock()
-	l.L.Named("GetWidth").Debugw("setting new width",
+	l.L.Named("GetWidth").Debug("setting new width",
 		zap.Int("old", l.width),
 	)
 	return l.width
@@ -223,7 +246,7 @@ func (l *Lexer) GetWidth() int {
 func (l *Lexer) SetWidth(width int) {
 	defer l.widthMutex.Unlock()
 	l.widthMutex.Lock()
-	l.L.Named("SetWidth").Debugw("setting new width",
+	l.L.Named("SetWidth").Debug("setting new width",
 		zap.Int("new", width),
 		zap.Int("old", l.width),
 	)
@@ -233,7 +256,7 @@ func (l *Lexer) SetWidth(width int) {
 func (l *Lexer) GetStart() int {
 	defer l.startMutex.Unlock()
 	l.startMutex.Lock()
-	l.L.Named("GetStart").Debugw("getting old start",
+	l.L.Named("GetStart").Debug("getting old start",
 		zap.Int("old", l.start),
 	)
 	return l.start
@@ -242,7 +265,7 @@ func (l *Lexer) GetStart() int {
 func (l *Lexer) SetStart(start int) {
 	defer l.startMutex.Unlock()
 	l.startMutex.Lock()
-	l.L.Named("SetStart").Debugw("setting new start",
+	l.L.Named("SetStart").Debug("setting new start",
 		zap.Int("new", start),
 		zap.Int("old", l.start),
 	)

lexer_test.go

@@ -6,8 +6,249 @@ import (
 	"code.ndumas.com/ndumas/wikilink-parser"
 )
 
-func Test_ObsidianWikilinks(t *testing.T) {
+func Test_ObsidianWikilinks_LinksIntext(t *testing.T) {
+	tcs := []struct {
+		name     string
+		in       string
+		expected []wikilink.Lexeme
+	}{
+		{
+			name: "wikilink",
+			in:   "this is a [[wikilink]]",
+			expected: []wikilink.Lexeme{
+				{Typ: wikilink.LexText, Val: "this is a "},
+				{Typ: wikilink.LexOpenLink, Val: "[["},
+				{Typ: wikilink.LexIdent, Val: "wikilink"},
+				{Typ: wikilink.LexCloseLink, Val: "]]"},
+				{Typ: wikilink.LexText, Val: ""},
+			},
+		},
+		{
+			name: "wikilink|display name",
+			in:   "this is a [[wikilink|display name]]",
+			expected: []wikilink.Lexeme{
+				{Typ: wikilink.LexText, Val: "this is a "},
+				{Typ: wikilink.LexOpenLink, Val: "[["},
+				{Typ: wikilink.LexIdent, Val: "wikilink"},
+				{Typ: wikilink.LexAlias, Val: "|"},
+				{Typ: wikilink.LexIdent, Val: "display name"},
+				{Typ: wikilink.LexCloseLink, Val: "]]"},
+				{Typ: wikilink.LexText, Val: ""},
+			},
+		},
+		{
+			name: "wikilink|display name|second pipe",
+			in:   "[[wikilink|display name|second pipe]]",
+			expected: []wikilink.Lexeme{
+				{Typ: wikilink.LexOpenLink, Val: "[["},
+				{Typ: wikilink.LexIdent, Val: "wikilink"},
+				{Typ: wikilink.LexAlias, Val: "|"},
+				{Typ: wikilink.LexIdent, Val: "display name"},
+				{Typ: wikilink.LexAlias, Val: "|"},
+				{Typ: wikilink.LexIdent, Val: "second pipe"},
+				{Typ: wikilink.LexCloseLink, Val: "]]"},
+				{Typ: wikilink.LexText, Val: ""},
+			},
+		},
+		{
+			name: "wikilink with numeric alias|420|second pipe",
+			in:   "[[wikilink|420|second pipe]]",
+			expected: []wikilink.Lexeme{
+				{Typ: wikilink.LexOpenLink, Val: "[["},
+				{Typ: wikilink.LexIdent, Val: "wikilink"},
+				{Typ: wikilink.LexAlias, Val: "|"},
+				{Typ: wikilink.LexIdent, Val: "420"},
+				{Typ: wikilink.LexAlias, Val: "|"},
+				{Typ: wikilink.LexIdent, Val: "second pipe"},
+				{Typ: wikilink.LexCloseLink, Val: "]]"},
+				{Typ: wikilink.LexText, Val: ""},
+			},
+		},
+		{
+			name: "wikilink with spaces in filename",
+			in:   "[[wikilink spaces]]",
+			expected: []wikilink.Lexeme{
+				{Typ: wikilink.LexOpenLink, Val: "[["},
+				{Typ: wikilink.LexIdent, Val: "wikilink spaces"},
+				{Typ: wikilink.LexCloseLink, Val: "]]"},
+				{Typ: wikilink.LexText, Val: ""},
+			},
+		},
+		{
+			name: "#heading",
+			in:   "[[#heading]]",
+			expected: []wikilink.Lexeme{
+				{Typ: wikilink.LexOpenLink, Val: "[["},
+				{Typ: wikilink.LexIdent, Val: ""},
+				{Typ: wikilink.LexHeading, Val: "#"},
+				{Typ: wikilink.LexIdent, Val: "heading"},
+				{Typ: wikilink.LexCloseLink, Val: "]]"},
+				{Typ: wikilink.LexText, Val: ""},
+			},
+		},
+		{
+			name: "wikilink#heading",
+			in:   "[[wikilink#heading]]",
+			expected: []wikilink.Lexeme{
+				{Typ: wikilink.LexOpenLink, Val: "[["},
+				{Typ: wikilink.LexIdent, Val: "wikilink"},
+				{Typ: wikilink.LexHeading, Val: "#"},
+				{Typ: wikilink.LexIdent, Val: "heading"},
+				{Typ: wikilink.LexCloseLink, Val: "]]"},
+				{Typ: wikilink.LexText, Val: ""},
+			},
+		},
+		{
+			name: "wikilink#heading|display name",
+			in:   "[[wikilink#heading|display name]]",
+			expected: []wikilink.Lexeme{
+				{Typ: wikilink.LexOpenLink, Val: "[["},
+				{Typ: wikilink.LexIdent, Val: "wikilink"},
+				{Typ: wikilink.LexHeading, Val: "#"},
+				{Typ: wikilink.LexIdent, Val: "heading"},
+				{Typ: wikilink.LexAlias, Val: "|"},
+				{Typ: wikilink.LexIdent, Val: "display name"},
+				{Typ: wikilink.LexCloseLink, Val: "]]"},
+				{Typ: wikilink.LexText, Val: ""},
+			},
+		},
+		{
+			name: "wikilink#heading|display name|second pipe",
+			in:   "[[wikilink#heading|display name|second pipe]]",
+			expected: []wikilink.Lexeme{
+				{Typ: wikilink.LexOpenLink, Val: "[["},
+				{Typ: wikilink.LexIdent, Val: "wikilink"},
+				{Typ: wikilink.LexHeading, Val: "#"},
+				{Typ: wikilink.LexIdent, Val: "heading"},
+				{Typ: wikilink.LexAlias, Val: "|"},
+				{Typ: wikilink.LexIdent, Val: "display name"},
+				{Typ: wikilink.LexAlias, Val: "|"},
+				{Typ: wikilink.LexIdent, Val: "second pipe"},
+				{Typ: wikilink.LexCloseLink, Val: "]]"},
+				{Typ: wikilink.LexText, Val: ""},
+			},
+		},
+		{
+			name: "wikilink with numeric aliases#heading|420|display name",
+			in:   "[[wikilink#heading|420|second pipe]]",
+			expected: []wikilink.Lexeme{
+				{Typ: wikilink.LexOpenLink, Val: "[["},
+				{Typ: wikilink.LexIdent, Val: "wikilink"},
+				{Typ: wikilink.LexHeading, Val: "#"},
+				{Typ: wikilink.LexIdent, Val: "heading"},
+				{Typ: wikilink.LexAlias, Val: "|"},
+				{Typ: wikilink.LexIdent, Val: "420"},
+				{Typ: wikilink.LexAlias, Val: "|"},
+				{Typ: wikilink.LexIdent, Val: "second pipe"},
+				{Typ: wikilink.LexCloseLink, Val: "]]"},
+				{Typ: wikilink.LexText, Val: ""},
+			},
+		},
+		{
+			name: "#^blockRef",
+			in:   "[[#^blockRef]]",
+			expected: []wikilink.Lexeme{
+				{Typ: wikilink.LexOpenLink, Val: "[["},
+				{Typ: wikilink.LexIdent, Val: ""},
+				{Typ: wikilink.LexBlockRef, Val: "#^"},
+				{Typ: wikilink.LexIdent, Val: "blockRef"},
+				{Typ: wikilink.LexCloseLink, Val: "]]"},
+				{Typ: wikilink.LexText, Val: ""},
+			},
+		},
+		{
+			name: "wikilink#^blockRef",
+			in:   "[[wikilink#^blockRef]]",
+			expected: []wikilink.Lexeme{
+				{Typ: wikilink.LexOpenLink, Val: "[["},
+				{Typ: wikilink.LexIdent, Val: "wikilink"},
+				{Typ: wikilink.LexBlockRef, Val: "#^"},
+				{Typ: wikilink.LexIdent, Val: "blockRef"},
+				{Typ: wikilink.LexCloseLink, Val: "]]"},
+				{Typ: wikilink.LexText, Val: ""},
+			},
+		},
+		{
+			name: "wikilink#^blockRef|display name",
+			in:   "[[wikilink#^blockRef|display name]]",
+			expected: []wikilink.Lexeme{
+				{Typ: wikilink.LexOpenLink, Val: "[["},
+				{Typ: wikilink.LexIdent, Val: "wikilink"},
+				{Typ: wikilink.LexBlockRef, Val: "#^"},
+				{Typ: wikilink.LexIdent, Val: "blockRef"},
+				{Typ: wikilink.LexAlias, Val: "|"},
+				{Typ: wikilink.LexIdent, Val: "display name"},
+				{Typ: wikilink.LexCloseLink, Val: "]]"},
+				{Typ: wikilink.LexText, Val: ""},
+			},
+		},
+		{
+			name: "wikilink#^blockRef|display name|second pipe",
+			in:   "[[wikilink#^blockRef|display name|second pipe]]",
+			expected: []wikilink.Lexeme{
+				{Typ: wikilink.LexOpenLink, Val: "[["},
+				{Typ: wikilink.LexIdent, Val: "wikilink"},
+				{Typ: wikilink.LexBlockRef, Val: "#^"},
+				{Typ: wikilink.LexIdent, Val: "blockRef"},
+				{Typ: wikilink.LexAlias, Val: "|"},
+				{Typ: wikilink.LexIdent, Val: "display name"},
+				{Typ: wikilink.LexAlias, Val: "|"},
+				{Typ: wikilink.LexIdent, Val: "second pipe"},
+				{Typ: wikilink.LexCloseLink, Val: "]]"},
+				{Typ: wikilink.LexText, Val: ""},
+			},
+		},
+		{
+			name: "wikilink with numeric aliases#^blockRef|420|second pipe",
+			in:   "[[wikilink#^blockRef|420|second pipe]]",
+			expected: []wikilink.Lexeme{
+				{Typ: wikilink.LexOpenLink, Val: "[["},
+				{Typ: wikilink.LexIdent, Val: "wikilink"},
+				{Typ: wikilink.LexBlockRef, Val: "#^"},
+				{Typ: wikilink.LexIdent, Val: "blockRef"},
+				{Typ: wikilink.LexAlias, Val: "|"},
+				{Typ: wikilink.LexIdent, Val: "420"},
+				{Typ: wikilink.LexAlias, Val: "|"},
+				{Typ: wikilink.LexIdent, Val: "second pipe"},
+				{Typ: wikilink.LexCloseLink, Val: "]]"},
+				{Typ: wikilink.LexText, Val: ""},
+			},
+		},
+	}
+	for _, tc := range tcs {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
 			// t.Parallel()
+			l := wikilink.Lex("testLexer", tc.in)
+			defer l.L.Sync()
+			if len(tc.expected) != len(l.Items) {
+				t.Logf("expected %d tokens, got %d\n", len(tc.expected), len(l.Items))
+				t.Fail()
+
+				return
+			}
+			for i, e := range tc.expected {
+				n := l.Items[i]
+				if e.Typ != n.Typ {
+					t.Logf("expected Type %s, received %s", e.Typ.String(), n.Typ.String())
+					t.Fail()
+
+					return
+				}
+
+				if e.Val != n.Val {
+					t.Logf("expected Value %q, received %q", e.Val, n.Val)
+					t.Fail()
+
+					return
+				}
+			}
+		})
+	}
+}
+
+func Test_ObsidianWikilinks_Basic(t *testing.T) {
+	t.Parallel()
 	tcs := []struct {
 		name     string
 		in       string
@@ -218,7 +459,7 @@ func Test_ObsidianWikilinks(t *testing.T) {
 	for _, tc := range tcs {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
+			// t.Parallel()
 			l := wikilink.Lex("testLexer", tc.in)
 			defer l.L.Sync()
 			if len(tc.expected) != len(l.Items) {