ndumas
/
wikilinks-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: ndumas:main
Branches Tags
ndumas:main
ndumas:dev
ndumas:v0.0.1
ndumas:v0.0.2
ndumas:v0.0.3
ndumas:v0.0.4
ndumas:v0.0.5
..
compare: ndumas:dev
Branches Tags
ndumas:main
ndumas:dev
ndumas:v0.0.1
ndumas:v0.0.2
ndumas:v0.0.3
ndumas:v0.0.4
ndumas:v0.0.5

No commits in common. 'main' and 'dev' have entirely different histories.
main ... dev

11 changed files with 451 additions and 502 deletions
Show Stats

2
.gitignore vendored
Unescape Escape View File

@ -2,5 +2,3 @@ node_modules/*
package*.json package*.json
dist/* dist/*
reports/* reports/*
lexer.log
parser.log

1
Makefile
Unescape Escape View File

@ -99,6 +99,7 @@ all: debug setup dep format lint test bench build dist
git-push: git-push:
git push origin main --tags git push origin main --tags
git push github main --tags
release-major: bump-major git-push release-major: bump-major git-push

17
cmd/demo/main.go
Unescape Escape View File

@ -1,17 +0,0 @@
package main
import (
"log"
"go.uber.org/zap/zapcore"
"code.ndumas.com/ndumas/wikilink-parser"
)
func main() {
l := wikilink.Lex("debugLexer", `this is a [[wikilink]]`, zapcore.InfoLevel)
for _, item := range l.Items {
item := item
log.Printf("%#+v\n", item)
}
}

8
go.mod
Unescape Escape View File

@ -2,15 +2,9 @@ module code.ndumas.com/ndumas/wikilink-parser
go 1.19 go 1.19
require ( require go.uber.org/zap v1.24.0
github.com/stretchr/testify v1.8.0
go.uber.org/zap v1.24.0
)
require ( require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
go.uber.org/atomic v1.7.0 // indirect go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect go.uber.org/multierr v1.6.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
) )

7
go.sum
Unescape Escape View File

@ -6,11 +6,8 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
@ -18,8 +15,4 @@ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

31
itemtype_string.go
Unescape Escape View File

@ -0,0 +1,31 @@
// Code generated by "stringer -type=ItemType"; DO NOT EDIT.
package wikilink
import "strconv"
func _() {
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
var x [1]struct{}
_ = x[ItemError-0]
_ = x[ItemEOF-1]
_ = x[ItemIdent-2]
_ = x[ItemOpenLink-3]
_ = x[ItemCloseLink-4]
_ = x[ItemHeading-5]
_ = x[ItemBlockRef-6]
_ = x[ItemAlias-7]
_ = x[ItemText-8]
}
const _ItemType_name = "ItemErrorItemEOFItemIdentItemOpenLinkItemCloseLinkItemHeadingItemBlockRefItemAliasItemText"
var _ItemType_index = [...]uint8{0, 9, 16, 25, 37, 50, 61, 73, 82, 90}
func (i ItemType) String() string {
if i < 0 || i >= ItemType(len(_ItemType_index)-1) {
return "ItemType(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _ItemType_name[_ItemType_index[i]:_ItemType_index[i+1]]
}

31
lexemetype_string.go
Unescape Escape View File

@ -1,31 +0,0 @@
// Code generated by "stringer -type=LexemeType"; DO NOT EDIT.
package wikilink
import "strconv"
func _() {
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
var x [1]struct{}
_ = x[LexError-0]
_ = x[LexEOF-1]
_ = x[LexIdent-2]
_ = x[LexOpenLink-3]
_ = x[LexCloseLink-4]
_ = x[LexHeading-5]
_ = x[LexBlockRef-6]
_ = x[LexAlias-7]
_ = x[LexText-8]
}
const _LexemeType_name = "LexErrorLexEOFLexIdentLexOpenLinkLexCloseLinkLexHeadingLexBlockRefLexAliasLexText"
var _LexemeType_index = [...]uint8{0, 8, 14, 22, 33, 45, 55, 66, 74, 81}
func (i LexemeType) String() string {
if i < 0 || i >= LexemeType(len(_LexemeType_index)-1) {
return "LexemeType(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _LexemeType_name[_LexemeType_index[i]:_LexemeType_index[i+1]]
}

136
lexer.go
Unescape Escape View File

@ -1,29 +1,26 @@
//go:generate stringer -type=LexemeType //go:generate stringer -type=ItemType
package wikilink package wikilink
import ( import (
"fmt" "fmt"
// "os"
"strings" "strings"
"sync" "sync"
// "unicode" // "unicode"
"unicode/utf8" "unicode/utf8"
"go.uber.org/zap" "go.uber.org/zap"
"go.uber.org/zap/zapcore"
) )
const ( const (
LexError LexemeType = iota ItemError ItemType = iota
LexEOF ItemEOF
LexIdent ItemIdent
LexOpenLink ItemOpenLink
LexCloseLink ItemCloseLink
LexHeading ItemHeading
LexBlockRef ItemBlockRef
LexAlias ItemAlias
LexText ItemText
) )
const ( const (
@ -38,41 +35,19 @@ const (
BlockRef = "#^" BlockRef = "#^"
) )
func Lex(name, input string, level zapcore.Level) *Lexer { func Lex(name, input string) *Lexer {
encoderCfg := zap.NewProductionEncoderConfig()
encoderCfg.EncodeTime = zapcore.ISO8601TimeEncoder
config := zap.Config{
Level: zap.NewAtomicLevelAt(level),
EncoderConfig: encoderCfg,
OutputPaths: []string{
"./lexer.log",
"stdout",
},
Encoding: "console",
ErrorOutputPaths: []string{
"stderr",
},
InitialFields: map[string]interface{}{
"lexer": name,
// "pid": os.Getpid(),
},
}
l := &Lexer{ l := &Lexer{
L: zap.Must(config.Build()).Named("lexer"), L: zap.NewExample().Sugar().Named("lexer"),
name: name, name: name,
input: input, input: input,
state: lexText, state: lexText,
Items: make([]Lexeme, 0), items: make(chan Item, 2),
} }
// go l.run() go l.run()
l.run()
return l return l
} }
/* shouldn't need this in non-concurrent implementation
func (l *Lexer) NextItem() Item { func (l *Lexer) NextItem() Item {
for { for {
select { select {
@ -91,7 +66,6 @@ func (l *Lexer) NextItem() Item {
} }
} }
} }
*/
func (l *Lexer) ignore() { func (l *Lexer) ignore() {
l.SetStart(l.pos) l.SetStart(l.pos)
@ -102,18 +76,15 @@ func (l *Lexer) backup() {
} }
type Lexer struct { type Lexer struct {
L *zap.Logger L *zap.SugaredLogger
name, input string name, input string
start, pos, width int start, pos, width int
state stateFn state stateFn
// Items chan Item items chan Item
Items []Lexeme
widthMutex, startMutex, posMutex, chanMutex sync.Mutex widthMutex, startMutex, posMutex, chanMutex sync.Mutex
} }
func (l *Lexer) peek() rune { func (l *Lexer) peek() rune {
L := l.L.Named("peek")
L.Debug("peeking")
r := l.next() r := l.next()
l.backup() l.backup()
@ -121,39 +92,22 @@ func (l *Lexer) peek() rune {
} }
func (l *Lexer) accept(valid string) bool { func (l *Lexer) accept(valid string) bool {
L := l.L.Named("accept").With(
zap.String("input", valid),
)
if strings.ContainsRune(valid, l.next()) { if strings.ContainsRune(valid, l.next()) {
L.Debug("matched input")
return true return true
} }
L.Debug("rejected input")
l.backup() l.backup()
return false return false
} }
func (l *Lexer) acceptRun(valid string) { func (l *Lexer) acceptRun(valid string) {
L := l.L.Named("acceptRun").With(
zap.String("input", valid),
)
L.Debug("scanning")
for strings.ContainsRune(valid, l.next()) { for strings.ContainsRune(valid, l.next()) {
} }
l.backup() l.backup()
} }
func (l *Lexer) emit(t LexemeType) { func (l *Lexer) emit(t ItemType) {
i := Lexeme{t, l.input[l.GetStart():l.GetPos()]}
L := l.L.Named("emit").With(
zap.String("item", i.String()),
)
L.Info("emitting lexeme")
l.Items = append(l.Items, i)
l.SetStart(l.GetPos())
/* original concurrent implementation
defer l.chanMutex.Unlock() defer l.chanMutex.Unlock()
l.chanMutex.Lock() l.chanMutex.Lock()
i := Item{t, l.input[l.GetStart():l.GetPos()]} i := Item{t, l.input[l.GetStart():l.GetPos()]}
@ -162,131 +116,99 @@ func (l *Lexer) emit(t LexemeType) {
zap.Int("width", l.GetWidth()), zap.Int("width", l.GetWidth()),
).Named("emit") ).Named("emit")
L.Debug("emitting item", L.Debugw("emitting item",
zap.String("item", i.String()), zap.String("item", i.String()),
) )
l.items <- i l.items <- i
*/ l.SetStart(l.GetPos())
} }
func (l *Lexer) errorf(format string, args ...interface{}) stateFn { func (l *Lexer) errorf(format string, args ...interface{}) stateFn {
L := l.L.Named("errorf") L := l.L.Named("errorf")
errorItem := Lexeme{ errorItem := Item{
LexError, ItemError,
fmt.Sprintf(format, args...), fmt.Sprintf(format, args...),
} }
L.Debug("emitting errorItem", L.Debugw("emitting errorItem",
zap.String("error", errorItem.String()), zap.String("error", errorItem.String()),
) )
l.Items = append(l.Items, errorItem) l.items <- errorItem
return nil return nil
} }
func (l *Lexer) next() rune { func (l *Lexer) next() rune {
var r rune var r rune
L := l.L.Named("next")
if l.GetPos() >= len(l.input) { if l.GetPos() >= len(l.input) {
L.Debug("end of input reached")
l.SetWidth(0) l.SetWidth(0)
return EOF return EOF
} }
r, width := utf8.DecodeRuneInString(l.input[l.GetPos():]) r, width := utf8.DecodeRuneInString(l.input[l.GetPos():])
L.Debug("found rune",
zap.String("rune", string(r)),
zap.Int("width", width),
)
l.SetWidth(width) l.SetWidth(width)
l.SetPos(l.GetPos() + l.GetWidth()) l.SetPos(l.GetPos() + l.GetWidth())
return r return r
} }
func (l *Lexer) run() { func (l *Lexer) run() {
for state := lexText; state != nil; {
state = state(l)
}
/* original concurrent implementation
defer l.chanMutex.Unlock() defer l.chanMutex.Unlock()
for state := lexText; state != nil; { for state := lexText; state != nil; {
state = state(l) state = state(l)
} }
l.chanMutex.Lock() l.chanMutex.Lock()
close(l.items) close(l.items)
*/
} }
func (l *Lexer) GetPos() int { func (l *Lexer) GetPos() int {
defer l.posMutex.Unlock() defer l.posMutex.Unlock()
l.posMutex.Lock() l.posMutex.Lock()
l.L.Named("GetPos").Debug("getting current position",
zap.Int("old", l.pos),
)
return l.pos return l.pos
} }
func (l *Lexer) SetPos(pos int) { func (l *Lexer) SetPos(pos int) {
defer l.posMutex.Unlock() defer l.posMutex.Unlock()
l.posMutex.Lock() l.posMutex.Lock()
l.L.Named("SetPos").Debug("setting new position",
zap.Int("new", pos),
zap.Int("old", l.pos),
)
l.pos = pos l.pos = pos
} }
func (l *Lexer) GetWidth() int { func (l *Lexer) GetWidth() int {
defer l.widthMutex.Unlock() defer l.widthMutex.Unlock()
l.widthMutex.Lock() l.widthMutex.Lock()
l.L.Named("GetWidth").Debug("setting new width",
zap.Int("old", l.width),
)
return l.width return l.width
} }
func (l *Lexer) SetWidth(width int) { func (l *Lexer) SetWidth(width int) {
defer l.widthMutex.Unlock() defer l.widthMutex.Unlock()
l.widthMutex.Lock() l.widthMutex.Lock()
l.L.Named("SetWidth").Debug("setting new width",
zap.Int("new", width),
zap.Int("old", l.width),
)
l.width = width l.width = width
} }
func (l *Lexer) GetStart() int { func (l *Lexer) GetStart() int {
defer l.startMutex.Unlock() defer l.startMutex.Unlock()
l.startMutex.Lock() l.startMutex.Lock()
l.L.Named("GetStart").Debug("getting old start",
zap.Int("old", l.start),
)
return l.start return l.start
} }
func (l *Lexer) SetStart(start int) { func (l *Lexer) SetStart(start int) {
defer l.startMutex.Unlock() defer l.startMutex.Unlock()
l.startMutex.Lock() l.startMutex.Lock()
l.L.Named("SetStart").Debug("setting new start",
zap.Int("new", start),
zap.Int("old", l.start),
)
l.start = start l.start = start
} }
type stateFn func(*Lexer) stateFn type stateFn func(*Lexer) stateFn
type LexemeType int type ItemType int
type Lexeme struct { type Item struct {
Typ LexemeType Typ ItemType
Val string Val string
} }
func (i Lexeme) String() string { func (i Item) String() string {
switch i.Typ { switch i.Typ {
case LexEOF: case ItemEOF:
return "EOF" return "EOF"
case LexError: case ItemError:
return i.Val return i.Val
} }

357
lexer_test.go
Unescape Escape View File

@ -3,314 +3,109 @@ package wikilink_test
import ( import (
"testing" "testing"
"github.com/stretchr/testify/assert"
"go.uber.org/zap/zapcore"
"code.ndumas.com/ndumas/wikilink-parser" "code.ndumas.com/ndumas/wikilink-parser"
) )
var SingleWikilink = []tc{ func Test_Lexer(t *testing.T) {
{ // t.Parallel()
name: "wikilink", tcs := []struct {
in: "[[wikilink]]", name string
expected: []wikilink.Lexeme{ in string
{Typ: wikilink.LexOpenLink, Val: "[["}, expected []wikilink.Item
{Typ: wikilink.LexIdent, Val: "wikilink"}, }{
{Typ: wikilink.LexCloseLink, Val: "]]"},
},
},
{
name: "wikilink|display name",
in: "[[wikilink|display name]]",
expected: []wikilink.Lexeme{
{Typ: wikilink.LexOpenLink, Val: "[["},
{Typ: wikilink.LexIdent, Val: "wikilink"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "display name"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
},
},
{
name: "wikilink|display name|second pipe",
in: "[[wikilink|display name|second pipe]]",
expected: []wikilink.Lexeme{
{Typ: wikilink.LexOpenLink, Val: "[["},
{Typ: wikilink.LexIdent, Val: "wikilink"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "display name"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "second pipe"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
},
},
{
name: "wikilink with numeric alias|420|second pipe",
in: "[[wikilink|420|second pipe]]",
expected: []wikilink.Lexeme{
{Typ: wikilink.LexOpenLink, Val: "[["},
{Typ: wikilink.LexIdent, Val: "wikilink"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "420"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "second pipe"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
},
},
{
name: "wikilink with spaces in filename",
in: "[[wikilink spaces]]",
expected: []wikilink.Lexeme{
{Typ: wikilink.LexOpenLink, Val: "[["},
{Typ: wikilink.LexIdent, Val: "wikilink spaces"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
},
},
{
name: "#heading",
in: "[[#heading]]",
expected: []wikilink.Lexeme{
{Typ: wikilink.LexOpenLink, Val: "[["},
{Typ: wikilink.LexIdent, Val: ""},
{Typ: wikilink.LexHeading, Val: "#"},
{Typ: wikilink.LexIdent, Val: "heading"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
},
},
{
name: "wikilink#heading",
in: "[[wikilink#heading]]",
expected: []wikilink.Lexeme{
{Typ: wikilink.LexOpenLink, Val: "[["},
{Typ: wikilink.LexIdent, Val: "wikilink"},
{Typ: wikilink.LexHeading, Val: "#"},
{Typ: wikilink.LexIdent, Val: "heading"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
},
},
{
name: "wikilink#heading|display name",
in: "[[wikilink#heading|display name]]",
expected: []wikilink.Lexeme{
{Typ: wikilink.LexOpenLink, Val: "[["},
{Typ: wikilink.LexIdent, Val: "wikilink"},
{Typ: wikilink.LexHeading, Val: "#"},
{Typ: wikilink.LexIdent, Val: "heading"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "display name"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
},
},
{ {
name: "wikilink#heading|display name|second pipe", name: "wikilink", in: "[[wikilink]]", expected: []wikilink.Item{
in: "[[wikilink#heading|display name|second pipe]]", {Typ: wikilink.ItemOpenLink, Val: "[["},
expected: []wikilink.Lexeme{ {Typ: wikilink.ItemIdent, Val: "wikilink"},
{Typ: wikilink.LexOpenLink, Val: "[["}, {Typ: wikilink.ItemCloseLink, Val: "]]"},
{Typ: wikilink.LexIdent, Val: "wikilink"},
{Typ: wikilink.LexHeading, Val: "#"},
{Typ: wikilink.LexIdent, Val: "heading"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "display name"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "second pipe"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
}, },
}, },
{ {
name: "wikilink with numeric aliases#heading|420|display name", name: "wikilink|display name", in: "[[wikilink|display name]]", expected: []wikilink.Item{
in: "[[wikilink#heading|420|second pipe]]", {Typ: wikilink.ItemOpenLink, Val: "[["},
expected: []wikilink.Lexeme{ {Typ: wikilink.ItemIdent, Val: "wikilink"},
{Typ: wikilink.LexOpenLink, Val: "[["}, {Typ: wikilink.ItemAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "wikilink"}, {Typ: wikilink.ItemIdent, Val: "display name"},
{Typ: wikilink.LexHeading, Val: "#"}, {Typ: wikilink.ItemCloseLink, Val: "]]"},
{Typ: wikilink.LexIdent, Val: "heading"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "420"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "second pipe"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
}, },
}, },
{ {
name: "#^blockRef", name: "wikilink|display name|second pipe", in: "[[wikilink|display name|second pipe]]", expected: []wikilink.Item{
in: "[[#^blockRef]]", {Typ: wikilink.ItemOpenLink, Val: "[["},
expected: []wikilink.Lexeme{ {Typ: wikilink.ItemIdent, Val: "wikilink"},
{Typ: wikilink.LexOpenLink, Val: "[["}, {Typ: wikilink.ItemAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: ""}, {Typ: wikilink.ItemIdent, Val: "display name"},
{Typ: wikilink.LexBlockRef, Val: "#^"}, {Typ: wikilink.ItemAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "blockRef"}, {Typ: wikilink.ItemIdent, Val: "second pipe"},
{Typ: wikilink.LexCloseLink, Val: "]]"}, {Typ: wikilink.ItemCloseLink, Val: "]]"},
}, },
}, },
{ {
name: "wikilink#^blockRef", name: "wikilink with numeric alias|420|second pipe", in: "[[wikilink|420|second pipe]]", expected: []wikilink.Item{
in: "[[wikilink#^blockRef]]", {Typ: wikilink.ItemOpenLink, Val: "[["},
expected: []wikilink.Lexeme{ {Typ: wikilink.ItemIdent, Val: "wikilink"},
{Typ: wikilink.LexOpenLink, Val: "[["}, {Typ: wikilink.ItemAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "wikilink"}, {Typ: wikilink.ItemIdent, Val: "420"},
{Typ: wikilink.LexBlockRef, Val: "#^"}, {Typ: wikilink.ItemAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "blockRef"}, {Typ: wikilink.ItemIdent, Val: "second pipe"},
{Typ: wikilink.LexCloseLink, Val: "]]"}, {Typ: wikilink.ItemCloseLink, Val: "]]"},
}, },
}, },
{ {
name: "wikilink#^blockRef|display name", name: "wikilink with spaces in filename", in: "[[wikilink spaces]]", expected: []wikilink.Item{
in: "[[wikilink#^blockRef|display name]]", {Typ: wikilink.ItemOpenLink, Val: "[["},
expected: []wikilink.Lexeme{ {Typ: wikilink.ItemIdent, Val: "wikilink spaces"},
{Typ: wikilink.LexOpenLink, Val: "[["}, {Typ: wikilink.ItemCloseLink, Val: "]]"},
{Typ: wikilink.LexIdent, Val: "wikilink"},
{Typ: wikilink.LexBlockRef, Val: "#^"},
{Typ: wikilink.LexIdent, Val: "blockRef"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "display name"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
}, },
}, },
{ {
name: "wikilink#^blockRef|display name|second pipe", name: "#heading", in: "[[#heading]]", expected: []wikilink.Item{
in: "[[wikilink#^blockRef|display name|second pipe]]", {Typ: wikilink.ItemOpenLink, Val: "[["},
expected: []wikilink.Lexeme{ {Typ: wikilink.ItemHeading, Val: "#"},
{Typ: wikilink.LexOpenLink, Val: "[["}, {Typ: wikilink.ItemIdent, Val: "heading"},
{Typ: wikilink.LexIdent, Val: "wikilink"}, {Typ: wikilink.ItemCloseLink, Val: "]]"},
{Typ: wikilink.LexBlockRef, Val: "#^"},
{Typ: wikilink.LexIdent, Val: "blockRef"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "display name"},
{Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "second pipe"},
{Typ: wikilink.LexCloseLink, Val: "]]"},
}, },
}, },
{ {
name: "wikilink with numeric aliases#^blockRef|420|second pipe", name: "wikilink#heading", in: "[[wikilink#heading]]", expected: []wikilink.Item{
in: "[[wikilink#^blockRef|420|second pipe]]", {Typ: wikilink.ItemOpenLink, Val: "[["},
expected: []wikilink.Lexeme{ {Typ: wikilink.ItemIdent, Val: "wikilink"},
{Typ: wikilink.LexOpenLink, Val: "[["}, {Typ: wikilink.ItemHeading, Val: "#"},
{Typ: wikilink.LexIdent, Val: "wikilink"}, {Typ: wikilink.ItemIdent, Val: "heading"},
{Typ: wikilink.LexBlockRef, Val: "#^"}, {Typ: wikilink.ItemCloseLink, Val: "]]"},
{Typ: wikilink.LexIdent, Val: "blockRef"}, },
{Typ: wikilink.LexAlias, Val: "|"}, },
{Typ: wikilink.LexIdent, Val: "420"}, /*
{Typ: wikilink.LexAlias, Val: "|"}, {name: "", in: "", expected: []wikilink.ItemType{}},
{Typ: wikilink.LexIdent, Val: "second pipe"}, {name: "wikilink#heading|display name", in: "[[wikilink#heading|display name]]", expected: []wikilink.ItemType{}},
{Typ: wikilink.LexCloseLink, Val: "]]"}, {name: "wikilink#heading|display name|second pipe", in: "[[wikilink#heading|display name|second pipe]]", expected: []wikilink.ItemType{}},
}, {name: "wikilink with numeric aliases#heading|420|display name", in: "[[wikilink#heading|420|second pipe]]", expected: []wikilink.ItemType{}},
}, {name: "^blockRef", in: "[[^blockRef]]", expected: []wikilink.ItemType{}},
} {name: "wikilink^blockRef", in: "[[wikilink^blockRef]]", expected: []wikilink.ItemType{}},
{name: "wikilink^blockRef|display name", in: "[[wikilink#^blockRef|display name]]", expected: []wikilink.ItemType{}},
func Test_ObsidianWikilinks_LinksEndOfMultiLineInput(t *testing.T) { {name: "wikilink^blockRef|display name|second pipe", in: "[[wikilink#^blockRef|display name|second pipe]]", expected: []wikilink.ItemType{}},
for _, tc := range SingleWikilink { {name: "wikilink with numeric aliases^blockRef|420|second pipe", in: "[[wikilink#^blockRef|420|second pipe]]", expected: []wikilink.ItemType{}},
mut, test := mutateTestCase( */
tc,
" test data please ignore.\nbling blonk more lines\nbling blong\nthis is a",
"",
[]wikilink.Lexeme{
{Typ: wikilink.LexText, Val: " test data please ignore.\n"},
{Typ: wikilink.LexText, Val: "bling blonk more lines\n"},
{Typ: wikilink.LexText, Val: "bling blong\n"},
{Typ: wikilink.LexText, Val: "this is a"},
},
[]wikilink.Lexeme{
{Typ: wikilink.LexText, Val: ""},
},
)
t.Run(mut.name, test)
}
}
func Test_ObsidianWikilinks_LinksStartOfMultiLineInput(t *testing.T) {
for _, tc := range SingleWikilink {
mut, test := mutateTestCase(
tc,
"",
" test data please ignore.\nbling blonk more lines\nbling blong\nthis is a",
[]wikilink.Lexeme{
{Typ: wikilink.LexText, Val: ""},
},
[]wikilink.Lexeme{
{Typ: wikilink.LexText, Val: " test data please ignore.\n"},
{Typ: wikilink.LexText, Val: "bling blonk more lines\n"},
{Typ: wikilink.LexText, Val: "bling blong\n"},
{Typ: wikilink.LexText, Val: "this is a"},
},
)
t.Run(mut.name, test)
} }
}
func Test_ObsidianWikilinks_LinksStartOfInput(t *testing.T) {
for _, tc := range SingleWikilink {
mut, test := mutateTestCase(
tc,
"",
" test data please ignore",
[]wikilink.Lexeme{
{Typ: wikilink.LexText, Val: ""},
},
[]wikilink.Lexeme{
{Typ: wikilink.LexText, Val: " test data please ignore"},
},
)
t.Run(mut.name, test)
}
}
func Test_ObsidianWikilinks_LinksEndOfInput(t *testing.T) { for _, tc := range tcs {
for _, tc := range SingleWikilink { tc := tc
mut, test := mutateTestCase( t.Run(tc.name, func(t *testing.T) {
tc, // t.Parallel()
"this is a ", l := wikilink.Lex("testLexer", tc.in)
"", defer l.L.Sync()
[]wikilink.Lexeme{ for _, e := range tc.expected {
{Typ: wikilink.LexText, Val: "this is a "}, n := l.NextItem()
}, if e.Typ != n.Typ {
[]wikilink.Lexeme{ t.Logf("expected Type %s, received %s", e.Typ.String(), n.Typ.String())
{Typ: wikilink.LexText, Val: ""}, t.Fail()
},
)
t.Run(mut.name, test)
} }
}
func Test_ObsidianWikilinks_Basic(t *testing.T) {
for _, tc := range SingleWikilink { if e.Val != n.Val {
mut, test := mutateTestCase( t.Logf("expected Value %q, received %q", e.Val, n.Val)
tc, t.Fail()
"",
"",
[]wikilink.Lexeme{
{Typ: wikilink.LexText, Val: ""},
},
[]wikilink.Lexeme{
{Typ: wikilink.LexText, Val: ""},
},
)
t.Run(mut.name, test)
} }
}
type tc struct {
name string
in string
expected []wikilink.Lexeme
}
func mutateTestCase(tc tc, prefix, suffix string, expectedPrefix, expectedSuffix []wikilink.Lexeme) (tc, func(t *testing.T)) {
tc.in = prefix + tc.in
tc.in = tc.in + suffix
if expectedPrefix != nil {
tc.expected = append(expectedPrefix, tc.expected...)
} }
if expectedSuffix != nil { })
tc.expected = append(tc.expected, expectedSuffix...)
}
return tc, func(t *testing.T) {
l := wikilink.Lex("testLexer", tc.in, zapcore.WarnLevel)
defer l.L.Sync()
assert.Equal(t, tc.expected, l.Items, "token stream mismatch")
} }
} }

275
race.log
Unescape Escape View File

@ -0,0 +1,275 @@
mkdir -p "/home/ndumas/work/wikilink-parser/reports"
mkdir -p "/home/ndumas/work/wikilink-parser/dist"
go clean code.ndumas.com/ndumas/wikilink-parser
rm -vrf "/home/ndumas/work/wikilink-parser/dist"/*
rm -vf "/home/ndumas/work/wikilink-parser/reports"/*
removed '/home/ndumas/work/wikilink-parser/reports/test.out'
go get -d -t code.ndumas.com/ndumas/wikilink-parser/...
go install golang.org/x/tools/cmd/stringer@latest
go generate
go test -race -v -tags "release" $(go list "code.ndumas.com/ndumas/wikilink-parser/..." | grep -v /vendor/) | tee "/home/ndumas/work/wikilink-parser/reports/test.out"
=== RUN Test_Lexer
=== RUN Test_Lexer/wikilink
2023/07/01 18:25:42 lexText
2023/07/01 18:25:42 lexOpenLink
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":2,"width":0,"item":"ItemOpenLink:\"[[\""}
2023/07/01 18:25:42 lexIdent
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":10,"width":1,"item":"ItemIdent:\"wikilink\""}
2023/07/01 18:25:42 lexCloseLink
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":12,"width":1,"item":"ItemCloseLink:\"]]\""}
=== RUN Test_Lexer/wikilink|display_name
2023/07/01 18:25:42 lexText
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":12,"width":0,"item":"ItemText:\"\""}
2023/07/01 18:25:42 lexText
2023/07/01 18:25:42 lexOpenLink
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":2,"width":0,"item":"ItemOpenLink:\"[[\""}
2023/07/01 18:25:42 lexIdent
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":10,"width":1,"item":"ItemIdent:\"wikilink\""}
2023/07/01 18:25:42 lexAlias
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":11,"width":1,"item":"ItemAlias:\"|\""}
2023/07/01 18:25:42 lexIdent
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":23,"width":1,"item":"ItemIdent:\"display name\""}
2023/07/01 18:25:42 lexCloseLink
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":25,"width":1,"item":"ItemCloseLink:\"]]\""}
=== RUN Test_Lexer/wikilink|display_name|second_pipe
2023/07/01 18:25:42 lexText
2023/07/01 18:25:42 lexOpenLink
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":2,"width":0,"item":"ItemOpenLink:\"[[\""}
2023/07/01 18:25:42 lexIdent
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":10,"width":1,"item":"ItemIdent:\"wikilink\""}
2023/07/01 18:25:42 lexAlias
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":11,"width":1,"item":"ItemAlias:\"|\""}
2023/07/01 18:25:42 lexIdent
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":23,"width":1,"item":"ItemIdent:\"display name\""}
2023/07/01 18:25:42 lexText
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":25,"width":0,"item":"ItemText:\"\""}
2023/07/01 18:25:42 lexAlias
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":24,"width":1,"item":"ItemAlias:\"|\""}
2023/07/01 18:25:42 lexIdent
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":35,"width":1,"item":"ItemIdent:\"second pipe\""}
2023/07/01 18:25:42 lexCloseLink
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":37,"width":1,"item":"ItemCloseLink:\"]]\""}
=== RUN Test_Lexer/wikilink_with_numeric_alias|420|second_pipe
2023/07/01 18:25:42 lexText
2023/07/01 18:25:42 lexOpenLink
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":2,"width":0,"item":"ItemOpenLink:\"[[\""}
2023/07/01 18:25:42 lexIdent
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":10,"width":1,"item":"ItemIdent:\"wikilink\""}
2023/07/01 18:25:42 lexAlias
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":11,"width":1,"item":"ItemAlias:\"|\""}
2023/07/01 18:25:42 lexIdent
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":14,"width":1,"item":"ItemIdent:\"420\""}
2023/07/01 18:25:42 lexAlias
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":15,"width":1,"item":"ItemAlias:\"|\""}
2023/07/01 18:25:42 lexIdent
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":26,"width":1,"item":"ItemIdent:\"second pipe\""}
2023/07/01 18:25:42 lexCloseLink
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":28,"width":1,"item":"ItemCloseLink:\"]]\""}
=== RUN Test_Lexer/wikilink_with_spaces_in_filename
2023/07/01 18:25:42 lexText
2023/07/01 18:25:42 lexOpenLink
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":2,"width":0,"item":"ItemOpenLink:\"[[\""}
2023/07/01 18:25:42 lexIdent
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":17,"width":1,"item":"ItemIdent:\"wikilink spaces\""}
2023/07/01 18:25:42 lexCloseLink
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":19,"width":1,"item":"ItemCloseLink:\"]]\""}
=== RUN Test_Lexer/#heading
2023/07/01 18:25:42 lexText
2023/07/01 18:25:42 lexOpenLink
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":2,"width":0,"item":"ItemOpenLink:\"[[\""}
2023/07/01 18:25:42 lexIdent
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":10,"width":1,"item":"ItemIdent:\"#heading\""}
lexer_test.go:100: expected Type ItemHeading, received ItemIdent
lexer_test.go:105: expected Value "#", received "#heading"
2023/07/01 18:25:42 lexCloseLink
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":12,"width":1,"item":"ItemCloseLink:\"]]\""}
lexer_test.go:100: expected Type ItemIdent, received ItemCloseLink
lexer_test.go:105: expected Value "heading", received "]]"
2023/07/01 18:25:42 lexText
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":12,"width":0,"item":"ItemText:\"\""}
lexer_test.go:100: expected Type ItemCloseLink, received ItemText
lexer_test.go:105: expected Value "]]", received ""
=== RUN Test_Lexer/wikilink#heading
2023/07/01 18:25:42 lexText
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":37,"width":0,"item":"ItemText:\"\""}
2023/07/01 18:25:42 lexText
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":28,"width":0,"item":"ItemText:\"\""}
2023/07/01 18:25:42 lexText
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":19,"width":0,"item":"ItemText:\"\""}
2023/07/01 18:25:42 lexText
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":12,"width":0,"item":"ItemText:\"\""}
2023/07/01 18:25:42 lexText
2023/07/01 18:25:42 lexOpenLink
2023/07/01 18:25:42 lexText
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":2,"width":0,"item":"ItemOpenLink:\"[[\""}
2023/07/01 18:25:42 lexIdent
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":10,"width":1,"item":"ItemIdent:\"wikilink\""}
==================
WARNING: DATA RACE
Read at 0x00c0000822b0 by goroutine 21:
code.ndumas.com/ndumas/wikilink-parser.lexText()
/home/ndumas/work/wikilink-parser/states.go:82 +0x69
code.ndumas.com/ndumas/wikilink-parser.(*Lexer).run()
/home/ndumas/work/wikilink-parser/lexer.go:149 +0x3a
code.ndumas.com/ndumas/wikilink-parser.Lex.func1()
/home/ndumas/work/wikilink-parser/lexer.go:45 +0x39
Previous write at 0x00c0000822b0 by goroutine 20:
code.ndumas.com/ndumas/wikilink-parser.lexOpenLink()
/home/ndumas/work/wikilink-parser/states.go:96 +0x86
code.ndumas.com/ndumas/wikilink-parser.(*Lexer).NextItem()
/home/ndumas/work/wikilink-parser/lexer.go:64 +0xd5
code.ndumas.com/ndumas/wikilink-parser_test.Test_Lexer.func1()
/home/ndumas/work/wikilink-parser/lexer_test.go:98 +0x18c
testing.tRunner()
/usr/lib/golang/src/testing/testing.go:1446 +0x216
testing.(*T).Run.func1()
/usr/lib/golang/src/testing/testing.go:1493 +0x47
Goroutine 21 (running) created at:
code.ndumas.com/ndumas/wikilink-parser.Lex()
/home/ndumas/work/wikilink-parser/lexer.go:45 +0x41a
code.ndumas.com/ndumas/wikilink-parser_test.Test_Lexer.func1()
/home/ndumas/work/wikilink-parser/lexer_test.go:95 +0x69
testing.tRunner()
/usr/lib/golang/src/testing/testing.go:1446 +0x216
testing.(*T).Run.func1()
/usr/lib/golang/src/testing/testing.go:1493 +0x47
Goroutine 20 (running) created at:
testing.(*T).Run()
/usr/lib/golang/src/testing/testing.go:1493 +0x75d
code.ndumas.com/ndumas/wikilink-parser_test.Test_Lexer()
/home/ndumas/work/wikilink-parser/lexer_test.go:93 +0x1086
testing.tRunner()
/usr/lib/golang/src/testing/testing.go:1446 +0x216
testing.(*T).Run.func1()
/usr/lib/golang/src/testing/testing.go:1493 +0x47
==================
==================
WARNING: DATA RACE
Write at 0x00c0000822b8 by goroutine 21:
code.ndumas.com/ndumas/wikilink-parser.(*Lexer).next()
/home/ndumas/work/wikilink-parser/lexer.go:142 +0xe8
code.ndumas.com/ndumas/wikilink-parser.lexText()
/home/ndumas/work/wikilink-parser/states.go:85 +0xc4
code.ndumas.com/ndumas/wikilink-parser.(*Lexer).run()
/home/ndumas/work/wikilink-parser/lexer.go:149 +0x3a
code.ndumas.com/ndumas/wikilink-parser.Lex.func1()
/home/ndumas/work/wikilink-parser/lexer.go:45 +0x39
Previous write at 0x00c0000822b8 by goroutine 20:
code.ndumas.com/ndumas/wikilink-parser.(*Lexer).next()
/home/ndumas/work/wikilink-parser/lexer.go:142 +0xe8
code.ndumas.com/ndumas/wikilink-parser.lexIdent()
/home/ndumas/work/wikilink-parser/states.go:31 +0x64
code.ndumas.com/ndumas/wikilink-parser.(*Lexer).NextItem()
/home/ndumas/work/wikilink-parser/lexer.go:64 +0xd5
code.ndumas.com/ndumas/wikilink-parser_test.Test_Lexer.func1()
/home/ndumas/work/wikilink-parser/lexer_test.go:98 +0x18c
testing.tRunner()
/usr/lib/golang/src/testing/testing.go:1446 +0x216
testing.(*T).Run.func1()
/usr/lib/golang/src/testing/testing.go:1493 +0x47
Goroutine 21 (running) created at:
code.ndumas.com/ndumas/wikilink-parser.Lex()
/home/ndumas/work/wikilink-parser/lexer.go:45 +0x41a
code.ndumas.com/ndumas/wikilink-parser_test.Test_Lexer.func1()
/home/ndumas/work/wikilink-parser/lexer_test.go:95 +0x69
testing.tRunner()
/usr/lib/golang/src/testing/testing.go:1446 +0x216
testing.(*T).Run.func1()
/usr/lib/golang/src/testing/testing.go:1493 +0x47
Goroutine 20 (running) created at:
testing.(*T).Run()
/usr/lib/golang/src/testing/testing.go:1493 +0x75d
code.ndumas.com/ndumas/wikilink-parser_test.Test_Lexer()
/home/ndumas/work/wikilink-parser/lexer_test.go:93 +0x1086
testing.tRunner()
/usr/lib/golang/src/testing/testing.go:1446 +0x216
testing.(*T).Run.func1()
/usr/lib/golang/src/testing/testing.go:1493 +0x47
==================
==================
WARNING: DATA RACE
Read at 0x00c0000822a8 by goroutine 21:
code.ndumas.com/ndumas/wikilink-parser.(*Lexer).emit()
/home/ndumas/work/wikilink-parser/lexer.go:109 +0x64
code.ndumas.com/ndumas/wikilink-parser.lexText()
/home/ndumas/work/wikilink-parser/states.go:88 +0xf4
code.ndumas.com/ndumas/wikilink-parser.(*Lexer).run()
/home/ndumas/work/wikilink-parser/lexer.go:149 +0x3a
code.ndumas.com/ndumas/wikilink-parser.Lex.func1()
/home/ndumas/work/wikilink-parser/lexer.go:45 +0x39
Previous write at 0x00c0000822a8 by goroutine 20:
code.ndumas.com/ndumas/wikilink-parser.(*Lexer).emit()
/home/ndumas/work/wikilink-parser/lexer.go:119 +0x4cf
code.ndumas.com/ndumas/wikilink-parser.lexOpenLink()
/home/ndumas/work/wikilink-parser/states.go:97 +0xa4
code.ndumas.com/ndumas/wikilink-parser.(*Lexer).NextItem()
/home/ndumas/work/wikilink-parser/lexer.go:64 +0xd5
code.ndumas.com/ndumas/wikilink-parser_test.Test_Lexer.func1()
/home/ndumas/work/wikilink-parser/lexer_test.go:98 +0x18c
testing.tRunner()
/usr/lib/golang/src/testing/testing.go:1446 +0x216
testing.(*T).Run.func1()
/usr/lib/golang/src/testing/testing.go:1493 +0x47
Goroutine 21 (running) created at:
code.ndumas.com/ndumas/wikilink-parser.Lex()
/home/ndumas/work/wikilink-parser/lexer.go:45 +0x41a
code.ndumas.com/ndumas/wikilink-parser_test.Test_Lexer.func1()
/home/ndumas/work/wikilink-parser/lexer_test.go:95 +0x69
testing.tRunner()
/usr/lib/golang/src/testing/testing.go:1446 +0x216
testing.(*T).Run.func1()
/usr/lib/golang/src/testing/testing.go:1493 +0x47
Goroutine 20 (running) created at:
testing.(*T).Run()
/usr/lib/golang/src/testing/testing.go:1493 +0x75d
code.ndumas.com/ndumas/wikilink-parser_test.Test_Lexer()
/home/ndumas/work/wikilink-parser/lexer_test.go:93 +0x1086
testing.tRunner()
/usr/lib/golang/src/testing/testing.go:1446 +0x216
testing.(*T).Run.func1()
/usr/lib/golang/src/testing/testing.go:1493 +0x47
==================
2023/07/01 18:25:42 lexHeading
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":20,"width":0,"item":"ItemText:\"wikilink#heading]]\""}
testing.go:1319: race detected during execution of test
--- FAIL: Test_Lexer (0.01s)
--- PASS: Test_Lexer/wikilink (0.00s)
--- PASS: Test_Lexer/wikilink|display_name (0.00s)
--- PASS: Test_Lexer/wikilink|display_name|second_pipe (0.00s)
--- PASS: Test_Lexer/wikilink_with_numeric_alias|420|second_pipe (0.00s)
--- PASS: Test_Lexer/wikilink_with_spaces_in_filename (0.00s)
--- FAIL: Test_Lexer/#heading (0.00s)
--- FAIL: Test_Lexer/wikilink#heading (0.00s)
panic: runtime error: slice bounds out of range [:21] with length 20 [recovered]
panic: runtime error: slice bounds out of range [:21] with length 20
goroutine 34 [running]:
testing.tRunner.func1.2({0x6e8160, 0xc0000e20a8})
/usr/lib/golang/src/testing/testing.go:1396 +0x372
testing.tRunner.func1()
/usr/lib/golang/src/testing/testing.go:1399 +0x5f0
panic({0x6e8160, 0xc0000e20a8})
/usr/lib/golang/src/runtime/panic.go:890 +0x262
code.ndumas.com/ndumas/wikilink-parser.(*Lexer).emit(0xc000082280, 0x5)
/home/ndumas/work/wikilink-parser/lexer.go:109 +0x506
code.ndumas.com/ndumas/wikilink-parser.lexHeading(0xc000082280)
/home/ndumas/work/wikilink-parser/states.go:58 +0xa5
code.ndumas.com/ndumas/wikilink-parser.(*Lexer).NextItem(0xc000082280)
/home/ndumas/work/wikilink-parser/lexer.go:64 +0xd6
code.ndumas.com/ndumas/wikilink-parser_test.Test_Lexer.func1(0xc000190b60)
/home/ndumas/work/wikilink-parser/lexer_test.go:98 +0x18d
testing.tRunner(0xc000190b60, 0xc00019cb40)
/usr/lib/golang/src/testing/testing.go:1446 +0x217
created by testing.(*T).Run
/usr/lib/golang/src/testing/testing.go:1493 +0x75e
FAIL code.ndumas.com/ndumas/wikilink-parser 0.024s
FAIL

50
states.go
Unescape Escape View File

@ -28,90 +28,78 @@ func isBlockRef(s string) bool {
func lexIdent(l *Lexer) stateFn { func lexIdent(l *Lexer) stateFn {
for { for {
L := l.L.Named("lexIdent") r := l.next()
l.L.Named("lexIdent").Debug("stepping through lexIdent",
zap.String("r", string(r)),
)
s := l.input[l.GetPos():] s := l.input[l.GetPos():]
L.Debug("stepping through lexIdent") if r == '\\' { // i think this will handle escape characters?
if s[0] == '\\' { // i think this will handle escape characters?
break break
} }
switch { switch {
case isCloseLink(s):
L.Debug("found CloseLink")
l.emit(LexIdent)
return lexCloseLink
case isBlockRef(s): case isBlockRef(s):
L.Debug("found BlockRef") l.emit(ItemIdent)
l.emit(LexIdent)
return lexBlockRef return lexBlockRef
case isAlias(s): case isAlias(s):
L.Debug("found Alias") l.emit(ItemIdent)
l.emit(LexIdent)
return lexAlias return lexAlias
case isCloseLink(s):
l.emit(ItemIdent)
return lexCloseLink
case isHeading(s): case isHeading(s):
L.Debug("found Heading") l.emit(ItemIdent)
l.emit(LexIdent)
return lexHeading return lexHeading
}
r := l.next()
L = l.L.With(
zap.String("rune", string(r)),
)
} }
}
return l.errorf("malformed link") return l.errorf("malformed link")
} }
func lexHeading(l *Lexer) stateFn { func lexHeading(l *Lexer) stateFn {
l.SetPos(l.GetPos() + len(Heading)) l.SetPos(l.GetPos() + len(Heading))
l.emit(LexHeading) l.emit(ItemHeading)
return lexIdent return lexIdent
} }
func lexBlockRef(l *Lexer) stateFn { func lexBlockRef(l *Lexer) stateFn {
l.SetPos(l.GetPos() + len(BlockRef)) l.SetPos(l.GetPos() + len(BlockRef))
l.emit(LexBlockRef) l.emit(ItemBlockRef)
return lexIdent return lexIdent
} }
func lexAlias(l *Lexer) stateFn { func lexAlias(l *Lexer) stateFn {
l.SetPos(l.GetPos() + len(Alias)) l.SetPos(l.GetPos() + len(Alias))
l.emit(LexAlias) l.emit(ItemAlias)
return lexIdent return lexIdent
} }
func lexText(l *Lexer) stateFn { func lexText(l *Lexer) stateFn {
L := l.L.Named("lexText")
for { for {
if isOpenLink(l.input[l.GetPos():]) { if isOpenLink(l.input[l.GetPos():]) {
L.Debug("found openLink")
l.emit(LexText)
return lexOpenLink return lexOpenLink
} }
r := l.next() r := l.next()
switch { switch {
case r == EOF: case r == EOF || r == '\n':
l.emit(LexText) l.emit(ItemText)
return nil return nil
case r == '\n':
l.emit(LexText)
return lexText
} }
} }
} }
func lexOpenLink(l *Lexer) stateFn { func lexOpenLink(l *Lexer) stateFn {
l.SetPos(l.GetPos() + len(OpenLink)) l.SetPos(l.GetPos() + len(OpenLink))
l.emit(LexOpenLink) l.emit(ItemOpenLink)
return lexIdent return lexIdent
} }
func lexCloseLink(l *Lexer) stateFn { func lexCloseLink(l *Lexer) stateFn {
l.SetPos(l.GetPos() + len(CloseLink)) l.SetPos(l.GetPos() + len(CloseLink))
l.emit(LexCloseLink) l.emit(ItemCloseLink)
return lexText return lexText
} }

Write Preview
Loading…
Cancel
Save