Compare commits

..

7 Commits
v0.0.3 ... main

@ -3,11 +3,13 @@ package main
import ( import (
"log" "log"
"go.uber.org/zap/zapcore"
"code.ndumas.com/ndumas/wikilink-parser" "code.ndumas.com/ndumas/wikilink-parser"
) )
func main() { func main() {
l := wikilink.Lex("debugLexer", `this is a [[wikilink]]`) l := wikilink.Lex("debugLexer", `this is a [[wikilink]]`, zapcore.InfoLevel)
for _, item := range l.Items { for _, item := range l.Items {
item := item item := item
log.Printf("%#+v\n", item) log.Printf("%#+v\n", item)

@ -2,9 +2,15 @@ module code.ndumas.com/ndumas/wikilink-parser
go 1.19 go 1.19
require go.uber.org/zap v1.24.0 require (
github.com/stretchr/testify v1.8.0
go.uber.org/zap v1.24.0
)
require ( require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
go.uber.org/atomic v1.7.0 // indirect go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect go.uber.org/multierr v1.6.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
) )

@ -6,8 +6,11 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
@ -15,4 +18,8 @@ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

@ -38,12 +38,12 @@ const (
BlockRef = "#^" BlockRef = "#^"
) )
func Lex(name, input string) *Lexer { func Lex(name, input string, level zapcore.Level) *Lexer {
encoderCfg := zap.NewProductionEncoderConfig() encoderCfg := zap.NewProductionEncoderConfig()
encoderCfg.EncodeTime = zapcore.ISO8601TimeEncoder encoderCfg.EncodeTime = zapcore.ISO8601TimeEncoder
config := zap.Config{ config := zap.Config{
Level: zap.NewAtomicLevelAt(zap.InfoLevel), Level: zap.NewAtomicLevelAt(level),
EncoderConfig: encoderCfg, EncoderConfig: encoderCfg,
OutputPaths: []string{ OutputPaths: []string{
"./lexer.log", "./lexer.log",
@ -150,7 +150,7 @@ func (l *Lexer) emit(t LexemeType) {
L := l.L.Named("emit").With( L := l.L.Named("emit").With(
zap.String("item", i.String()), zap.String("item", i.String()),
) )
L.Debug("emitting lexeme") L.Info("emitting lexeme")
l.Items = append(l.Items, i) l.Items = append(l.Items, i)
l.SetStart(l.GetPos()) l.SetStart(l.GetPos())
/* original concurrent implementation /* original concurrent implementation

@ -3,10 +3,13 @@ package wikilink_test
import ( import (
"testing" "testing"
"github.com/stretchr/testify/assert"
"go.uber.org/zap/zapcore"
"code.ndumas.com/ndumas/wikilink-parser" "code.ndumas.com/ndumas/wikilink-parser"
) )
var testCases = []tc{ var SingleWikilink = []tc{
{ {
name: "wikilink", name: "wikilink",
in: "[[wikilink]]", in: "[[wikilink]]",
@ -14,7 +17,6 @@ var testCases = []tc{
{Typ: wikilink.LexOpenLink, Val: "[["}, {Typ: wikilink.LexOpenLink, Val: "[["},
{Typ: wikilink.LexIdent, Val: "wikilink"}, {Typ: wikilink.LexIdent, Val: "wikilink"},
{Typ: wikilink.LexCloseLink, Val: "]]"}, {Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
}, },
}, },
{ {
@ -26,7 +28,6 @@ var testCases = []tc{
{Typ: wikilink.LexAlias, Val: "|"}, {Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "display name"}, {Typ: wikilink.LexIdent, Val: "display name"},
{Typ: wikilink.LexCloseLink, Val: "]]"}, {Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
}, },
}, },
{ {
@ -40,7 +41,6 @@ var testCases = []tc{
{Typ: wikilink.LexAlias, Val: "|"}, {Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "second pipe"}, {Typ: wikilink.LexIdent, Val: "second pipe"},
{Typ: wikilink.LexCloseLink, Val: "]]"}, {Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
}, },
}, },
{ {
@ -54,7 +54,6 @@ var testCases = []tc{
{Typ: wikilink.LexAlias, Val: "|"}, {Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "second pipe"}, {Typ: wikilink.LexIdent, Val: "second pipe"},
{Typ: wikilink.LexCloseLink, Val: "]]"}, {Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
}, },
}, },
{ {
@ -64,7 +63,6 @@ var testCases = []tc{
{Typ: wikilink.LexOpenLink, Val: "[["}, {Typ: wikilink.LexOpenLink, Val: "[["},
{Typ: wikilink.LexIdent, Val: "wikilink spaces"}, {Typ: wikilink.LexIdent, Val: "wikilink spaces"},
{Typ: wikilink.LexCloseLink, Val: "]]"}, {Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
}, },
}, },
{ {
@ -76,7 +74,6 @@ var testCases = []tc{
{Typ: wikilink.LexHeading, Val: "#"}, {Typ: wikilink.LexHeading, Val: "#"},
{Typ: wikilink.LexIdent, Val: "heading"}, {Typ: wikilink.LexIdent, Val: "heading"},
{Typ: wikilink.LexCloseLink, Val: "]]"}, {Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
}, },
}, },
{ {
@ -88,7 +85,6 @@ var testCases = []tc{
{Typ: wikilink.LexHeading, Val: "#"}, {Typ: wikilink.LexHeading, Val: "#"},
{Typ: wikilink.LexIdent, Val: "heading"}, {Typ: wikilink.LexIdent, Val: "heading"},
{Typ: wikilink.LexCloseLink, Val: "]]"}, {Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
}, },
}, },
{ {
@ -102,7 +98,6 @@ var testCases = []tc{
{Typ: wikilink.LexAlias, Val: "|"}, {Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "display name"}, {Typ: wikilink.LexIdent, Val: "display name"},
{Typ: wikilink.LexCloseLink, Val: "]]"}, {Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
}, },
}, },
{ {
@ -118,7 +113,6 @@ var testCases = []tc{
{Typ: wikilink.LexAlias, Val: "|"}, {Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "second pipe"}, {Typ: wikilink.LexIdent, Val: "second pipe"},
{Typ: wikilink.LexCloseLink, Val: "]]"}, {Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
}, },
}, },
{ {
@ -134,7 +128,6 @@ var testCases = []tc{
{Typ: wikilink.LexAlias, Val: "|"}, {Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "second pipe"}, {Typ: wikilink.LexIdent, Val: "second pipe"},
{Typ: wikilink.LexCloseLink, Val: "]]"}, {Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
}, },
}, },
{ {
@ -146,7 +139,6 @@ var testCases = []tc{
{Typ: wikilink.LexBlockRef, Val: "#^"}, {Typ: wikilink.LexBlockRef, Val: "#^"},
{Typ: wikilink.LexIdent, Val: "blockRef"}, {Typ: wikilink.LexIdent, Val: "blockRef"},
{Typ: wikilink.LexCloseLink, Val: "]]"}, {Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
}, },
}, },
{ {
@ -158,7 +150,6 @@ var testCases = []tc{
{Typ: wikilink.LexBlockRef, Val: "#^"}, {Typ: wikilink.LexBlockRef, Val: "#^"},
{Typ: wikilink.LexIdent, Val: "blockRef"}, {Typ: wikilink.LexIdent, Val: "blockRef"},
{Typ: wikilink.LexCloseLink, Val: "]]"}, {Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
}, },
}, },
{ {
@ -172,7 +163,6 @@ var testCases = []tc{
{Typ: wikilink.LexAlias, Val: "|"}, {Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "display name"}, {Typ: wikilink.LexIdent, Val: "display name"},
{Typ: wikilink.LexCloseLink, Val: "]]"}, {Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
}, },
}, },
{ {
@ -188,7 +178,6 @@ var testCases = []tc{
{Typ: wikilink.LexAlias, Val: "|"}, {Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "second pipe"}, {Typ: wikilink.LexIdent, Val: "second pipe"},
{Typ: wikilink.LexCloseLink, Val: "]]"}, {Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
}, },
}, },
{ {
@ -204,13 +193,54 @@ var testCases = []tc{
{Typ: wikilink.LexAlias, Val: "|"}, {Typ: wikilink.LexAlias, Val: "|"},
{Typ: wikilink.LexIdent, Val: "second pipe"}, {Typ: wikilink.LexIdent, Val: "second pipe"},
{Typ: wikilink.LexCloseLink, Val: "]]"}, {Typ: wikilink.LexCloseLink, Val: "]]"},
{Typ: wikilink.LexText, Val: ""},
}, },
}, },
} }
func Test_ObsidianWikilinks_LinksEndOfMultiLineInput(t *testing.T) {
for _, tc := range SingleWikilink {
mut, test := mutateTestCase(
tc,
" test data please ignore.\nbling blonk more lines\nbling blong\nthis is a",
"",
[]wikilink.Lexeme{
{Typ: wikilink.LexText, Val: " test data please ignore.\n"},
{Typ: wikilink.LexText, Val: "bling blonk more lines\n"},
{Typ: wikilink.LexText, Val: "bling blong\n"},
{Typ: wikilink.LexText, Val: "this is a"},
},
[]wikilink.Lexeme{
{Typ: wikilink.LexText, Val: ""},
},
)
t.Run(mut.name, test)
}
}
func Test_ObsidianWikilinks_LinksStartOfMultiLineInput(t *testing.T) {
for _, tc := range SingleWikilink {
mut, test := mutateTestCase(
tc,
"",
" test data please ignore.\nbling blonk more lines\nbling blong\nthis is a",
[]wikilink.Lexeme{
{Typ: wikilink.LexText, Val: ""},
},
[]wikilink.Lexeme{
{Typ: wikilink.LexText, Val: " test data please ignore.\n"},
{Typ: wikilink.LexText, Val: "bling blonk more lines\n"},
{Typ: wikilink.LexText, Val: "bling blong\n"},
{Typ: wikilink.LexText, Val: "this is a"},
},
)
t.Run(mut.name, test)
}
}
func Test_ObsidianWikilinks_LinksStartOfInput(t *testing.T) { func Test_ObsidianWikilinks_LinksStartOfInput(t *testing.T) {
for _, tc := range testCases { for _, tc := range SingleWikilink {
mut, test := mutateTestCase( mut, test := mutateTestCase(
tc, tc,
"", "",
@ -228,7 +258,7 @@ func Test_ObsidianWikilinks_LinksStartOfInput(t *testing.T) {
} }
func Test_ObsidianWikilinks_LinksEndOfInput(t *testing.T) { func Test_ObsidianWikilinks_LinksEndOfInput(t *testing.T) {
for _, tc := range testCases { for _, tc := range SingleWikilink {
mut, test := mutateTestCase( mut, test := mutateTestCase(
tc, tc,
"this is a ", "this is a ",
@ -236,16 +266,17 @@ func Test_ObsidianWikilinks_LinksEndOfInput(t *testing.T) {
[]wikilink.Lexeme{ []wikilink.Lexeme{
{Typ: wikilink.LexText, Val: "this is a "}, {Typ: wikilink.LexText, Val: "this is a "},
}, },
nil, []wikilink.Lexeme{
{Typ: wikilink.LexText, Val: ""},
},
) )
t.Run(mut.name, test) t.Run(mut.name, test)
} }
} }
func Test_ObsidianWikilinks_Basic(t *testing.T) { func Test_ObsidianWikilinks_Basic(t *testing.T) {
// t.Parallel()
for _, tc := range testCases { for _, tc := range SingleWikilink {
mut, test := mutateTestCase( mut, test := mutateTestCase(
tc, tc,
"", "",
@ -253,7 +284,9 @@ func Test_ObsidianWikilinks_Basic(t *testing.T) {
[]wikilink.Lexeme{ []wikilink.Lexeme{
{Typ: wikilink.LexText, Val: ""}, {Typ: wikilink.LexText, Val: ""},
}, },
nil, []wikilink.Lexeme{
{Typ: wikilink.LexText, Val: ""},
},
) )
t.Run(mut.name, test) t.Run(mut.name, test)
} }
@ -276,32 +309,8 @@ func mutateTestCase(tc tc, prefix, suffix string, expectedPrefix, expectedSuffix
} }
return tc, func(t *testing.T) { return tc, func(t *testing.T) {
// t.Parallel() l := wikilink.Lex("testLexer", tc.in, zapcore.WarnLevel)
l := wikilink.Lex("testLexer", tc.in)
defer l.L.Sync() defer l.L.Sync()
if len(tc.expected) != len(l.Items) { assert.Equal(t, tc.expected, l.Items, "token stream mismatch")
t.Logf("expected %d tokens, got %d\n", len(tc.expected), len(l.Items))
t.Logf("expected items: %#v\n", tc.expected)
t.Logf("raw items: %#v\n", l.Items)
t.Fail()
return
}
for i, e := range tc.expected {
n := l.Items[i]
if e.Typ != n.Typ {
t.Logf("expected Type %s, received %s", e.Typ.String(), n.Typ.String())
t.Fail()
return
}
if e.Val != n.Val {
t.Logf("expected Value %q, received %q", e.Val, n.Val)
t.Fail()
return
}
}
} }
} }

@ -1,275 +0,0 @@
mkdir -p "/home/ndumas/work/wikilink-parser/reports"
mkdir -p "/home/ndumas/work/wikilink-parser/dist"
go clean code.ndumas.com/ndumas/wikilink-parser
rm -vrf "/home/ndumas/work/wikilink-parser/dist"/*
rm -vf "/home/ndumas/work/wikilink-parser/reports"/*
removed '/home/ndumas/work/wikilink-parser/reports/test.out'
go get -d -t code.ndumas.com/ndumas/wikilink-parser/...
go install golang.org/x/tools/cmd/stringer@latest
go generate
go test -race -v -tags "release" $(go list "code.ndumas.com/ndumas/wikilink-parser/..." | grep -v /vendor/) | tee "/home/ndumas/work/wikilink-parser/reports/test.out"
=== RUN Test_Lexer
=== RUN Test_Lexer/wikilink
2023/07/01 18:25:42 lexText
2023/07/01 18:25:42 lexOpenLink
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":2,"width":0,"item":"ItemOpenLink:\"[[\""}
2023/07/01 18:25:42 lexIdent
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":10,"width":1,"item":"ItemIdent:\"wikilink\""}
2023/07/01 18:25:42 lexCloseLink
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":12,"width":1,"item":"ItemCloseLink:\"]]\""}
=== RUN Test_Lexer/wikilink|display_name
2023/07/01 18:25:42 lexText
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":12,"width":0,"item":"ItemText:\"\""}
2023/07/01 18:25:42 lexText
2023/07/01 18:25:42 lexOpenLink
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":2,"width":0,"item":"ItemOpenLink:\"[[\""}
2023/07/01 18:25:42 lexIdent
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":10,"width":1,"item":"ItemIdent:\"wikilink\""}
2023/07/01 18:25:42 lexAlias
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":11,"width":1,"item":"ItemAlias:\"|\""}
2023/07/01 18:25:42 lexIdent
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":23,"width":1,"item":"ItemIdent:\"display name\""}
2023/07/01 18:25:42 lexCloseLink
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":25,"width":1,"item":"ItemCloseLink:\"]]\""}
=== RUN Test_Lexer/wikilink|display_name|second_pipe
2023/07/01 18:25:42 lexText
2023/07/01 18:25:42 lexOpenLink
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":2,"width":0,"item":"ItemOpenLink:\"[[\""}
2023/07/01 18:25:42 lexIdent
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":10,"width":1,"item":"ItemIdent:\"wikilink\""}
2023/07/01 18:25:42 lexAlias
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":11,"width":1,"item":"ItemAlias:\"|\""}
2023/07/01 18:25:42 lexIdent
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":23,"width":1,"item":"ItemIdent:\"display name\""}
2023/07/01 18:25:42 lexText
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":25,"width":0,"item":"ItemText:\"\""}
2023/07/01 18:25:42 lexAlias
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":24,"width":1,"item":"ItemAlias:\"|\""}
2023/07/01 18:25:42 lexIdent
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":35,"width":1,"item":"ItemIdent:\"second pipe\""}
2023/07/01 18:25:42 lexCloseLink
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":37,"width":1,"item":"ItemCloseLink:\"]]\""}
=== RUN Test_Lexer/wikilink_with_numeric_alias|420|second_pipe
2023/07/01 18:25:42 lexText
2023/07/01 18:25:42 lexOpenLink
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":2,"width":0,"item":"ItemOpenLink:\"[[\""}
2023/07/01 18:25:42 lexIdent
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":10,"width":1,"item":"ItemIdent:\"wikilink\""}
2023/07/01 18:25:42 lexAlias
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":11,"width":1,"item":"ItemAlias:\"|\""}
2023/07/01 18:25:42 lexIdent
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":14,"width":1,"item":"ItemIdent:\"420\""}
2023/07/01 18:25:42 lexAlias
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":15,"width":1,"item":"ItemAlias:\"|\""}
2023/07/01 18:25:42 lexIdent
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":26,"width":1,"item":"ItemIdent:\"second pipe\""}
2023/07/01 18:25:42 lexCloseLink
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":28,"width":1,"item":"ItemCloseLink:\"]]\""}
=== RUN Test_Lexer/wikilink_with_spaces_in_filename
2023/07/01 18:25:42 lexText
2023/07/01 18:25:42 lexOpenLink
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":2,"width":0,"item":"ItemOpenLink:\"[[\""}
2023/07/01 18:25:42 lexIdent
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":17,"width":1,"item":"ItemIdent:\"wikilink spaces\""}
2023/07/01 18:25:42 lexCloseLink
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":19,"width":1,"item":"ItemCloseLink:\"]]\""}
=== RUN Test_Lexer/#heading
2023/07/01 18:25:42 lexText
2023/07/01 18:25:42 lexOpenLink
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":2,"width":0,"item":"ItemOpenLink:\"[[\""}
2023/07/01 18:25:42 lexIdent
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":10,"width":1,"item":"ItemIdent:\"#heading\""}
lexer_test.go:100: expected Type ItemHeading, received ItemIdent
lexer_test.go:105: expected Value "#", received "#heading"
2023/07/01 18:25:42 lexCloseLink
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":12,"width":1,"item":"ItemCloseLink:\"]]\""}
lexer_test.go:100: expected Type ItemIdent, received ItemCloseLink
lexer_test.go:105: expected Value "heading", received "]]"
2023/07/01 18:25:42 lexText
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":12,"width":0,"item":"ItemText:\"\""}
lexer_test.go:100: expected Type ItemCloseLink, received ItemText
lexer_test.go:105: expected Value "]]", received ""
=== RUN Test_Lexer/wikilink#heading
2023/07/01 18:25:42 lexText
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":37,"width":0,"item":"ItemText:\"\""}
2023/07/01 18:25:42 lexText
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":28,"width":0,"item":"ItemText:\"\""}
2023/07/01 18:25:42 lexText
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":19,"width":0,"item":"ItemText:\"\""}
2023/07/01 18:25:42 lexText
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":12,"width":0,"item":"ItemText:\"\""}
2023/07/01 18:25:42 lexText
2023/07/01 18:25:42 lexOpenLink
2023/07/01 18:25:42 lexText
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":2,"width":0,"item":"ItemOpenLink:\"[[\""}
2023/07/01 18:25:42 lexIdent
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":10,"width":1,"item":"ItemIdent:\"wikilink\""}
==================
WARNING: DATA RACE
Read at 0x00c0000822b0 by goroutine 21:
code.ndumas.com/ndumas/wikilink-parser.lexText()
/home/ndumas/work/wikilink-parser/states.go:82 +0x69
code.ndumas.com/ndumas/wikilink-parser.(*Lexer).run()
/home/ndumas/work/wikilink-parser/lexer.go:149 +0x3a
code.ndumas.com/ndumas/wikilink-parser.Lex.func1()
/home/ndumas/work/wikilink-parser/lexer.go:45 +0x39
Previous write at 0x00c0000822b0 by goroutine 20:
code.ndumas.com/ndumas/wikilink-parser.lexOpenLink()
/home/ndumas/work/wikilink-parser/states.go:96 +0x86
code.ndumas.com/ndumas/wikilink-parser.(*Lexer).NextItem()
/home/ndumas/work/wikilink-parser/lexer.go:64 +0xd5
code.ndumas.com/ndumas/wikilink-parser_test.Test_Lexer.func1()
/home/ndumas/work/wikilink-parser/lexer_test.go:98 +0x18c
testing.tRunner()
/usr/lib/golang/src/testing/testing.go:1446 +0x216
testing.(*T).Run.func1()
/usr/lib/golang/src/testing/testing.go:1493 +0x47
Goroutine 21 (running) created at:
code.ndumas.com/ndumas/wikilink-parser.Lex()
/home/ndumas/work/wikilink-parser/lexer.go:45 +0x41a
code.ndumas.com/ndumas/wikilink-parser_test.Test_Lexer.func1()
/home/ndumas/work/wikilink-parser/lexer_test.go:95 +0x69
testing.tRunner()
/usr/lib/golang/src/testing/testing.go:1446 +0x216
testing.(*T).Run.func1()
/usr/lib/golang/src/testing/testing.go:1493 +0x47
Goroutine 20 (running) created at:
testing.(*T).Run()
/usr/lib/golang/src/testing/testing.go:1493 +0x75d
code.ndumas.com/ndumas/wikilink-parser_test.Test_Lexer()
/home/ndumas/work/wikilink-parser/lexer_test.go:93 +0x1086
testing.tRunner()
/usr/lib/golang/src/testing/testing.go:1446 +0x216
testing.(*T).Run.func1()
/usr/lib/golang/src/testing/testing.go:1493 +0x47
==================
==================
WARNING: DATA RACE
Write at 0x00c0000822b8 by goroutine 21:
code.ndumas.com/ndumas/wikilink-parser.(*Lexer).next()
/home/ndumas/work/wikilink-parser/lexer.go:142 +0xe8
code.ndumas.com/ndumas/wikilink-parser.lexText()
/home/ndumas/work/wikilink-parser/states.go:85 +0xc4
code.ndumas.com/ndumas/wikilink-parser.(*Lexer).run()
/home/ndumas/work/wikilink-parser/lexer.go:149 +0x3a
code.ndumas.com/ndumas/wikilink-parser.Lex.func1()
/home/ndumas/work/wikilink-parser/lexer.go:45 +0x39
Previous write at 0x00c0000822b8 by goroutine 20:
code.ndumas.com/ndumas/wikilink-parser.(*Lexer).next()
/home/ndumas/work/wikilink-parser/lexer.go:142 +0xe8
code.ndumas.com/ndumas/wikilink-parser.lexIdent()
/home/ndumas/work/wikilink-parser/states.go:31 +0x64
code.ndumas.com/ndumas/wikilink-parser.(*Lexer).NextItem()
/home/ndumas/work/wikilink-parser/lexer.go:64 +0xd5
code.ndumas.com/ndumas/wikilink-parser_test.Test_Lexer.func1()
/home/ndumas/work/wikilink-parser/lexer_test.go:98 +0x18c
testing.tRunner()
/usr/lib/golang/src/testing/testing.go:1446 +0x216
testing.(*T).Run.func1()
/usr/lib/golang/src/testing/testing.go:1493 +0x47
Goroutine 21 (running) created at:
code.ndumas.com/ndumas/wikilink-parser.Lex()
/home/ndumas/work/wikilink-parser/lexer.go:45 +0x41a
code.ndumas.com/ndumas/wikilink-parser_test.Test_Lexer.func1()
/home/ndumas/work/wikilink-parser/lexer_test.go:95 +0x69
testing.tRunner()
/usr/lib/golang/src/testing/testing.go:1446 +0x216
testing.(*T).Run.func1()
/usr/lib/golang/src/testing/testing.go:1493 +0x47
Goroutine 20 (running) created at:
testing.(*T).Run()
/usr/lib/golang/src/testing/testing.go:1493 +0x75d
code.ndumas.com/ndumas/wikilink-parser_test.Test_Lexer()
/home/ndumas/work/wikilink-parser/lexer_test.go:93 +0x1086
testing.tRunner()
/usr/lib/golang/src/testing/testing.go:1446 +0x216
testing.(*T).Run.func1()
/usr/lib/golang/src/testing/testing.go:1493 +0x47
==================
==================
WARNING: DATA RACE
Read at 0x00c0000822a8 by goroutine 21:
code.ndumas.com/ndumas/wikilink-parser.(*Lexer).emit()
/home/ndumas/work/wikilink-parser/lexer.go:109 +0x64
code.ndumas.com/ndumas/wikilink-parser.lexText()
/home/ndumas/work/wikilink-parser/states.go:88 +0xf4
code.ndumas.com/ndumas/wikilink-parser.(*Lexer).run()
/home/ndumas/work/wikilink-parser/lexer.go:149 +0x3a
code.ndumas.com/ndumas/wikilink-parser.Lex.func1()
/home/ndumas/work/wikilink-parser/lexer.go:45 +0x39
Previous write at 0x00c0000822a8 by goroutine 20:
code.ndumas.com/ndumas/wikilink-parser.(*Lexer).emit()
/home/ndumas/work/wikilink-parser/lexer.go:119 +0x4cf
code.ndumas.com/ndumas/wikilink-parser.lexOpenLink()
/home/ndumas/work/wikilink-parser/states.go:97 +0xa4
code.ndumas.com/ndumas/wikilink-parser.(*Lexer).NextItem()
/home/ndumas/work/wikilink-parser/lexer.go:64 +0xd5
code.ndumas.com/ndumas/wikilink-parser_test.Test_Lexer.func1()
/home/ndumas/work/wikilink-parser/lexer_test.go:98 +0x18c
testing.tRunner()
/usr/lib/golang/src/testing/testing.go:1446 +0x216
testing.(*T).Run.func1()
/usr/lib/golang/src/testing/testing.go:1493 +0x47
Goroutine 21 (running) created at:
code.ndumas.com/ndumas/wikilink-parser.Lex()
/home/ndumas/work/wikilink-parser/lexer.go:45 +0x41a
code.ndumas.com/ndumas/wikilink-parser_test.Test_Lexer.func1()
/home/ndumas/work/wikilink-parser/lexer_test.go:95 +0x69
testing.tRunner()
/usr/lib/golang/src/testing/testing.go:1446 +0x216
testing.(*T).Run.func1()
/usr/lib/golang/src/testing/testing.go:1493 +0x47
Goroutine 20 (running) created at:
testing.(*T).Run()
/usr/lib/golang/src/testing/testing.go:1493 +0x75d
code.ndumas.com/ndumas/wikilink-parser_test.Test_Lexer()
/home/ndumas/work/wikilink-parser/lexer_test.go:93 +0x1086
testing.tRunner()
/usr/lib/golang/src/testing/testing.go:1446 +0x216
testing.(*T).Run.func1()
/usr/lib/golang/src/testing/testing.go:1493 +0x47
==================
2023/07/01 18:25:42 lexHeading
{"level":"debug","logger":"lexer.emit","msg":"emitting item","pos":20,"width":0,"item":"ItemText:\"wikilink#heading]]\""}
testing.go:1319: race detected during execution of test
--- FAIL: Test_Lexer (0.01s)
--- PASS: Test_Lexer/wikilink (0.00s)
--- PASS: Test_Lexer/wikilink|display_name (0.00s)
--- PASS: Test_Lexer/wikilink|display_name|second_pipe (0.00s)
--- PASS: Test_Lexer/wikilink_with_numeric_alias|420|second_pipe (0.00s)
--- PASS: Test_Lexer/wikilink_with_spaces_in_filename (0.00s)
--- FAIL: Test_Lexer/#heading (0.00s)
--- FAIL: Test_Lexer/wikilink#heading (0.00s)
panic: runtime error: slice bounds out of range [:21] with length 20 [recovered]
panic: runtime error: slice bounds out of range [:21] with length 20
goroutine 34 [running]:
testing.tRunner.func1.2({0x6e8160, 0xc0000e20a8})
/usr/lib/golang/src/testing/testing.go:1396 +0x372
testing.tRunner.func1()
/usr/lib/golang/src/testing/testing.go:1399 +0x5f0
panic({0x6e8160, 0xc0000e20a8})
/usr/lib/golang/src/runtime/panic.go:890 +0x262
code.ndumas.com/ndumas/wikilink-parser.(*Lexer).emit(0xc000082280, 0x5)
/home/ndumas/work/wikilink-parser/lexer.go:109 +0x506
code.ndumas.com/ndumas/wikilink-parser.lexHeading(0xc000082280)
/home/ndumas/work/wikilink-parser/states.go:58 +0xa5
code.ndumas.com/ndumas/wikilink-parser.(*Lexer).NextItem(0xc000082280)
/home/ndumas/work/wikilink-parser/lexer.go:64 +0xd6
code.ndumas.com/ndumas/wikilink-parser_test.Test_Lexer.func1(0xc000190b60)
/home/ndumas/work/wikilink-parser/lexer_test.go:98 +0x18d
testing.tRunner(0xc000190b60, 0xc00019cb40)
/usr/lib/golang/src/testing/testing.go:1446 +0x217
created by testing.(*T).Run
/usr/lib/golang/src/testing/testing.go:1493 +0x75e
FAIL code.ndumas.com/ndumas/wikilink-parser 0.024s
FAIL

@ -92,9 +92,12 @@ func lexText(l *Lexer) stateFn {
} }
r := l.next() r := l.next()
switch { switch {
case r == EOF || r == '\n': case r == EOF:
l.emit(LexText) l.emit(LexText)
return nil return nil
case r == '\n':
l.emit(LexText)
return lexText
} }
} }
} }

Loading…
Cancel
Save