obsidian-markdown/lexer_test.go

package markdown_test

import (
	"testing"

	"github.com/stretchr/testify/assert"
	"go.uber.org/zap/zapcore"

	"code.ndumas.com/ndumas/obsidian-markdown"
)

var wikilinkWithEscapeCharacters = []tc{
	{
		name: "wikilink with escaped close link",
		in:   `[[wiki\]\]link]]`,
		expected: []markdown.Lexeme{
			{Typ: markdown.LexOpenLink, Val: "[["},
			{Typ: markdown.LexIdent, Val: `wiki\]\]link`},
			{Typ: markdown.LexCloseLink, Val: "]]"},
		},
	},
	{
		name: "wikilink with partial escaped close link",
		in:   `[[wiki\]link]]`,
		expected: []markdown.Lexeme{
			{Typ: markdown.LexOpenLink, Val: "[["},
			{Typ: markdown.LexIdent, Val: `wiki\]link`},
			{Typ: markdown.LexCloseLink, Val: "]]"},
		},
	},
	{
		name: "wikilink with escaped open link",
		in:   `[[wiki\[\[link]]`,
		expected: []markdown.Lexeme{
			{Typ: markdown.LexOpenLink, Val: "[["},
			{Typ: markdown.LexIdent, Val: `wiki\[\[link`},
			{Typ: markdown.LexCloseLink, Val: "]]"},
		},
	},
	{
		name: "wikilink with partial escaped open link",
		in:   `[[wiki\[link]]`,
		expected: []markdown.Lexeme{
			{Typ: markdown.LexOpenLink, Val: "[["},
			{Typ: markdown.LexIdent, Val: `wiki\[link`},
			{Typ: markdown.LexCloseLink, Val: "]]"},
		},
	},
	{
		name: "wikilink with escaped alias",
		in:   `[[wiki\|link]]`,
		expected: []markdown.Lexeme{
			{Typ: markdown.LexOpenLink, Val: "[["},
			{Typ: markdown.LexIdent, Val: `wiki\|link`},
			{Typ: markdown.LexCloseLink, Val: "]]"},
		},
	},
	{
		name: "wikilink with escaped blockref",
		in:   `[[wiki\#\^link]]`,
		expected: []markdown.Lexeme{
			{Typ: markdown.LexOpenLink, Val: "[["},
			{Typ: markdown.LexIdent, Val: `wiki\#\^link`},
			{Typ: markdown.LexCloseLink, Val: "]]"},
		},
	},
	{
		name: "wikilink with partial escaped blockref",
		in:   `[[wiki\^link]]`,
		expected: []markdown.Lexeme{
			{Typ: markdown.LexOpenLink, Val: "[["},
			{Typ: markdown.LexIdent, Val: `wiki\^link`},
			{Typ: markdown.LexCloseLink, Val: "]]"},
		},
	},
	{
		name: "wikilink with escaped header",
		in:   `[[wiki\#link]]`,
		expected: []markdown.Lexeme{
			{Typ: markdown.LexOpenLink, Val: "[["},
			{Typ: markdown.LexIdent, Val: `wiki\#link`},
			{Typ: markdown.LexCloseLink, Val: "]]"},
		},
	},
}

var singleWikilink = []tc{
	{
		name: "wikilink",
		in:   "[[wikilink]]",
		expected: []markdown.Lexeme{
			{Typ: markdown.LexOpenLink, Val: "[["},
			{Typ: markdown.LexIdent, Val: "wikilink"},
			{Typ: markdown.LexCloseLink, Val: "]]"},
		},
	},
	{
		name: "wikilink|display name",
		in:   "[[wikilink|display name]]",
		expected: []markdown.Lexeme{
			{Typ: markdown.LexOpenLink, Val: "[["},
			{Typ: markdown.LexIdent, Val: "wikilink"},
			{Typ: markdown.LexAlias, Val: "|"},
			{Typ: markdown.LexIdent, Val: "display name"},
			{Typ: markdown.LexCloseLink, Val: "]]"},
		},
	},
	{
		name: "wikilink|display name|second pipe",
		in:   "[[wikilink|display name|second pipe]]",
		expected: []markdown.Lexeme{
			{Typ: markdown.LexOpenLink, Val: "[["},
			{Typ: markdown.LexIdent, Val: "wikilink"},
			{Typ: markdown.LexAlias, Val: "|"},
			{Typ: markdown.LexIdent, Val: "display name"},
			{Typ: markdown.LexAlias, Val: "|"},
			{Typ: markdown.LexIdent, Val: "second pipe"},
			{Typ: markdown.LexCloseLink, Val: "]]"},
		},
	},
	{
		name: "wikilink with numeric alias|420|second pipe",
		in:   "[[wikilink|420|second pipe]]",
		expected: []markdown.Lexeme{
			{Typ: markdown.LexOpenLink, Val: "[["},
			{Typ: markdown.LexIdent, Val: "wikilink"},
			{Typ: markdown.LexAlias, Val: "|"},
			{Typ: markdown.LexIdent, Val: "420"},
			{Typ: markdown.LexAlias, Val: "|"},
			{Typ: markdown.LexIdent, Val: "second pipe"},
			{Typ: markdown.LexCloseLink, Val: "]]"},
		},
	},
	{
		name: "wikilink with spaces in filename",
		in:   "[[wikilink spaces]]",
		expected: []markdown.Lexeme{
			{Typ: markdown.LexOpenLink, Val: "[["},
			{Typ: markdown.LexIdent, Val: "wikilink spaces"},
			{Typ: markdown.LexCloseLink, Val: "]]"},
		},
	},
	{
		name: "#heading",
		in:   "[[#heading]]",
		expected: []markdown.Lexeme{
			{Typ: markdown.LexOpenLink, Val: "[["},
			{Typ: markdown.LexIdent, Val: ""},
			{Typ: markdown.LexHeading, Val: "#"},
			{Typ: markdown.LexIdent, Val: "heading"},
			{Typ: markdown.LexCloseLink, Val: "]]"},
		},
	},
	{
		name: "wikilink#heading",
		in:   "[[wikilink#heading]]",
		expected: []markdown.Lexeme{
			{Typ: markdown.LexOpenLink, Val: "[["},
			{Typ: markdown.LexIdent, Val: "wikilink"},
			{Typ: markdown.LexHeading, Val: "#"},
			{Typ: markdown.LexIdent, Val: "heading"},
			{Typ: markdown.LexCloseLink, Val: "]]"},
		},
	},
	{
		name: "wikilink#heading|display name",
		in:   "[[wikilink#heading|display name]]",
		expected: []markdown.Lexeme{
			{Typ: markdown.LexOpenLink, Val: "[["},
			{Typ: markdown.LexIdent, Val: "wikilink"},
			{Typ: markdown.LexHeading, Val: "#"},
			{Typ: markdown.LexIdent, Val: "heading"},
			{Typ: markdown.LexAlias, Val: "|"},
			{Typ: markdown.LexIdent, Val: "display name"},
			{Typ: markdown.LexCloseLink, Val: "]]"},
		},
	},
	{
		name: "wikilink#heading|display name|second pipe",
		in:   "[[wikilink#heading|display name|second pipe]]",
		expected: []markdown.Lexeme{
			{Typ: markdown.LexOpenLink, Val: "[["},
			{Typ: markdown.LexIdent, Val: "wikilink"},
			{Typ: markdown.LexHeading, Val: "#"},
			{Typ: markdown.LexIdent, Val: "heading"},
			{Typ: markdown.LexAlias, Val: "|"},
			{Typ: markdown.LexIdent, Val: "display name"},
			{Typ: markdown.LexAlias, Val: "|"},
			{Typ: markdown.LexIdent, Val: "second pipe"},
			{Typ: markdown.LexCloseLink, Val: "]]"},
		},
	},
	{
		name: "wikilink with numeric aliases#heading|420|display name",
		in:   "[[wikilink#heading|420|second pipe]]",
		expected: []markdown.Lexeme{
			{Typ: markdown.LexOpenLink, Val: "[["},
			{Typ: markdown.LexIdent, Val: "wikilink"},
			{Typ: markdown.LexHeading, Val: "#"},
			{Typ: markdown.LexIdent, Val: "heading"},
			{Typ: markdown.LexAlias, Val: "|"},
			{Typ: markdown.LexIdent, Val: "420"},
			{Typ: markdown.LexAlias, Val: "|"},
			{Typ: markdown.LexIdent, Val: "second pipe"},
			{Typ: markdown.LexCloseLink, Val: "]]"},
		},
	},
	{
		name: "#^blockRef",
		in:   "[[#^blockRef]]",
		expected: []markdown.Lexeme{
			{Typ: markdown.LexOpenLink, Val: "[["},
			{Typ: markdown.LexIdent, Val: ""},
			{Typ: markdown.LexBlockRef, Val: "#^"},
			{Typ: markdown.LexIdent, Val: "blockRef"},
			{Typ: markdown.LexCloseLink, Val: "]]"},
		},
	},
	{
		name: "wikilink#^blockRef",
		in:   "[[wikilink#^blockRef]]",
		expected: []markdown.Lexeme{
			{Typ: markdown.LexOpenLink, Val: "[["},
			{Typ: markdown.LexIdent, Val: "wikilink"},
			{Typ: markdown.LexBlockRef, Val: "#^"},
			{Typ: markdown.LexIdent, Val: "blockRef"},
			{Typ: markdown.LexCloseLink, Val: "]]"},
		},
	},
	{
		name: "wikilink#^blockRef|display name",
		in:   "[[wikilink#^blockRef|display name]]",
		expected: []markdown.Lexeme{
			{Typ: markdown.LexOpenLink, Val: "[["},
			{Typ: markdown.LexIdent, Val: "wikilink"},
			{Typ: markdown.LexBlockRef, Val: "#^"},
			{Typ: markdown.LexIdent, Val: "blockRef"},
			{Typ: markdown.LexAlias, Val: "|"},
			{Typ: markdown.LexIdent, Val: "display name"},
			{Typ: markdown.LexCloseLink, Val: "]]"},
		},
	},
	{
		name: "wikilink#^blockRef|display name|second pipe",
		in:   "[[wikilink#^blockRef|display name|second pipe]]",
		expected: []markdown.Lexeme{
			{Typ: markdown.LexOpenLink, Val: "[["},
			{Typ: markdown.LexIdent, Val: "wikilink"},
			{Typ: markdown.LexBlockRef, Val: "#^"},
			{Typ: markdown.LexIdent, Val: "blockRef"},
			{Typ: markdown.LexAlias, Val: "|"},
			{Typ: markdown.LexIdent, Val: "display name"},
			{Typ: markdown.LexAlias, Val: "|"},
			{Typ: markdown.LexIdent, Val: "second pipe"},
			{Typ: markdown.LexCloseLink, Val: "]]"},
		},
	},
	{
		name: "wikilink with numeric aliases#^blockRef|420|second pipe",
		in:   "[[wikilink#^blockRef|420|second pipe]]",
		expected: []markdown.Lexeme{
			{Typ: markdown.LexOpenLink, Val: "[["},
			{Typ: markdown.LexIdent, Val: "wikilink"},
			{Typ: markdown.LexBlockRef, Val: "#^"},
			{Typ: markdown.LexIdent, Val: "blockRef"},
			{Typ: markdown.LexAlias, Val: "|"},
			{Typ: markdown.LexIdent, Val: "420"},
			{Typ: markdown.LexAlias, Val: "|"},
			{Typ: markdown.LexIdent, Val: "second pipe"},
			{Typ: markdown.LexCloseLink, Val: "]]"},
		},
	},
}

func Test_ObsidianWikilinks_TextWithEscapeCharacters(t *testing.T) {
	for _, tc := range singleWikilink {
		tc.name = "escape characters preceding " + tc.name
		mut, test := mutateTestCase(
			tc,
			`foo\[\[not a link, but this is`,
			"",
			[]markdown.Lexeme{
				{Typ: markdown.LexText, Val: `foo\[\[not a link, but this is`},
			},
			[]markdown.Lexeme{
				{Typ: markdown.LexText, Val: ""},
			},
		)
		t.Run(mut.name, test)
	}

	for _, tc := range singleWikilink {
		tc.name = "escape characters following " + tc.name
		mut, test := mutateTestCase(
			tc,
			"",
			`foo\[\[not a link, but this is`,
			[]markdown.Lexeme{
				{Typ: markdown.LexText, Val: ""},
			},
			[]markdown.Lexeme{
				{Typ: markdown.LexText, Val: `foo\[\[not a link, but this is`},
			},
		)
		t.Run(mut.name, test)
	}
}

func Test_ObsidianWikilinks_EscapeCharacters(t *testing.T) {
	for _, tc := range wikilinkWithEscapeCharacters {
		mut, test := mutateTestCase(
			tc,
			"",
			"",
			[]markdown.Lexeme{
				{Typ: markdown.LexText, Val: ""},
			},
			[]markdown.Lexeme{
				{Typ: markdown.LexText, Val: ""},
			},
		)
		t.Run(mut.name, test)
	}
}

func Test_ObsidianWikilinks_LinksEndOfMultiLineInput(t *testing.T) {
	for _, tc := range singleWikilink {
		mut, test := mutateTestCase(
			tc,
			" test data please ignore.\nbling blonk more lines\nbling blong\nthis is a",
			"",

			[]markdown.Lexeme{
				{Typ: markdown.LexText, Val: " test data please ignore.\n"},
				{Typ: markdown.LexText, Val: "bling blonk more lines\n"},
				{Typ: markdown.LexText, Val: "bling blong\n"},
				{Typ: markdown.LexText, Val: "this is a"},
			},
			[]markdown.Lexeme{
				{Typ: markdown.LexText, Val: ""},
			},
		)
		t.Run(mut.name, test)
	}
}

func Test_ObsidianWikilinks_LinksStartOfMultiLineInput(t *testing.T) {
	for _, tc := range singleWikilink {
		mut, test := mutateTestCase(
			tc,
			"",
			" test data please ignore.\nbling blonk more lines\nbling blong\nthis is a",

			[]markdown.Lexeme{
				{Typ: markdown.LexText, Val: ""},
			},
			[]markdown.Lexeme{
				{Typ: markdown.LexText, Val: " test data please ignore.\n"},
				{Typ: markdown.LexText, Val: "bling blonk more lines\n"},
				{Typ: markdown.LexText, Val: "bling blong\n"},
				{Typ: markdown.LexText, Val: "this is a"},
			},
		)
		t.Run(mut.name, test)
	}
}

func Test_ObsidianWikilinks_LinksStartOfInput(t *testing.T) {
	for _, tc := range singleWikilink {
		mut, test := mutateTestCase(
			tc,
			"",
			" test data please ignore",

			[]markdown.Lexeme{
				{Typ: markdown.LexText, Val: ""},
			},
			[]markdown.Lexeme{
				{Typ: markdown.LexText, Val: " test data please ignore"},
			},
		)
		t.Run(mut.name, test)
	}
}

func Test_ObsidianWikilinks_LinksEndOfInput(t *testing.T) {
	for _, tc := range singleWikilink {
		mut, test := mutateTestCase(
			tc,
			"this is a ",
			"",
			[]markdown.Lexeme{
				{Typ: markdown.LexText, Val: "this is a "},
			},
			[]markdown.Lexeme{
				{Typ: markdown.LexText, Val: ""},
			},
		)
		t.Run(mut.name, test)
	}
}

func Test_ObsidianWikilinks_Basic(t *testing.T) {

	for _, tc := range singleWikilink {
		mut, test := mutateTestCase(
			tc,
			"",
			"",
			[]markdown.Lexeme{
				{Typ: markdown.LexText, Val: ""},
			},
			[]markdown.Lexeme{
				{Typ: markdown.LexText, Val: ""},
			},
		)
		t.Run(mut.name, test)
	}
}

type tc struct {
	name     string
	in       string
	expected []markdown.Lexeme
}

func mutateTestCase(tc tc, prefix, suffix string, expectedPrefix, expectedSuffix []markdown.Lexeme) (tc, func(t *testing.T)) {
	tc.in = prefix + tc.in
	tc.in = tc.in + suffix
	if expectedPrefix != nil {
		tc.expected = append(expectedPrefix, tc.expected...)
	}
	if expectedSuffix != nil {
		tc.expected = append(tc.expected, expectedSuffix...)
	}

	return tc, func(t *testing.T) {
		l := markdown.Lex("testLexer", tc.in, zapcore.WarnLevel)
		defer l.L.Sync()
		assert.Equal(t, tc.expected, l.Items, "token stream mismatch")
	}
}