sivukhin / godjot

Djot parser written in Go
MIT License
23 stars 3 forks source link

Question: Failing tests? #3

Closed acheong08 closed 9 months ago

acheong08 commented 9 months ago

I'm getting a lot of failing tests with go test ./....

```go ? github.com/sivukhin/godjot [no test files] ? github.com/sivukhin/godjot/html_writer [no test files] ok github.com/sivukhin/godjot/djot_parser 0.161s --- FAIL: TestSimpleText (0.00s) djot_tokenizer_test.go:13: [{DocumentBlock 8 0 0 } {ParagraphBlock 6 0 0 } {None 0 0 6 } {StrongInline 2 6 7 0xc0000683c0} {None 0 7 12 } {StrongInlineClose -2 12 13 } {None 0 13 14 } {ParagraphBlockClose -6 14 14 } {DocumentBlockClose -8 14 14 }] djot_tokenizer_test.go:14: Error Trace: /tmp/godjot/djot_tokenizer/djot_tokenizer_test.go:14 Error: Not equal: expected: {DocumentBlock 8 0 0 },{ParagraphBlock 6 0 0 },{None 0 0 6 },{StrongInline 2 6 7 },{None 0 7 12 },{StrongInlineClose -2 12 13 },{None 0 13 14 },{ParagraphBlockClose -6 14 14 },{DocumentBlockClose -8 14 14 } actual : {DocumentBlock 8 0 0 },{ParagraphBlock 6 0 0 },{None 0 0 6 },{StrongInline 2 6 7 0xc0000683c0},{None 0 7 12 },{StrongInlineClose -2 12 13 },{None 0 13 14 },{ParagraphBlockClose -6 14 14 },{DocumentBlockClose -8 14 14 } Diff: --- Expected +++ Actual @@ -27,3 +27,6 @@ End: (int) 7, - Attributes: (*tokenizer.Attributes)() + Attributes: (*tokenizer.Attributes)({ + Keys: ([]string) , + Map: (map[string]string) + }) }, Test: TestSimpleText --- FAIL: TestSimpleLink (0.00s) djot_tokenizer_test.go:48: Error Trace: /tmp/godjot/djot_tokenizer/djot_tokenizer_test.go:48 Error: Not equal: expected: {DocumentBlock 9 0 0 },{ParagraphBlock 7 0 0 },{SpanInline 2 0 1 },{None 0 1 2 },{SpanInlineClose -2 2 3 },{LinkUrlInline 2 3 4 },{None 0 4 5 },{LinkUrlInlineClose -2 5 6 },{ParagraphBlockClose -7 6 6 },{DocumentBlockClose -9 6 6 } actual : {DocumentBlock 9 0 0 },{ParagraphBlock 7 0 0 },{SpanInline 2 0 1 0xc000068f80},{None 0 1 2 },{SpanInlineClose -2 2 3 },{LinkUrlInline 2 3 4 0xc000068fa0},{None 0 4 5 },{LinkUrlInlineClose -2 5 6 },{ParagraphBlockClose -7 6 6 },{DocumentBlockClose -9 6 6 } Diff: --- Expected +++ Actual @@ -20,3 +20,6 @@ End: (int) 1, - Attributes: (*tokenizer.Attributes)() + Attributes: (*tokenizer.Attributes)({ + Keys: ([]string) , + Map: (map[string]string) + }) }, @@ -41,3 +44,6 @@ End: (int) 4, - Attributes: (*tokenizer.Attributes)() + Attributes: (*tokenizer.Attributes)({ + Keys: ([]string) , + Map: (map[string]string) + }) }, Test: TestSimpleLink --- FAIL: TestSimpleLinkWithNewline (0.00s) djot_tokenizer_test.go:64: Error Trace: /tmp/godjot/djot_tokenizer/djot_tokenizer_test.go:64 Error: Not equal: expected: {DocumentBlock 9 0 0 },{ParagraphBlock 7 0 0 },{SpanInline 2 0 1 },{None 0 1 13 },{SpanInlineClose -2 13 14 },{LinkUrlInline 2 14 15 },{None 0 15 74 },{LinkUrlInlineClose -2 74 75 },{ParagraphBlockClose -7 75 75 },{DocumentBlockClose -9 75 75 } actual : {DocumentBlock 11 0 0 },{ParagraphBlock 9 0 0 },{SpanInline 2 0 1 0xc000069040},{None 0 1 13 },{SpanInlineClose -2 13 14 },{LinkUrlInline 4 14 15 0xc000069060},{None 0 15 61 },{SmartSymbolInline 0 61 62 },{None 0 62 74 },{LinkUrlInlineClose -4 74 75 },{ParagraphBlockClose -9 75 75 },{DocumentBlockClose -11 75 75 } Diff: --- Expected +++ Actual @@ -1,5 +1,5 @@ -(tokenizer.TokenList[github.com/sivukhin/godjot/djot_tokenizer.DjotToken]) (len=10) { +(tokenizer.TokenList[github.com/sivukhin/godjot/djot_tokenizer.DjotToken]) (len=12) { (tokenizer.Token[github.com/sivukhin/godjot/djot_tokenizer.DjotToken]) { Type: (djot_tokenizer.DjotToken) 3, - JumpToPair: (int) 9, + JumpToPair: (int) 11, Start: (int) 0, @@ -10,3 +10,3 @@ Type: (djot_tokenizer.DjotToken) 21, - JumpToPair: (int) 7, + JumpToPair: (int) 9, Start: (int) 0, @@ -20,3 +20,6 @@ End: (int) 1, - Attributes: (*tokenizer.Attributes)() + Attributes: (*tokenizer.Attributes)({ + Keys: ([]string) , + Map: (map[string]string) + }) }, @@ -38,5 +41,22 @@ Type: (djot_tokenizer.DjotToken) 37, - JumpToPair: (int) 2, + JumpToPair: (int) 4, Start: (int) 14, End: (int) 15, + Attributes: (*tokenizer.Attributes)({ + Keys: ([]string) , + Map: (map[string]string) + }) + }, + (tokenizer.Token[github.com/sivukhin/godjot/djot_tokenizer.DjotToken]) { + Type: (djot_tokenizer.DjotToken) 0, + JumpToPair: (int) 0, + Start: (int) 15, + End: (int) 61, + Attributes: (*tokenizer.Attributes)() + }, + (tokenizer.Token[github.com/sivukhin/godjot/djot_tokenizer.DjotToken]) { + Type: (djot_tokenizer.DjotToken) 67, + JumpToPair: (int) 0, + Start: (int) 61, + End: (int) 62, Attributes: (*tokenizer.Attributes)() @@ -46,3 +66,3 @@ JumpToPair: (int) 0, - Start: (int) 15, + Start: (int) 62, End: (int) 74, @@ -52,3 +72,3 @@ Type: (djot_tokenizer.DjotToken) 36, - JumpToPair: (int) -2, + JumpToPair: (int) -4, Start: (int) 74, @@ -59,3 +79,3 @@ Type: (djot_tokenizer.DjotToken) 20, - JumpToPair: (int) -7, + JumpToPair: (int) -9, Start: (int) 75, @@ -66,3 +86,3 @@ Type: (djot_tokenizer.DjotToken) 2, - JumpToPair: (int) -9, + JumpToPair: (int) -11, Start: (int) 75, Test: TestSimpleLinkWithNewline --- FAIL: TestMathVerbatim (0.00s) djot_tokenizer_test.go:80: Error Trace: /tmp/godjot/djot_tokenizer/djot_tokenizer_test.go:80 Error: Not equal: expected: {DocumentBlock 6 0 0 },{ParagraphBlock 4 0 0 },{VerbatimInline 2 0 3 },{None 0 3 8 },{VerbatimInlineClose -2 8 9 },{ParagraphBlockClose -4 9 9 },{DocumentBlockClose -6 9 9 } actual : {DocumentBlock 6 0 0 },{ParagraphBlock 4 0 0 },{VerbatimInline 2 0 3 0xc000069120},{None 0 3 8 },{VerbatimInlineClose -2 8 9 },{ParagraphBlockClose -4 9 9 },{DocumentBlockClose -6 9 9 } Diff: --- Expected +++ Actual @@ -20,3 +20,10 @@ End: (int) 3, - Attributes: (*tokenizer.Attributes)() + Attributes: (*tokenizer.Attributes)({ + Keys: ([]string) (len=1) { + (string) (len=15) "$DisplayMathKey" + }, + Map: (map[string]string) (len=1) { + (string) (len=15) "$DisplayMathKey": (string) "" + } + }) }, Test: TestMathVerbatim --- FAIL: TestVerbatim (0.00s) djot_tokenizer_test.go:93: [{DocumentBlock 10 0 0 } {ParagraphBlock 8 0 0 } {VerbatimInline 2 0 2 0xc0000691c0} {None 0 2 43 } {VerbatimInlineClose -2 43 45 } {SmartSymbolInline 0 45 46 } {VerbatimInline 2 46 47 0xc0000691e0} {None 0 47 96 } {VerbatimInlineClose -2 96 97 } {ParagraphBlockClose -8 97 97 } {DocumentBlockClose -10 97 97 }] djot_tokenizer_test.go:94: Error Trace: /tmp/godjot/djot_tokenizer/djot_tokenizer_test.go:94 Error: Not equal: expected: {DocumentBlock 10 0 0 },{ParagraphBlock 8 0 0 },{VerbatimInline 2 0 2 },{None 0 2 37 },{VerbatimInlineClose -2 37 39 },{None 0 39 40 },{VerbatimInline 2 40 41 },{None 0 41 84 },{VerbatimInlineClose -2 84 85 },{ParagraphBlockClose -8 85 85 },{DocumentBlockClose -10 85 85 } actual : {DocumentBlock 10 0 0 },{ParagraphBlock 8 0 0 },{VerbatimInline 2 0 2 0xc0000691c0},{None 0 2 43 },{VerbatimInlineClose -2 43 45 },{SmartSymbolInline 0 45 46 },{VerbatimInline 2 46 47 0xc0000691e0},{None 0 47 96 },{VerbatimInlineClose -2 96 97 },{ParagraphBlockClose -8 97 97 },{DocumentBlockClose -10 97 97 } Diff: --- Expected +++ Actual @@ -20,3 +20,6 @@ End: (int) 2, - Attributes: (*tokenizer.Attributes)() + Attributes: (*tokenizer.Attributes)({ + Keys: ([]string) , + Map: (map[string]string) + }) }, @@ -26,3 +29,3 @@ Start: (int) 2, - End: (int) 37, + End: (int) 43, Attributes: (*tokenizer.Attributes)() @@ -32,4 +35,4 @@ JumpToPair: (int) -2, - Start: (int) 37, - End: (int) 39, + Start: (int) 43, + End: (int) 45, Attributes: (*tokenizer.Attributes)() @@ -37,6 +40,6 @@ (tokenizer.Token[github.com/sivukhin/godjot/djot_tokenizer.DjotToken]) { - Type: (djot_tokenizer.DjotToken) 0, + Type: (djot_tokenizer.DjotToken) 67, JumpToPair: (int) 0, - Start: (int) 39, - End: (int) 40, + Start: (int) 45, + End: (int) 46, Attributes: (*tokenizer.Attributes)() @@ -46,5 +49,8 @@ JumpToPair: (int) 2, - Start: (int) 40, - End: (int) 41, - Attributes: (*tokenizer.Attributes)() + Start: (int) 46, + End: (int) 47, + Attributes: (*tokenizer.Attributes)({ + Keys: ([]string) , + Map: (map[string]string) + }) }, @@ -53,4 +59,4 @@ JumpToPair: (int) 0, - Start: (int) 41, - End: (int) 84, + Start: (int) 47, + End: (int) 96, Attributes: (*tokenizer.Attributes)() @@ -60,4 +66,4 @@ JumpToPair: (int) -2, - Start: (int) 84, - End: (int) 85, + Start: (int) 96, + End: (int) 97, Attributes: (*tokenizer.Attributes)() @@ -67,4 +73,4 @@ JumpToPair: (int) -8, - Start: (int) 85, - End: (int) 85, + Start: (int) 97, + End: (int) 97, Attributes: (*tokenizer.Attributes)() @@ -74,4 +80,4 @@ JumpToPair: (int) -10, - Start: (int) 85, - End: (int) 85, + Start: (int) 97, + End: (int) 97, Attributes: (*tokenizer.Attributes)() Test: TestVerbatim --- FAIL: TestNestedEmphasis (0.00s) djot_tokenizer_test.go:111: Error Trace: /tmp/godjot/djot_tokenizer/djot_tokenizer_test.go:111 Error: Not equal: expected: {DocumentBlock 10 0 0 },{ParagraphBlock 8 0 0 },{EmphasisInline 6 0 1 },{EmphasisInline 4 1 2 },{EmphasisInline 2 2 3 },{None 0 3 6 },{EmphasisInlineClose -2 6 7 },{EmphasisInlineClose -4 7 8 },{EmphasisInlineClose -6 8 9 },{ParagraphBlockClose -8 9 9 },{DocumentBlockClose -10 9 9 } actual : {DocumentBlock 10 0 0 },{ParagraphBlock 8 0 0 },{EmphasisInline 6 0 1 0xc0000692c0},{EmphasisInline 4 1 2 0xc0000692e0},{EmphasisInline 2 2 3 0xc000069300},{None 0 3 6 },{EmphasisInlineClose -2 6 7 },{EmphasisInlineClose -4 7 8 },{EmphasisInlineClose -6 8 9 },{ParagraphBlockClose -8 9 9 },{DocumentBlockClose -10 9 9 } Diff: --- Expected +++ Actual @@ -20,3 +20,6 @@ End: (int) 1, - Attributes: (*tokenizer.Attributes)() + Attributes: (*tokenizer.Attributes)({ + Keys: ([]string) , + Map: (map[string]string) + }) }, @@ -27,3 +30,6 @@ End: (int) 2, - Attributes: (*tokenizer.Attributes)() + Attributes: (*tokenizer.Attributes)({ + Keys: ([]string) , + Map: (map[string]string) + }) }, @@ -34,3 +40,6 @@ End: (int) 3, - Attributes: (*tokenizer.Attributes)() + Attributes: (*tokenizer.Attributes)({ + Keys: ([]string) , + Map: (map[string]string) + }) }, Test: TestNestedEmphasis --- FAIL: TestUnmatchedEmphasis (0.00s) djot_tokenizer_test.go:128: Error Trace: /tmp/godjot/djot_tokenizer/djot_tokenizer_test.go:128 Error: Not equal: expected: {DocumentBlock 8 0 0 },{ParagraphBlock 6 0 0 },{None 0 0 23 },{VerbatimInline 2 23 24 },{None 0 24 25 },{VerbatimInlineClose -2 25 26 },{None 0 26 37 },{ParagraphBlockClose -6 37 37 },{DocumentBlockClose -8 37 37 } actual : {DocumentBlock 8 0 0 },{ParagraphBlock 6 0 0 },{None 0 0 23 },{VerbatimInline 2 23 24 0xc0000693e0},{None 0 24 25 },{VerbatimInlineClose -2 25 26 },{None 0 26 37 },{ParagraphBlockClose -6 37 37 },{DocumentBlockClose -8 37 37 } Diff: --- Expected +++ Actual @@ -27,3 +27,6 @@ End: (int) 24, - Attributes: (*tokenizer.Attributes)() + Attributes: (*tokenizer.Attributes)({ + Keys: ([]string) , + Map: (map[string]string) + }) }, Test: TestUnmatchedEmphasis FAIL FAIL github.com/sivukhin/godjot/djot_tokenizer 0.003s --- FAIL: TestAssert (0.00s) debug_test.go:10: Error Trace: /tmp/godjot/tokenizer/debug_test.go:10 Error: func (assert.PanicTestFunc)(0x580e00) should panic Panic value: Test: TestAssert FAIL FAIL github.com/sivukhin/godjot/tokenizer 0.002s FAIL ```

Are these expected?

Many of them are simply because tokenizer.Attributes is instantiated with nil fields while the tests expect just a nil while others expect completely different start/ends

acheong08 commented 9 months ago

Sidenote:

https://github.com/sivukhin/godjot/blob/cc87ac44176a59311ae8b80ec30c9c6bde89e4ed/tokenizer/attributes.go#L4-L7

This can be simplified to just the map.

diff --git a/tokenizer/attributes.go b/tokenizer/attributes.go
index 2d3f4cd..9d1f1eb 100644
--- a/tokenizer/attributes.go
+++ b/tokenizer/attributes.go
@@ -2,7 +2,6 @@ package tokenizer

 // Attributes - is a map with support for deterministic order of enumeration
 type Attributes struct {
-       Keys []string
        Map  map[string]string
 }

@@ -12,7 +11,7 @@ func (a *Attributes) Size() int {
        if a == nil {
                return 0
        }
-       return len(a.Keys)
+       return len(a.Map)
 }

 func (a *Attributes) Append(key, value string) *Attributes {
@@ -25,9 +24,6 @@ func (a *Attributes) Append(key, value string) *Attributes {
 }

 func (a *Attributes) Set(key, value string) *Attributes {
-       if _, ok := a.Map[key]; !ok {
-               a.Keys = append(a.Keys, key)
-       }
        if a.Map == nil {
                a.Map = make(map[string]string)
        }
@@ -52,8 +48,8 @@ func (a *Attributes) Get(key string) string {

 func (a *Attributes) MergeWith(other *Attributes) *Attributes {
        if other != nil {
-               for _, key := range other.Keys {
-                       a.Set(key, other.Get(key))
+               for key, value := range other.Map {
+                       a.Set(key, value)
                }
        }
        return a
@@ -64,8 +60,8 @@ func (a *Attributes) Entries() []AttributeEntry {
                return nil
        }
        entries := make([]AttributeEntry, 0, len(a.Map))
-       for _, key := range a.Keys {
-               entries = append(entries, AttributeEntry{Key: key, Value: a.Get(key)})
+       for key, value := range a.Map {
+               entries = append(entries, AttributeEntry{Key: key, Value: value})
        }
        return entries
 }
@@ -75,8 +71,8 @@ func (a *Attributes) GoMap() map[string]string {
                return nil
        }
        entries := make(map[string]string)
-       for _, key := range a.Keys {
-               entries[key] = a.Get(key)
+       for key, value := range a.Map {
+               entries[key] = value
        }
        return entries
 }
sivukhin commented 9 months ago

@acheong08, yes, sure - I fixed tests in the commit: https://github.com/sivukhin/godjot/commit/4ea464d83d5e95118b525230bc493f2c7c29a763

I wanted attributes to be deterministic - so I need to manually maintain keys order because Go randomize map iteration order (which is good in some cases but it's kind of annoying if we want to have deterministic result - which we do here)

Feel free to reopen or file new issue if there will be still problems with the tests.

acheong08 commented 9 months ago

Thanks!