Single or Multiple

[pixie79]

Hi,

Thanks for the work on this module looks great, I do have a couple of questions.

I am trying to write a data check and I need to be able to distinguish between the following different type of json record.

Case 1

{"name": "Paul","Age": 25,"Location": "USA"} 

Case 2

[
{"name": "Paul","Age": 25,"Location": "USA"} ,
{"name": "Fred","Age": 27,"Location": "UK"} ,
{"name": "John","Age": 20,"Location": "UAE"} 
]

Case 3

{"name": "Paul","Age": 25,"Location": "USA"} 
{"name": "Fred","Age": 27,"Location": "UK"} 
{"name": "John","Age": 20,"Location": "UAE"} 

Case 1 is the standard simple approach but if the data comes through in either Case 2 or 3 I need a way to identify this and then be able to iterate over this using a range so that I can not only use Get but also use the single string line for other data.

Any ideas on the best way to do this?

Mark

[pixie79]

With a bit of playing around I got nearly the output needed using the following code:

https://go.dev/play/p/zQoFPc0l8Vi

package main

import (
    "bufio"
    "fmt"
    "strings"

    "github.com/ohler55/ojg"
    "github.com/ohler55/ojg/jp"
    "github.com/ohler55/ojg/oj"
)

var (
    event1 = `{"name": "Paul","Age": 25,"Location": "USA"}`
    event2 = `[
    {"name": "Paul","Age": 25,"Location": "USA"},
    {"name": "Fred","Age": 27,"Location": "UK"},
    {"name": "John","Age": 20,"Location": "UAE"}
    ]`
    event3 = `{"name": "Paul","Age": 25,"Location": "USA"}
    {"name": "Fred","Age": 27,"Location": "UK"}
    {"name": "John","Age": 20,"Location": "UAE"}`
)

func getJsonStrings(data string) []string {
    var result []string
    obj, err := oj.ParseString(data)
    if err != nil {
        fmt.Println("going to try and parse as json lines")
        obj = convertJsonLines(data)
    }
    switch obj.(type) {
    case []interface{}:
        for _, v := range obj.([]interface{}) {
            parse := localParse(v)
            var b strings.Builder
            if err := oj.Write(&b, parse, &ojg.Options{Sort: true}); err != nil {
                panic(err)
            }
            result = append(result, b.String())
        }
    case map[string]interface{}:
        parse := localParse(obj)
        var b strings.Builder
        if err := oj.Write(&b, parse, &ojg.Options{Sort: true}); err != nil {
            panic(err)
        }
        result = append(result, b.String())
    default:
        panic("unknown type")
    }
    return result
}

func localParse(obj any) any {
    x, err := jp.ParseString("$.name")
    if err != nil {
        fmt.Println(fmt.Sprintf("error parsing json: %s", err))
    }
    result := x.Get(obj)
    return result
}

func convertJsonLines(data string) any {
    var (
        payload []string
    )
    scanner := bufio.NewScanner(strings.NewReader(data))
    for scanner.Scan() {
        payload = append(payload, scanner.Text())
    }

    if err := scanner.Err(); err != nil {
        fmt.Printf("error occurred: %v\n", err)
    }
    result := strings.Join(payload, ",")
    obj, err := oj.ParseString("[" + result + "]")
    if err != nil {
        fmt.Println(fmt.Sprintf("error parsing jsonlines: %s", err))
    }
    return obj
}

func main() {
    var tests = []struct {
        name  string
        event string
        topic []string
    }{
        {"single json event", event1, []string{"Paul"}},
        {"map of json events", event2, []string{"Paul", "Fred", "John"}},
        {"multiple json line events", event3, []string{"Paul", "Fred", "John"}},
    }
    {
        for _, tt := range tests {
            fmt.Println(fmt.Sprintf("running test: %s", tt.name))
            result := getJsonStrings(tt.event)
            fmt.Println(fmt.Sprintf("result: %s", result))

            if result[0] != tt.topic[0] {
                fmt.Println(fmt.Sprintf("got %s, want %s", result, tt.topic))
            } else {
                fmt.Println("success correct result")
            }
            fmt.Println("")
        }
    }
}

But it looks like i am having an issue using x.Get(obj) which returns any when cast to string it get [data] rather than data?

How do i resolve that?

Is there a better way to rewrite this?

[pixie79]

Well with a little more playing and a bit of trim I think the following is one way of achiving my result not sure it if the best though.

package main

import (
    "bufio"
    "fmt"
    "strings"

    "github.com/ohler55/ojg"
    "github.com/ohler55/ojg/jp"
    "github.com/ohler55/ojg/oj"
)

var (
    event1 = `{"name": "Paul","Age": 25,"Location": { "country": "USA", "city": "New York" }}`
    event2 = `[
    {"name": "Paul","Age": 25,"Location": { "country": "USA", "city": "New York" }},
    {"name": "Fred","Age": 27,"Location": { "country": "UK", "city": "London" }},
    {"name": "John","Age": 20,"Location": { "country": "UAE", "city": "Dubai" }}
    ]`
    event3 = `{"name": "Paul","Age": 25,"Location": { "country": "USA", "city": "New York" }}
    {"name": "Fred","Age": 27,"Location": { "country": "UK", "city": "London" }}
    {"name": "John","Age": 20,"Location": { "country": "UAE", "city": "Dubai" }}`
)

func jsonConversion(data string) interface{} {
    obj, err := oj.ParseString(data)
    if err != nil {
        fmt.Println("Attempting to parse as json lines...")
        return convertJsonLines(data)
    }
    return obj
}

func writeJson(parsedData interface{}) string {
    var writer strings.Builder
    if err := oj.Write(&writer, parsedData, &ojg.Options{Sort: true}); err != nil {
        panic(err)
    }
    fmt.Println(writer.String())
    return writer.String()
}

func getJSONStrings(data string, jsonString string) []string {
    var result []string
    jsonObj := jsonConversion(data)

    switch parsedObj := jsonObj.(type) {
    case []interface{}:
        result = appendItems(result, jsonString, parsedObj...)
    case map[string]interface{}:
        result = appendItem(result, jsonString, parsedObj)
    default:
        panic("Unknown type")
    }
    return result
}

func appendItems(slice []string, jsonString string, items ...interface{}) []string {
    for _, item := range items {
        slice = appendItem(slice, jsonString, item)
    }
    return slice
}

func appendItem(slice []string, jsonString string, item interface{}) []string {
    jsonStr := writeJson(localParse(item, jsonString))
    trimmedStr := trimEdges(jsonStr)
    return append(slice, trimmedStr)
}

func trimEdges(jsonString string) string {
    return strings.Trim(
        strings.Trim(jsonString, "\"]"),
        "[\"",
    )
}

func localParse(obj interface{}, jsonPath string) interface{} {
    path, _ := jp.ParseString(jsonPath)
    return path.Get(obj)
}

func convertJsonLines(data string) interface{} {
    var payload []string
    scanner := bufio.NewScanner(strings.NewReader(data))
    for scanner.Scan() {
        payload = append(payload, scanner.Text())
    }

    if err := scanner.Err(); err != nil {
        fmt.Printf("Error occurred: %v\n", err)
    }

    combinedLines := fmt.Sprintf("[%s]", strings.Join(payload, ","))
    obj, err := oj.ParseString(combinedLines)
    if err != nil {
        fmt.Printf("Error parsing jsonlines: %s\n", err)
    }
    return obj
}

func main() {
    var tests = []struct {
        name  string
        event string
        jsonPath string
        topic []string
    }{
        {"single json event - name", event1, "$.name", []string{"Paul"}},
        {"map of json events - name", event2, "$.name", []string{"Paul", "Fred", "John"}},
        {"multiple json line events - name", event3, "$.name", []string{"Paul", "Fred", "John"}},
        {"single json event - location", event1, "$.Location", []string{"{\"city\":\"New York\",\"country\":\"USA\"}"}},
        {"map of json events - location", event2, "$.Location", []string{"{\"city\":\"New York\",\"country\":\"USA\"}", "{\"city\":\"London\",\"country\":\"UK\"}", "{\"city\":\"Dubai\",\"country\":\"UAE\"}"}},
        {"multiple json line events - location", event3, "$.Location", []string{"{\"city\":\"New York\",\"country\":\"USA\"}", "{\"city\":\"London\",\"country\":\"UK\"}", "{\"city\":\"Dubai\",\"country\":\"UAE\"}"}},
    }
    {
        for _, tt := range tests {
            fmt.Println(fmt.Sprintf("running test: %s", tt.name))
            result := getJSONStrings(tt.event, tt.jsonPath)
            fmt.Println(fmt.Sprintf("result: %s", result))

            if result[0] != tt.topic[0] {
                fmt.Println(fmt.Sprintf("got %s, want %s", result, tt.topic))
            } else {
                fmt.Println("success correct result")
            }
            fmt.Println("")
        }
    }
}

https://go.dev/play/p/ZZvRD9y8lXZ

[ohler55]

My apologies for taking so very long to reply. Work and then a vacation were a little too distracting. If you are still stuck I'd be glad to explain a different and simpler approach.

[pixie79]

Hi,

No worries, I think I have it working now just but more than happy to learn a different approach especially if it is simpler.

Thanks

Mark

[ohler55]

This is another approach:

package main

import (
    "fmt"

    "github.com/ohler55/ojg/jp"
    "github.com/ohler55/ojg/oj"
)

var (
    event1 = `{"name": "Paul","Age": 25,"Location": "USA"}`
    event2 = `[
    {"name": "Paul","Age": 25,"Location": "USA"},
    {"name": "Fred","Age": 27,"Location": "UK"},
    {"name": "John","Age": 20,"Location": "UAE"}
    ]`
    event3 = `{"name": "Paul","Age": 25,"Location": "USA"}
    {"name": "Fred","Age": 27,"Location": "UK"}
    {"name": "John","Age": 20,"Location": "UAE"}`
)

func main() {
    var tests = []struct {
        name  string
        event string
        topic []string
    }{
        {"single json event", event1, []string{"Paul"}},
        {"map of json events", event2, []string{"Paul", "Fred", "John"}},
        {"multiple json line events", event3, []string{"Paul", "Fred", "John"}},
    }
top:
    for _, tt := range tests {
        fmt.Printf("running test: %s\n", tt.name)
        result := getJsonStrings(tt.event)
        fmt.Printf("result: %s\n", result)

        if len(tt.topic) != len(result) {
            fmt.Printf("got %s, want %s\n", result, tt.topic)
            continue
        }
        for i, name := range tt.topic {
            if name != result[i] {
                fmt.Printf("got %s, want %s\n", result, tt.topic)
                continue top
            }
        }
        fmt.Println("success correct result")
    }
}

func getJsonStrings(data string) (result []string) {
    _ = oj.MustParseString(data, func(v any) bool {
        if list, ok := v.([]any); ok {
            for _, v2 := range list {
                if name, _ := jp.C("name").First(v2).(string); 0 < len(name) {
                    result = append(result, name)
                }
            }
        } else {
            if name, _ := jp.C("name").First(v).(string); 0 < len(name) {
                result = append(result, name)
            }
        }
        return false
    })
    return
}

[ohler55]

Did the example help? If so can this be closed?