Flinkcompare

[Johnwickdev]

package com.example.flink.core;

import org.apache.flink.streaming.api.functions.co.KeyedCoProcessFunction; import org.apache.flink.util.Collector; import org.bson.Document;

public class FlinkCompareAndProcess extends KeyedCoProcessFunction<String, Document, String, Document> {

@Override
public void processElement1(Document mongoDoc, Context context, Collector<Document> out) throws Exception {
    // No action needed here for the moment, since we're focusing on the second stream for comparison
}

@Override
public void processElement2(String jsonString, Context context, Collector<Document> out) throws Exception {
    // Convert the string to a Document
    Document jsonDoc = Document.parse(jsonString);

    // Compare JSON document (from local source) with the MongoDB document
    Document deltaDoc = new Document();

    // Get the common key for comparison
    String key = jsonDoc.getString("CSPA");
    deltaDoc.put("CSPA", key);
    deltaDoc.put("date", jsonDoc.getString("date"));
    deltaDoc.put("version", jsonDoc.getString("version"));
    deltaDoc.put("layout_name", jsonDoc.getString("layout_name"));

    // Process document field
    Document document = (Document) jsonDoc.get("document");
    Document deltaDocument = new Document();

    // Example of processing a single field. You'd need to repeat this for each field in your document.
    for (String field : document.keySet()) {
        if (!mongoDoc.containsKey(field) || !mongoDoc.get(field).equals(document.get(field))) {
            deltaDocument.put(field, "Changed from " + mongoDoc.get(field) + " to " + document.get(field));
        }
    }

    if (!deltaDocument.isEmpty()) {
        deltaDoc.put("document", deltaDocument);
        deltaDoc.put("Operation", "update"); // Marking it as an update operation
        deltaDoc.put("Timestamp", System.currentTimeMillis());
        out.collect(deltaDoc); // This will be collected and written to the delta collection
    }
}

}