CellID does not pull the entire ID

[g0Idfinger]

I'm trying to get the data from the velo integration, however the cell id is cell_id=NC.CQ7HMMAG35HO2 when it should be something like cell_id=NC.CQ7HMMAG35HO2-CQ7JC9G6TLLJC

Using Chrome Inspect tool I was able to hardcode the cell ID in the veloapi.js and it seemed to work. the cell_id seams to be pretty dynamic and always changing. my java skills are poor at best.

[cgosec]

Hi, are you able to give me the console output?

Did you have the Blauhaunt Artifact running multiple times?

[g0Idfinger]

I have the hunt only ran once: here is the console error: the get notebook api call returns "{"total_rows":"-1"} when I take the url and run it in the browser I get the same, the cellID looks to be off and missing the "update data"

although this is looking for the monitoring data, we did start monitoring artifact but decided to just use the regular artifact. which isn't pulling in data.

[g0Idfinger]

after some troubleshooting this modified veloAPI.js allowed me to collect the hunt data:

let artifactName = "Custom.Windows.EventLogs.Blauhaunt"
let monitoringArtifact = "Custom.Windows.Events.Blauhaunt"
let url = window.location.origin
let header = {}
checkForVelociraptor()

function selectionModal(title, selectionList) {
    // remove duplicates from selectionList
    selectionList = [...new Set(selectionList)]
    let modal = new Promise((resolve, reject) => {
        // create modal
        let modal = document.createElement("div");
        modal.id = "modal";
        modal.className = "modal";
        let modalContent = document.createElement("div");
        modalContent.className = "modal-content";
        let modalHeader = document.createElement("h2");
        modalHeader.innerHTML = title;
        modalContent.appendChild(modalHeader);
        let modalBody = document.createElement("div");
        modalBody.className = "modal-body";
        selectionList.forEach(option => {
            let notebookButton = document.createElement("button");
            notebookButton.innerHTML = option;
            notebookButton.onclick = function () {
                modal.remove();
                return option;
            }
            modalBody.appendChild(notebookButton);
        });
        modalContent.appendChild(modalBody);
        modal.appendChild(modalContent);
        document.body.appendChild(modal);
        // show modal
        modal.style.display = "block";
        // close modal when clicked outside of it
        window.onclick = function (event) {
            if (event.target === modal) {
                modal.remove();
                return null;
            }
        }
    });
    return modal;
}

function getNotebook(huntID) {
    let notebooks = []
    fetch(url + '/api/v1/GetHunt?hunt_id=' + huntID, {headers: header}).then(response => {
        return response.json()
    }).then(data => {
        let artifacts = data.artifacts;
        let notebookID = ""
        artifacts.forEach(artifact => {
            notebookID = "N." + huntID
            if (artifact === artifactName) {
                notebooks.push(notebookID);
            }
        });
        if (notebooks.length === 0) {
            return;
        }
        // if there are more notebooks wit the artifact name, show a modal to select the notebook to use
        if (notebooks.length > 1) {
            selectionModal("Select Notebook", notebooks).then(selectedNotebook => {
                if (selectedNotebook === null) {
                    return;
                }
                getCells(selectedNotebook);
            });
        } else {
            getCells(notebooks[0]);
        }
    });
}

function getCells(notebookID) {
    fetch(url + `/api/v1/GetNotebooks?notebook_id=${notebookID}&include_uploads=true`, {headers: header}).then(response => {
        // get the X-Csrf-Token form the header of the response
        localStorage.setItem('csrf-token', response.headers.get("X-Csrf-Token"))
        return response.json()
    }).then(data => {
        let cells = data.items;
        if (cells.length > 1) {
            let cellIDs = {}
            cells.forEach(cell => {
                cell.cell_metadata.forEach(metadata => {
                    let suffix = ""
                    let i = 0
                    while (cellIDs[metadata.cell_id + suffix] !== undefined) {
                    }
                    cellIDs[metadata.cell_id + suffix] = {cell_id: metadata.cell_id, version: metadata.timestamp};
                });
            });
            selectionModal("Select Cell", cellIDs.keys()).then(selectedCell => {
                if (selectedCell === null) {
                    return;
                }
                updateData(notebookID, cellIDs[selectedCell].cell_id, cellIDs[selectedCell].version, localStorage.getItem('csrf-token'));
            });
        }
        cells.forEach(cell => {
            cell.cell_metadata.forEach(metadata => {
                updateData(notebookID, metadata.cell_id, metadata.timestamp, localStorage.getItem('csrf-token'));
            });
        });
    });
}

function updateData(notebookID, cellID, version, csrf_token) {
    header["X-Csrf-Token"] = csrf_token
    fetch(url + '/api/v1/UpdateNotebookCell', {
        method: 'POST',
        headers: header,
        body: JSON.stringify({
            "notebook_id": notebookID,
            "cell_id": cellID,
            "env": [{"key": "ArtifactName", "value": artifactName}],
            "input": "\n/*\n# BLAUHAUNT\n*/\nSELECT * FROM source(artifact=\"" + artifactName + "\")\n",
            "type": "vql"
        })
    }).then(response => {
        return response.json()
    }).then(data => {
        let fullCellID = `${cellID}-${data.current_version}`;
        loadData(notebookID, fullCellID, version);
    });
}

let dataRows = []

function loadData(notebookID, fullCellID, version, startRow = 0, toRow = 1000, retryCount = 9, delay = 100) {
    fetch(url + `/api/v1/GetTable?notebook_id=${notebookID}&client_id=&cell_id=${fullCellID}&table_id=1&TableOptions=%7B%7D&Version=${version}&start_row=${startRow}&rows=${toRow}&sort_direction=false`,
        { headers: header }
    ).then(response => {
        if (response.status === 429) {
            // Retry after reducing toRow
            if (retryCount > 0) {
                let newToRow = toRow;
                let newStartRow = startRow;
                if (retryCount === 9) {
                    newToRow = 500;
                } else if (retryCount === 8) {
                    newToRow = 250;
                } else if (retryCount === 7) {
                    newToRow = 100;
                } else if (retryCount === 6) {
                    newToRow = 50;
                } else if (retryCount === 5) {
                    newToRow = 25;
                } else if (retryCount === 4) {
                    newToRow = 10;
                } else if (retryCount === 3) {
                    newToRow = 5;
                } else if (retryCount === 2) {
                    newToRow = 1;
                } else {
                    newStartRow += 1; // Increment startRow by 1 for each retry with toRow = 1
                }
                return new Promise(resolve => setTimeout(resolve, delay))
                    .then(() => loadData(notebookID, fullCellID, version, newStartRow, newToRow, retryCount - 1, delay));
            } else {
                throw new Error('Too many requests. Retry limit exceeded.');
            }
        } else {
            return response.json();
        }
    }).then(data => {
        if (data.rows === undefined) {
            return;
        }
        let dataRows = [];
        data.rows.forEach(row => {
            row = row.cell;
            let entry = {};
            for (let i = 0; i < row.length; i++) {
                if (data.columns[i] === "LogonTimes") {
                    entry[data.columns[i]] = JSON.parse(row[i]);
                } else {
                    entry[data.columns[i]] = row[i];
                }
            }
            dataRows.push(JSON.stringify(entry));
        });
        // show loading spinner
        document.getElementById("loading").style.display = "block";
        processJSONUpload(dataRows.join("\n")).then(() => {
            document.getElementById("loading").style.display = "none";
        });
        // if there are more rows, load them
        if (data.total_rows > startRow + toRow) {
            // Continue from where it left off
            loadData(notebookID, fullCellID, version, startRow + toRow, 1000);
        }
        storeDataToIndexDB(header["Grpc-Metadata-Orgid"]);
    }).catch(error => {
        console.error('Error fetching data:', error);
        // Handle error, show message, retry, etc.
    });
}

function getHunts(orgID) {
    url = window.location.origin
    fetch(url + '/api/v1/ListHunts?count=2000&offset=0&summary=true&user_filter=', {headers: header}).then(response => {
        return response.json()
    }).then(data => {
        let hunts = data.items;
        hunts.forEach(hunt => {
            getNotebook(hunt.hunt_id);
        });
    })
}

function updateClientInfoData(clientInfoNotebook, cellID, version, csrf_token) {
    header["X-Csrf-Token"] = csrf_token
    fetch(url + '/api/v1/UpdateNotebookCell', {
        method: 'POST',
        headers: header,
        body: JSON.stringify({
            "notebook_id": clientInfoNotebook,
            "cell_id": cellID,
            "env": [{"key": "ArtifactName", "value": artifactName}],
            "input": "SELECT * FROM clients()\n",
            "type": "vql"
        })
    }).then(response => {
        return response.json()
    }).then(data => {
        let fullCellID = `${cellID}-${data.current_version}`;
        loadFromClientInfoCell(clientInfoNotebook, fullCellID, version);
    });
}

function getClientInfoFromVelo() {
    fetch(url + '/api/v1/GetNotebooks?count=1000&offset=0', {headers: header}).then(response => {
        localStorage.setItem('csrf-token', response.headers.get("X-Csrf-Token"))
        return response.json()
    }).then(data => {
        let notebooks = data.items;
        if (!notebooks) {
            return false;
        }
        let clientInfoNotebook = ""
        notebooks.forEach(notebook => {
            let notebookID = notebook.notebook_id;
            notebook.cell_metadata.forEach(metadata => {
                let cellID = metadata.cell_id;
                fetch(url + `/api/v1/GetNotebookCell?notebook_id=${notebookID}&cell_id=${fullCellID}`, {headers: header}).then(response => {
                    return response.json()
                }).then(data => {
                    let query = data.input;
                    if (query.trim().toLowerCase() === 'select * from clients()') {
                        clientInfoNotebook = notebookID
                        let version = metadata.timestamp
                        updateClientInfoData(clientInfoNotebook, fullCellID, version, localStorage.getItem('csrf-token'));
                    }
                });
            });
        });
    });
}

function loadFromClientInfoCell(notebookID, fullCellID, version, startRow = 0, toRow = 1000) {
    //let fullCellID = `${cellID}-${version}`;
    fetch(url + `/api/v1/GetTable?notebook_id=${notebookID}&client_id=&cell_id=${fullCellID}&table_id=1&TableOptions=%7B%7D&Version=${version}&start_row=${startRow}&rows=${toRow}&sort_direction=false`,
        {headers: header}
    ).then(response => {
        return response.json()
    }).then(data => {
        clientIDs = []
        let clientRows = []
        data.rows.forEach(row => {
            row = row.cell;
            let entry = {}
            for (i = 0; i < row.length; i++) {
                let value = null
                try {
                    value = JSON.parse(row[i]);
                } catch (e) {
                    value = row[i];
                }
                entry[data.columns[i]] = value;
            }
            clientRows.push(JSON.stringify(entry));
            console.debug(entry)
            clientIDs.push(entry["client_id"]);
        });
        // show loading spinner
        loadClientInfo(clientRows.join("\n"))
        caseData.clientIDs = clientIDs;
        // if there are more rows, load them
        if (data.total_rows > toRow) {
            loadFromClientInfoCell(notebookID, cellID, version, startRow + toRow, toRow + 1000);
        }
    });

}

function getFromMonitoringArtifact() {
    let notebookIDStart = "N.E." + monitoringArtifact
    console.log("checking for monitoring artifact data...")
    // iterate over notebooks to find the one with the monitoring artifact
    // check if caseData has clientMonitoringLatestUpdate set
    if (caseData.clientMonitoringLatestUpdate === undefined) {
        caseData.clientMonitoringLatestUpdate = {}
    }
    caseData.clientIDs.forEach(clientID => {
        console.debug("checking monitoring artifact for clientID: " + clientID)
        let latestUpdate = caseData.clientMonitoringLatestUpdate[clientID] || 0;
        fetch(url + `/api/v1/GetTable?client_id=${clientID}&artifact=${monitoringArtifact}&type=CLIENT_EVENT&start_time=${latestUpdate}&end_time=9999999999&rows=10000`, {
            headers: header
        }).then(response => {
            return response.json()
        }).then(data => {
            console.debug("monitoring data for clientID: " + clientID)
            console.debug(data)
            if (data.rows === undefined) {
                return;
            }
            let rows = data.rows;
            let serverTimeIndex = data.columns.indexOf("_ts");
            let monitoringData = []
            let maxUpdatedTime = 0;
            rows.forEach(row => {
                console.debug(`row time: ${row.cell[serverTimeIndex]}, lastUpdatedTime: ${latestUpdate}`)
                if (row.cell[serverTimeIndex] > latestUpdate) {
                    if (row.cell[serverTimeIndex] > maxUpdatedTime) {
                        console.debug("updating maxUpdatedTime to" + row.cell[serverTimeIndex])
                        maxUpdatedTime = row.cell[serverTimeIndex];
                    }
                    let entry = {}
                    row.cell.forEach((cell, i) => {
                        try {
                            cell = JSON.parse(cell);
                        } catch (e) {
                        }
                        if (data.columns[i] === "LogonTimes") {
                            // if the column is LogonTimes is not an array, make it one
                            if (!Array.isArray(cell)) {
                                cell = [cell];
                            }
                        }
                        entry[data.columns[i]] = cell;
                    });
                    if (entry) {
                        console.debug(entry)
                        monitoringData.push(JSON.stringify(entry));
                    }
                }
            });
            caseData.clientMonitoringLatestUpdate[clientID] = maxUpdatedTime;
            if (monitoringData.length > 0) {
                console.debug("monitoring data for clientID: " + clientID + " is being processed with " + monitoringData.length + " entries")
                processJSONUpload(monitoringData.join("\n")).then(() => {
                    console.log("monitoring data processed");
                    storeDataToIndexDB(header["Grpc-Metadata-Orgid"]);
                });
            }
        });
    });
}

function changeBtn(replaceBtn, text, ordID) {
    let newBtn = document.createElement("button");
    // get child btn from replaceBtn and copy the classes to the new btn
    newBtn.className = replaceBtn.children[0].className;
    replaceBtn.innerHTML = ""
    newBtn.innerText = text;
    newBtn.addEventListener("click", evt => {
        evt.preventDefault()
        getClientInfoFromVelo();
        getHunts(ordID);
    });
    replaceBtn.appendChild(newBtn)
}

function loadDataFromDB(orgID) {
    // check if casedata with orgID is already in indexedDB
    retrieveDataFromIndexDB(orgID);
}

function syncFromMonitoringArtifact() {
    return setInterval(getFromMonitoringArtifact, 60000);
}

function stopMonitoringAync(id) {
    clearInterval(id);
}

function createSyncBtn() {
    let syncBtn = document.createElement("input");
    /*
    <div class="form-check form-switch ms-2">
                        <input class="form-check-input" id="darkSwitch" type="checkbox">
                        <label class="form-check-label" for="darkSwitch">Dark Mode</label>
                    </div>
     */
    // add classes to make it a bootstrap toggle button
    syncBtn.className = "form-check-input";
    syncBtn.type = "checkbox";
    syncBtn.id = "syncBtn";
    let syncLabel = document.createElement("label");
    syncLabel.className = "form-check-label";
    syncLabel.innerText = "Life Data";
    syncLabel.setAttribute("for", "syncBtn");
    syncBtn.addEventListener("click", evt => {
        let syncID = syncFromMonitoringArtifact();
        evt.target.innerText = "Stop";
        evt.target.removeEventListener("click", evt);
        evt.target.addEventListener("click", evt => {
            stopMonitoringAync(syncID);
            evt.target.innerText = "Life Data";
            evt.target.removeEventListener("click", evt);
            evt.target.addEventListener("click", evt);
        });
    });
    let wrapper = document.createElement("div");
    wrapper.className = "form-check form-switch ms-2";
    wrapper.appendChild(syncBtn);
    wrapper.appendChild(syncLabel);
    document.getElementById("casesBtnGrp").innerHTML = "";
    document.getElementById("casesBtnGrp").appendChild(wrapper);
}

function checkForVelociraptor() {
    fetch(url + '/api/v1/GetUserUITraits', {headers: header}).then(response => {
        return response.json()
    }).then(data => {
        let orgID = data.interface_traits.org;
        if (orgID === undefined) {
            console.log("No ordID available. Running in standalone mode... may you try to select an organization.");
            return;
        }
        header = {"Grpc-Metadata-Orgid": orgID}
        // hide the Upload button
        let replaceBtn = document.getElementById("dataBtnWrapper");
        changeBtn(replaceBtn, "Load " + orgID, orgID);
        loadDataFromDB(orgID);
        createSyncBtn()
        //getHunts(orgID);
    }).catch(error => {
        console.log("seems to be not connected to Velociraptor.");
    });
}

[cgosec]

Hi, sorry for my delay.

By quick checking your code it seems like there is an issue when the rows for the last request do not match is that correct? Also you altered the way the cell_id is versioned.

Were you able to identify the concrete issue you were facing in your scenario?

I will try to follow along your approach as good as I can when I have some minutes free.

Thanks for your contribution!

[g0Idfinger]

Honestly, I used AI to help fix it, it pulls the data now, I also added in some error checking as I found that some records would cause issues, that's the part where it pulls less records until it finds the bad record and skips it. the code above works, I'm sure it can be better, but I don't know JS at all.