omkarcloud / botasaurus

The All in One Framework to build Awesome Scrapers.
https://www.omkar.cloud/botasaurus/
MIT License
1.16k stars 104 forks source link

a doubt #44

Closed ConnorMAD closed 4 months ago

ConnorMAD commented 5 months ago

How can I get response responses on the traffic network using Botsaurus?

Chetan11-dev commented 5 months ago

use this code to spy on xhr


    var originalSend = XMLHttpRequest.prototype.send

    // Override the send method
    XMLHttpRequest.prototype.send = function () {
        // Keep a reference to the XMLHttpRequest instance
        var xhr = this

        // Function to handle the state change
        function onReadyStateChange() {
            if (xhr.readyState === 4) { // Check if the request is complete
                try {
                        // Attempt to parse JSON response
                        var json = JSON.parse(xhr.responseText)
                        // Log the URL and JSON response to the console
                        console.log("URL: " + xhr.responseURL)
                        console.log("Response: ", json)
                        window.data = json

                } catch (e) {
                    // If response is not JSON, log an error
                    console.error("Could not parse JSON response for URL: " + xhr.responseURL)
                }
            }
        }

        // Add the event listener for 'readystatechange'
        this.addEventListener('readystatechange', onReadyStateChange)

        // Call the original send method
        originalSend.apply(this, arguments)
    }
Chetan11-dev commented 5 months ago

Not Currently, it's in backlog, I except to do it this year. Till then you can use code like execute_js and run similar code


(function () {

    if (window.hasRun) {
        return
    }
    window.hasRun = true

    // Store the original fetch function in a variable
    const originalFetch = window.fetch

    window.fetch = async function (...args) {

        if (true) {
            try {
                // Call the original fetch function
                const response = await originalFetch.apply(this, args)

                // Clone the response to not interfere with the original processing
                const clonedResponse = response.clone()

                // Read the response as JSON and log it
                clonedResponse.json().then(json => {
                        console.log(json)
                })

                // Return the original response
                return response
            } catch (error) {
                console.error('Error in fetch interceptor:', error)
                throw error // Re-throw the error for proper error handling
            }
        }
        else {
            // Call the original fetch function for other URLs
            return originalFetch.apply(this, args)
        }
    }

})()```
ConnorMAD commented 5 months ago

ty