ultrafunkamsterdam
commented
4 years ago @breixopd
That is not how you usually do it..
Here's a quick and REAL dirty snippet i just wrote, i wouldn't recommend ever using this in production
# ultrafunkamsterdam | undetected-chromedriver | log network requests
import threading
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
import json
import pprint
import undetected_chromedriver as uc
capabilities = DesiredCapabilities.CHROME
capabilities["goog:loggingPrefs"] = {"performance": "ALL"}
driver = uc.Chrome(desired_capabilities=capabilities)
def process_browser_logs_for_network_events(logs):
for entry in logs:
log = json.loads(entry["message"])["message"]
if (
"Network.response" in log["method"]
or "Network.request" in log["method"]
or "Network.webSocket" in log["method"]
):
yield log
def print_logs(fn):
log = fn()
for ev in process_browser_logs_for_network_events(log):
pprint.pprint(ev)
# yeah, i know, i was lazy
sites = [ s for s in """
http://www.staggeringbeauty.com/
http://www.fallingfalling.com/
http://benedictcumberbatchgenerator.tumblr.com/
https://www.makeuseof.com/tag/9-hilarious-websites-to-visit-when-you-need-to-kill-time/
https://findtheinvisiblecow.com/
https://weirdorconfusing.com/
https://www.pointerpointer.com/
http://dontevenreply.com/
https://www.thewikigame.com/
http://www.donothingfor2minutes.com/
http://www.randomthingstodo.com/
""".splitlines() if s ]
for site in sites:
handle = threading.Timer(1, print_logs, args=(lambda: driver.get_log('performance'),))
handle.start()
driver.get(site)
OUTPUT:
{ "method": "Network.requestWillBeSent",
"params": { "documentURL": "http://www.staggeringbeauty.com/",
"frameId": "04585B830B1EF2B1D3B9DB229DFD253F",
"hasUserGesture": False,
"initiator": {"type": "other"},
"loaderId": "4612FE9C61094E1E16491EEA95048287",
"request": { "headers": { "Upgrade-Insecure-Requests": "1",
"User-Agent": "Mozilla/5.0 "
"(Windows NT "
"10.0; Win64; "
"x64) "
"AppleWebKit/537.36 "
"(KHTML, like "
"Gecko) "
"Chrome/87.0.4280.66 "
"Safari/537.36"},
"initialPriority": "VeryHigh",
"method": "GET",
"mixedContentType": "none",
"referrerPolicy": "no-referrer-when-downgrade",
"url": "http://www.staggeringbeauty.com/"},
"requestId": "4612FE9C61094E1E16491EEA95048287",
"timestamp": 123924.817263,
"type": "Document",
"wallTime": 1606334989.121815}}
{ "method": "Network.requestWillBeSentExtraInfo",
"params": { "associatedCookies": [],
"headers": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"Accept-Encoding": "gzip, deflate",
"Accept-Language": "en-US,en;q=0.9",
"Connection": "keep-alive",
"Host": "www.staggeringbeauty.com",
"Upgrade-Insecure-Requests": "1",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; "
"Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) "
"Chrome/87.0.4280.66 "
"Safari/537.36"},
"requestId": "4612FE9C61094E1E16491EEA95048287"}}
{ "method": "Network.responseReceivedExtraInfo",
"params": { "blockedCookies": [],
"headers": { "Accept-Ranges": "bytes",
"Cache-Control": "max-age=600",
"Connection": "Upgrade, Keep-Alive",
"Content-Encoding": "gzip",
"Content-Length": "1193",
"Content-Type": "text/html",
"Date": "Wed, 25 Nov 2020 20:09:50 GMT",
"ETag": ""af5-4c84b646fe600-gzip"",
"Expires": "Wed, 25 Nov 2020 20:19:50 GMT",
"Keep-Alive": "timeout=2, max=100",
"Last-Modified": "Tue, 28 Aug 2012 03:54:00 "
"GMT",
"Server": "Apache",
"Upgrade": "h2",
"Vary": "Accept-Encoding,User-Agent"},
"headersText": "HTTP/1.1 200 OK\r\n"
"Date: Wed, 25 Nov 2020 20:09:50 GMT\r\n"
"Server: Apache\r\n"
"Upgrade: h2\r\n"
"Connection: Upgrade, Keep-Alive\r\n"
"Last-Modified: Tue, 28 Aug 2012 03:54:00 "
"GMT\r\n"
"ETag: "af5-4c84b646fe600-gzip"\r\n"
"Accept-Ranges: bytes\r\n"
"Cache-Control: max-age=600\r\n"
"Expires: Wed, 25 Nov 2020 20:19:50 GMT\r\n"
"Vary: Accept-Encoding,User-Agent\r\n"
"Content-Encoding: gzip\r\n"
"Content-Length: 1193\r\n"
"Keep-Alive: timeout=2, max=100\r\n"
"Content-Type: text/html\r\n"
"\r\n",
"requestId": "4612FE9C61094E1E16491EEA95048287"}}
{ "method": "Network.responseReceived",
"params": { "frameId": "04585B830B1EF2B1D3B9DB229DFD253F",
"loaderId": "4612FE9C61094E1E16491EEA95048287",
"requestId": "4612FE9C61094E1E16491EEA95048287",
"response": { "connectionId": 36,
"connectionReused": False,
"encodedDataLength": 423,
"fromDiskCache": False,
"fromPrefetchCache": False,
"fromServiceWorker": False,
"headers": { "Accept-Ranges": "bytes",
"Cache-Control": "max-age=600",
"Connection": "Upgrade, "
"Keep-Alive",
"Content-Encoding": "gzip",
"Content-Length": "1193",
"Content-Type": "text/html",
"Date": "Wed, 25 Nov 2020 "
"20:09:50 GMT",
"ETag": ""af5-4c84b646fe600-gzip"",
"Expires": "Wed, 25 Nov 2020 "
"20:19:50 GMT",
"Keep-Alive": "timeout=2, "
"max=100",
"Last-Modified": "Tue, 28 Aug "
"2012 "
"03:54:00 "
"GMT",
"Server": "Apache",
"Upgrade": "h2",
"Vary": "Accept-Encoding,User-Agent"},
"headersText": "HTTP/1.1 200 OK\r\n"
"Date: Wed, 25 Nov 2020 "
"20:09:50 GMT\r\n"
"Server: Apache\r\n"
"Upgrade: h2\r\n"
"Connection: Upgrade, "
"Keep-Alive\r\n"
"Last-Modified: Tue, 28 Aug "
"2012 03:54:00 GMT\r\n"
"ETag: "
""af5-4c84b646fe600-gzip"\r\n"
"Accept-Ranges: bytes\r\n"
"Cache-Control: "
"max-age=600\r\n"
"Expires: Wed, 25 Nov 2020 "
"20:19:50 GMT\r\n"
"Vary: "
"Accept-Encoding,User-Agent\r\n"
"Content-Encoding: gzip\r\n"
"Content-Length: 1193\r\n"
"Keep-Alive: timeout=2, "
"max=100\r\n"
"Content-Type: text/html\r\n"
"\r\n",
"mimeType": "text/html",
"protocol": "http/1.1",
"remoteIPAddress": "75.119.198.160",
"remotePort": 80,
"requestHeaders": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"Accept-Encoding": "gzip, "
"deflate",
"Accept-Language": "en-US,en;q=0.9",
"Connection": "keep-alive",
"Host": "www.staggeringbeauty.com",
"Upgrade-Insecure-Requests": "1",
"User-Agent": "Mozilla/5.0 "
"(Windows "
"NT "
"10.0; "
"Win64; "
"x64) "
"AppleWebKit/537.36 "
"(KHTML, "
"like "
"Gecko) "
"Chrome/87.0.4280.66 "
"Safari/537.36"},
"requestHeadersText": "GET / HTTP/1.1\r\n"
"Host: "
"www.staggeringbeauty.com\r\n"
"Connection: "
"keep-alive\r\n"
"Upgrade-Insecure-Requests: "
"1\r\n"
"User-Agent: "
"Mozilla/5.0 (Windows "
"NT 10.0; Win64; x64) "
"AppleWebKit/537.36 "
"(KHTML, like Gecko) "
"Chrome/87.0.4280.66 "
"Safari/537.36\r\n"
"Accept: "
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9\r\n"
"Accept-Encoding: "
"gzip, deflate\r\n"
"Accept-Language: "
"en-US,en;q=0.9\r\n",
"responseTime": 1606334989493.78,
"securityState": "insecure",
"status": 200,
"statusText": "OK",
"timing": { "connectEnd": 181.506,
"connectStart": 0,
"dnsEnd": 0,
"dnsStart": 0,
"proxyEnd": -1,
"proxyStart": -1,
"pushEnd": 0,
"pushStart": 0,
"receiveHeadersEnd": 370.815,
"requestTime": 123924.819199,
"sendEnd": 181.685,
"sendStart": 181.557,
"sslEnd": -1,
"sslStart": -1,
"workerFetchStart": -1,
"workerReady": -1,
"workerRespondWithSettled": -1,
"workerStart": -1},
"url": "http://www.staggeringbeauty.com/"},
"timestamp": 123925.190892,
"type": "Document"}}
{ "method": "Network.requestWillBeSent",
"params": { "documentURL": "http://www.staggeringbeauty.com/",
"frameId": "04585B830B1EF2B1D3B9DB229DFD253F",
"hasUserGesture": False,
"initiator": { "lineNumber": 5,
"type": "parser",
"url": "http://www.staggeringbeauty.com/"},
"loaderId": "4612FE9C61094E1E16491EEA95048287",
"request": { "headers": { "Referer": "http://www.staggeringbeauty.com/",
"User-Agent": "Mozilla/5.0 "
"(Windows NT "
"10.0; Win64; "
"x64) "
"AppleWebKit/537.36 "
"(KHTML, like "
"Gecko) "
"Chrome/87.0.4280.66 "
"Safari/537.36"},
"initialPriority": "High",
"method": "GET",
"mixedContentType": "none",
"referrerPolicy": "no-referrer-when-downgrade",
"url": "http://www.staggeringbeauty.com/lib/paper.js"},
"requestId": "11620.2",
"timestamp": 123925.218207,
"type": "Script",
"wallTime": 1606334989.522785}}
{ "method": "Network.requestWillBeSent",
"params": { "documentURL": "http://www.staggeringbeauty.com/",
"frameId": "04585B830B1EF2B1D3B9DB229DFD253F",
"hasUserGesture": False,
"initiator": { "lineNumber": 7,
"type": "parser",
"url": "http://www.staggeringbeauty.com/"},
"loaderId": "4612FE9C61094E1E16491EEA95048287",
"request": { "headers": { "Referer": "http://www.staggeringbeauty.com/",
"User-Agent": "Mozilla/5.0 "
"(Windows NT "
"10.0; Win64; "
"x64) "
"AppleWebKit/537.36 "
"(KHTML, like "
"Gecko) "
"Chrome/87.0.4280.66 "
"Safari/537.36"},
"initialPriority": "High",
"method": "GET",
"mixedContentType": "none",
"referrerPolicy": "no-referrer-when-downgrade",
"url": "http://www.staggeringbeauty.com/lib/Tween.js"},
"requestId": "11620.3",
"timestamp": 123925.218471,
"type": "Script",
"wallTime": 1606334989.52305}}
{ "method": "Network.requestWillBeSent",
"params": { "documentURL": "http://www.staggeringbeauty.com/",
"frameId": "04585B830B1EF2B1D3B9DB229DFD253F",
"hasUserGesture": False,
"initiator": { "lineNumber": 8,
"type": "parser",
"url": "http://www.staggeringbeauty.com/"},
"loaderId": "4612FE9C61094E1E16491EEA95048287",
"request": { "headers": { "Referer": "http://www.staggeringbeauty.com/",
"User-Agent": "Mozilla/5.0 "
"(Windows NT "
"10.0; Win64; "
"x64) "
"AppleWebKit/537.36 "
"(KHTML, like "
"Gecko) "
"Chrome/87.0.4280.66 "
"Safari/537.36"},
"initialPriority": "High",
"method": "GET",
"mixedContentType": "none",
"referrerPolicy": "no-referrer-when-downgrade",
"url": "http://www.staggeringbeauty.com/lib/traer.js"},
"requestId": "11620.4",
"timestamp": 123925.218655,
"type": "Script",
"wallTime": 1606334989.523234}}
{ "method": "Network.requestWillBeSent",
"params": { "documentURL": "http://www.staggeringbeauty.com/",
"frameId": "04585B830B1EF2B1D3B9DB229DFD253F",
"hasUserGesture": False,
"initiator": { "lineNumber": 10,
"type": "parser",
"url": "http://www.staggeringbeauty.com/"},
"loaderId": "4612FE9C61094E1E16491EEA95048287",
"request": { "headers": { "Referer": "http://www.staggeringbeauty.com/",
"User-Agent": "Mozilla/5.0 "
"(Windows NT "
"10.0; Win64; "
"x64) "
"AppleWebKit/537.36 "
"(KHTML, like "
"Gecko) "
"Chrome/87.0.4280.66 "
"Safari/537.36"},
"initialPriority": "High",
"method": "GET",
"mixedContentType": "none",
"referrerPolicy": "no-referrer-when-downgrade",
"url": "http://www.staggeringbeauty.com/lib/soundmanager2-nodebug-jsmin.js"},
"requestId": "11620.5",
"timestamp": 123925.218906,
"type": "Script",
"wallTime": 1606334989.523485}}
{ "method": "Network.requestWillBeSent",
"params": { "documentURL": "http://www.staggeringbeauty.com/",
"frameId": "04585B830B1EF2B1D3B9DB229DFD253F",
"hasUserGesture": False,
"initiator": { "lineNumber": 12,
"type": "parser",
"url": "http://www.staggeringbeauty.com/"},
"loaderId": "4612FE9C61094E1E16491EEA95048287",
"request": { "headers": { "Referer": "http://www.staggeringbeauty.com/",
"User-Agent": "Mozilla/5.0 "
"(Windows NT "
"10.0; Win64; "
"x64) "
"AppleWebKit/537.36 "
"(KHTML, like "
"Gecko) "
"Chrome/87.0.4280.66 "
"Safari/537.36"},
"initialPriority": "High",
"method": "GET",
"mixedContentType": "none",
"referrerPolicy": "no-referrer-when-downgrade",
"url": "http://www.staggeringbeauty.com/src/jiggler.js"},
"requestId": "11620.6",
"timestamp": 123925.219131,
"type": "Script",
"wallTime": 1606334989.52371}}
{ "method": "Network.requestWillBeSent",
"params": { "documentURL": "http://www.staggeringbeauty.com/",
"frameId": "04585B830B1EF2B1D3B9DB229DFD253F",
"hasUserGesture": False,
"initiator": { "lineNumber": 15,
"type": "parser",
"url": "http://www.staggeringbeauty.com/"},
"loaderId": "4612FE9C61094E1E16491EEA95048287",
"request": { "headers": { "Referer": "http://www.staggeringbeauty.com/",
"User-Agent": "Mozilla/5.0 "
"(Windows NT "
"10.0; Win64; "
"x64) "
"AppleWebKit/537.36 "
"(KHTML, like "
"Gecko) "
"Chrome/87.0.4280.66 "
"Safari/537.36"},
"initialPriority": "VeryHigh",
"method": "GET",
"mixedContentType": "none",
"referrerPolicy": "no-referrer-when-downgrade",
"url": "http://www.staggeringbeauty.com/src/style.css"},
"requestId": "11620.7",
"timestamp": 123925.219418,
"type": "Stylesheet",
"wallTime": 1606334989.523997}}
{ "method": "Network.requestWillBeSentExtraInfo",
"params": { "associatedCookies": [],
"headers": { "Accept": "*/*",
"Accept-Encoding": "gzip, deflate", ...```
jd174
Raidus
Im trying to see each individual request while using this like in normal Selenium to sort by method, etc but it says
for request in driver.requests: AttributeError: 'WebDriver' object has no attribute 'requests'Is there something else i should do to sort requests? And how do i print headers? Cookies works fine by doing
.get_cookies()but headers doesnt work whichever way i try?