from seleniumwire.undetected_chromedriver.v2 import Chrome, ChromeOptions
Heres the modified request interceptor:
def interceptor(request):
global dHistory
if request.url and (expected_url == request.url or parse.quote(expected_url, safe=":/") == request.url):
print(get_methods(request))
dom = getDomain2(request.url)
print("dom:", dom)
if dom in dHistory:
lastref= dHistory[dom][-1]
else:
lastref=dom
dHistory[dom] = [dom]
del request.headers['Referer'] # Remember to delete the header first
print("headers before:")
for k in request.headers:
print(k, request.headers[k])
print()
request.headers['Referer'] = lastref # Spoof the referer
print()
print("headers after:")
for k in request.headers:
print(k, request.headers[k])
print()
Search Url is: https://www.novelbed.com/s/"global survival"
class Represents an HTTP request.
delattr Implement delattr(self, name).
dir Default dir() implementation.
eq Return self==value.
format Default object formatter.
ge Return self>=value.
getattribute Return getattr(self, name).
gt Return self>value.
hash Return hash(self).
init Initialise a new Request object. Args: method: The request method - G
init_subclass This method is called when a class is subclassed. The default implementation does nothing
le Return self<=value.
lt Return self<value.
ne Return self!=value.
new Create and return a new object. See help(type) for accurate signature.
reduce Helper for pickle.
reduce_ex Helper for pickle.
repr None
setattr Implement setattr(self, name, value).
sizeof Size of object in memory, in bytes.
str None
subclasshook Abstract classes can override this to customize issubclass(). This is invoked early on by
abort Convenience method for signalling that this request is to be terminated with a spe
create_response Create a response object and attach it to this request.
None
dom: https://www.novelbed.comheaders before:
sec-ch-ua "Chromium";v="110", "Not A(Brand";v="24", "Google Chrome";v="110"
sec-ch-ua-mobile ?0
sec-ch-ua-platform "Windows"
upgrade-insecure-requests 1
user-agent Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36
accept text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,/;q=0.8,application/signed-exchange;v=b3;q=0.7
sec-fetch-site none
sec-fetch-mode navigate
sec-fetch-user ?1
sec-fetch-dest document
accept-encoding gzip, deflate, br
accept-language en-GB,en-US;q=0.9,en;q=0.8
cookie _ga=GA1.1.9721537.1672933628; night_mode=0; __cf_bm=IFoTT.fwgSTukCvU4ZZLbbl77YPZV2tSQWoQq5FzzlM-1675881416-0-Ad50PIPBI6O1N49X4IUcd7eFFnbxXzd22qNooc7b75JlNNc7ucqjypGnVrmXAZSOwk9SI/sPu8pp2BiUcEfc2YsPrsVPtMmR8Ph81IUfhsJX4eER9MQZ5K2CZDR/ZlPNBzQtUBswzk1gktLrBcnB6GI=; _ga_NWFK4QKDYJ=GS1.1.1675881411.17.1.1675882252.0.0.0; novelbed_session=eyJpdiI6IjhqallaTkN1NExDclpRbDVabGJhSnc9PSIsInZhbHVlIjoiUDRrdCtNQ1VjQ3luZGY0WGVQRXdNeXkzNkpNdnlZVUl4MlpHVGtIRVQ1V29xVDhtNlJoa0Vwd0lHbnBZV0gwZUNOZm5WT1grcCtDdjk3V0FhNW9HN09lQXZqYUx6eUs1UHlBVWJMUW5zVXozdHJueGhKVktBZ2pGUVBacTd1a0oiLCJtYWMiOiJlODM2NWE3YTgxODVmMGU4Nzk3YzMzOTgyZDIyM2MxYjI2NDcwNzhiMDA5OTVmMWY4MDE5MzliNDI3NTA3MWViIiwidGFnIjoiIn0%3D; b8VVU3PNHx118liHBxKbyoDl4x3uoxqQVbcYWJe8=eyJpdiI6Ijh4ZDRtY3puVXJxTk1iMkVNMWdRa0E9PSIsInZhbHVlIjoiMVFPbGlJTnVxK0YwMFhvYkFBeUkzeDIwNjAxUFZTcittbGJGaFo5N0N1SnRxdE55R1NWN0h4NnVzNXo0YVNvNkhTdlZFeGo1bXQwYW0yY0VJbDVub0FiN2ZEWFpFY1JZZWhQZW10czhIWm1LRkI1ZnVOd3J3a1UwelJTOVJaeklOcS9ZSDhUbWFURTVuU3hXbERJb1hDN3RCVHJWWWhjRE1PcDBBYXd4UW5yMkVXNnFqOVltQ1JhbFRJSi9McnMzTFdMTkhzYTVlTENUVW9MMnMvYjBoWmRmR29wVGwvNVJoWDJ2cWp2NkRSSnBNOUpYVG9rTUE1Q1RyZzYyTEpOVjMrOWoreXl2NGc1SDFGVmFvcW13VnR5UEM3blFCd0hNSDM3b0lWVU9aV2t2WGU5Q3F2NDlDbDR3YmVBS3hqc0Q1dGFCRDJ2MXA5VkpDZ1BGYndGVVB5ZHUxMDVRdUhNay85YWkxY2pxM1NDZjdmdXY3bm9Hc0RzRXFFQWtEc1o3VkdmYUc5QU5NdnkxWnFGN3hvUGhBaC9pQkVNcUNQcUVmSTRwcmJHN0h2WTB6NkxOV1hDYzFQaHAzY29HNkVQNHhFajhtWWJ6a1dRYnVsWXJaMU1lbmhMSVBZLzlUSWN1SHVSTlNtejk3ZjZ3NWJvV3Z3YmZNc0V1M3JvU1IrQjNrWStzUXJ5ZVJ1TTNSejQ2U3hVS1lRPT0iLCJtYWMiOiJkNjA3ZGNlNWNiYzVhMjQ3ZWVlNWQ4MGE4MWQzMzJjNDJmM2ZkNzU1NTJjNjkzMGJhNDIzN2UzNzkzZTg2ZmI2IiwidGFnIjoiIn0%3D
When I open chrome and go into the console, document.referrer shows up as '', but it clearly showing as set in program output. Any ideas ? If I do this web nav manually obviously works.
I've tried older ver. Updated my chrome and tried v2 as well (which I'm currently on). Nothing zilch. Any ideas folks?
So, heres the import:
from seleniumwire.undetected_chromedriver.v2 import Chrome, ChromeOptions
Heres the modified request interceptor:
def interceptor(request): global dHistory if request.url and (expected_url == request.url or parse.quote(expected_url, safe=":/") == request.url): print(get_methods(request)) dom = getDomain2(request.url) print("dom:", dom) if dom in dHistory: lastref= dHistory[dom][-1] else: lastref=dom dHistory[dom] = [dom] del request.headers['Referer'] # Remember to delete the header first print("headers before:") for k in request.headers: print(k, request.headers[k]) print() request.headers['Referer'] = lastref # Spoof the referer print() print("headers after:") for k in request.headers: print(k, request.headers[k]) print()
results are:
Search Url is: https://www.novelbed.com/s/"global survival" class Represents an HTTP request. delattr Implement delattr(self, name). dir Default dir() implementation. eq Return self==value. format Default object formatter. ge Return self>=value. getattribute Return getattr(self, name). gt Return self>value. hash Return hash(self). init Initialise a new Request object. Args: method: The request method - G init_subclass This method is called when a class is subclassed. The default implementation does nothing le Return self<=value. lt Return self<value. ne Return self!=value. new Create and return a new object. See help(type) for accurate signature. reduce Helper for pickle. reduce_ex Helper for pickle. repr None setattr Implement setattr(self, name, value). sizeof Size of object in memory, in bytes. str None subclasshook Abstract classes can override this to customize issubclass(). This is invoked early on by abort Convenience method for signalling that this request is to be terminated with a spe create_response Create a response object and attach it to this request. None dom: https://www.novelbed.com headers before: sec-ch-ua "Chromium";v="110", "Not A(Brand";v="24", "Google Chrome";v="110" sec-ch-ua-mobile ?0 sec-ch-ua-platform "Windows" upgrade-insecure-requests 1 user-agent Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 accept text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,/;q=0.8,application/signed-exchange;v=b3;q=0.7 sec-fetch-site none sec-fetch-mode navigate sec-fetch-user ?1 sec-fetch-dest document accept-encoding gzip, deflate, br accept-language en-GB,en-US;q=0.9,en;q=0.8 cookie _ga=GA1.1.9721537.1672933628; night_mode=0; __cf_bm=IFoTT.fwgSTukCvU4ZZLbbl77YPZV2tSQWoQq5FzzlM-1675881416-0-Ad50PIPBI6O1N49X4IUcd7eFFnbxXzd22qNooc7b75JlNNc7ucqjypGnVrmXAZSOwk9SI/sPu8pp2BiUcEfc2YsPrsVPtMmR8Ph81IUfhsJX4eER9MQZ5K2CZDR/ZlPNBzQtUBswzk1gktLrBcnB6GI=; _ga_NWFK4QKDYJ=GS1.1.1675881411.17.1.1675882252.0.0.0; novelbed_session=eyJpdiI6IjhqallaTkN1NExDclpRbDVabGJhSnc9PSIsInZhbHVlIjoiUDRrdCtNQ1VjQ3luZGY0WGVQRXdNeXkzNkpNdnlZVUl4MlpHVGtIRVQ1V29xVDhtNlJoa0Vwd0lHbnBZV0gwZUNOZm5WT1grcCtDdjk3V0FhNW9HN09lQXZqYUx6eUs1UHlBVWJMUW5zVXozdHJueGhKVktBZ2pGUVBacTd1a0oiLCJtYWMiOiJlODM2NWE3YTgxODVmMGU4Nzk3YzMzOTgyZDIyM2MxYjI2NDcwNzhiMDA5OTVmMWY4MDE5MzliNDI3NTA3MWViIiwidGFnIjoiIn0%3D; b8VVU3PNHx118liHBxKbyoDl4x3uoxqQVbcYWJe8=eyJpdiI6Ijh4ZDRtY3puVXJxTk1iMkVNMWdRa0E9PSIsInZhbHVlIjoiMVFPbGlJTnVxK0YwMFhvYkFBeUkzeDIwNjAxUFZTcittbGJGaFo5N0N1SnRxdE55R1NWN0h4NnVzNXo0YVNvNkhTdlZFeGo1bXQwYW0yY0VJbDVub0FiN2ZEWFpFY1JZZWhQZW10czhIWm1LRkI1ZnVOd3J3a1UwelJTOVJaeklOcS9ZSDhUbWFURTVuU3hXbERJb1hDN3RCVHJWWWhjRE1PcDBBYXd4UW5yMkVXNnFqOVltQ1JhbFRJSi9McnMzTFdMTkhzYTVlTENUVW9MMnMvYjBoWmRmR29wVGwvNVJoWDJ2cWp2NkRSSnBNOUpYVG9rTUE1Q1RyZzYyTEpOVjMrOWoreXl2NGc1SDFGVmFvcW13VnR5UEM3blFCd0hNSDM3b0lWVU9aV2t2WGU5Q3F2NDlDbDR3YmVBS3hqc0Q1dGFCRDJ2MXA5VkpDZ1BGYndGVVB5ZHUxMDVRdUhNay85YWkxY2pxM1NDZjdmdXY3bm9Hc0RzRXFFQWtEc1o3VkdmYUc5QU5NdnkxWnFGN3hvUGhBaC9pQkVNcUNQcUVmSTRwcmJHN0h2WTB6NkxOV1hDYzFQaHAzY29HNkVQNHhFajhtWWJ6a1dRYnVsWXJaMU1lbmhMSVBZLzlUSWN1SHVSTlNtejk3ZjZ3NWJvV3Z3YmZNc0V1M3JvU1IrQjNrWStzUXJ5ZVJ1TTNSejQ2U3hVS1lRPT0iLCJtYWMiOiJkNjA3ZGNlNWNiYzVhMjQ3ZWVlNWQ4MGE4MWQzMzJjNDJmM2ZkNzU1NTJjNjkzMGJhNDIzN2UzNzkzZTg2ZmI2IiwidGFnIjoiIn0%3D
headers after: sec-ch-ua "Chromium";v="110", "Not A(Brand";v="24", "Google Chrome";v="110" sec-ch-ua-mobile ?0 sec-ch-ua-platform "Windows" upgrade-insecure-requests 1 user-agent Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 accept text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,/;q=0.8,application/signed-exchange;v=b3;q=0.7 sec-fetch-site none sec-fetch-mode navigate sec-fetch-user ?1 sec-fetch-dest document accept-encoding gzip, deflate, br accept-language en-GB,en-US;q=0.9,en;q=0.8 cookie _ga=GA1.1.9721537.1672933628; night_mode=0; __cf_bm=IFoTT.fwgSTukCvU4ZZLbbl77YPZV2tSQWoQq5FzzlM-1675881416-0-Ad50PIPBI6O1N49X4IUcd7eFFnbxXzd22qNooc7b75JlNNc7ucqjypGnVrmXAZSOwk9SI/sPu8pp2BiUcEfc2YsPrsVPtMmR8Ph81IUfhsJX4eER9MQZ5K2CZDR/ZlPNBzQtUBswzk1gktLrBcnB6GI=; _ga_NWFK4QKDYJ=GS1.1.1675881411.17.1.1675882252.0.0.0; novelbed_session=eyJpdiI6IjhqallaTkN1NExDclpRbDVabGJhSnc9PSIsInZhbHVlIjoiUDRrdCtNQ1VjQ3luZGY0WGVQRXdNeXkzNkpNdnlZVUl4MlpHVGtIRVQ1V29xVDhtNlJoa0Vwd0lHbnBZV0gwZUNOZm5WT1grcCtDdjk3V0FhNW9HN09lQXZqYUx6eUs1UHlBVWJMUW5zVXozdHJueGhKVktBZ2pGUVBacTd1a0oiLCJtYWMiOiJlODM2NWE3YTgxODVmMGU4Nzk3YzMzOTgyZDIyM2MxYjI2NDcwNzhiMDA5OTVmMWY4MDE5MzliNDI3NTA3MWViIiwidGFnIjoiIn0%3D; b8VVU3PNHx118liHBxKbyoDl4x3uoxqQVbcYWJe8=eyJpdiI6Ijh4ZDRtY3puVXJxTk1iMkVNMWdRa0E9PSIsInZhbHVlIjoiMVFPbGlJTnVxK0YwMFhvYkFBeUkzeDIwNjAxUFZTcittbGJGaFo5N0N1SnRxdE55R1NWN0h4NnVzNXo0YVNvNkhTdlZFeGo1bXQwYW0yY0VJbDVub0FiN2ZEWFpFY1JZZWhQZW10czhIWm1LRkI1ZnVOd3J3a1UwelJTOVJaeklOcS9ZSDhUbWFURTVuU3hXbERJb1hDN3RCVHJWWWhjRE1PcDBBYXd4UW5yMkVXNnFqOVltQ1JhbFRJSi9McnMzTFdMTkhzYTVlTENUVW9MMnMvYjBoWmRmR29wVGwvNVJoWDJ2cWp2NkRSSnBNOUpYVG9rTUE1Q1RyZzYyTEpOVjMrOWoreXl2NGc1SDFGVmFvcW13VnR5UEM3blFCd0hNSDM3b0lWVU9aV2t2WGU5Q3F2NDlDbDR3YmVBS3hqc0Q1dGFCRDJ2MXA5VkpDZ1BGYndGVVB5ZHUxMDVRdUhNay85YWkxY2pxM1NDZjdmdXY3bm9Hc0RzRXFFQWtEc1o3VkdmYUc5QU5NdnkxWnFGN3hvUGhBaC9pQkVNcUNQcUVmSTRwcmJHN0h2WTB6NkxOV1hDYzFQaHAzY29HNkVQNHhFajhtWWJ6a1dRYnVsWXJaMU1lbmhMSVBZLzlUSWN1SHVSTlNtejk3ZjZ3NWJvV3Z3YmZNc0V1M3JvU1IrQjNrWStzUXJ5ZVJ1TTNSejQ2U3hVS1lRPT0iLCJtYWMiOiJkNjA3ZGNlNWNiYzVhMjQ3ZWVlNWQ4MGE4MWQzMzJjNDJmM2ZkNzU1NTJjNjkzMGJhNDIzN2UzNzkzZTg2ZmI2IiwidGFnIjoiIn0%3D Referer https://www.novelbed.com
lastref: https://www.novelbed.com
When I open chrome and go into the console, document.referrer shows up as '', but it clearly showing as set in program output. Any ideas ? If I do this web nav manually obviously works.
I've tried older ver. Updated my chrome and tried v2 as well (which I'm currently on). Nothing zilch. Any ideas folks?