tlcaputi
commented
4 years ago I needed to figure out a quick fix for this same problem, and I ended up writing this. It's not the most exact or beautifully written piece of code in the world, but it works for my purposes. Maybe it'll work for yours.
# MIT License
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
import datetime
from time import sleep
def archive_url(
url,
timeout=100,
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'
):
"""Submits a URL to WebArchive's Save Page Now Feature (working as of 2020-07-31 on Python 3.6.10)
Keyword arguments:
url -- The url you want to archive
timeout -- Max number of seconds you're willing to wait
user_agent -- You can pass a custom user agent here
"""
# POST Request
headers = {
'authority': 'web.archive.org',
'cache-control': 'max-age=0',
'upgrade-insecure-requests': '1',
'origin': 'https://web.archive.org',
'content-type': 'application/x-www-form-urlencoded',
'user-agent': user_agent,
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'sec-fetch-site': 'same-origin',
'sec-fetch-mode': 'navigate',
'sec-fetch-user': '?1',
'sec-fetch-dest': 'document',
'referer': 'https://web.archive.org/save',
'accept-language': 'en-US,en;q=0.9,de;q=0.8',
}
data = {
'url': url,
'capture_all': 'on'
}
r = requests.post(f'https://web.archive.org/save/{url}', headers=headers, data=data)
# BS4 get SCRIPTS and find watchJob arguments
soup = BeautifulSoup(r.content, 'html.parser')
scripts = soup.find_all("script")
job_id = None
for script in scripts:
string = script.string
if string and "watchJob" in string:
args_string_list = string.strip().split('"')
job_id = args_string_list[1]
break
assert job_id is not None, "Couldn't find job_id in html"
# Request status of the job
out_url = None
was_pending = False
wait_time = 0
while wait_time < timeout:
r = requests.get(f"https://web.archive.org/save/status/{job_id}?_t={datetime.datetime.now().timestamp()}", headers=headers)
rj = r.json()
if rj.get('status', 'none') == "pending":
was_pending = True
if rj.get('status', 'none') == "success":
original_url = rj.get('original_url', 'none')
ext_url = f"/web/{rj['timestamp']}/{rj['original_url']}"
out_url = urljoin('https://web.archive.org', ext_url)
break
seconds_to_wait = int(r.headers.get("Retry-After", 5))
print(f"[{wait_time} seconds elapsed] Waiting for archive to complete...")
wait_time += seconds_to_wait
sleep(seconds_to_wait)
assert out_url is not None, f"Process did not complete after {timeout} seconds"
out = {
"original_url": original_url,
"archive_url": out_url,
"from_cache": was_pending == False
}
return out
if __name__ == "__main__":
url = "https://ultimateframedata.com/"
print(archive_url(url))
eggplants
palewire
zakyum
dannguyen
When I was running this code: