nfriedly / node-unblocker

Web proxy for evading internet censorship, and general-purpose Node.js library for proxying and rewriting remote webpages
https://www.npmjs.com/package/unblocker
GNU Affero General Public License v3.0
455 stars 875 forks source link

how to bypass cloudflare Human Veriication #262

Closed bobwatcherx closed 2 months ago

bobwatcherx commented 2 months ago

i make program . for scrape website . but error . node unblocker canot scrape cloudflare

you can test my code and how to solve this thx

from fastapi import FastAPI
from fastapi.responses import JSONResponse
import requests
from bs4 import BeautifulSoup

app = FastAPI()

@app.get("/scrape")
async def scrape_jable():
    url = "https://bobwatcherx-unblockjapan.hf.space/proxy/https://jable.tv/videos/venx-170/"
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
    }
    response = requests.get(url, headers=headers)
    if response.status_code == 200:
        soup = BeautifulSoup(response.content, "html.parser")
        video_tag = soup.find("video", id="player")
        script_tag = soup.find("script", text=lambda t: "var hlsUrl =" in t).string

        if video_tag and script_tag:
            poster = video_tag.get("poster")
            hls_url = script_tag.split("var hlsUrl = '")[1].split("'")[0]
            return JSONResponse(content={"poster": poster, "hls_url": hls_url})

    return JSONResponse(content={"error": "Failed to scrape data"}, status_code=500)
nfriedly commented 2 months ago

Sorry, I don't know the answer for that.