ultrafunkamsterdam / undetected-chromedriver

Custom Selenium Chromedriver | Zero-Config | Passes ALL bot mitigation systems (like Distil / Imperva/ Datadadome / CloudFlare IUAM)
https://github.com/UltrafunkAmsterdam/undetected-chromedriver
GNU General Public License v3.0
9.64k stars 1.14k forks source link

Trying to replicate these steps for Java #1992

Open Serimert90 opened 1 month ago

Serimert90 commented 1 month ago

Hello, thanks for this beautiful library. It worked most of the time. It is really appreciated.

I am reading source code of this and i want to ask about cdc_ replacement string in driver.exe. Are you doing anything beside replacing all cdc_ 23 length stuff with random?

Aside from driver.exe modification, i found these stuff,

options.addArguments("--disable-blink-features=AutomationControlled");
options.addArguments("--disable-dev-shm-usage");
options.addArguments("--no-sandbox");
options.setExperimentalOption("excludeSwitches", new String[] { "enable-automation" });
options.setExperimentalOption("useAutomationExtension", false);
-setting user agent to that version's real user agent value
-and make sure navigator.webdriver returns false. 

also new scripts on new page and remove scripts on new page

addScriptsOnNewPage.put("source", """
                            Object.defineProperty(Navigator.prototype, 'webdriver', {
                                    set: undefined,
                                    enumerable: true,
                                    configurable: true,
                                    get: new Proxy(
                                        Object.getOwnPropertyDescriptor(Navigator.prototype, 'webdriver').get,
                                        { apply: (target, thisArg, args) => {
                                            // emulate getter call validation
                                            Reflect.apply(target, thisArg, args);
                                            return false;
                                        }}
                                    )
                                });
                            let objectToInspect = window,
                            result = [];
                            while(objectToInspect !== null)
                                { result = result.concat(Object.getOwnPropertyNames(objectToInspect));
                                objectToInspect = Object.getPrototypeOf(objectToInspect); }
                            result.forEach(p => p.match(/.+_.+_(Array|Promise|Symbol)/ig)
                                &&delete window[p]&&console.log('removed',p))
                        """);
                ((ChromeDriver) driver).executeCdpCommand("Page.addScriptToEvaluateOnNewDocument", addScriptsOnNewPage);

-------------------------------
Map<String, Object> removeScriptsOnNewPage = new HashMap<>();
                removeScriptsOnNewPage.put("identifier", "1");
                ((ChromeDriver) driver).executeCdpCommand("Page.removeScriptToEvaluateOnNewDocument", removeScriptsOnNewPage);

Is there anything i am missing here?

Serimert90 commented 1 month ago

I achieved better than this one. In fact my java driver "https://www.browserscan.net/bot-detection" passes while this library MAY not. PM me for the code.

Serimert90 commented 2 weeks ago

Before reading the sample code, please keep in mind,

import org.junit.jupiter.api.Test;
import org.openqa.selenium.By;
import org.openqa.selenium.JavascriptExecutor;
import org.openqa.selenium.PageLoadStrategy;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeDriverService;
import org.openqa.selenium.chrome.ChromeOptions;

import java.time.Duration;
import java.util.HashMap;
import java.util.Map;

import static org.junit.jupiter.api.Assertions.assertTrue;

public class UndetectedChromeDriverTest {

    private WebDriver driver;
    private JavascriptExecutor jsExecutor;
    private ChromeDriverService chromeService = ChromeDriverService.createDefaultService();

    private void initStealthChromeDriver(PageLoadStrategy pageLoadStrategy) {
        // U need to specify the location for chrome driver before initializing web driver as below
        // System.setProperty("webdriver.chrome.driver", location);

        driver = new ChromeDriver(chromeService, getStealthChromeOptions(pageLoadStrategy, true));
        jsExecutor = (JavascriptExecutor) driver;
        try {
            jsExecutor.executeScript("Object.defineProperty(navigator, 'webdriver', {get: () => false})");
            Map<String, Object> addScriptsOnNewPage = new HashMap<>();
            addScriptsOnNewPage.put("source", BOT_DETECTION_PREVENTION_SCRIP);
            ((ChromeDriver) driver).executeCdpCommand("Page.addScriptToEvaluateOnNewDocument", addScriptsOnNewPage);
            Map<String, Object> removeScriptsOnNewPage = new HashMap<>();
            removeScriptsOnNewPage.put("identifier", "1");
            ((ChromeDriver) driver).executeCdpCommand("Page.removeScriptToEvaluateOnNewDocument", removeScriptsOnNewPage);
        } catch (Exception e) {
            //logger.error("Error while executing JsAndCdpCommands", e);
        }
    }

    private void initStealthChromeDriverWithReconnect(PageLoadStrategy desiredPageLoadStrategy,
                                                                String startAddress, Duration waitTime) {
        initStealthChromeDriver(PageLoadStrategy.NONE);
        driver.get(startAddress);
        chromeService.stop();
        reAttachBrowser(desiredPageLoadStrategy, waitTime);
    }

    /**
     * Use this if you need to detach and reconnect in the middle of page navigation,
     * use initStealthChromeDriverWithReconnect if first page requires captcha bypass.
     * @param desiredPageLoadStrategy .
     * @param waitTime .
     */
    private void detachAndReattachToChromeBrowser(PageLoadStrategy desiredPageLoadStrategy, Duration waitTime) {
        chromeService.stop();
        reAttachBrowser(desiredPageLoadStrategy, waitTime);
    }

    private void reAttachBrowser(PageLoadStrategy desiredPageLoadStrategy, Duration waitTime) {
        waitThread(waitTime);
        ChromeOptions optionsInner = new ChromeOptions();
        optionsInner.setPageLoadStrategy(desiredPageLoadStrategy);
        optionsInner.setExperimentalOption("debuggerAddress", "127.0.0.1:9022");
        chromeService = ChromeDriverService.createDefaultService();
        driver = new ChromeDriver(chromeService, optionsInner);
        jsExecutor = (JavascriptExecutor) driver;
    }

    private ChromeOptions getStealthChromeOptions(PageLoadStrategy pageLoadStrategy, boolean disableWebSecurity) {
        ChromeOptions options = new ChromeOptions();

        options.setPageLoadStrategy(pageLoadStrategy);
        options.addArguments("--start-maximized");
        if (disableWebSecurity) // sometimes needed, try and find out
            options.addArguments("--disable-web-security");
        options.addArguments("--disable-blink-features");
        options.addArguments("--disable-blink-features=AutomationControlled");
        options.addArguments("--disable-dev-shm-usage");
        options.addArguments("--no-sandbox");

        // this and installed version must match.
        String userAgent =  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
                    "(KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36";
        options.addArguments("--user-agent=" + userAgent);
        //options.addArguments("--headless=new");

        options.setExperimentalOption("excludeSwitches", new String[] { "enable-automation" });
        options.setExperimentalOption("useAutomationExtension", false);
        options.setExperimentalOption("detach", true);
        options.addArguments("--remote-debugging-port=9022");

        options.addArguments("--user-data-dir=C:/DEV/selenium/datadir");
        return options;
    }

    @Test
    public void testStealthDriver() {
        System.setProperty("webdriver.chrome.driver",
                "C:\\chromedriver.exe");
        initStealthChromeDriverWithReconnect(PageLoadStrategy.NORMAL,"https://www.browserscan.net/bot-detection",
                Duration.ofSeconds(4));
        assertTrue(driver.findElement(By.tagName("body")).getText().contains("Test Results:\nNormal"));
        int a = 0;
    }

    public void waitThread(Duration duration) {
        try {Thread.sleep(duration.toMillis());} catch (InterruptedException ignored) {}
    }

    public static final String BOT_DETECTION_PREVENTION_SCRIP = """
                            Object.defineProperty(Navigator.prototype, 'webdriver', {
                                    set: undefined,
                                    enumerable: true,
                                    configurable: true,
                                    get: new Proxy(
                                        Object.getOwnPropertyDescriptor(Navigator.prototype, 'webdriver').get,
                                        { apply: (target, thisArg, args) => {
                                            // emulate getter call validation
                                            Reflect.apply(target, thisArg, args);
                                            return false;
                                        }}
                                    )
                                });
                            let objectToInspect = window,
                            result = [];
                            while(objectToInspect !== null)
                                { result = result.concat(Object.getOwnPropertyNames(objectToInspect));
                                objectToInspect = Object.getPrototypeOf(objectToInspect); }
                            result.forEach(p => p.match(/.+_.+_(Array|Promise|Symbol)/ig)
                                &&delete window[p]&&console.log('removed',p))
                        """;
}