miyakogi / pyppeteer

Headless chrome/chromium automation library (unofficial port of puppeteer)
Other
3.57k stars 371 forks source link

Issue with Evaluate #210

Open BiatuAutMiahn opened 5 years ago

BiatuAutMiahn commented 5 years ago

Here is my script:

import asyncio
import json
import pyppeteer
from pyppeteer import launch

bills=[{
        "Alias": "Piedmont Natural Gas",
        "URI": "https://www.piedmontng.com/youraccount/onlineservices/billinformation.aspx",
        "Username": "<Snip>",
        "Password": "<Snip>",
        "UsernameSelector": "#ctl00_BodyCopy_Login1_LoginView1_Login1_UserName",
        "PasswordSelector": "#ctl00_BodyCopy_Login1_LoginView1_Login1_Password",
        "SubmitSelector": "#ctl00_BodyCopy_Login1_LoginView1_Login1_BtnLogin",
        "BillFunc": '''() => {var t=document.getElementById("ctl00_BodyCopy_BillingInformationForm1_lblYesterday").parentElement.parentElement.parentElement;return bal=$(t).find('td:contains("Account Balance as of")').next("td").text(),datedue=$(t).find('td:contains("Current Charges Past Due After")').next("td").text().replace(/(\d{2})\/(\d{2})\/(\d{4})/g,'$3.$1.$2'),pastdue=$(t).find('td:contains("Total Past Due Balance")').next("td").text(),{DueDate:datedue,CurrentBalance:bal,PastBalance:pastdue}}'''
       },
       {
        "Alias": "Magnolia Rental Property Management",
        "URI": "https://secure.rentecdirect.com/tenants/t/tstatement.php",
        "Username": "<Snip>",
        "Password": "<Snip>",
        "UsernameSelector": "#username",
        "PasswordSelector": "#password",
        "SubmitSelector": "#Login",
        "BillFunc": '''() => {
            var csvJSON = function(csv) {
                var lines = csv.split("\n");
                var result = [];
                var headers = lines[0].split(",");
                lines.map(function(line, indexLine) {
                    if (indexLine < 1) return;
                    var obj = {};
                    var currentline = line.split(",");
                    headers.map(function(header, indexHeader) {
                        obj[header] = currentline[indexHeader];
                    });
                    result.push(obj);
                });
                result.pop();
                return result
            };
            var e = null;
            $.ajax({
                url: "tcsv_download.php?date_begin=2019-01-01&date_end=2019-04-30",
                type: "get",
                dataType: "html",
                async: 0,
                cache: 0,
                success: function(a) {
                    e = csvJSON(a)
                }
            });
            due = e[e.length - 1].Date.replace(/(\d{2})\/\d{2}\/(\d{4})/g, "$2.$1.01");
            bal = Number(-e[e.length - 1].Balance).toFixed(2);
            return {
                DueDate: due,
                CurrentBalance: "NaN",
                PastBalance: bal
            }
        }'''
       },
       {
        "Alias": "",
        "URI": "",
        "Username": "",
        "Password": "",
        "UsernameSelector": "",
        "PasswordSelector": "",
        "SubmitSelector": "",
        "BillFunc": '''() => {
            return {
                DueDate: due,
                CurrentBalance: bal,
                PastBalance: pdbal,
            }
         }'''
       },
       {
        "Alias": "",
        "URI": "",
        "Username": "",
        "Password": "",
        "UsernameSelector": "",
        "PasswordSelector": "",
        "SubmitSelector": "",
        "BillFunc": '''() => {
            return {
                DueDate: due,
                CurrentBalance: bal,
                PastBalance: pdbal,
            }
         }'''
       },
       {
        "Alias": "",
        "URI": "",
        "Username": "",
        "Password": "",
        "UsernameSelector": "",
        "PasswordSelector": "",
        "SubmitSelector": "",
        "BillFunc": '''() => {
            return {
                DueDate: due,
                CurrentBalance: bal,
                PastBalance: pdbal,
            }
         }'''
       }]

async def getBill(browser,n):
    page = await browser.newPage()
    await page.goto(n['URI'])
    await page.waitForSelector("body",{'timeout': 60000})
    await page.type(n['UsernameSelector'],n['Username'])
    await page.type(n['PasswordSelector'],n['Password'])
    await page.click(n['SubmitSelector'])
    await page.waitForSelector("body",{'timeout': 60000})
    if page.url is not n['URI']:
        await page.goto(n['URI'])
        await page.waitForSelector("body",{'timeout': 60000})
    #await page.waitForNavigation({'timeout': 300000})
    bill=await page.evaluate(n['BillFunc'])
    print(bill)

async def main():
    pyppeteer.DEBUG = True
    browser = await launch(headless=False)
    await getBill(browser,bills[1])

asyncio.get_event_loop().run_until_complete(main())

For some reason, when running the evaluate on the "BillFunc" script, I get this error:

[E:pyppeteer.execution_context] Protocol error (Runtime.releaseObject): Cannot find context with specified id
Traceback (most recent call last):
  File "G:\My Drive\Dev\Python\Fin.py", line 139, in <module>
    asyncio.get_event_loop().run_until_complete(main())
  File "C:\Program Files (x86)\Python37-32\lib\asyncio\base_events.py", line 584, in run_until_complete
    return future.result()
  File "G:\My Drive\Dev\Python\Fin.py", line 135, in main
    await getBill(browser,bills[1])
  File "G:\My Drive\Dev\Python\Fin.py", line 128, in getBill
    bill=await page.evaluate(n['BillFunc'])
  File "C:\Program Files (x86)\Python37-32\lib\site-packages\pyppeteer\page.py", line 1186, in evaluate
    return await frame.evaluate(pageFunction, *args, force_expr=force_expr)
  File "C:\Program Files (x86)\Python37-32\lib\site-packages\pyppeteer\frame_manager.py", line 309, in evaluate
    pageFunction, *args, force_expr=force_expr)
  File "C:\Program Files (x86)\Python37-32\lib\site-packages\pyppeteer\execution_context.py", line 54, in evaluate
    pageFunction, *args, force_expr=force_expr)
  File "C:\Program Files (x86)\Python37-32\lib\site-packages\pyppeteer\execution_context.py", line 113, in evaluateHandle
    helper.getExceptionMessage(exceptionDetails)))
pyppeteer.errors.ElementHandleError: Evaluation failed: SyntaxError: Invalid or unexpected token

Process returned 1 (0x1)        execution time : 6.240 s
Press any key to continue . . .

I have tried to debug it many times, running the script in the chrome console manually provides the result, however the Evaluate fails. And it seems to fail on the nested function, cause as soon as I take the nested function out...it works again but does not provide the desired result.

SeverinAlexB commented 4 years ago

I have a similar bug although I don't know if its related.

await self.page.evaluate('''
       await new Promise((resolve, reject) => {
            var totalHeight = 0;
            var distance = 100;
            var timer = setInterval(() => {
                var scrollHeight = document.body.scrollHeight;
                window.scrollBy(0, distance);
                totalHeight += distance;

                if(totalHeight >= scrollHeight){
                    clearInterval(timer);
                    resolve();
                }
            }, 100);
        });
    '''
)

Exception:

  File "/Users/severinbuhler/git/ecom_scraper_puppeter/crawler/scraper.py", line 47, in request
    await self._scroll_to_bottom()
  File "/Users/severinbuhler/git/ecom_scraper_puppeter/crawler/scraper.py", line 16, in _scroll_to_bottom
    return await self.page.evaluate(
  File "/Users/severinbuhler/git/ecom_scraper_puppeter/venv/lib/python3.8/site-packages/pyppeteer/page.py", line 1158, in evaluate
    return await frame.evaluate(pageFunction, *args, force_expr=force_expr)
  File "/Users/severinbuhler/git/ecom_scraper_puppeter/venv/lib/python3.8/site-packages/pyppeteer/frame_manager.py", line 294, in evaluate
    return await context.evaluate(
  File "/Users/severinbuhler/git/ecom_scraper_puppeter/venv/lib/python3.8/site-packages/pyppeteer/execution_context.py", line 54, in evaluate
    handle = await self.evaluateHandle(
  File "/Users/severinbuhler/git/ecom_scraper_puppeter/venv/lib/python3.8/site-packages/pyppeteer/execution_context.py", line 113, in evaluateHandle
    raise ElementHandleError('Evaluation failed: {}'.format(
pyppeteer.errors.ElementHandleError: Evaluation failed: SyntaxError: Unexpected token new

Process finished with exit code 1