Comportamentos distintos em execuções de uma mesma coleta

Comportamento Esperado

Uma instância de coleta é efetuada localmente, sem apresentar erros. E quando passada para a infra do MP, se comporta da mesma maneira, finalizando corretamente sem interrupções.

Comportamento Atual

A coleta indicada na seção seguinte executou bem localmente mas retornou erros ao ser executada no MP. E a cada execução, o comportamento da coleta apresenta mudanças. Os novos erros geralmente se referem a Xpaths não encontrados, mas presentes na página em questão (como pode ser verificado a partir dos screenshots da coleta e na tentativa de reprodução do comportamento manualmente).

Especificações da Coleta

Arquivo de configuração: { "source_name": "[Realiza\u00e7\u00e3o-F01] Coleta de Despesas - Relat\u00f3rios e Contas P\u00fablicas de Luz", "base_url": "https://e-gov.betha.com.br/transparencia/01037-132/recursos.faces?mun=e7QgmIkky-ywyWyWbpGr7_da3er16J-5", "obey_robots": false, "data_path": "/datalake/ufmg/webcrawlerc01/realizacaof01/luz/Despesas/Relatorios_e_Contas_publicas", "request_type": "GET", "form_request_type": "POST", "antiblock_download_delay": 2, "antiblock_autothrottle_enabled": false, "antiblock_autothrottle_start_delay": 2, "antiblock_autothrottle_max_delay": 10, "antiblock_ip_rotation_enabled": false, "antiblock_ip_rotation_type": "tor", "antiblock_max_reqs_per_ip": 10, "antiblock_max_reuse_rounds": 10, "antiblock_proxy_list": "", "antiblock_user_agent_rotation_enabled": false, "antiblock_reqs_per_user_agent": 100, "antiblock_user_agents_list": "", "antiblock_insert_cookies_enabled": false, "antiblock_cookies_list": "", "captcha": "none", "has_webdriver": false, "webdriver_path": null, "img_xpath": null, "sound_xpath": null, "dynamic_processing": true, "explore_links": false, "link_extractor_max_depth": null, "link_extractor_allow_url": null, "link_extractor_allow_domains": null, "link_extractor_tags": null, "link_extractor_attrs": null, "link_extractor_check_type": false, "link_extractor_process_value": "", "download_files": false, "download_files_allow_url": null, "download_files_allow_extensions": null, "download_files_allow_domains": null, "download_files_tags": null, "download_files_attrs": null, "download_files_process_value": "", "download_files_check_large_content": true, "download_imgs": false, "steps": "{\"step\":\"root\",\"depth\":0,\"children\":[{\"step\":\"espere\",\"depth\":1,\"arguments\":{\"segundos\":\"4\"}},{\"step\":\"screenshot\",\"depth\":1,\"arguments\":{}},{\"step\":\"atribuicao\",\"depth\":1,\"target\":\"contas_publicas\",\"source\":{\"call\":{\"step\":\"objeto\",\"arguments\":{\"objeto\":\"\\\"/html/body/div/div[3]/div/div/div/form/div/div/div/ul/li[14]/div[2]/div[2]/ul/li/a\\\"\"}}}},{\"step\":\"clique\",\"depth\":1,\"arguments\":{\"elemento\":\"contas_publicas\"}},{\"step\":\"espere\",\"depth\":1,\"arguments\":{\"segundos\":\"4\"}},{\"step\":\"screenshot\",\"depth\":1,\"arguments\":{}},{\"step\":\"para_cada\",\"depth\":1,\"iterator\":\"opcao\",\"children\":[{\"step\":\"selecione\",\"depth\":2,\"arguments\":{\"xpath\":\"\\\"/html/body/div[1]/div[1]/div/div/div/form/div[1]/div[1]/span/div[1]/div[1]/div/label/span/select\\\"\",\"opcao\":\"opcao\"}},{\"step\":\"espere\",\"depth\":2,\"arguments\":{\"segundos\":\"2\"}},{\"step\":\"imprime\",\"depth\":2,\"arguments\":{\"texto\":\"f\\\"Coletando dados de {opcao}:\\\"\"}},{\"step\":\"screenshot\",\"depth\":2,\"arguments\":{}},{\"step\":\"para_cada\",\"depth\":2,\"iterator\":\"ano\",\"children\":[{\"step\":\"selecione\",\"depth\":3,\"arguments\":{\"xpath\":\"\\\"/html/body/div[1]/div[1]/div/div/div/form/div[1]/div[1]/span/div[1]/div[2]/div[1]/label/span/select\\\"\",\"opcao\":\"ano\"}},{\"step\":\"clique\",\"depth\":3,\"arguments\":{\"elemento\":\"\\\"/html/body/div[1]/div[1]/div/div/div/form/div[1]/div[1]/span/div[2]/span[2]/input\\\"\"}},{\"step\":\"espere\",\"depth\":3,\"arguments\":{\"segundos\":\"4\"}},{\"step\":\"atribuicao\",\"depth\":3,\"target\":\"num_paginas\",\"source\":{\"call\":{\"step\":\"objeto\",\"arguments\":{\"objeto\":\"1\"}}}},{\"step\":\"para_cada\",\"depth\":3,\"iterator\":\"ultima_pagina\",\"children\":[{\"step\":\"clique\",\"depth\":4,\"arguments\":{\"elemento\":\"ultima_pagina\"}},{\"step\":\"espere\",\"depth\":4,\"arguments\":{\"segundos\":\"8\"}},{\"step\":\"screenshot\",\"depth\":4,\"arguments\":{}},{\"step\":\"imprime\",\"depth\":4,\"arguments\":{\"texto\":\"\\\"Extraindo n\u00famero de p\u00e1ginas!\\\"\"}},{\"step\":\"atribuicao\",\"depth\":4,\"target\":\"num_paginas\",\"source\":{\"call\":{\"step\":\"extrai_texto\",\"arguments\":{\"xpath\":\"\\\"/html/body/div[1]/div[1]/div/div/div/form/div[1]/div[1]/div[2]/div[2]/div/div/a[last()]\\\"\"}}}},{\"step\":\"clique\",\"depth\":4,\"arguments\":{\"elemento\":\"\\\"/html/body/div[1]/div[1]/div/div/div/form/div[1]/div[1]/div[2]/div[2]/div/div/a[1]\\\"\"}},{\"step\":\"espere\",\"depth\":4,\"arguments\":{\"segundos\":\"4\"}}],\"iterable\":{\"call\":{\"step\":\"localiza_elementos\",\"arguments\":{\"xpath\":\"\\\"/html/body/div[1]/div[1]/div/div/div/form/div[1]/div[1]/div[2]/div[2]/div/div/a[text()='\u00daltima']\\\"\"}}}},{\"step\":\"para_cada\",\"depth\":3,\"iterator\":\"pagina\",\"children\":[{\"step\":\"para_cada\",\"depth\":4,\"iterator\":\"pagina_numero\",\"children\":[{\"step\":\"clique\",\"depth\":5,\"arguments\":{\"elemento\":\"pagina_numero\"}}],\"iterable\":{\"call\":{\"step\":\"localiza_elementos\",\"arguments\":{\"xpath\":\"f\\\"/html/body/div[1]/div[1]/div/div/div/form/div[1]/div[1]/div[2]/div[2]/div/div/a[text()='{pagina+1}']\\\"\"}}}},{\"step\":\"espere\",\"depth\":4,\"arguments\":{\"segundos\":\"4\"}},{\"step\":\"imprime\",\"depth\":4,\"arguments\":{\"texto\":\"f\\\"Ano {ano} : Pagina {pagina+1}\\\"\"}},{\"step\":\"screenshot\",\"depth\":4,\"arguments\":{}},{\"step\":\"para_cada\",\"depth\":4,\"iterator\":\"anexo\",\"children\":[{\"step\":\"clique\",\"depth\":5,\"arguments\":{\"elemento\":\"anexo\"}},{\"step\":\"espere\",\"depth\":5,\"arguments\":{\"segundos\":\"2\"}},{\"step\":\"para_cada\",\"depth\":5,\"iterator\":\"botao_baixar\",\"children\":[{\"step\":\"clique\",\"depth\":6,\"arguments\":{\"elemento\":\"botao_baixar\"}},{\"step\":\"espere\",\"depth\":6,\"arguments\":{\"segundos\":\"4\"}},{\"step\":\"clique\",\"depth\":6,\"arguments\":{\"elemento\":\"\\\"/html/body/div[1]/div[1]/div/div/div/form/div[6]/div/div/a\\\"\"}},{\"step\":\"espere\",\"depth\":6,\"arguments\":{\"segundos\":\"2\"}},{\"step\":\"imprime\",\"depth\":6,\"arguments\":{\"texto\":\"f\\\"Baixando anexo {anexo}\\\"\"}}],\"iterable\":{\"call\":{\"step\":\"localiza_elementos\",\"arguments\":{\"xpath\":\"\\\"/html/body/div[1]/div[1]/div/div/div/form/div[6]/div/div/div/div/div/div[2]/table/tbody/tr[1]/td[2]/a\\\"\"}}}}],\"iterable\":{\"call\":{\"step\":\"localiza_elementos\",\"arguments\":{\"xpath\":\"\\\"/html/body/div[1]/div[1]/div/div/div/form/div[1]/div[1]/div[2]/div[1]/div[3]/table/tbody/tr[*]/td[7]/div[1]/a\\\"\"}}}}],\"iterable\":{\"call\":{\"step\":\"repete\",\"arguments\":{\"vezes\":\"int(num_paginas)\"}}}},{\"step\":\"clique\",\"depth\":3,\"arguments\":{\"elemento\":\"\\\"/html/body/div[1]/div[1]/div/div/div/form/div[1]/div[1]/span/div[2]/span[1]/input\\\"\"}},{\"step\":\"espere\",\"depth\":3,\"arguments\":{\"segundos\":\"4\"}}],\"iterable\":{\"call\":{\"step\":\"opcoes\",\"arguments\":{\"xpath\":\"\\\"/html/body/div[1]/div[1]/div/div/div/form/div[1]/div[1]/span/div[1]/div[2]/div[1]/label/span/select\\\"\"}}}}],\"iterable\":{\"call\":{\"step\":\"objeto\",\"arguments\":{\"objeto\":\"[\\\"Anexos anuais\\\"]\"}}}},{\"step\":\"selecione\",\"depth\":1,\"arguments\":{\"xpath\":\"\\\"/html/body/div[1]/div[1]/div/div/div/form/div[1]/div[1]/span/div[1]/div[1]/div/label/span/select\\\"\",\"opcao\":\"\\\"Lei de Responsabilidade Fiscal\\\"\"}},{\"step\":\"espere\",\"depth\":1,\"arguments\":{\"segundos\":\"2\"}},{\"step\":\"para_cada\",\"depth\":1,\"iterator\":\"ano\",\"children\":[{\"step\":\"selecione\",\"depth\":2,\"arguments\":{\"xpath\":\"\\\"/html/body/div[1]/div[1]/div/div/div/form/div[1]/div[1]/span/div[1]/div[2]/div[1]/label/span/select\\\"\",\"opcao\":\"ano\"}},{\"step\":\"espere\",\"depth\":2,\"arguments\":{\"segundos\":\"4\"}},{\"step\":\"para_cada\",\"depth\":2,\"iterator\":\"grupo\",\"children\":[{\"step\":\"selecione\",\"depth\":3,\"arguments\":{\"xpath\":\"\\\"/html/body/div[1]/div[1]/div/div/div/form/div[1]/div[1]/span/div[1]/div[3]/div/label/span/select\\\"\",\"opcao\":\"grupo\"}},{\"step\":\"espere\",\"depth\":3,\"arguments\":{\"segundos\":\"4\"}},{\"step\":\"clique\",\"depth\":3,\"arguments\":{\"elemento\":\"\\\"/html/body/div[1]/div[1]/div/div/div/form/div[1]/div[1]/span/div[2]/span[2]/input\\\"\"}},{\"step\":\"espere\",\"depth\":3,\"arguments\":{\"segundos\":\"6\"}},{\"step\":\"screenshot\",\"depth\":3,\"arguments\":{}},{\"step\":\"atribuicao\",\"depth\":3,\"target\":\"num_paginas\",\"source\":{\"call\":{\"step\":\"objeto\",\"arguments\":{\"objeto\":\"1\"}}}},{\"step\":\"para_cada\",\"depth\":3,\"iterator\":\"ultima_pagina\",\"children\":[{\"step\":\"clique\",\"depth\":4,\"arguments\":{\"elemento\":\"ultima_pagina\"}},{\"step\":\"espere\",\"depth\":4,\"arguments\":{\"segundos\":\"8\"}},{\"step\":\"screenshot\",\"depth\":4,\"arguments\":{}},{\"step\":\"imprime\",\"depth\":4,\"arguments\":{\"texto\":\"\\\"Extraindo n\u00famero de p\u00e1ginas!\\\"\"}},{\"step\":\"atribuicao\",\"depth\":4,\"target\":\"num_paginas\",\"source\":{\"call\":{\"step\":\"extrai_texto\",\"arguments\":{\"xpath\":\"\\\"/html/body/div[1]/div[1]/div/div/div/form/div[1]/div[1]/div[2]/div[2]/div/div/a[last()]\\\"\"}}}},{\"step\":\"clique\",\"depth\":4,\"arguments\":{\"elemento\":\"\\\"/html/body/div[1]/div[1]/div/div/div/form/div[1]/div[1]/div[2]/div[2]/div/div/a[1]\\\"\"}},{\"step\":\"espere\",\"depth\":4,\"arguments\":{\"segundos\":\"4\"}}],\"iterable\":{\"call\":{\"step\":\"localiza_elementos\",\"arguments\":{\"xpath\":\"\\\"/html/body/div[1]/div[1]/div/div/div/form/div[1]/div[1]/div[2]/div[2]/div/div/a[text()='\u00daltima']\\\"\"}}}},{\"step\":\"imprime\",\"depth\":3,\"arguments\":{\"texto\":\"f\\\"{ano} : {num_paginas} p\u00e1ginas\\\"\"}},{\"step\":\"para_cada\",\"depth\":3,\"iterator\":\"pagina\",\"children\":[{\"step\":\"para_cada\",\"depth\":4,\"iterator\":\"pagina_numero\",\"children\":[{\"step\":\"clique\",\"depth\":5,\"arguments\":{\"elemento\":\"pagina_numero\"}},{\"step\":\"espere\",\"depth\":5,\"arguments\":{\"segundos\":\"4\"}}],\"iterable\":{\"call\":{\"step\":\"localiza_elementos\",\"arguments\":{\"xpath\":\"f\\\"/html/body/div[1]/div[1]/div/div/div/form/div[1]/div[1]/div[2]/div[2]/div/div/a[text()='{pagina+1}']\\\"\"}}}},{\"step\":\"imprime\",\"depth\":4,\"arguments\":{\"texto\":\"f\\\"{grupo} : Pagina {pagina+1}\\\"\"}},{\"step\":\"para_cada\",\"depth\":4,\"iterator\":\"anexo\",\"children\":[{\"step\":\"clique\",\"depth\":5,\"arguments\":{\"elemento\":\"anexo\"}},{\"step\":\"espere\",\"depth\":5,\"arguments\":{\"segundos\":\"2\"}},{\"step\":\"screenshot\",\"depth\":5,\"arguments\":{}},{\"step\":\"clique\",\"depth\":5,\"arguments\":{\"elemento\":\"\\\"/html/body/div[1]/div[1]/div/div/div/form/div[6]/div/div/div/div/div/div[2]/table/tbody/tr/td[2]/a\\\"\"}},{\"step\":\"espere\",\"depth\":5,\"arguments\":{\"segundos\":\"4\"}},{\"step\":\"clique\",\"depth\":5,\"arguments\":{\"elemento\":\"\\\"/html/body/div[1]/div[1]/div/div/div/form/div[6]/div/div/a\\\"\"}},{\"step\":\"espere\",\"depth\":5,\"arguments\":{\"segundos\":\"2\"}},{\"step\":\"imprime\",\"depth\":5,\"arguments\":{\"texto\":\"f\\\"Baixando anexo {anexo}\\\"\"}}],\"iterable\":{\"call\":{\"step\":\"localiza_elementos\",\"arguments\":{\"xpath\":\"\\\"/html/body/div[1]/div[1]/div/div/div/form/div[1]/div[1]/div[2]/div[1]/div[3]/table/tbody/tr[*]/td[8]/div[1]/a\\\"\"}}}}],\"iterable\":{\"call\":{\"step\":\"repete\",\"arguments\":{\"vezes\":\"int(num_paginas)\"}}}},{\"step\":\"clique\",\"depth\":3,\"arguments\":{\"elemento\":\"\\\"/html/body/div[1]/div[1]/div/div/div/form/div[1]/div[1]/span/div[2]/span[1]/input\\\"\"}},{\"step\":\"espere\",\"depth\":3,\"arguments\":{\"segundos\":\"4\"}}],\"iterable\":{\"call\":{\"step\":\"objeto\",\"arguments\":{\"objeto\":\"[\\\"RGF\\\", \\\"RREO\\\"]\"}}}}],\"iterable\":{\"call\":{\"step\":\"opcoes\",\"arguments\":{\"xpath\":\"\\\"/html/body/div[1]/div[1]/div/div/div/form/div[1]/div[1]/span/div[1]/div[2]/div[1]/label/span/select\\\"\"}}}}]}", "encoding_detection_method": 1, "templated_url_parameter_handlers": [], "static_form_parameter_handlers": [], "templated_url_response_handlers": [], "static_form_response_handlers": [], "crawler_id": 782, "instance_id": "164512155772972" }

MPMG-DCC-UFMG / C01