Crawler test - Githubissues

Alhajras commented 11 months ago

all links = [ "/", "/mobile/separate_desktop", "/mobile/desktop_with_AMP_as_mobile", "/mobile/separate_desktop_with_different_h1", "/mobile/separate_desktop_with_different_title", "/mobile/separate_desktop_with_different_wordcount", "/mobile/separate_desktop_with_different_links_in", "/mobile/separate_desktop_with_different_links_out", "/mobile/separate_desktop_with_mobile_not_subdomain", "/mobile/desktop_with_self_canonical_mobile_and_amp", "/mobile/separate_mobile_with_mobile_not_subdomain", "/mobile/dynamic", "/mobile/responsive", "/mobile/no_mobile_configuration", "/mobile/other_desktop_that_links_to_the_same_mobile_pages", "/mobile/amp_with_separate_mobile", "/mobile/responsive_with_amp", "/mobile/amp_with_responsive", "/mobile/no_mobile_with_amp", "/mobile/amp_with_no_mobile", "/mobile/amp_no_references", "/mobile/amp_as_desktop_amp_and_mobile", "/mobile/separate_amp_with_self_canonical", "/mobile/separate_desktop_irregular_media", "/mobile/separate_desktop_response_header_alt", "/description_tags/description_with_whitespace", "/description_tags/missing_description", "/description_tags/no_description_nosnippet", "/description_tags/duplicate_description", "/description_tags/duplicate_description/foo", "/description_tags/duplicate_description_and_noindex", "/description_tags/duplicate_description_and_noindex/foo", "/description_tags/description_over_max", "/description_tags/short_meta_description", "/description_tags/description_http_equiv", "/encoding/page_titles_character_encoded", "/encoding/url_with_foreign_characters/בלהבלה", "/encoding/url_with_foreign_characters/すべての単語が高校程度の辞書に載っている", "/encoding/url_with_foreign_characters/pchnąć-w-tę-łódź-jeża-lub-ośm-skrzyń-fig", "/encoding/url_with_foreign_characters/Шеф-взъярён-тчк-щипцы-с-эхом-гудбай-Жюль", "/encoding/url_with_foreign_characters/Zwölf-große-Boxkämpfer-jagen-Viktor-quer-über-den-Sylter-Deich", "/encoding/url_with_foreign_characters/Fabio-me-exige-sin-tapujos-que-añada-cerveza-al-whisky", "/encoding/url_with_foreign_characters/﴿محمد-رسول-الله-والذين-معه-أشداء", "/encoding/url_with_foreign_characters/γράμματα-του-ισπανικού-αλφαβήτου-καθώς", "/encoding/url_with_foreign_characters/asød-æada-ådjghf-gägfd-asödsads", "/encoding/double_encoded_url/Zw%25C3%25B6lf-gro%25C3%259Fe-Boxk%25C3%25A4mpfer-jagen-Viktor-quer-%25C3%25BCber-den-Sylter-Deich", "/encoding/inconsistent_character_encoding", "http://www.søkbar.no", "/encoding/url/encoded_hashbang%23abc", "/titles/title_with_whitespace", "/titles/empty_title", "/titles/missing_title", "/titles/duplicate_title", "/titles/duplicate_title/foo", "/titles/duplicate_title/foo", "/titles/duplicate_title/bar", "/titles/duplicate_title/baz", "/titles/duplicate_title_and_noindex/bat", "/titles/duplicate_title_and_noindex/bak", "/titles/title_over_max", "/titles/title_warning", "/titles/page_title_length_n", "/titles/page_title_width_n", "/titles/page_title_leading_trailing_spaces", "/titles/double_triple_quadruple_spaces", "/titles/svg_title", "/titles/forced_double_triple_quadruple_spaces", "/robots_protocol/robots_excluded", "/robots_protocol/deepcrawl_excluded", "/robots_protocol/robots_excluded_duplicate_description", "/robots_protocol/robots_excluded_meta_noindex", "/robots_protocol/deepcrawl_ua_disallow/foo", "/robots_protocol/user_excluded", "/robots_protocol/meta_nofollow", "/robots_protocol/meta_noarchive", "/robots_protocol/meta_noindex", "/robots_protocol/meta_noindex_uppercase", "/robots_protocol/x_robots_tag_noindex", "/robots_protocol/page_allowed_with_robots", "/robots_protocol/robots_noindexed", "/robots_protocol/robots_noindex_conflict", "/robots_protocol/robots_excluded_blank_line", "/robots_protocol/robots_noindexed_and_robots_disallowed", "/robots_protocol/allowed_same_length", "/robots_protocol/allowed_shorter", "/robots_protocol/allowed_longer", "/robots_protocol/meta_robots_and_x_robots_conflict", "/robots_protocol/robots_meta_none", "/robots_protocol/robots_meta_noodp_noydir_none_noindex", "/robots_protocol/robots_meta_multiple_tags_noindex_nofollow", "/robots_protocol/x_robots_multiple_directives", "/robots_protocol/multiple_robots_directive_meta_tag", "/robots_protocol/multiple_googlebot_directive_meta_tag", "/robots_protocol/non_200_with_noindex", "/robots_protocol/canonicalised_with_noindex", "/robots_protocol/canonicalised_with_non_200", "/redirects/redirect_1", "/redirects/redirect_2", "/redirects/redirect_3_302", "/redirects/redirect_4_307", "/redirects/disallowed_redirect", "/redirects/redirect_chain_allowed", "/redirects/disallowed_redirect_target_redirect", "/redirects/infinite_redirect", "/redirects/two_step_redirect_loop_1", "/redirects/external_redirect", "/redirects/invalid_redirect", "/redirects/meta_redirect_1", "/redirects/meta_redirect_2", "/redirects/meta_redirect_3", "/redirects/infinite_meta_redirect", "/redirects/external_meta_redirect", "/redirects/invalid_meta_redirect", "/redirects/header_refresh_redirect", "/redirects/redirect_to_404", "/redirects/url_redirect_chains", "/redirects/redirect_content", "/redirects/external_redirect_chain1", "/redirects/redirect_300", "/redirects/redirect_303", "/redirects/redirect_304", "/redirects/redirect_305", "/redirects/redirect_306", "/redirects/redirect_308", "/javascript/window-location-internal", "/javascript/window-location-external", "/javascript/window-location-function-absolute", "/javascript/window-location-function-relative", "/links/broken_links_internal", "/links/broken_links_external", "/links/max_external_links", "/links/page_with_external_links", "/links/nofollowed_page", "/links/nofollow_link_with_nofollowed_backlinks", "/links/relative_link/a/b", "/links/relative_link_with_base/a/b", "/links/image_links", "/links/non_default_language", "/links/meta_refresh", "/links/header_refresh", "/links/external_links_to_disallwed_urls", "/links/non_standard_links", "/links/repeated_external_links", "/links/repeated_internal_links", "/links/links_with_quote_variations", "/links/whitespace_in_links", "/links/comma_separated_attributes", "/links/nofollow_and_follow_link", "//crawler-test.com/links/relative_protocol_page", "/javascript/window-location-onchange", "/javascript/window-open", "/javascript/onmousedown", "/javascript/concatenatedlink", "/javascript/data-hreflink", "/javascript/push_state", "/urls/double_slash//one", "/urls/double_slash////two", "/urls/double_slash//////three", "/urls/double_slash//////four//", "/urls/parameter_1_1?parameter_1=foo", "/urls/parameter_1_2?parameter_x=x&parameter_1=foo", "/urls/parameter_1_3?parameter_x=x&parameter_1=foo&parameter_y=y", "/urls/parameter_2_1?parameter_1=foo", "/urls/parameter_2_2?parameter_x=x&parameter_1=foo", "/urls/parameter_2_3?parameter_x=x&parameter_1=foo&parameter_y=y", "/urls/parameter?parameter_1=foo&parameter_1=bar", "/urls/parameter?parameter_1=foo&parameter_1=foo", "/urls/url_with_spaces/URL with spaces", "/urls/url_with_trailing_space/%20", "/urls/url_with_encoded_trailing_space/ ", "/urls/duplication_types/", "/urls/duplication_types", "/urls/duplication_types?tracking=yes", "/urls/duplication_types/index.html", "/urls/duplication_types/duplication_types/", "/urls/DUPLICATION_TYPES", "/urls/links_to_malformed_urls", "/urls/paginated_page", "/urls/unlinked_paginated_page", "/urls/paginated_and_noindex_page", "/urls/links_to_non_html_filetypes", "/urls/pages_with_hreflang", "/urls/page_with_hreflang_header_ok", "/urls/page_with_hreflang_header_not_ok", "/urls/duplicate_page", "/urls/duplicate_page/foo", "/urls/duplicate_page/bar", "/urls/duplicate_page/baz", "/urls/with_session_id?sessionID=qNVyaAoX5CE5L7hqR4WD", "/urls/page_url_length_n", "/urls/url_with_fragment#fragment", "/url/URL%2c_with_encoded_reserved_character", "/url/URL_with_encodedunreserved%63haracter", "/url/with_encoded_space ", "/url/with_encoded_ó_character", "/urls/directory_index/", "/urls/directory_index/index.htm", "/urls/directory_index/default.htm", "/urls/directory_index/index.html", "/urls/directory_index/index", "/infinite/", "/relative_base_source", "/one/two/three/four", "/one/two/three/four/five", "/one/two/three/four/five/six", "/one/two/three/four/five/six/seven", "/path/1/path/2", "/path/1/path/2/path/3", "/urls/multiple_slashes///200_404", "/urls/multiple_slashes///404_200", "https://crawler-test.com//urls/double_slash/disallowed_start", "/urls/double_slash//disallowed_middle", "/urls/double_slash/disallowed_end//", "?parameter-on-hostname-root=parameter-value", "?removed_parameter=1&retained_parameter=1", "/url/url_with:colon", "relativeurl:withcolon", "/canonical_tags/relative_root_canonical_tag", "/canonical_tags/relative_canonical_tag", "/canonical_tags/canonical_tag", "/canonical_tags/canonical_tag_uppercase", "/canonical_tags/canonical_tag/2", "/canonical_tags/canonical_tag/3", "/canonical_tags/canonical_tag/4", "/canonical_tags/canonical_tag/5", "/canonical_tags/canonical_tag/6", "/canonical_tags/canonical_tag/7", "/canonical_tags/canonical_tag/8", "/canonical_tags/canonical_tag/9", "/canonical_tags/canonical_tag/10", "/canonical_tags/canonical_tag/11", "/canonical_tags/canonical_tag/12", "/canonical_tags/canonical_tag/13", "/canonical_tags/canonical_tag/14", "/canonical_tags/canonical_tag/15", "/canonical_tags/canonical_tag/16", "/canonical_tags/canonical_tag/17", "/canonical_tags/canonical_tag/18", "/canonical_tags/canonical_tag/19", "/canonical_tags/canonical_tag/20", "/canonical_tags/canonical_tag/21", "/canonical_tags/canonical_tag/22", "/canonical_tags/canonical_tag/23", "/canonical_tags/canonical_tag/24", "/canonical_tags/canonical_tag/25", "/canonical_tags/canonical_tag_like_page", "/canonical_tags/canonical_duplicate_description", "/canonical_tags/canonical_tag_in_header", "/canonical_tags/canonical_tag_outside_head", "/canonical_tags/canonical_tag_html_header_conflict", "/canonical_tags/canonical_tag_html_conflict", "/canonical_tags/page_with_external_canonical", "/canonical_tags/page_without_canonical_tag", "/canonical_tags/unlinked_canonical", "/canonical_tags/canonical_tag_og_url_conflict", "/canonical_tags/canonical_tag_with_self_reference", "/canonical_tags/canonicalised_to_disallowed_url", "/canonical_tags/unlinked_canonical_header", "/canonical_tags/non_head_canonical", "/canonical_tags/non_head_canonical_link", "/canonical_tags/non_head_canonical_link_2", "/canonical_tags/canonical_port_80", "/canonical_tags/canonical_port_443", "/canonical_tags/canonical_port_8080", "/canonical_tags/canonical_url_encoded_vs_non_encoded/café", "/canonical_tags/canonical_url_encoded_vs_non_encoded/🍺/ist", "/canonical_tags/canonical_prameter_key_is_case_sensitive?key=value", "/canonical_tags/canonical_prameter_value_is_case_sensitive?key=value", "/canonical_tags/canonical_url_fragments#url-fragment", "/canonical_tags/canonical_different_parameter_order?key2=value2&key=value", "/canonical_tags/canonical_hostname_case_insensitive", "/canonical_tags/canonical_protocol_case_insensitive", "/canonical_tags/canonical_path_is_case_sensitive", "/canonical_tags/canonical_url_with_slash", "/canonical_tags/canonical_trailing_dot", "/status_codes/status_100", "/status_codes/status_101", "/status_codes/status_102", "/status_codes/status_200", "/status_codes/status_201", "/status_codes/status_202", "/status_codes/status_203", "/status_codes/status_204", "/status_codes/status_205", "/status_codes/status_206", "/status_codes/status_207", "/status_codes/status_226", "/status_codes/status_400", "/status_codes/status_401", "/status_codes/status_402", "/status_codes/status_403", "/status_codes/status_404", "/status_codes/status_405", "/status_codes/status_406", "/status_codes/status_407", "/status_codes/status_408", "/status_codes/status_409", "/status_codes/status_410", "/status_codes/status_411", "/status_codes/status_412", "/status_codes/status_413", "/status_codes/status_414", "/status_codes/status_415", "/status_codes/status_416", "/status_codes/status_417", "/status_codes/status_418", "/status_codes/status_419", "/status_codes/status_420", "/status_codes/status_421", "/status_codes/status_422", "/status_codes/status_423", "/status_codes/status_424", "/status_codes/status_426", "/status_codes/status_428", "/status_codes/status_429", "/status_codes/status_431", "/status_codes/status_440", "/status_codes/status_444", "/status_codes/status_449", "/status_codes/status_450", "/status_codes/status_451", "/status_codes/status_494", "/status_codes/status_495", "/status_codes/status_496", "/status_codes/status_497", "/status_codes/status_498", "/status_codes/status_499", "/status_codes/status_500", "/status_codes/status_501", "/status_codes/status_502", "/status_codes/status_503", "/status_codes/status_504", "/status_codes/status_505", "/status_codes/status_506", "/status_codes/status_507", "/status_codes/status_508", "/status_codes/status_509", "/status_codes/status_510", "/status_codes/status_511", "/status_codes/status_520", "/status_codes/status_598", "/status_codes/status_599", "/social_tags/open_graph_tags", "/social_tags/twitter_card_page/1", "/social_tags/twitter_card_page/2", "/social_tags/og_no_twitter", "/social_tags/max_twitter_card_description_length", "/content/custom_text", "/content/error_page", "/content/above_min_content_volume", "/content/no_h1", "/content/h1_in_img", "/content/mult_h1", "/content/page_html_size_n", "/content/page_content_size_n", "/content/meta_content_type_text_html", "/content/meta_content_type_malformed", "/content/word_count_100_words", "/content/word_count_number", "/content/word_count_hyphenated", "/content/word_count_symbols", "/content/word_count_script", "/content/custom_extraction_text", "/content/multiple_titles_and_descriptions", "/content/title_with_newline_quote_doublequote_and_comma_characters", "/content/header_content_type_malformed", "/javascript/window-location-onclick", "/javascript/onclick-reveals-element-programmatically-added-onclick", "/javascript/ajax-return-data", "/javascript/dynamically-inserted-text", "/javascript/dynamically-inserted-text-meta-data", "/javascript/dynamically-inserted-nofollow", "/javascript/onload-added-title", "/javascript/onload-inserted-canonical", "/javascript/dialog_window", "/javascript/alert_box", "/javascript/ad_script", "/javascript/analytics_script", "/javascript/renderer_timeout/1", "/javascript/renderer_timeout/2", "/javascript/renderer_timeout/3", "/javascript/renderer_timeout/4", "/javascript/renderer_timeout/5", "/content/non_secure_form_fields_text", "/content/non_secure_form_fields_email", "/content/non_secure_form_fields_search", "/content/non_secure_form_fields_number", "/content/non_secure_form_fields_tel", "/content/non_secure_form_fields_url", "/content/non_secure_form_fields_textarea", "/content/non_secure_form_fields_password_and_cc", "/other/crawler_user_agent", "/other/crawler_ip_address", "/other/conflicting_language_tags", "/other/page_load_time_n", "/other/crawler_request_headers", "/other/expiring_page_for_removed_test/1690103191.5602536", "/other/duplicated_body_content_1", "/other/duplicated_body_content_2", "/other/string_width/512/string", "/other/script_tag_contents", "/other/noodp_noydir_tags", "/other/page_with_hsts_headers", "https://subdomain.crawler-test.com", "https://invalid.crawler-test.com", "http://crawler-test.com/", "http://crawler-test.com", "https://crawler-test.com", "/other/in_web_linking", "/other/in_web_linked", "/other/typo_in_head", "/other/unfinished_tag_in_head", "/other/non_head_tag_in_head", "/other/link_tag_in_body", "/other/basic_auth" ]

Alhajras commented 11 months ago

Second run

721.9988715648651 {23834: 28, 23832: 44, 23830: 46, 23831: 150, 23828: 204, 23833: 4}

Alhajras commented 11 months ago

All link in the original page are: 415 Visited: 275 found links but never visited: 675 those links where not fisited: 214

Alhajras / webscraper

Crawler test #16