File Match Rule no filtering unwanted result

[kenkenchow]

I want to filter out the path that end with search.asp* but no luck. May I know how to fix it with below gopa.yml?

file_match_rule: must: prefix: [] contain: [] suffix: [] should: prefix: [] contain: [] suffix: [] must_not: prefix:

• /search.asp
• search.asp contain:
• /search.asp
• search.asp suffix:
• /search.asp
• search.asp

[medcl]

may i got the full config?

[kenkenchow]

cluster: name: gopa

node.name: node1

path.data: data path.logs: log path.certs: test/cert

cookie_secret: YOUR-GOPA-SECRET

allow_multi_instance: true max_num_of_instances: 1

tls_enabled: false

logging.level: debug

network: host: 127.0.0.1

cluster.seeds:

• 127.0.0.1:10000

modules:

• name: pipeline enabled: true runners:

  • name: checker enabled: true input_queue: check max_go_routine: 5 threshold_in_ms: 0 timeout_in_ms: 5000 default_config: start: joint: init_task enabled: true process:
    • joint: url_normalization enabled: true parameters: follow_all_domain: false follow_sub_domain: true max_filename_length: 1000
    • joint: url_filter enabled: true parameters: url_match_rule: must_not: prefix:
      • "mailto:"
      • "data:image/"
      • "#"
      • "javascript:" contain:
      • "crond/run"
      • "login"
      • "logout" suffix: [] must: prefix: [] contain: [] suffix: [] should: prefix: [] contain: [] suffix: [] host_match_rule: must: prefix: [] contain: [] suffix: [] should: prefix: [] contain:
      • xxxxx.xxxx.com suffix: [] must_not: prefix: [] contain: [] suffix: [] file_match_rule: must: prefix: [] contain: [] suffix: [] should: prefix: [] contain: [] suffix: [] must_not: prefix: [] contain:
      • search
      • search.asp suffix: [] file_ext_match_rule: should: prefix: [] contain: [] suffix: [] must: prefix: [] contain: [] suffix: [] must_not: contain: [zip, exe, jar, js, css, rar, gz, zip, bmp, jpeg, gif,svg, png, jpg, apk] prefix: [] suffix: []
    • joint: filter_check enabled: true parameters: filter_key: check_filter
    • joint: task_deduplication enabled: true end: joint: save_task enabled: true parameters: is_create: true
  • name: fetch enabled: true input_queue: fetch max_go_routine: 1 threshold_in_ms: 0 timeout_in_ms: 30000 default_config: start: joint: init_task enabled: true process:
    • joint: url_normalization enabled: true parameters: follow_all_domain: false follow_sub_domain: true max_filename_length: 1000
    • joint: ignore_timeout parameters: ignore_timeout_after_count: 500 enabled: true
    • joint: chrome_fetch enabled: false parameters: chrome_host: http://127.0.0.1:9223 save_screenshot: true compress_enabled: true timeout_in_seconds: 60 screenshot_quality: 50
    • joint: fetch enabled: true parameters: timeout_in_seconds: 100
    • joint: parse enabled: true parameters: dispatch_links: true save_images: true max_depth: 100 max_breadth: 1
    • joint: extract enabled: false parameters: html_block: your_tag_name1: ".tag_class" your_tag_name2: "#tag_id"
    • joint: html2text parameters: remove_nonscript: true merge_whitespace: false enabled: true
    • joint: hash enabled: true
    • joint: content_deduplication enabled: true
    • joint: update_check_time enabled: true parameters: decelerate_steps: "24h,48h,72h,168h,360h,720h" accelerate_steps: "24h,12h,6h,3h,1h30m,45m,30m,20m,10m"
    • joint: lang_detect enabled: true
    • joint : save_snapshot_fs enabled: false
    • joint : save_snapshot_db enabled: true parameters: compress_enabled: true bucket: "Snapshot" max_revision: 5
    • joint: index enabled: true end: joint: save_task enabled: true keep_404: false keep_redirected: false

• name: cluster enabled: true

• name: dispatch enabled: true failed_retry: false new_task: true update_task: true max_concurrent_fetch_tasks: 10 fetch_size: 1000 get_host_config_by_url: false

• name: api enabled: true

• name: web enabled: true auth: enabled: false oauth_provider: github oauth_authorize_url: https://github.com/login/oauth/authorize oauth_token_url: https://github.com/login/oauth/access_token client_id: 850d747174ace88ce889 client_secret: 3d437b64e06371d6f62769320438d3dfc95a8d8e authorized_admin:

  • medcl
  • your_github_login_name

• name: filter enabled: true

• name: queue enabled: true

• name: index enabled: true ui: enabled: true elasticsearch: endpoint: http://localhost:9200 index_prefix: gopa- username: elastic password: changeme

• name: elastic enabled: true store_enabled: true orm_enabled: true elasticsearch: endpoint: http://localhost:9200 index_prefix: gopa- username: elastic password: changeme

• name: stats enabled: true

• name: statsd enabled: false host: 127.0.0.1 port: 8125 namespace: gopa. interval_in_seconds: 1

plugins:

• name: chrome enabled: true command: "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" debug_port: 9223