File Match Rule no filtering unwanted result

[kenkenchow]

I want to filter out the path that end with search.asp* but no luck. May I know how to fix it with below gopa.yml?

file_match_rule:
must:
prefix: []
contain: []
suffix: []
should:
prefix: []
contain: []
suffix: []
must_not:
prefix:
- /search.asp
- search.asp
contain:
- /search.asp
- search.asp
suffix:
- /search.asp
- search.asp

[medcl]

may i got the full config?

[kenkenchow]

cluster:
name: gopa
#node.name: node1

path.data: data
path.logs: log
path.certs: test/cert

cookie_secret: YOUR-GOPA-SECRET

allow_multi_instance: true
max_num_of_instances: 1

tls_enabled: false

logging.level: debug

network:
host: 127.0.0.1

cluster.seeds:

• 127.0.0.1:10000

modules:

• name: pipeline
  enabled: true
  runners:

  • name: checker
    enabled: true
    input_queue: check
    max_go_routine: 5
    threshold_in_ms: 0
    timeout_in_ms: 5000
    default_config:
    start:
    joint: init_task
    enabled: true
    process:
    - joint: url_normalization
    enabled: true
    parameters:
    follow_all_domain: false
    follow_sub_domain: true
    max_filename_length: 1000
    - joint: url_filter
    enabled: true
    parameters:
    url_match_rule:
    must_not:
    prefix:
    - "mailto:"
    - "data:image/"
    - "#"
    - "javascript:"
    contain:
    - "crond/run"
    - "login"
    - "logout"
    suffix: []
    must:
    prefix: []
    contain: []
    suffix: []
    should:
    prefix: []
    contain: []
    suffix: []
    host_match_rule:
    must:
    prefix: []
    contain: []
    suffix: []
    should:
    prefix: []
    contain:
    - xxxxx.xxxx.com
    suffix: []
    must_not:
    prefix: []
    contain: []
    suffix: []
    file_match_rule:
    must:
    prefix: []
    contain: []
    suffix: []
    should:
    prefix: []
    contain: []
    suffix: []
    must_not:
    prefix: []
    contain:
    - search
    - search.asp
    suffix: []
    file_ext_match_rule:
    should:
    prefix: []
    contain: []
    suffix: []
    must:
    prefix: []
    contain: []
    suffix: []
    must_not:
    contain: [zip, exe, jar, js, css, rar, gz, zip, bmp, jpeg, gif,svg, png, jpg, apk]
    prefix: []
    suffix: []
    - joint: filter_check
    enabled: true
    parameters:
    filter_key: check_filter
    - joint: task_deduplication
    enabled: true
    end:
    joint: save_task
    enabled: true
    parameters:
    is_create: true
  • name: fetch
    enabled: true
    input_queue: fetch
    max_go_routine: 1
    threshold_in_ms: 0
    timeout_in_ms: 30000
    default_config:
    start:
    joint: init_task
    enabled: true
    process:
    - joint: url_normalization
    enabled: true
    parameters:
    follow_all_domain: false
    follow_sub_domain: true
    max_filename_length: 1000
    - joint: ignore_timeout
    parameters:
    ignore_timeout_after_count: 500
    enabled: true
    - joint: chrome_fetch
    enabled: false
    parameters:
    chrome_host: http://127.0.0.1:9223
    save_screenshot: true
    compress_enabled: true
    timeout_in_seconds: 60
    screenshot_quality: 50
    - joint: fetch
    enabled: true
    parameters:
    timeout_in_seconds: 100
    - joint: parse
    enabled: true
    parameters:
    dispatch_links: true
    save_images: true
    max_depth: 100
    max_breadth: 1
    - joint: extract
    enabled: false
    parameters:
    html_block:
    your_tag_name1: ".tag_class"
    your_tag_name2: "#tag_id"
    - joint: html2text
    parameters:
    remove_nonscript: true
    merge_whitespace: false
    enabled: true
    - joint: hash
    enabled: true
    - joint: content_deduplication
    enabled: true
    - joint: update_check_time
    enabled: true
    parameters:
    decelerate_steps: "24h,48h,72h,168h,360h,720h"
    accelerate_steps: "24h,12h,6h,3h,1h30m,45m,30m,20m,10m"
    - joint: lang_detect
    enabled: true
    - joint : save_snapshot_fs
    enabled: false
    - joint : save_snapshot_db
    enabled: true
    parameters:
    compress_enabled: true
    bucket: "Snapshot"
    max_revision: 5
    - joint: index
    enabled: true
    end:
    joint: save_task
    enabled: true
    keep_404: false
    keep_redirected: false

• name: cluster
  enabled: true

• name: dispatch
  enabled: true
  failed_retry: false
  new_task: true
  update_task: true
  max_concurrent_fetch_tasks: 10
  fetch_size: 1000
  get_host_config_by_url: false

• name: api
  enabled: true

• name: web
  enabled: true
  auth:
  enabled: false
  oauth_provider: github
  oauth_authorize_url: https://github.com/login/oauth/authorize
  oauth_token_url: https://github.com/login/oauth/access_token
  client_id: 850d747174ace88ce889
  client_secret: 3d437b64e06371d6f62769320438d3dfc95a8d8e
  authorized_admin:
  - medcl
  - your_github_login_name

• name: filter
  enabled: true

• name: queue
  enabled: true

• name: index
  enabled: true
  ui:
  enabled: true
  elasticsearch:
  endpoint: http://localhost:9200
  index_prefix: gopa-
  username: elastic
  password: changeme

• name: elastic
  enabled: true
  store_enabled: true
  orm_enabled: true
  elasticsearch:
  endpoint: http://localhost:9200
  index_prefix: gopa-
  username: elastic
  password: changeme

• name: stats
  enabled: true

• name: statsd
  enabled: false
  host: 127.0.0.1
  port: 8125
  namespace: gopa.
  interval_in_seconds: 1

plugins:

• name: chrome
  enabled: true
  command: "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
  debug_port: 9223