id: robots-txt-endpoint

info:
  name: robots.txt endpoint prober
  author: CasperGN,pdteam,TheZakMan
  severity: info
  metadata:
    max-request: 1
    verified: true
  tags: miscellaneous,misc,generic,discovery

http:
  - method: GET
    path:
      - "{{BaseURL}}/robots.txt"

    extractors:
      - type: regex
        name: endpoints
        group: 1
        regex:
          - '(?i:(?:Allow|Disallow):\s*(/[\.a-zA-Z0-9/_-]+/?))'

    matchers-condition: and
    matchers:
      - type: word
        part: body
        words:
          - 'User-agent:'

      - type: word
        part: content_type
        words:
          - text/plain

      - type: status
        status:
          - 200
# digest: 490a0046304402202954b12f564fb4fe2c1a8806f87edbc65991a3993c2bd86422d77c871100b4e60220029b78a01a1929151d2a554765c2c517078708cf1536aec0d73d51186784b004:922c64590222798bb761d5b6d8e72950