curl use --tcp-fastopen & DOH

ZeroDot1 commented 4 years ago

If --tcp-fastopen https://en.wikipedia.org/wiki/TCP_Fast_Open is used, the results can be loaded faster. Another thing would be to use DNS over HTTPS for example, this could improve security.

Examples:

curl --tcp-fastopen --tcp-nodelay exampledomain.com curl --doh-url https://cloudflare-dns.com/dns-query exampledomain.com

ZeroDot1 commented 4 years ago

OK, I just tried --tcp-fastopen, and it works much faster with my slow internet connection. I recommend implementing this with the following example.

SHODAN(){
## Using API Key
local URL_SHODAN="https://api.shodan.io/shodan/host/search?key=" ## Using API Shodan
  if [[ ! -z "$SHODAN_API" ]];then
    echo -e  "${PADDING}${YELLOW}${PADDING}⍥${PADDING}${RESET}Shodan${RESET}${DPADDING}\t\t[${GREEN} ✔ ${RESET}]"
      MAKEFILE=$(curl --tcp-fastopen --tcp-nodelay --silent --request GET --url "${URL_SHODAN}${SHODAN_API}&query=hostname:${DOMAIN}" | jq --raw-output -r '.matches[] |.hostnames[]' | sort -u > ${OUT_SHODAN})
  else
    echo -e  "${PADDING}${YELLOW}${PADDING}⍥${PADDING}${RESET}Shodan${RESET}${DPADDING}\t\t[${RED} ✕ ${RESET}]"

  fi
}

VIRUSTOTAL(){
## Using API Key
local URL_VIRUSTOTAL="https://www.virustotal.com/vtapi/v2/domain/report?apikey=" ## Using API Virus Total
  if [[ ! -z "$VIRUSTOTAL" ]];then
    echo -e  "${PADDING}${YELLOW}${PADDING}⍥${PADDING}${RESET}Virustotal${RESET}${DPADDING}\t[${GREEN} ✔ ${RESET}]"
      MAKEFILE=`curl --tcp-fastopen --tcp-nodelay --silent --request GET --url "${URL_VIRUSTOTAL}${VIRUSTOTAL}&domain=${DOMAIN}" | jq --raw-output -r '.subdomains[]?' | sort -u > ${OUT_VIRUSTOTAL}`
  #COUNT=$(cat output/vt | wc  -l  )
  else
  echo -e  "${PADDING}${YELLOW}${PADDING}⍥${PADDING}${RESET}Virustotal${RESET}${DPADDING}\t[${RED} ✕ ${RESET}]"

  fi
}

BINARYEDGE(){
## Using API Key

local URL_BINARY="https://api.binaryedge.io/v2/query/domains/subdomain/"
  if [[ ! -z "$BINARYEDGE" ]];then
    echo -e  "${PADDING}${YELLOW}${PADDING}⍥${PADDING}${RESET}Binaryedge${RESET}${DPADDING}\t[${GREEN} ✔ ${RESET}]"
      MAKEFILE=`curl --tcp-fastopen --tcp-nodelay --silent "${URL_BINARY}${DOMAIN}" -H 'X-Key:'${BINARYEDGE}''| jq --raw-output -r '.events[]?' | sort -u > ${OUT_BINARYEDGE}  `
  else
    echo -e  "${PADDING}${YELLOW}${PADDING}⍥${PADDING}${RESET}Binaryedge${RESET}${DPADDING}\t[${RED} ✕ ${RESET}]"
  fi
}

SECURITY_TRAILS(){
## Using API Key
local URL_STRAILS="https://api.securitytrails.com/v1/domain/"
  if [[ ! -z "$SECURITY_TRAILS" ]];then
    echo -e  "${PADDING}${YELLOW}${PADDING}⍥${PADDING}${RESET}Securitytrails${RESET}${DPADDING}\t[${GREEN} ✔ ${RESET}]"
  #rm -rf ${OUT_STRAILS}
      MAKEFILE=`curl --tcp-fastopen --tcp-nodelay --silent --request GET --url "${URL_STRAILS}${DOMAIN}/subdomains?apikey=${SECURITY_TRAILS}" | jq --raw-output -r '.subdomains[]' | sort -u > ${OUT_STRAILS}  `
  sed -i s/$/.${DOMAIN}/ ${OUT_STRAILS}
  ## SUFFIX DOMAIN
  else
    echo -e  "${PADDING}${YELLOW}${PADDING}⍥${PADDING}${RESET}Securitytrails${RESET}${DPADDING}\t[${RED} ✕ ${RESET}]"
  fi

}

CENSYS(){
## Using API Key
local _CENSYS="lib/censys/censys_subdomain_finder.py"
  if [[ ! -z "$CENSYS_API" ]] && [[ ! -z "$CENSYS_SECRET" ]] ;then
    echo -e  "${PADDING}${YELLOW}${PADDING}⍥${PADDING}${RESET}Censys${RESET}${DPADDING}\t\t[${GREEN} ✔ ${RESET}]"
      MAKEFILE=$(python ${_CENSYS} --censys-api-id ${CENSYS_API} --censys-api-secret ${CENSYS_SECRET} ${DOMAIN}> ${OUT_CENSYS})
  else
    echo -e  "${PADDING}${YELLOW}${PADDING}⍥${PADDING}${RESET}Censys${RESET}${DPADDING}\t\t[${RED} ✕ ${RESET}]"
  fi
}

CERTSPOTTER(){
local URL_CERTSPOTER="https://api.certspotter.com/v1/issuances?domain="
#if [[ ! -z "$VIRUSTOTAL" ]];then
  echo -e  "${PADDING}${YELLOW}${PADDING}⍥${PADDING}${RESET}Certspotter${RESET}${DPADDING}\t[${GREEN} ✔ ${RESET}]"
    curl --tcp-fastopen --tcp-nodelay --silent --request GET --url "${URL_CERTSPOTER}${DOMAIN}&include_subdomains=true&expand=dns_names" | jq --raw-output -r '.[].dns_names[]' | sed 's/\*\.//g' | tr -d "\"" | sort -u > ${OUT_CERTSPOTTER}
}

THREATMINER(){
local URL_THREATMINER="https://api.threatminer.org/v2/domain.php?q="
  echo -e  "${PADDING}${YELLOW}${PADDING}⍥${PADDING}${RESET}Threatminer${RESET}${DPADDING}\t[${GREEN} ✔ ${RESET}]"
  curl --tcp-fastopen --tcp-nodelay --silent --request GET --url "${URL_THREATMINER}${DOMAIN}&rt=5" | jq --raw-output -r '.results[]' | sort -u > ${OUT_THREATMINER}
}

BUFFEROVER(){
local URL_BUFFEROVER="dns.bufferover.run/dns?q="
  echo -e  "${PADDING}${YELLOW}${PADDING}⍥${PADDING}${RESET}Bufferover${RESET}${DPADDING}\t[${GREEN} ✔ ${RESET}]"
   curl --tcp-fastopen --tcp-nodelay --silent --request GET --url "${URL_BUFFEROVER}.${DOMAIN}&rt=5" | jq --raw-output '.FDNS_A[]' | awk '{print $1}' | sed -e 's/^.*,//g' | sort -u > ${OUT_BUFFEROVER}
}

HACKERTARGET(){
local URL_HACKERTARGET="https://api.hackertarget.com/hostsearch/?q="
  echo -e  "${PADDING}${YELLOW}${PADDING}⍥${PADDING}${RESET}Hackertarget${RESET}${DPADDING}\t[${GREEN} ✔ ${RESET}]"
  curl --tcp-fastopen --tcp-nodelay --silent --request GET --url "${URL_HACKERTARGET}${DOMAIN}" |  sed 's/,.*//' | sort -u  > ${OUT_HACKERTARGET}
}

ENTRUST(){
local URL_ENTRUST="https://ctsearch.entrust.com/api/v1/certificates?fields=subjectDN&domain="
  echo -e  "${PADDING}${YELLOW}${PADDING}⍥${PADDING}${RESET}Entrust${RESET}${DPADDING}\t\t[${GREEN} ✔ ${RESET}]"
   curl --tcp-fastopen --tcp-nodelay --silent --request GET --url "${URL_ENTRUST}${DOMAIN}&includeExpired=false&exactMatch=false&limit=5000" | jq --raw-output -r '.[].subjectDN' | sed 's/,.*//' | sed 's/\*\.//g' |  sed 's/cn=//g' | sort -u > ${OUT_ENTRUST}
}

FINDSUBDOMAIN(){
local _FINDSUBDOMAIN="https://findsubdomains.com/search/subdomains?domain="
   curl --tcp-fastopen --tcp-nodelay --silent ${_FINDSUBDOMAIN}"${DOMAIN}&page=1&per_page=100&domain=${DOMAIN}" | sed 's/\\//g' | grep -Po '(?<=data-target=").*?(?=")' > ${OUT_FINDSUBDOMAIN}
  echo -e  "${PADDING}${YELLOW}${PADDING}⍥${PADDING}${RESET}Findsubdomain${RESET}${DPADDING}\t[${GREEN} ✔ ${RESET}]"

}

THREATCROWD(){
local URL_THREATCROWD="https://threatcrowd.org/searchApi/v2/domain/report/?domain="
  echo -e  "${PADDING}${YELLOW}${PADDING}⍥${PADDING}${RESET}Threatcrowd${RESET}${DPADDING}\t[${GREEN} ✔ ${RESET}]"
    curl --tcp-fastopen --tcp-nodelay --silent --request GET --url  "${URL_THREATCROWD}${DOMAIN}" | jq --raw-output -r '.subdomains[]' | sort -u > ${OUT_THREATCROWD}
}

RIDDLER(){

local URL_RIDDLER="https://riddler.io/search/exportcsv?q=pld:"
  echo -e  "${PADDING}${YELLOW}${PADDING}⍥${PADDING}${RESET}Riddler${RESET}${DPADDING}\t\t[${GREEN} ✔ ${RESET}]"
    wget -q "${URL_RIDDLER}${DOMAIN}" --output-document=${OUT_FRIDDLER}
       cat ${OUT_FRIDDLER} | tail -n +3 | awk -F, '{print $6}' > ${OUT_RIDDLER} && rm ${OUT_FRIDDLER}
}

WEBARCHIVE(){
  echo -e  "${PADDING}${YELLOW}${PADDING}⍥${PADDING}${RESET}Webarchive${RESET}${DPADDING}\t[${GREEN} ✔ ${RESET}]"
      curl --tcp-fastopen --tcp-nodelay --silent "http://web.archive.org/cdx/search/cdx?url=*.${DOMAIN}/*&output=text&fl=original&collapse=urlkey" | sed -e 's_https*://__' -e "s/\/.*//" -e 's/:.*//' -e 's/^www\.//' | sed "/@/d" | sed -e 's/\.$//' | sort -u > ${OUT_WEBARCHIVE}
}

DNSDUMPSTER(){
local URL_DNS="https://dnsdumpster.com"
  echo -e  "${PADDING}${YELLOW}${PADDING}⍥${PADDING}${RESET}Dnsdumpster${RESET}${DPADDING}\t[${GREEN} ✔ ${RESET}]"
        local CSRF=$(curl -s ${URL_DNS} | grep -P "csrfmiddlewaretoken" | grep -Po '(?<=value=")[^"]*(?=")')
        MAKE=$(curl --tcp-fastopen --tcp-nodelay -s --cookie "csrftoken=$CSRF" -H "Referer: ${URL_DNS}" --data  "csrfmiddlewaretoken=$CSRF&targetip=${DOMAIN}" ${URL_DNS} | grep -Po '<td class="col-md-4">\K[^<]*' > ${OUT_DNSDUMPSTER})
}

CERTSH(){
local URL_CERTSH="https://crt.sh\?q\="
  echo -e  "${PADDING}${YELLOW}${PADDING}⍥${PADDING}${RESET}Certsh${RESET}${DPADDING}\t\t[${GREEN} ✔ ${RESET}]"
    curl --tcp-fastopen --tcp-nodelay -s https://crt.sh\?q\=%.${DOMAIN} | awk -v pattern="<TD>.*${DOMAIN}" '$0 ~ pattern {gsub("<[^>]*>","");gsub(//,""); print}' | sort -u | sed 's/    //' > ${OUT_CRTSH}
 }

screetsec commented 4 years ago

That's a good idea, if I had some free time I would try it out first and checkit. Oh ya areating a pull request too :))

ZeroDot1 commented 4 years ago

Hi @Screetsec, I checked everything and created a pull request. I hope everything works.

screetsec / Sudomy

curl use --tcp-fastopen & DOH #17