Add headers for Umbraco CMS and Oracle Commerce.

[righettod]

Hi,

🎯 This PR add the headers to remove for the following software:

• Umbraco CMS.
• Oracle Commerce.

🌎 Source: I developer a script to search for HTTP response headers disclosing information using the majestic top site.

💻 Execution overview:

📖 Code:

import requests
import re
import shutil
import os
import sys
from multiprocessing import Pool
from termcolor import colored
from colorama import just_fix_windows_console

# pip install requests termcolor colorama

def loadOSHPHeaders():
    resp = requests.get("https://owasp.org/www-project-secure-headers/ci/headers_remove.json")
    return resp.json()["headers"]

def loadCommonsHeaders():
    resp = requests.get("https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers")
    headers = []
    for h in re.findall(r'/en-US/docs/Web/HTTP/Headers/([A-Za-z0-9-]+)', resp.text):
        headers.append(h)
    return headers

def loadLocalSkipHeaders():
    headers = []
    with open("local-skip.txt", mode="r", encoding="utf-8") as f:
        lines = f.read().splitlines()
    for line in lines:
        h = line.strip(" \r\n\t").lower()
        if h not in headers:
            headers.append(h)
    return headers

def loadDomains(know_headers):
    domains = []
    with open("majestic_million.csv", mode="r", encoding="utf-8") as f:
        lines = f.read().splitlines()
    for line in lines:
        domains.append(line.split(",")[2] + "#" + know_headers)
    return domains

def hunt(v):
    parts = v.split("#")
    domain = parts[0]
    ref = parts[1].split(";")
    print(f"\r=> {domain[:50]:<60}", end="", flush=True)
    try:
        resp = requests.get(f"https://{domain}", allow_redirects=True, timeout=5)
        for hd in resp.headers:
            hv = resp.headers[hd]
            if not toSkip(hd, hv,  ref):
                store(f"{hd}: {hv}", domain)
    except:
        pass
    return "OK"

def store(value, domain):
    data_file = f"data/{domain.replace('.','-')}.txt"
    with open(data_file, "a", encoding="utf-8") as f:
        f.write(f"{value}\n")

def toSkip(hdn, hdv, ref):
    hd = hdn.lower().strip(" \n\r\t")
    hv = str(hdv).strip(" \n\r\t").lower()
    expr = ["-amz-", "-azure", "p3p", "cache", "download", "request", "-goog-", "-ua-",
            "varnish", "envoy", "github", "origin", "correlation", "drupal", "allow-ranges",
            "adblock", "-fb", "cf-", "proxy", "city", "country", "postal", "region", "cluster", "zone"
            "code", "akamai", "-arc", "ki-", "cdn", "fastly", "location", "cgi", "device", "runtime",
            "trace", "state", "ttl", "-id", "route", "hacker", "rate", "limit",
            "time", "edge", "msn", "cnection", "feature-policy", "iinfo", "employment",
            "content-security-policy", "mime", "eventid", "recruiting", "terms-of-service", "encoding", "content"
            "status", "date", "-served-", "proto", "-dc", "-ac", "-auto-", "age", "endpoints", "via", "key-pins",
            "x-li-", "md5", "-geo", "trackingid"]
    check1 = (hv in ["miss", "hit", "cached", "off", "on", "expired", "true", "false", "none", "1", "0"])
    check2 = False
    for exp in expr:
        if exp in hd:
            check2 = True
            break
    check3 = (hd in ref)
    check4 = ("-" not in hd)
    check5 = hv.isdigit()
    mustSkip = (check1 or check2 or check3 or check4 or check5)
    return mustSkip

def printStep(msg, color="yellow"):
    print(colored(msg, color, attrs=['bold']))

if __name__ == '__main__':
    TAKE_FIRST = 50000
    THREAD_COUNT = 25
    if len(sys.argv) > 1:
        TAKE_FIRST = int(sys.argv[1])
    just_fix_windows_console()
    printStep("\U0001F4BB Initialize the workspace ...")
    dfolder = "data"
    if os.path.exists(dfolder):
        shutil.rmtree(dfolder, ignore_errors=True)
    os.makedirs(dfolder)
    printStep("\U0001F4E5 Loading resources ...")
    know_headers = ";".join(loadOSHPHeaders()) + ";" + ";".join(loadCommonsHeaders()) + ";" + ";".join(loadLocalSkipHeaders()) + ";"
    domains = loadDomains(know_headers.lower())
    domains.sort()
    inputs = domains[:TAKE_FIRST]
    printStep(f"\U0001F50E Hunting using first {TAKE_FIRST} domains ...", "cyan")
    with Pool(processes=THREAD_COUNT, maxtasksperchild=THREAD_COUNT) as pool:
        outputs = pool.imap_unordered(hunt, inputs)
        outputs_size = sum(1 for _ in outputs)
        printStep(f"\r\U0001F197 Processed: {outputs_size:<40}", "green")

Thanks in advance 😃

[riramar]

Thanks!