search

complexorganizations / content-blocker

✔️ Content Blocker is a robust web filtering project aimed at enhancing online privacy and security.
Other
16 stars 2 forks source link

To block, enter a valid domain. #70

Open ghost opened 3 years ago

ghost commented 3 years ago

Please do not report subdomain www.example.com, just report the domain with evidence example.com

Prajwal-Koirala commented 7 months ago 
func startScraping() {
    // Advertisement && Tracking
    advertisement := []string{
        "https://raw.githubusercontent.com/AdAway/adaway.github.io/master/hosts.txt",
        "https://raw.githubusercontent.com/DRSDavidSoft/additional-hosts/master/domains/blacklist/adservers-and-trackers.txt",
        "https://raw.githubusercontent.com/Ewpratten/youtube_ad_blocklist/master/blocklist.txt",
        "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/UncheckyAds/hosts",
        "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.2o7Net/hosts",
        "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts",
        "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Spam/hosts",
        "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/GoodbyeAds-Samsung-Adblock-Extension/hosts",
        "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/GoodbyeAds-Spotify-AdBlock-Extension/hosts",
        "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/GoodbyeAds-YouTube-Adblock-Extension/hosts",
        "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/GoodbyeAds-Xiaomi-Extension/hosts",
        "https://raw.githubusercontent.com/HorusTeknoloji/TR-PhishingList/master/url-lists.txt",
        "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/AmazonFireTV.txt",
        "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SessionReplay.txt",
        "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV.txt",
        "https://raw.githubusercontent.com/PolishFiltersTeam/KADhosts/master/KADhosts.txt",
        "https://raw.githubusercontent.com/RooneyMcNibNug/pihole-stuff/master/SNAFU.txt",
        "https://raw.githubusercontent.com/Spam404/lists/master/main-blacklist.txt",
        "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts",
        "https://raw.githubusercontent.com/Ultimate-Hosts-Blacklist/cameleon_at_sysctl.org/master/domains.list",
        "https://raw.githubusercontent.com/VeleSila/yhosts/master/hosts",
        "https://raw.githubusercontent.com/anudeepND/blacklist/master/adservers.txt",
        "https://raw.githubusercontent.com/anudeepND/youtubeadsblacklist/master/domainlist.txt",
        "https://raw.githubusercontent.com/bigdargon/hostsVN/master/hosts",
        "https://raw.githubusercontent.com/cbuijs/shallalist/master/adv/domains",
        "https://raw.githubusercontent.com/cbuijs/shallalist/master/tracker/domains",
        "https://raw.githubusercontent.com/d3ward/toolz/master/src/d3host.txt",
        "https://raw.githubusercontent.com/jdlingyu/ad-wars/master/hosts",
        "https://raw.githubusercontent.com/jerryn70/GoodbyeAds/master/Hosts/GoodbyeAds-Ultra.txt",
        "https://raw.githubusercontent.com/justdomains/blocklists/master/lists/adguarddns-justdomains.txt",
        "https://raw.githubusercontent.com/justdomains/blocklists/master/lists/easylist-justdomains.txt",
        "https://raw.githubusercontent.com/justdomains/blocklists/master/lists/easyprivacy-justdomains.txt",
        "https://raw.githubusercontent.com/kboghdady/youTube_ads_4_pi-hole/master/youtubelist.txt",
        "https://raw.githubusercontent.com/lightswitch05/hosts/master/docs/lists/ads-and-tracking-extended.txt",
        "https://raw.githubusercontent.com/lightswitch05/hosts/master/docs/lists/tracking-aggressive-extended.txt",
        "https://raw.githubusercontent.com/matomo-org/referrer-spam-blacklist/master/spammers.txt",
        "https://raw.githubusercontent.com/notracking/hosts-blocklists/master/hostnames.txt",
        "https://raw.githubusercontent.com/ookangzheng/dbl-oisd-nl/master/dbl.txt",
        "https://raw.githubusercontent.com/tiuxo/hosts/master/ads",
        "https://raw.githubusercontent.com/yous/YousList/master/hosts.txt",
        "https://block.energized.pro/ultimate/formats/domains.txt",
        "https://raw.githubusercontent.com/blocklistproject/Lists/master/tracking.txt",
        "https://raw.githubusercontent.com/blocklistproject/Lists/master/ads.txt",
        "https://raw.githubusercontent.com/blocklistproject/Lists/master/piracy.txt",
        "https://raw.githubusercontent.com/259095/someonewhocares/main/list",
        "https://raw.githubusercontent.com/badmojr/1Hosts/master/Xtra/domains.txt",
        "https://block.energized.pro/extensions/xtreme/formats/domains.txt",
        "https://raw.githubusercontent.com/allendema/noplaylist/main/NoPlayList.txt",
        "https://raw.githubusercontent.com/anthony-wang/PiHoleBlocklist/master/hosts1.txt",
        "https://raw.githubusercontent.com/ftpmorph/ftprivacy/master/blocklists/xiaomi-ads-tracking.txt",
        "https://raw.githubusercontent.com/ftpmorph/ftprivacy/master/blocklists/huawei-trackers.txt",
        "https://raw.githubusercontent.com/ftpmorph/ftprivacy/master/blocklists/smart-tv-ads-tracking.txt",
        "https://raw.githubusercontent.com/ftpmorph/ftprivacy/master/blocklists/smartphone-ads-tracking.txt",
        "https://raw.githubusercontent.com/ftpmorph/ftprivacy/master/blocklists/spotify-ads-tracking.txt",
        "https://raw.githubusercontent.com/ftpmorph/ftprivacy/master/blocklists/windows10-spying-erm-i-mean-telemetry-lol.txt",
        "https://raw.githubusercontent.com/hemiipatu/PiHoleBlocklists/master/blocklists/advertisement.txt",
        "https://raw.githubusercontent.com/mhhakim/pihole-blocklist/master/list.txt",
        "https://raw.githubusercontent.com/mhxion/pornaway/master/hosts/porn_ads.txt",
        "https://raw.githubusercontent.com/migueldemoura/ublock-umatrix-rulesets/master/Hosts/ads-tracking",
        "https://raw.githubusercontent.com/xlimit91/xlimit91-block-list/master/blacklist.txt",
        "https://raw.githubusercontent.com/ookangzheng/dbl-oisd-nl/master/hosts.txt",
        "https://raw.githubusercontent.com/mkb2091/blockconvert/master/output/domains.txt",
        "https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Ads",
        "https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Tracking",
        "https://raw.githubusercontent.com/RooneyMcNibNug/pihole-stuff/master/SNAFU.txt",
    }
    // Malicious
    malicious := []string{
        "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareHosts.txt",
        "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/CoinBlockerList/hosts",
        "https://raw.githubusercontent.com/piwik/referrer-spam-blacklist/master/spammers.txt",
        "https://raw.githubusercontent.com/justdomains/blocklists/master/lists/nocoin-justdomains.txt",
        "https://raw.githubusercontent.com/BlackJack8/iOSAdblockList/master/Regular%20Hosts.txt",
        "https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/gambling/hosts",
        "https://raw.githubusercontent.com/StevenBlack/hosts/master/extensions/fakenews/hosts",
        "https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt",
        "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/android-tracking.txt",
        "https://raw.githubusercontent.com/lightswitch05/hosts/master/docs/lists/hate-and-junk-extended.txt",
        "https://raw.githubusercontent.com/blocklistproject/Lists/master/crypto.txt",
        "https://raw.githubusercontent.com/blocklistproject/Lists/master/drugs.txt",
        "https://raw.githubusercontent.com/blocklistproject/Lists/master/malware.txt",
        "https://raw.githubusercontent.com/blocklistproject/Lists/master/gambling.txt",
        "https://raw.githubusercontent.com/blocklistproject/Lists/master/ransomware.txt",
        "https://raw.githubusercontent.com/blocklistproject/Lists/master/smart-tv.txt",
        "https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews/hosts",
        "https://badmojr.github.io/1Hosts/Pro/domains.txt",
        "https://raw.githubusercontent.com/missdeer/blocklist/master/toblock-without-shorturl.lst",
        "https://gitlab.com/andryou/block/raw/master/senpai-strict-domains",
        "https://gitlab.com/andryou/block/raw/master/chibi-strict-domains",
        "https://gitlab.com/andryou/block/raw/master/kouhai-strict-domains",
        "https://gitlab.com/curben/urlhaus-filter/-/raw/master/urlhaus-filter-domains.txt",
        "https://raw.githubusercontent.com/bongochong/CombinedPrivacyBlockLists/master/newhosts-final.hosts",
        "https://raw.githubusercontent.com/durablenapkin/scamblocklist/master/hosts.txt",
        "https://raw.githubusercontent.com/ftpmorph/ftprivacy/master/blocklists/avg-avast-data-mining-full-block.txt",
        "https://raw.githubusercontent.com/furkun/ProtectorHosts/main/hosts",
        "https://raw.githubusercontent.com/hemiipatu/PiHoleBlocklists/master/blocklists/fraudulent.txt",
        "https://raw.githubusercontent.com/hemiipatu/PiHoleBlocklists/master/blocklists/malware.txt",
        "https://raw.githubusercontent.com/hemiipatu/PiHoleBlocklists/master/blocklists/ransomware.txt",
        "https://raw.githubusercontent.com/hemiipatu/PiHoleBlocklists/master/blocklists/scam.txt",
        "https://raw.githubusercontent.com/herrbischoff/trackers/master/domains.txt",
        "https://raw.githubusercontent.com/migueldemoura/ublock-umatrix-rulesets/master/Hosts/malware",
        "https://raw.githubusercontent.com/Strappazzon/teleme7ry/master/rules.txt",
        "https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Bloat",
        "https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Malware",
        "https://raw.githubusercontent.com/nextdns/cname-cloaking-blocklist/master/domains",
        "https://raw.githubusercontent.com/rimu/no-qanon/master/etc_hosts.txt",
    }
    // Social Engineering
    socialEngineering := []string{
        "https://raw.githubusercontent.com/MetaMask/eth-phishing-detect/master/src/hosts.txt",
        "https://raw.githubusercontent.com/tg12/pihole-phishtank-list/master/list/phish_domains.txt",
        "https://raw.githubusercontent.com/blocklistproject/Lists/master/abuse.txt",
        "https://raw.githubusercontent.com/blocklistproject/Lists/master/fraud.txt",
        "https://raw.githubusercontent.com/blocklistproject/Lists/master/scam.txt",
        "https://blocklist.cyberthreatcoalition.org/vetted/domain.txt",
        "https://raw.githubusercontent.com/sk-cat/fluffy-blocklist/main/domains",
        "https://phishing.army/download/phishing_army_blocklist_extended.txt",
        "https://raw.githubusercontent.com/hemiipatu/PiHoleBlocklists/master/blocklists/phishing.txt",
        "https://raw.githubusercontent.com/merkleID/covid-domains/master/full-domains-list.txt",
        "https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-domains-ACTIVE.txt",
        "https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Scam",
    }
    // Adult content
    explicit := []string{
        "https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/porn/hosts",
        "https://block.energized.pro/porn/formats/domains.txt",
        "https://raw.githubusercontent.com/Bon-Appetit/porn-domains/master/block.txt",
        "https://raw.githubusercontent.com/mhhakim/pihole-blocklist/master/porn.txt",
    }
Prajwal-Koirala commented 7 months ago 
package main

import (
    "fmt"
    "log"
    "strings"

    "github.com/domainr/whois"
)

func checkDomainRegistration(domain string) bool {
    request, err := whois.NewRequest(domain)
    if err != nil {
        log.Println("Error creating request: ", err)
        return false
    }
    response, err := whois.DefaultClient.Fetch(request)
    if err != nil {
        log.Println("Error fetching WHOIS info: ", err)
        return false
    }
    whoisInfo := response.String()
    // Return true if the domain is registered
    return !strings.Contains(whoisInfo, "No match")
}

func main() {
    domain := "prajwalkoirala.com" // Replace with the domain you want to query
    fmt.Println(checkDomainRegistration(domain))
}

Valid way to validate a domain registration in golang.

Prajwal-Koirala commented 7 months ago 
package main

import (
    "fmt"
    "log"
    "net"
    "strings"

    "github.com/domainr/whois"
    "github.com/likexian/whois-parser"
    "golang.org/x/net/publicsuffix"
    "github.com/openrdap/rdap"
)

func checkDomainRegistration(domain string) bool {
    request, err := whois.NewRequest(domain)
    if err != nil {
        log.Println("Error creating request: ", err)
        return false
    }
    response, err := whois.DefaultClient.Fetch(request)
    if err != nil {
        log.Println("Error fetching WHOIS info: ", err)
        return false
    }
    whoisInfo := response.String()
    fmt.Println(whoisInfo)
    // Return true if the domain is registered
    return !strings.Contains(whoisInfo, "No match")
}

// Get the domain from a given domain with subdomain
func getDomainFromDomainWithSubdomain(content string) string {
    domain, err := publicsuffix.EffectiveTLDPlusOne(content)
    if err != nil {
        log.Fatalln(err)
    }
    return domain
}

// Make sure it's not an IP address.
func checkIPAddress(ip string) bool {
    return net.ParseIP(ip) != nil
}

// Check domain registration
func checkDomainRegistrationWithWhois(domain string) bool {
    result, err := whoisparser.Parse(domain)
    fmt.Println(result)
    if err == nil {
        // Print the domain status
        fmt.Println(result.Domain.Status)

        // Print the domain created date
        fmt.Println(result.Domain.CreatedDate)

        // Print the domain expiration date
        fmt.Println(result.Domain.ExpirationDate)

        // Print the registrar name
        fmt.Println(result.Registrar.Name)

        // Print the registrant name
        fmt.Println(result.Registrant.Name)

        // Print the registrant email address
        fmt.Println(result.Registrant.Email)
    }
    return false
}

// Check domain registration
func QueryDomainWithRDAP(domain string) {
    client := &rdap.Client{}
    domainInfo, err := client.QueryDomain(domain)

    if err != nil {
        fmt.Println("Error querying domain: ", err)
        return
    }

    fmt.Printf("Handle=%s Domain=%s\n", domainInfo.Handle, domainInfo.LDHName)
}

func main() {
    domain := "google.com" // Replace with the domain you want to query
    //domain = getDomainFromDomainWithSubdomain(domain)
    //
    //fmt.Println(domain)
    //fmt.Println(checkDomainRegistration(domain))
    //fmt.Println("----")
    fmt.Println(checkIPAddress(domain))
    fmt.Println(checkDomainRegistrationWithWhois(domain))
    QueryDomainWithRDAP(domain)
}