v2

[fabriziosalmi]

main.py

from datetime import datetime

# Import checks
from checks import (
    check_domain_breach, check_domain_expiration, check_ssl_cert, check_dns_blacklist,
    check_domainsblacklists_blacklist, check_hsts, check_xss_protection, check_redirect_chains,
    check_pagespeed_performances, check_website_load_time, check_rate_limiting, check_cdn,
    check_brotli_compression, check_deprecated_libraries, check_clientside_rendering,
    check_mixed_content, check_content_type_headers, check_internationalization, check_floc,
    check_amp_compatibility, check_robot_txt, check_sitemap, check_favicon, check_alt_tags,
    check_open_graph_protocol, check_semantic_markup, check_ad_and_tracking, check_privacy_protected_whois,
    check_privacy_exposure
)

# Initialize an error log
error_log = []

def log_error(message):
    """A simple function to log errors for later use."""
    error_log.append(message)
    print(message)  # This will also print the error in the console

# Read websites from external file
with open('websites.txt', 'r') as f:
    websites = [line.strip() for line in f.readlines()]

# Initialize Markdown report
report_md = "# Websites Monitor\n"

# Read the project description and usage instructions
with open('project_description.md', 'r') as f:
    report_md += f"{f.read()}\n"

with open('usage_instructions.md', 'r') as f:
    report_md += f"{f.read()}\n"

# Initialize the table
report_md += "\n## Monitoring Checks\n"
report_md += "| Check Type | " + " | ".join(websites) + " |\n"
report_md += "|------------|" + "---|" * len(websites) + "\n"

# List of check functions and their human-readable names
check_functions = [
    ("Domain breach", check_domain_breach),
    # ... [other checks]
    ("Privacy Exposure", check_privacy_exposure),
]

# Populate the table with check results
for check_name, check_func in check_functions:
    report_md += f"| {check_name} | "
    for website in websites:
        try:
            result = check_func(website)
            report_md += f"{result} | "
        except Exception as e:
            err_msg = f"Error occurred with {check_name} for {website}: {e}"
            log_error(err_msg)
            report_md += "⚪ | "  # Add a default grey indicator for errors
    report_md += "\n"

# Timestamp
current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
report_md += f"\n---\nLast Updated: {current_time}\n"

# Write the Markdown report to a file
with open("README.md", "w") as f:
    f.write(report_md)

# Exit with a non-zero code if errors were encountered
if error_log:
    exit(1)

[fabriziosalmi]

check_domain_breach.py

import requests

def check_domain_breach(website):
    domain_breach = "🟢"  # Start with a default "safe" status.

    try:
        breach_url = f"https://breachdirectory.com/api/domain/{website}"
        response = requests.get(breach_url)

        if response.status_code != 200:
            print(f"Error occurred while fetching breach data for {website}. Status Code: {response.status_code}")
            domain_breach = "🔘"  # Assume "grey" due to an API error.
        else:
            data = response.json()

            # If the domain was found in any breaches, set the status to "red".
            if data.get('found') and data['found'] == True:
                domain_breach = "🔴"

    except Exception as e:
        print(f"An error occurred while checking breach data for {website}: {e}")
        domain_breach = "⚪"  # Set "grey" due to a processing error.

    return domain_breach

[fabriziosalmi]

check_domain_expiraion.py

from datetime import datetime
import whois

def check_domain_expiration(domain):
    """
    Check the expiration date of a domain.

    Args:
    - domain (str): The domain name to be checked.

    Returns:
    - str: "🟢 (X days left)" if the domain has more than 30 days to expire,
           "🟡 (X days left)" if the domain has between 15 to 30 days to expire,
           "🔴 (X days left)" if the domain has less than 15 days to expire,
           "⚪" for other errors, where X is the number of days until expiration.
    """

    def get_days_to_expire(exp_date):
        """Calculate the days remaining for expiration."""
        if not exp_date:
            return None
        if isinstance(exp_date, list):
            exp_date = exp_date[0]
        return (exp_date - datetime.now()).days

    try:
        w = whois.whois(domain)
        days_to_expire = get_days_to_expire(w.expiration_date)

        if days_to_expire is None:
            print(f"Couldn't retrieve expiration details for {domain}.")
            return "🔴"
        elif days_to_expire < 15:
            return f"🔴 ({days_to_expire} days left)"
        elif days_to_expire < 30:
            return f"🟡 ({days_to_expire} days left)"
        else:
            return f"🟢 ({days_to_expire} days left)"
    except Exception as e:
        print(f"An error occurred while checking domain expiration for {domain}: {e}")
        return "⚪"

[fabriziosalmi]

check_ssl_cert.py

import ssl
import socket
from datetime import datetime

def check_ssl_cert(host, port=443):
    """
    Check the SSL certificate of a given host for its validity period.

    Args:
        host (str): The hostname to check.
        port (int, optional): The port number. Defaults to 443 (standard HTTPS port).

    Returns:
        str: 
            - "🟢 (X days left)" if the certificate is valid and has more than 30 days left.
            - "🟡 (X days left)" if the certificate is valid but has 30 days or fewer left.
            - "🔴" if the certificate is expired or there's an SSL related error.
            where X is the number of days left for the SSL certificate to expire.
    """
    context = ssl.create_default_context()

    try:
        with socket.create_connection((host, port)) as conn:
            with context.wrap_socket(conn, server_hostname=host) as sock:
                cert = sock.getpeercert()

        cert_expiry = datetime.strptime(cert['notAfter'], r"%b %d %H:%M:%S %Y %Z")
        days_to_expire = (cert_expiry - datetime.utcnow()).days

        if days_to_expire <= 0:
            return "🔴"
        elif days_to_expire <= 30:
            return f"🟡 ({days_to_expire} days left)"
        else:
            return f"🟢 ({days_to_expire} days left)"

    except (ssl.SSLError, ssl.CertificateError):
        return "🔴"
    except Exception as e:
        print(f"Unexpected error while checking SSL certificate for {host} on port {port}: {e}")
        return "🔴"

[fabriziosalmi]

check_dns_blacklist.py

import dns.resolver

def check_dns_blacklist(domain):
    """
    Check if a domain is blacklisted in known DNS-based blacklists.

    Args:
    - domain (str): The domain name to be checked.

    Returns:
    - str: "🟢" if the domain is not in any blacklist,
           "🔴" if the domain is found in a blacklist.
    """

    # Set of DNS blacklists
    blacklists = {
        "zen.spamhaus.org",
        "bl.spamcop.net"
        # ... You can add more blacklists here
    }

    for blacklist in blacklists:
        try:
            dns.resolver.query(f"{domain}.{blacklist}", 'A')
            return "🔴"
        except dns.resolver.NXDOMAIN:
            continue  # The domain is not blacklisted in this blacklist
        except (dns.resolver.NoAnswer, dns.resolver.Timeout, dns.resolver.NoNameservers):
            continue  # Ignore these errors for the sake of the check
    return "🟢"

[fabriziosalmi]

check_domainsblacklists_blacklist.py

import requests

def check_domainsblacklists_blacklist(domain):
    url = "https://get.domainsblacklists.com/blacklist.txt"

    try:
        response = requests.get(url, stream=True, timeout=10)
        response.raise_for_status()

        # We'll use an iterative approach to prevent loading the entire list into memory.
        # This will search the file line by line.
        for line in response.iter_lines(decode_unicode=True):
            if line.strip() == domain:
                return "🔴"
        return "🟢"
    except requests.RequestException:
        return "⚪"  # Return gray if there's an error in fetching or processing.