generate_fqdn.sh

[fabriziosalmi]

#!/bin/bash

setup_environment() {
    echo "Setup script"

    # Detect package manager
    if command -v apt-get &>/dev/null; then
        PACKAGE_MANAGER="apt-get"
        UPDATE_CMD="sudo apt-get update"
        INSTALL_CMD="sudo apt-get install -y"
    else
        echo "Unsupported package manager. Exiting."
        exit 1
    fi

    # Update and install prerequisites
    $UPDATE_CMD
    $INSTALL_CMD python3

    # Link python3 to python (for Ubuntu)
    sudo ln -s /usr/bin/python3 /usr/bin/python

    python3 -m ensurepip --upgrade
    pip3 install --no-cache-dir --upgrade pip setuptools tldextract tqdm

    # Install other necessary packages
    for package in pv ncftp; do
        if ! $INSTALL_CMD $package; then
            echo "Failed to install '$package' using $PACKAGE_MANAGER."
            exit 1
        fi
    done
}

download_blacklists() {
    echo "Download blacklists"

    while IFS= read -r url; do
        local random_filename=$(uuidgen | tr -dc '[:alnum:]')
        if ! wget -q --progress=bar:force -O "$random_filename.fqdn.list" "$url"; then
            echo "Failed to download: $url"
        fi
    done < "blacklists.fqdn.urls"
}

aggregate_blacklists() {
    echo "Aggregate blacklists"

    for file in *.fqdn.list; do
        sudo cat "$file" >> aggregated.fqdn.list
    done

    sudo cat aggregated.fqdn.list | sort -u > all.fqdn.blacklist
    sudo rm ./*.fqdn.list
}

sanitize_blacklists() {
    echo "Sanitize blacklists"

    mv all.fqdn.blacklist input.txt
    python sanitize.py
    mv output.txt all.fqdn.blacklist

    echo "Remove whitelisted domains"
    mv all.fqdn.blacklist blacklist.txt
    python whitelist.py
    mv filtered_blacklist.txt all.fqdn.blacklist
}

create_compressed_file() {
    echo "Create compressed file"
    if ! tar -czf all.fqdn.blacklist.tar.gz "all.fqdn.blacklist"; then
        echo "Error: Failed to create the tar.gz file."
        exit 1
    fi

    total_lines_new=$(cat all.fqdn.blacklist | wc -l)
    echo "Total domains: $total_lines_new."
}

# Execute functions
setup_environment
download_blacklists
aggregate_blacklists
sanitize_blacklists
create_compressed_file

[fabriziosalmi]

#!/bin/bash

# Constants
BLACKLIST_URL_FILE="blacklists.fqdn.urls"
AGGREGATED_LIST="aggregated.fqdn.list"
BLACKLIST_OUTPUT="all.fqdn.blacklist"
COMPRESSED_BLACKLIST="all.fqdn.blacklist.tar.gz"
REQUIRED_PACKAGES=(pv ncftp python3)

# Error Handling
trap 'echo "An error occurred. Exiting."; exit 1' ERR

setup_environment() {
    echo "Setting up the environment..."

    # Detect package manager
    if command -v apt-get &>/dev/null; then
        PACKAGE_MANAGER="apt-get"
        UPDATE_CMD="sudo apt-get update"
        INSTALL_CMD="sudo apt-get install -y"
    else
        echo "Unsupported package manager. Exiting."
        exit 1
    fi

    # Update and install prerequisites
    $UPDATE_CMD
    $INSTALL_CMD "${REQUIRED_PACKAGES[@]}"

    # Link python3 to python (for Ubuntu)
    if [[ ! -f "/usr/bin/python" ]]; then
        sudo ln -s /usr/bin/python3 /usr/bin/python
    fi

    python3 -m ensurepip --upgrade
    pip3 install --no-cache-dir --upgrade pip setuptools tldextract tqdm
}

download_blacklists() {
    echo "Downloading blacklists..."

    while IFS= read -r url; do
        local random_filename=$(uuidgen | tr -dc '[:alnum:]')
        if ! wget -q --progress=bar:force -O "$random_filename.fqdn.list" "$url"; then
            echo "Failed to download: $url"
        fi
    done < "$BLACKLIST_URL_FILE"
}

aggregate_blacklists() {
    echo "Aggregating blacklists..."

    cat *.fqdn.list | sort -u > "$BLACKLIST_OUTPUT"
    rm -f *.fqdn.list
}

sanitize_blacklists() {
    echo "Sanitizing blacklists..."
    python sanitize.py < "$BLACKLIST_OUTPUT" > sanitized.tmp

    echo "Removing whitelisted domains..."
    python whitelist.py < sanitized.tmp > "$BLACKLIST_OUTPUT"

    rm sanitized.tmp
}

create_compressed_file() {
    echo "Compressing blacklist file..."

    tar -czf "$COMPRESSED_BLACKLIST" "$BLACKLIST_OUTPUT" || {
        echo "Error: Failed to create the tar.gz file."
        exit 1
    }

    total_lines_new=$(wc -l < "$BLACKLIST_OUTPUT")
    echo "Total domains: $total_lines_new."
}

# Execute functions
setup_environment
download_blacklists
aggregate_blacklists
sanitize_blacklists
create_compressed_file

[fabriziosalmi]

#!/bin/bash

# Constants
readonly BLACKLIST_URL_FILE="blacklists.fqdn.urls"
readonly AGGREGATED_LIST="aggregated.fqdn.list"
readonly BLACKLIST_OUTPUT="all.fqdn.blacklist"
readonly COMPRESSED_BLACKLIST="all.fqdn.blacklist.tar.gz"
readonly REQUIRED_PACKAGES=(pv ncftp python3)

# Error Handling
trap 'echo "An error occurred. Exiting."; exit 1' ERR

setup_environment() {
    echo "Setting up the environment..."

    # Detect package manager
    if command -v apt-get &>/dev/null; then
        PACKAGE_MANAGER="apt-get"
        UPDATE_CMD="sudo apt-get update"
        INSTALL_CMD="sudo apt-get install -y"
    else
        echo "Unsupported package manager. Exiting."
        exit 1
    fi

    # Update and install prerequisites
    $UPDATE_CMD
    $INSTALL_CMD "${REQUIRED_PACKAGES[@]}"

    # Link python3 to python (for Ubuntu)
    if ! command -v python &>/dev/null; then
        sudo ln -s /usr/bin/python3 /usr/bin/python
    fi

    python3 -m ensurepip --upgrade
    pip3 install --no-cache-dir --upgrade pip setuptools tldextract tqdm
}

download_blacklists() {
    echo "Downloading blacklists..."

    while IFS= read -r url; do
        local random_filename=$(uuidgen | tr -dc '[:alnum:]')
        if ! wget -q --progress=bar:force -O "$random_filename.fqdn.list" "$url"; then
            echo "Failed to download: $url"
        fi
    done < "$BLACKLIST_URL_FILE"
}

aggregate_blacklists() {
    echo "Aggregating blacklists..."

    cat *.fqdn.list | sort -u > "$BLACKLIST_OUTPUT"
    rm -f *.fqdn.list
}

sanitize_blacklists() {
    echo "Sanitizing blacklists..."
    local temp_file
    temp_file=$(mktemp)
    python sanitize.py < "$BLACKLIST_OUTPUT" > "$temp_file"

    echo "Removing whitelisted domains..."
    python whitelist.py < "$temp_file" > "$BLACKLIST_OUTPUT"

    rm "$temp_file"
}

create_compressed_file() {
    echo "Compressing blacklist file..."

    tar -czf "$COMPRESSED_BLACKLIST" "$BLACKLIST_OUTPUT" || {
        echo "Error: Failed to create the tar.gz file."
        exit 1
    }

    total_lines_new=$(wc -l < "$BLACKLIST_OUTPUT")
    echo "Total domains: $total_lines_new."
}

# Execute functions
setup_environment
download_blacklists
aggregate_blacklists
sanitize_blacklists
create_compressed_file

[fabriziosalmi]

#!/bin/bash

# Constants
readonly BLACKLIST_URL_FILE="blacklists.fqdn.urls"
readonly AGGREGATED_LIST="aggregated.fqdn.list"
readonly BLACKLIST_OUTPUT="all.fqdn.blacklist"
readonly COMPRESSED_BLACKLIST="all.fqdn.blacklist.tar.gz"
readonly REQUIRED_PACKAGES=(pv ncftp python3)

# Error Handling
trap 'echo "An error occurred during the process: $? . Exiting." ; exit 1' ERR

# Check if a command exists
command_exists() {
    command -v "$1" &> /dev/null
}

setup_environment() {
    echo "Setting up the environment..."

    # Detect package manager
    if command_exists apt-get; then
        PACKAGE_MANAGER="apt-get"
        UPDATE_CMD="sudo apt-get update"
        INSTALL_CMD="sudo apt-get install -y"
    else
        echo "Unsupported package manager. Exiting."
        exit 1
    fi

    # Update and install prerequisites
    echo "Updating packages..."
    $UPDATE_CMD | pv -N "Updating packages" -s 1 -l > /dev/null
    echo "Installing required packages..."
    $INSTALL_CMD "${REQUIRED_PACKAGES[@]}" | pv -N "Installing" -l > /dev/null

    # Link python3 to python (for Ubuntu)
    if ! command_exists python; then
        sudo ln -s /usr/bin/python3 /usr/bin/python
    fi

    python3 -m ensurepip --upgrade
    pip3 install --no-cache-dir --upgrade pip setuptools tldextract tqdm
}

download_blacklists() {
    echo "Downloading blacklists..."
    local count=$(wc -l < "$BLACKLIST_URL_FILE")

    while IFS= read -r url; do
        local random_filename=$(uuidgen | tr -dc '[:alnum:]')
        echo "Downloading from: $url"
        wget -q --progress=bar:force -O "$random_filename.fqdn.list" "$url" 2>&1 | pv -N "$url" -l -s $count > /dev/null || {
            echo "Failed to download: $url"
        }
    done < "$BLACKLIST_URL_FILE"
}

aggregate_blacklists() {
    echo "Aggregating blacklists..."

    cat *.fqdn.list | pv -N "Aggregating" -l | sort -u > "$BLACKLIST_OUTPUT"
    rm -f *.fqdn.list
}

sanitize_blacklists() {
    echo "Sanitizing blacklists..."
    local temp_file
    temp_file=$(mktemp)
    python sanitize.py < "$BLACKLIST_OUTPUT" | pv -N "Sanitizing" -l > "$temp_file"

    echo "Removing whitelisted domains..."
    python whitelist.py < "$temp_file" | pv -N "Whitelisting" -l > "$BLACKLIST_OUTPUT"

    rm "$temp_file"
}

create_compressed_file() {
    echo "Compressing blacklist file..."

    tar -czf "$COMPRESSED_BLACKLIST" "$BLACKLIST_OUTPUT" | pv -N "Compressing" -s $(du -sb "$BLACKLIST_OUTPUT" | awk '{print $1}') > /dev/null || {
        echo "Error: Failed to create the tar.gz file."
        exit 1
    }

    total_lines_new=$(wc -l < "$BLACKLIST_OUTPUT")
    echo "Total domains: $total_lines_new."
}

# Execute functions
setup_environment
download_blacklists
aggregate_blacklists
sanitize_blacklists
create_compressed_file

[fabriziosalmi]

#!/bin/bash

echo "Setup script"

# Check if a command exists
command_exists() {
    command -v "$1" &> /dev/null
}

setup_environment() {
    # Detect package manager
    if command_exists apt-get; then
        PACKAGE_MANAGER="apt-get"
        UPDATE_CMD="sudo apt-get update"
        INSTALL_CMD="sudo apt-get install -y"
    elif command_exists apk; then
        PACKAGE_MANAGER="apk"
        UPDATE_CMD="sudo apk update"
        INSTALL_CMD="sudo apk add --no-cache"
    else
        echo "Unsupported package manager. Exiting."
        exit 1
    fi

    # Update and install prerequisites
    $UPDATE_CMD
    $INSTALL_CMD python3

    # Link python3 to python (for Ubuntu, since Alpine doesn't have python2 by default)
    if [ "$PACKAGE_MANAGER" == "apt-get" ] && ! command_exists python; then
        sudo ln -s /usr/bin/python3 /usr/bin/python
    fi

    python3 -m ensurepip --upgrade
    pip3 install --no-cache-dir --upgrade pip setuptools tldextract tqdm

    # Install pv and ncftp based on the detected package manager
    for package in pv ncftp; do
        if ! $INSTALL_CMD $package; then
            echo "Failed to install '$package' using $PACKAGE_MANAGER."
            exit 1
        fi
    done
}

download_blacklists() {
    local LISTS="blacklists.fqdn.urls"
    echo "Download blacklists"

    while IFS= read -r url; do
        echo "Blacklist: $url"
        random_filename=$(uuidgen | tr -dc '[:alnum:]')
        if ! wget -q --progress=bar:force -O "$random_filename.fqdn.list" "$url"; then
            echo "Failed to download: $url"
        fi
    done < "$LISTS"
}

aggregate_blacklists() {
    echo "Aggregate blacklists"
    > aggregated.fqdn.list

    for file in *.fqdn.list; do
        cat "$file" >> aggregated.fqdn.list
    done

    sort -u aggregated.fqdn.list > all.fqdn.blacklist
    echo "Remove source files"
    rm ./*.fqdn.list
}

sanitize_and_whitelist() {
    echo "Sanitize blacklists"
    mv all.fqdn.blacklist input.txt
    python sanitize.py
    mv output.txt all.fqdn.blacklist

    echo "Remove whitelisted domains"
    mv all.fqdn.blacklist blacklist.txt
    python whitelist.py
    mv filtered_blacklist.txt all.fqdn.blacklist
    rm blacklist.txt input.txt
}

compress_file() {
    echo "Create compressed file"
    if ! tar -czf all.fqdn.blacklist.tar.gz "all.fqdn.blacklist"; then
        echo "Error: Failed to create the tar.gz file."
        exit 1
    fi

    total_lines_new=$(wc -l < all.fqdn.blacklist)
    echo "Total domains: $total_lines_new."
}

# Execute functions
setup_environment
download_blacklists
aggregate_blacklists
sanitize_and_whitelist
compress_file