:zap: Compression for the page responses of the search engine

What would you like to share?

Provide compressed page responses for all the pages requested by the client from the search engine server. By doing these change, this improves the performance of the search engine by reducing the time it takes to load the pages.

The changes that should be made for this change should be done in the two files lib.rs located in the src directory under the codebase (websurfx directory) and the Cargo.toml file located under the codebase (websurfx directory). The changes that should be made in these files is shown below:

lib.rs

//! This main library module provides the functionality to provide and handle the Tcp server
//! and register all the routes for the `websurfx` meta search engine website.

#![forbid(unsafe_code, clippy::panic)]
#![deny(missing_docs, clippy::missing_docs_in_private_items, clippy::perf)]
#![warn(clippy::cognitive_complexity, rust_2018_idioms)]

pub mod cache;
pub mod config;
pub mod engines;
pub mod handler;
pub mod models;
pub mod results;
pub mod server;
pub mod templates;

use std::net::TcpListener;

use crate::server::router;

use actix_cors::Cors;
use actix_files as fs;
use actix_governor::{Governor, GovernorConfigBuilder};
-use actix_web::{dev::Server, http::header, middleware::Logger, web, App, HttpServer};
+use actix_web::{
+   dev::Server,
+   http::header,
+   middleware::{Compress, Logger},
+   web, App, HttpServer,
+};
use cache::cacher::{Cacher, SharedCache};
use config::parser::Config;
use handler::{file_path, FileType};

/// Runs the web server on the provided TCP listener and returns a `Server` instance.
///
/// # Arguments
///
/// * `listener` - A `TcpListener` instance representing the address and port to listen on.
///
/// # Returns
///
/// Returns a `Result` containing a `Server` instance on success, or an `std::io::Error` on failure.
///
/// # Example
///
/// ```rust
/// use std::net::TcpListener;
/// use websurfx::{config::parser::Config, run, cache::cacher::create_cache};
///
/// #[tokio::main]
/// async fn main(){
///     let config = Config::parse(true).unwrap();
///     let listener = TcpListener::bind("127.0.0.1:8080").expect("Failed to bind address");
///     let cache = create_cache(&config).await;
///     let server = run(listener,config,cache).expect("Failed to start server");
/// }
/// ```
pub fn run(
    listener: TcpListener,
    config: Config,
    cache: impl Cacher + 'static,
) -> std::io::Result<Server> {
    let public_folder_path: &str = file_path(FileType::Theme)?;

    let cloned_config_threads_opt: u8 = config.threads;

    let cache = web::Data::new(SharedCache::new(cache));

    let server = HttpServer::new(move || {
        let cors: Cors = Cors::default()
            .allow_any_origin()
            .allowed_methods(vec!["GET"])
            .allowed_headers(vec![
                header::ORIGIN,
                header::CONTENT_TYPE,
                header::REFERER,
                header::COOKIE,
            ]);

        App::new()
+           // Compress the responses provided by the server for the client requests.
+           .wrap(Compress::default())
            .wrap(Logger::default()) // added logging middleware for logging.
            .app_data(web::Data::new(config.clone()))
            .app_data(cache.clone())
            .wrap(cors)
            .wrap(Governor::new(
                &GovernorConfigBuilder::default()
                    .per_second(config.rate_limiter.time_limit as u64)
                    .burst_size(config.rate_limiter.number_of_requests as u32)
                    .finish()
                    .unwrap(),
            ))
            // Serve images and static files (css and js files).
            .service(
                fs::Files::new("/static", format!("{}/static", public_folder_path))
                    .show_files_listing(),
            )
            .service(
                fs::Files::new("/images", format!("{}/images", public_folder_path))
                    .show_files_listing(),
            )
            .service(router::robots_data) // robots.txt
            .service(router::index) // index page
            .service(server::routes::search::search) // search page
            .service(router::about) // about page
            .service(router::settings) // settings page
            .default_service(web::route().to(router::not_found)) // error page
    })
    .workers(cloned_config_threads_opt as usize)
    // Start server on 127.0.0.1 with the user provided port number. for example 127.0.0.1:8080.
    .listen(listener)?
    .run();
    Ok(server)
}

Cargo.toml

[package]
name = "websurfx"
version = "1.6.8"
edition = "2021"
description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind."
repository = "https://github.com/neon-mmd/websurfx"
license = "AGPL-3.0"

[[bin]]
name = "websurfx"
test = true
bench = false
path = "src/bin/websurfx.rs"

[dependencies]
reqwest = {version="0.11.22", default-features=false, features=["rustls-tls","brotli", "gzip"]}
tokio = {version="1.32.0",features=["rt-multi-thread","macros"], default-features = false}
serde = {version="1.0.190", default-features=false, features=["derive"]}
serde_json = {version="1.0.108", default-features=false}
maud = {version="0.25.0", default-features=false, features=["actix-web"]}
scraper = {version="0.18.1", default-features = false}
-actix-web = {version="4.4.0", features = ["cookies", "macros"], default-features=false}
+actix-web = {version="4.4.0", features = ["cookies", "macros", "compress-brotli"], default-features=false}
actix-files = {version="0.6.2", default-features=false}
actix-cors = {version="0.6.4", default-features=false}
fake-useragent = {version="0.1.3", default-features=false}
env_logger = {version="0.10.0", default-features=false}
log = {version="0.4.20", default-features=false}
mlua = {version="0.9.1", features=["luajit", "vendored"], default-features=false}
redis = {version="0.23.3", features=["tokio-comp","connection-manager"], default-features = false, optional = true}
blake3 = {version="1.5.0", default-features=false}
error-stack = {version="0.4.0", default-features=false, features=["std"]}
async-trait = {version="0.1.73", default-features=false}
regex = {version="1.9.4", features=["perf"], default-features = false}
smallvec = {version="1.11.0", features=["union", "serde"], default-features=false}
futures = {version="0.3.28", default-features=false}
dhat = {version="0.3.2", optional = true, default-features=false}
mimalloc = { version = "0.1.38", default-features = false }
async-once-cell = {version="0.5.3", default-features=false}
actix-governor = {version="0.5.0", default-features=false}
mini-moka = { version="0.10", optional = true, default-features=false, features=["sync"]}

[dev-dependencies]
rusty-hook = {version="^0.11.2", default-features=false}
criterion = {version="0.5.1", default-features=false}
tempfile = {version="3.8.0", default-features=false}

[build-dependencies]
lightningcss = {version="1.0.0-alpha.50", default-features=false, features=["grid"]}
minify-js = {version="0.5.6", default-features=false}

[profile.dev]
opt-level = 0
debug = true
split-debuginfo = 'unpacked'
debug-assertions = true
overflow-checks = true
lto = false
panic = 'unwind'
incremental = true
codegen-units = 256
rpath = false

[profile.release]
opt-level = 3
debug = false # This should only be commented when testing with dhat profiler
# debug = 1 # This should only be uncommented when testing with dhat profiler
split-debuginfo = '...'
debug-assertions = false
overflow-checks = false
lto = true
panic = 'abort'
incremental = false
codegen-units = 1
rpath = false
strip = "debuginfo"

[features]
default = ["memory-cache"]
dhat-heap = ["dep:dhat"] 
memory-cache = ["dep:mini-moka"]
redis-cache = ["dep:redis"]

[!Note]

To get started contributing make sure to read the contributing.md file for the guidlines on how to contribute in this project

To contribute first fork this project by following this video tutorial if you are not familliar with process and add your changes and make a pull request with the changes to this repository and if you are new to GitHub then follow this video tutorial to get started contributing :slightly_smiling_face: .

Do you want to work on this issue?

None

Additional information

No response

neon-mmd / websurfx