search

drakkan / sftpgo

Full-featured and highly configurable SFTP, HTTP/S, FTP/S and WebDAV server - S3, Google Cloud Storage, Azure Blob
https://sftpgo.com
GNU Affero General Public License v3.0
9.36k stars 728 forks source link

Decrease msg size to match pkg/sftp client maxMsgSize constant #1708

Closed tpoxa closed 3 months ago

tpoxa commented 3 months ago

⚠️ This issue respects the following points: ⚠️

Bug description

pkg/sftp client unable to grab large directory list

please check https://github.com/pkg/sftp/issues/591#issuecomment-2173082373 for the details

Steps to reproduce

  1. Install the sftp server using sftpgo 2.6
  2. Create a directory with 100k objects in it. (file names should be long, like uuids or sth)

  3. Use the next client code

    conf := &ssh.ClientConfig{}
conf.HostKeyCallback = ssh.InsecureIgnoreHostKey()

conf.User = "ftp_user_1"
conf.Auth = append(conf.Auth, ssh.Password("ftp_user_1"))

hostname := fmt.Sprintf("%s:%s", "localhost", "2022")

conn, err := ssh.Dial("tcp", hostname, conf)
if err != nil {
log.Fatal("dial ", err)
}

client, err := sftppkg.NewClient(conn)
if err != nil {
log.Fatal("new client", err)
}
defer client.Close()

files, err := client.ReadDir("/docker/registry/v2/repositories/asdf/library/library/gitlab")
if err != nil {
log.Fatal("read dir ", err)
}
fmt.Println(len(files))
  4. See the error main.go:33]: read dir connection lost

Expected behavior

No error. List of 100k objects.

SFTPGo version

2.6.0

Data provider

ghcr.io/drakkan/sftpgo:v2.6.0-plugins@sha256:3b90adefa53611427cfeda58bf9f251b01290e867bed9b1850fda3e12b18a107

Installation method

Community Docker image

Configuration

{
  "common": {
    "idle_timeout": 15,
    "upload_mode": 0,
    "actions": {
      "execute_on": [],
      "execute_sync": [],
      "hook": ""
    },
    "setstat_mode": 0,
    "rename_mode": 0,
    "resume_max_size": 0,
    "temp_path": "",
    "proxy_protocol": 0,
    "proxy_allowed": [],
    "proxy_skipped": [],
    "startup_hook": "",
    "post_connect_hook": "",
    "post_disconnect_hook": "",
    "data_retention_hook": "",
    "max_total_connections": 0,
    "max_per_host_connections": 20,
    "allowlist_status": 0,
    "allow_self_connections": 0,
    "umask": "",
    "server_version": "",
    "metadata": {
      "read": 0
    },
    "defender": {
      "enabled": false,
      "driver": "memory",
      "ban_time": 30,
      "ban_time_increment": 50,
      "threshold": 15,
      "score_invalid": 2,
      "score_valid": 1,
      "score_limit_exceeded": 3,
      "score_no_auth": 0,
      "observation_time": 30,
      "entries_soft_limit": 100,
      "entries_hard_limit": 150
    },
    "rate_limiters": [
      {
        "average": 0,
        "period": 1000,
        "burst": 1,
        "type": 2,
        "protocols": [
          "SSH",
          "FTP",
          "DAV",
          "HTTP"
        ],
        "generate_defender_events": false,
        "entries_soft_limit": 100,
        "entries_hard_limit": 150
      }
    ]
  },
  "acme": {
    "domains": [],
    "email": "",
    "key_type": "4096",
    "certs_path": "certs",
    "ca_endpoint": "https://acme-v02.api.letsencrypt.org/directory",
    "renew_days": 30,
    "http01_challenge": {
      "port": 80,
      "proxy_header": "",
      "webroot": ""
    },
    "tls_alpn01_challenge": {
      "port": 0
    }
  },
  "sftpd": {
    "bindings": [
      {
        "port": 2022,
        "address": "",
        "apply_proxy_config": true
      }
    ],
    "max_auth_tries": 0,
    "host_keys": [],
    "host_certificates": [],
    "host_key_algorithms": [],
    "moduli": [],
    "kex_algorithms": [],
    "ciphers": [],
    "macs": [],
    "public_key_algorithms": [],
    "trusted_user_ca_keys": [],
    "revoked_user_certs_file": "",
    "login_banner_file": "",
    "enabled_ssh_commands": [
      "md5sum",
      "sha1sum",
      "sha256sum",
      "cd",
      "pwd",
      "scp"
    ],
    "keyboard_interactive_authentication": true,
    "keyboard_interactive_auth_hook": "",
    "password_authentication": true,
    "folder_prefix": ""
  },
  "ftpd": {
    "bindings": [
      {
        "port": 0,
        "address": "",
        "apply_proxy_config": true,
        "tls_mode": 0,
        "tls_session_reuse": 0,
        "certificate_file": "",
        "certificate_key_file": "",
        "min_tls_version": 12,
        "force_passive_ip": "",
        "passive_ip_overrides": [],
        "passive_host": "",
        "client_auth_type": 0,
        "tls_cipher_suites": [],
        "passive_connections_security": 0,
        "active_connections_security": 0,
        "ignore_ascii_transfer_type": 0,
        "debug": false
      }
    ],
    "banner_file": "",
    "active_transfers_port_non_20": true,
    "passive_port_range": {
      "start": 50000,
      "end": 50100
    },
    "disable_active_mode": false,
    "enable_site": false,
    "hash_support": 0,
    "combine_support": 0,
    "certificate_file": "",
    "certificate_key_file": "",
    "ca_certificates": [],
    "ca_revocation_lists": []
  },
  "webdavd": {
    "bindings": [
      {
        "port": 0,
        "address": "",
        "enable_https": false,
        "certificate_file": "",
        "certificate_key_file": "",
        "min_tls_version": 12,
        "client_auth_type": 0,
        "tls_cipher_suites": [],
        "tls_protocols": [],
        "prefix": "",
        "proxy_allowed": [],
        "client_ip_proxy_header": "",
        "client_ip_header_depth": 0,
        "disable_www_auth_header": false
      }
    ],
    "certificate_file": "",
    "certificate_key_file": "",
    "ca_certificates": [],
    "ca_revocation_lists": [],
    "cors": {
      "enabled": false,
      "allowed_origins": [],
      "allowed_methods": [],
      "allowed_headers": [],
      "exposed_headers": [],
      "allow_credentials": false,
      "max_age": 0,
      "options_passthrough": false,
      "options_success_status": 0,
      "allow_private_network": false
    },
    "cache": {
      "users": {
        "expiration_time": 0,
        "max_size": 50
      },
      "mime_types": {
        "enabled": true,
        "max_size": 1000,
        "custom_mappings": []
      }
    }
  },
  "data_provider": {
    "driver": "sqlite",
    "name": "sftpgo.db",
    "host": "",
    "port": 0,
    "username": "",
    "password": "",
    "sslmode": 0,
    "disable_sni": false,
    "target_session_attrs": "",
    "root_cert": "",
    "client_cert": "",
    "client_key": "",
    "connection_string": "",
    "sql_tables_prefix": "",
    "track_quota": 2,
    "delayed_quota_update": 0,
    "pool_size": 0,
    "users_base_dir": "/srv/sftpgo/data",
    "actions": {
      "execute_on": [],
      "execute_for": [],
      "hook": ""
    },
    "external_auth_hook": "",
    "external_auth_scope": 0,
    "pre_login_hook": "",
    "post_login_hook": "",
    "post_login_scope": 0,
    "check_password_hook": "",
    "check_password_scope": 0,
    "password_hashing": {
      "bcrypt_options": {
        "cost": 10
      },
      "argon2_options": {
        "memory": 65536,
        "iterations": 1,
        "parallelism": 2
      },
      "algo": "bcrypt"
    },
    "password_validation": {
      "admins": {
        "min_entropy": 0
      },
      "users": {
        "min_entropy": 0
      }
    },
    "password_caching": true,
    "update_mode": 0,
    "create_default_admin": false,
    "naming_rules": 5,
    "is_shared": 0,
    "node": {
      "host": "",
      "port": 0,
      "proto": "http"
    },
    "backups_path": "/srv/sftpgo/backups"
  },
  "httpd": {
    "bindings": [
      {
        "port": 8080,
        "address": "",
        "enable_web_admin": true,
        "enable_web_client": true,
        "enable_rest_api": true,
        "enabled_login_methods": 0,
        "enable_https": false,
        "certificate_file": "",
        "certificate_key_file": "",
        "min_tls_version": 12,
        "client_auth_type": 0,
        "tls_cipher_suites": [],
        "tls_protocols": [],
        "proxy_allowed": [],
        "client_ip_proxy_header": "",
        "client_ip_header_depth": 0,
        "hide_login_url": 0,
        "render_openapi": true,
        "oidc": {
          "client_id": "",
          "client_secret": "",
          "client_secret_file": "",
          "config_url": "",
          "redirect_base_url": "",
          "scopes": [
            "openid",
            "profile",
            "email"
          ],
          "username_field": "",
          "role_field": "",
          "implicit_roles": false,
          "custom_fields": [],
          "insecure_skip_signature_check": false,
          "debug": false
        },
        "security": {
          "enabled": false,
          "allowed_hosts": [],
          "allowed_hosts_are_regex": false,
          "hosts_proxy_headers": [],
          "https_redirect": false,
          "https_host": "",
          "https_proxy_headers": [],
          "sts_seconds": 0,
          "sts_include_subdomains": false,
          "sts_preload": false,
          "content_type_nosniff": false,
          "content_security_policy": "",
          "permissions_policy": "",
          "cross_origin_opener_policy": ""
        },
        "branding": {
          "web_admin": {
            "name": "",
            "short_name": "",
            "favicon_path": "",
            "logo_path": "",
            "login_image_path": "",
            "disclaimer_name": "",
            "disclaimer_path": "",
            "default_css": [],
            "extra_css": []
          },
          "web_client": {
            "name": "",
            "short_name": "",
            "favicon_path": "",
            "logo_path": "",
            "login_image_path": "",
            "disclaimer_name": "",
            "disclaimer_path": "",
            "default_css": [],
            "extra_css": []
          }
        }
      }
    ],
    "templates_path": "templates",
    "static_files_path": "static",
    "openapi_path": "openapi",
    "web_root": "",
    "certificate_file": "",
    "certificate_key_file": "",
    "ca_certificates": [],
    "ca_revocation_lists": [],
    "signing_passphrase": "",
    "signing_passphrase_file": "",
    "token_validation": 0,
    "max_upload_file_size": 0,
    "cors": {
      "enabled": false,
      "allowed_origins": [],
      "allowed_methods": [],
      "allowed_headers": [],
      "exposed_headers": [],
      "allow_credentials": false,
      "max_age": 0,
      "options_passthrough": false,
      "options_success_status": 0,
      "allow_private_network": false
    },
    "setup": {
      "installation_code": "",
      "installation_code_hint": "Installation code"
    },
    "hide_support_link": false
  },
  "telemetry": {
    "bind_port": 0,
    "bind_address": "127.0.0.1",
    "enable_profiler": false,
    "auth_user_file": "",
    "certificate_file": "",
    "certificate_key_file": "",
    "min_tls_version": 12,
    "tls_cipher_suites": [],
    "tls_protocols": []
  },
  "http": {
    "timeout": 20,
    "retry_wait_min": 2,
    "retry_wait_max": 30,
    "retry_max": 3,
    "ca_certificates": [],
    "certificates": [],
    "skip_tls_verify": false,
    "headers": []
  },
  "command": {
    "timeout": 30,
    "env": [],
    "commands": []
  },
  "kms": {
    "secrets": {
      "url": "",
      "master_key": "",
      "master_key_path": ""
    }
  },
  "mfa": {
    "totp": [
      {
        "name": "Default",
        "issuer": "SFTPGo",
        "algo": "sha1"
      }
    ]
  },
  "smtp": {
    "host": "",
    "port": 587,
    "from": "",
    "user": "",
    "password": "",
    "auth_type": 0,
    "encryption": 0,
    "domain": "",
    "templates_path": "templates",
    "debug": 0,
    "oauth2": {
      "provider": 0,
      "tenant": "",
      "client_id": "",
      "client_secret": "",
      "refresh_token": ""
    }
  },
  "plugins": []
}

Relevant log output

No response

What are you using SFTPGo for?

Private user, home usecase (home backup/VPS)

Additional info

Thank you

drakkan commented 3 months ago

Please re-open if it also happens with the latest stable release

tpoxa commented 3 months ago

Ok, I'll try, thanks. I have not seen any mention of that in the latest two release notes.

drakkan commented 3 months ago

Release notes and docs may be lacking. I don't have time for everything, sorry. I give priority to requests from paying users who help keep the project alive