Old commit nukes entire data folders for active models

[HistoricTheater]

Latest commit 66a368e has nuked my ENTIRE onlyfans folder,all content in the "messages" and "posts" folder for the active models has been deleted. Models previously downloaded but no longer subbed to are untouched. The metadata base is properly filled out however, no content is downloaded after running the script. Deleting the metadata makes the script download the content again, but running the script a second time removes the data completly upon updating Also setting "auto_profile_choice" to 0 or 1 breaks the script and requires user input

config.json

{
  "info": {
    "version": 8.0
  },
  "settings": {
    "auto_site_choice": "onlyfans",
    "export_type": "json",
    "profile_directories": [
      ".profiles"
    ],
    "max_threads": -1,
    "min_drive_space": 0,
    "helpers": {
      "renamer": true,
      "reformat_media": true,
      "downloader": true,
      "delete_empty_directories": false
    },
    "webhooks": {
      "global_webhooks": [],
      "global_status": true,
      "auth_webhook": {
        "succeeded": {
          "webhooks": [],
          "status": null,
          "hide_sensitive_info": true
        },
        "failed": {
          "webhooks": [],
          "status": null,
          "hide_sensitive_info": true
        }
      },
      "download_webhook": {
        "succeeded": {
          "webhooks": [],
          "status": null,
          "hide_sensitive_info": true
        }
      }
    },
    "exit_on_completion": true,
    "infinite_loop": false,
    "loop_timeout": 0,
    "dynamic_rules_link": "https://raw.githubusercontent.com/DATAHOARDERS/dynamic-rules/main/onlyfans.json",
    "proxies": [],
    "cert": "",
    "random_string": "f9c2f430730811ecaf3aa85e45a229e9"
  },
  "supported": {
    "onlyfans": {
      "settings": {
        "auto_profile_choice": "0",
        "auto_model_choice": true,
        "auto_media_choice": "0",
        "auto_api_choice": true,
        "browser": {
          "auth": false
        },
        "jobs": {
          "scrape": {
            "subscriptions": true,
            "paid_content": true
          },
          "metadata": {
            "posts": true,
            "comments": true
          }
        },
        "download_directories": [
          ".sites"
        ],
        "file_directory_format": "{site_name}/{model_username}/{api_type}/{value}/{media_type}",
        "filename_format": "{filename}.{ext}",
        "metadata_directories": [
          ".sites"
        ],
        "metadata_directory_format": "{site_name}/{model_username}/Metadata",
        "delete_legacy_metadata": false,
        "text_length": 255,
        "video_quality": "source",
        "overwrite_files": false,
        "date_format": "%d-%m-%Y",
        "ignored_keywords": [],
        "ignore_type": "",
        "blacklists": [
          "Blacklisted"
        ],
        "webhook": false
      }
    },
    "fansly": {
      "settings": {
        "auto_profile_choice": [],
        "auto_model_choice": false,
        "auto_media_choice": "",
        "auto_api_choice": true,
        "browser": {
          "auth": true
        },
        "jobs": {
          "scrape": {
            "subscriptions": true,
            "paid_content": true
          },
          "metadata": {
            "posts": true,
            "comments": true
          }
        },
        "download_directories": [
          ".sites"
        ],
        "file_directory_format": "{site_name}/{model_username}/{api_type}/{value}/{media_type}",
        "filename_format": "{filename}.{ext}",
        "metadata_directories": [
          ".sites"
        ],
        "metadata_directory_format": "{site_name}/{model_username}/Metadata",
        "delete_legacy_metadata": false,
        "text_length": 255,
        "video_quality": "source",
        "overwrite_files": false,
        "date_format": "%d-%m-%Y",
        "ignored_keywords": [],
        "ignore_type": "",
        "blacklists": [],
        "webhook": true
      }
    },
    "starsavn": {
      "settings": {
        "auto_profile_choice": [],
        "auto_model_choice": false,
        "auto_media_choice": "",
        "auto_api_choice": true,
        "browser": {
          "auth": true
        },
        "jobs": {
          "scrape": {
            "subscriptions": true,
            "paid_content": true
          },
          "metadata": {
            "posts": true,
            "comments": true
          }
        },
        "download_directories": [
          ".sites"
        ],
        "file_directory_format": "{site_name}/{model_username}/{api_type}/{value}/{media_type}",
        "filename_format": "{filename}.{ext}",
        "metadata_directories": [
          ".sites"
        ],
        "metadata_directory_format": "{site_name}/{model_username}/Metadata",
        "delete_legacy_metadata": false,
        "text_length": 255,
        "video_quality": "source",
        "overwrite_files": false,
        "date_format": "%d-%m-%Y",
        "ignored_keywords": [],
        "ignore_type": "",
        "blacklists": [],
        "webhook": true
      }
    }
  }
}

[rileypollard12]

Get this error too

[Site3op]

The commit from 5 hours ago will also nuke your folders OF and Fansly.

[DIGITALCRIMINAL]

You don't have to delete the metadata, just set overwrite_files to true

[DIGITALCRIMINAL]

I don't know how you guys are nuking entire folders. I used the exact same config. Are you all using Windows?

[HistoricTheater]

You don't have to delete the metadata, just set overwrite_files to true

That would cause potential hundreds of gigs of wasted writes everyday,which isnt wanted and it hasent been needed previously

I don't know how you guys are nuking entire folders. I used the exact same config. Are you all using Windows?

Archlinux Python 3.10.1 using the latest requirements.txt All i did was git pull and run the script as usual,i went to check the rip and all the data was gone from every active sub i have.

[rileypollard12]

I’m using windows, I’ll try to use/document more of what is happening but folders were empty and had to delete meta data to get it to download.

[der-pepe]

I'm, using Python 3.10.1 on Linux, and have the exact same issues with the latest available package (downloaded with the updater.py script) whole directories get erased and maybe there's one or two files with the newest content, but yeah, all the old files gone (some of them aren't even available anymore on the OF platform, so quite a loss) Is there some way we can help debuging this?

[der-pepe]

Also, quick note, deleting the metadata and re-running the script doesn't always download all the content again, sometimes you need to perform several runs in order to get some pics/vids the previous runs missed... (maybe unrelated, but still)

[DIGITALCRIMINAL]

You can recover them with software if it's truly lost.

For debugging, I use VSCODE and pretty much add a break point at the start of start_ofd.py and follow the scripts execution until you notice files missing.

This is strange behaviour and shouldn't happen...

[DIGITALCRIMINAL]

I'm on my Windows machine and I've recreated the issue

[DIGITALCRIMINAL]

I stopped the script from deleting files, sorry about this... I used the program Recuva to recover deleted files

https://github.com/DIGITALCRIMINALS/OnlyFans/commit/21e4e5684fe2f98ca4320b8715fad6b131dca1bd

[DIGITALCRIMINAL]

If a model deleted their account and you can no longer download their content, email me their usernames and I will see if I can help. (I've mainly only got OnlyFans)

[Gamma-Velorum]

Jesus christ, glad I put off cloning the latest commit and only did so after the fix.

[rileypollard12]

Luckily I always move the downloaded files out of the folder 😂

[E0911CD45B19686]

That would've been a huge bummer if I didn't have snapshots enabled 🙀

before after

[E0911CD45B19686]

Possibly the same or a similar issue? I found ~2000 files that weren't deleted, but had a file size of exactly 741 bytes. Happened to free and paid images and videos, for models that I'm currently subscribed to and not subscribed to anymore.

I can't say for certain this was a recent or related problem because I didn't verify my backups until just now.

If you only restored missing files, check again to see that you still didn't have any data loss.

[rileypollard12]

Let’s take this as a lesion to back up our data people!

[icemouton]

Just to make sure this had no effect on linux ?

[DIGITALCRIMINAL]

Just to make sure this had no effect on linux ?

I'm using Arch Linux and nothing was deleted on mine, but on Windows 99.9% :disappointed:

If you used any commits from this https://github.com/DIGITALCRIMINALS/OnlyFans/commit/af3f6b793dde384221255205646255989802d2d1 to https://github.com/DIGITALCRIMINALS/OnlyFans/commit/65cb5a1eeab9a57d472494b5d7f9fb6a8505bc2f you may have gotten rekted.

The problem was me comparing a path string to a Path object. If they weren't the same, it would make the script delete the file.

[E0911CD45B19686]

Just to make sure this had no effect on linux ?

I'm running inside docker on linux, it affected me (see above comments).

[Animal6311]

I still have this problem and I have the latest commit :(

[DIGITALCRIMINAL]

I still have this problem and I have the latest commit :(

Can't happen again because I removed the code that removed files... Make sure you're using the latest I just updated it.

[Animal6311]

I still have this problem and I have the latest commit :(

Can't happen again because I removed the code that removed files... Make sure you're using the latest I just updated it.

Alright, I'll update and try again. Thanks

[Luke-L]

have the problems with the current release been fixed??

[DIGITALCRIMINAL]

The script will no longer delete any files. We'll put everything that needs to be deleted into a trash folder which resides in the __user_data__ folder.

https://github.com/DIGITALCRIMINALS/OnlyFans/tree/Test