FILE_EXISTS/FILE_EXISTS_IN_GAME fails to detect large files on Windows

[Argent77]

Both FILE_EXISTS and FILE_EXISTS_IN_GAME return false for files of size >= 1 GB. I could reproduce this bug on Windows only. Mac OS X isn't affected. I haven't tested on Linux, but I doubt it will be affected as well.

The bug is currently affecting mods that try to detect non-modmerge'd games.

[FredrikLindgren]

This is likely due to the 32-bitness of the Windows binary. FILE_EXISTS_IN_GAME is supposed to check for too-large files and return true on the appropriate exception, but maybe the string/bytes thing has resulted in a different exception being raised. FILE_EXISTS is probably running into the same limitation Blaze reported during his zipload work: files over a GB raise an exception, which is turned into boolean false.

I'll see what I can do.

[Argent77]

I did a few tests for FILE_EXISTS, based on zipload code from Blaze. The LargeFile extension seems to work fine on Windows.

diff -Naur weidu-orig/src/case_ins_linux.ml weidu/src/case_ins_linux.ml
--- weidu-orig/src/case_ins_linux.ml    2017-02-22 12:07:24 +0100
+++ weidu/src/case_ins_linux.ml 2017-08-18 12:02:11 +0200
@@ -22,6 +22,7 @@
 let unix_opendir s = Unix.opendir (String.lowercase (backslash_to_slash s)) ;;
 let unix_rename s d = Unix.rename (String.lowercase (backslash_to_slash s)) (String.lowercase (backslash_to_slash d));;
 let unix_rmdir s = Unix.rmdir (String.lowercase (backslash_to_slash s));;
+let unix_stat64 s = Unix.LargeFile.stat (String.lowercase (backslash_to_slash s)) ;;

 let sys_readdir s = Sys.readdir (String.lowercase (backslash_to_slash s));;

diff -Naur weidu-orig/src/case_ins_mac.ml weidu/src/case_ins_mac.ml
--- weidu-orig/src/case_ins_mac.ml  2017-02-22 12:07:24 +0100
+++ weidu/src/case_ins_mac.ml   2017-08-18 12:02:31 +0200
@@ -19,6 +19,7 @@
 let unix_opendir s = Unix.opendir (backslash_to_slash s) ;;
 let unix_rename s d = Unix.rename (backslash_to_slash s) (backslash_to_slash d);;
 let unix_rmdir s = Unix.rmdir (backslash_to_slash s);;
+let unix_stat64 s = Unix.LargeFile.stat (backslash_to_slash s) ;;

 let sys_readdir s = Sys.readdir (backslash_to_slash s);;

diff -Naur weidu-orig/src/case_ins_win.ml weidu/src/case_ins_win.ml
--- weidu-orig/src/case_ins_win.ml  2017-02-22 12:07:24 +0100
+++ weidu/src/case_ins_win.ml   2017-08-18 12:02:43 +0200
@@ -15,6 +15,7 @@
 let unix_opendir s = Unix.opendir s ;;
 let unix_rename s d = Unix.rename s d;;
 let unix_rmdir s = Unix.rmdir s;;
+let unix_stat64 s = Unix.LargeFile.stat s ;;

 let sys_readdir s = Sys.readdir s;;

diff -Naur weidu-orig/src/tppe.ml weidu/src/tppe.ml
--- weidu-orig/src/tppe.ml  2017-06-21 17:49:38 +0200
+++ weidu/src/tppe.ml   2017-08-18 13:03:37 +0200
@@ -24,7 +24,7 @@
     end
   else begin
 (*    log_and_print "\nbigg_file_exist special case not triggered: %s\n" file ; *)
-    test := file_size file >= 0
+    test := file_size_large file >= Int64.zero
   end ;
   !test

diff -Naur weidu-orig/src/util.ml weidu/src/util.ml
--- weidu-orig/src/util.ml  2017-06-21 17:49:38 +0200
+++ weidu/src/util.ml   2017-08-18 13:06:19 +0200
@@ -339,7 +339,13 @@
     stats.Unix.st_size
   with _ ->  -1)

-let file_exists name = (file_size name >= 0)
+let file_size_large name =
+  (try
+    let stats = Case_ins.unix_stat64 name in
+    stats.Unix.LargeFile.st_size
+  with _ ->  -1L)
+
+let file_exists name = (file_size_large name >= Int64.zero)

 let is_directory name =
   (try

[FredrikLindgren]

This should fix FILE_EXISTS_IN_GAME too, right? Since it uses file_exists internally (the file-too-big handling was, of course, only for biffed files). I'll probably forego setting up a test environment and just apply this. Thanks for the help!

[Argent77]

No. FILE_EXISTS_IN_GAME appears to be handled differently.

For files between 16MB and 1 GB size it returns false in current and past WeiDU versions and prints the following message:

ERROR: [./override/test.tis] has size 16777216: TOO BIG FOR WEIDU (max 16777211)
ERROR: error loading [./override/test.tis]

Files greater 1 GB still return false, but don't produce any error messages.

With LargeFile code added FILE_EXISTS_IN_GAME behaves the same, but still prints the following message for files > 1 GB:

ERROR: error loading [./override/test.tis]

[FredrikLindgren]

Fixed. You can now check if files up to Int64.max_int in size exist with both FILE_EXISTS and FILE_EXISTS_IN_GAME.