Closed zjplab closed 5 years ago
Ok. I have solved this problem myself. This error arises when you try to compile fastBPE( a dependency of fairseq) in windows. In order to solve this, you have to make modifications on:
distutils
's cygwinccompiler.py
First, git clone fastBPE source code to your local directory. Change line 15:
#include <mman.c>
to
#include "mman.c"
Add mman.h
to fastBPE/fastBPE directory:
/*
* sys/mman.h
* mman-win32
*/
#ifndef _SYS_MMAN_H_
#define _SYS_MMAN_H_
#ifndef _WIN32_WINNT // Allow use of features specific to Windows XP or later.
#define _WIN32_WINNT 0x0501 // Change this to the appropriate value to target other versions of Windows.
#endif
/* All the headers include this file. */
#ifndef _MSC_VER
#include <_mingw.h>
#endif
#include <sys/types.h>
#ifdef __cplusplus
extern "C" {
#endif
#define PROT_NONE 0
#define PROT_READ 1
#define PROT_WRITE 2
#define PROT_EXEC 4
#define MAP_FILE 0
#define MAP_SHARED 1
#define MAP_PRIVATE 2
#define MAP_TYPE 0xf
#define MAP_FIXED 0x10
#define MAP_ANONYMOUS 0x20
#define MAP_ANON MAP_ANONYMOUS
#define MAP_FAILED ((void *)-1)
/* Flags for msync. */
#define MS_ASYNC 1
#define MS_SYNC 2
#define MS_INVALIDATE 4
void* mmap(void *addr, size_t len, int prot, int flags, int fildes, off_t off);
int munmap(void *addr, size_t len);
int mprotect(void *addr, size_t len, int prot);
int msync(void *addr, size_t len, int flags);
int mlock(const void *addr, size_t len);
int munlock(const void *addr, size_t len);
#ifdef __cplusplus
};
#endif
#endif /* _SYS_MMAN_H_ */
and also mman.c
#include <windows.h>
#include <errno.h>
#include <io.h>
#include "mman.h"
#ifndef FILE_MAP_EXECUTE
#define FILE_MAP_EXECUTE 0x0020
#endif /* FILE_MAP_EXECUTE */
static int __map_mman_error(const DWORD err, const int deferr)
{
if (err == 0)
return 0;
//TODO: implement
return err;
}
static DWORD __map_mmap_prot_page(const int prot)
{
DWORD protect = 0;
if (prot == PROT_NONE)
return protect;
if ((prot & PROT_EXEC) != 0)
{
protect = ((prot & PROT_WRITE) != 0) ?
PAGE_EXECUTE_READWRITE : PAGE_EXECUTE_READ;
}
else
{
protect = ((prot & PROT_WRITE) != 0) ?
PAGE_READWRITE : PAGE_READONLY;
}
return protect;
}
static DWORD __map_mmap_prot_file(const int prot)
{
DWORD desiredAccess = 0;
if (prot == PROT_NONE)
return desiredAccess;
if ((prot & PROT_READ) != 0)
desiredAccess |= FILE_MAP_READ;
if ((prot & PROT_WRITE) != 0)
desiredAccess |= FILE_MAP_WRITE;
if ((prot & PROT_EXEC) != 0)
desiredAccess |= FILE_MAP_EXECUTE;
return desiredAccess;
}
void* mmap(void *addr, size_t len, int prot, int flags, int fildes, off_t off)
{
HANDLE fm, h;
void * map = MAP_FAILED;
#ifdef _MSC_VER
#pragma warning(push)
#pragma warning(disable: 4293)
#endif
const DWORD dwFileOffsetLow = (sizeof(off_t) <= sizeof(DWORD)) ?
(DWORD)off : (DWORD)(off & 0xFFFFFFFFL);
const DWORD dwFileOffsetHigh = (sizeof(off_t) <= sizeof(DWORD)) ?
(DWORD)0 : (DWORD)((off >> 32) & 0xFFFFFFFFL);
const DWORD protect = __map_mmap_prot_page(prot);
const DWORD desiredAccess = __map_mmap_prot_file(prot);
const off_t maxSize = off + (off_t)len;
const DWORD dwMaxSizeLow = (sizeof(off_t) <= sizeof(DWORD)) ?
(DWORD)maxSize : (DWORD)(maxSize & 0xFFFFFFFFL);
const DWORD dwMaxSizeHigh = (sizeof(off_t) <= sizeof(DWORD)) ?
(DWORD)0 : (DWORD)((maxSize >> 32) & 0xFFFFFFFFL);
#ifdef _MSC_VER
#pragma warning(pop)
#endif
errno = 0;
if (len == 0
/* Unsupported flag combinations */
|| (flags & MAP_FIXED) != 0
/* Usupported protection combinations */
|| prot == PROT_EXEC)
{
errno = EINVAL;
return MAP_FAILED;
}
h = ((flags & MAP_ANONYMOUS) == 0) ?
(HANDLE)_get_osfhandle(fildes) : INVALID_HANDLE_VALUE;
if ((flags & MAP_ANONYMOUS) == 0 && h == INVALID_HANDLE_VALUE)
{
errno = EBADF;
return MAP_FAILED;
}
fm = CreateFileMapping(h, NULL, protect, dwMaxSizeHigh, dwMaxSizeLow, NULL);
if (fm == NULL)
{
errno = __map_mman_error(GetLastError(), EPERM);
return MAP_FAILED;
}
map = MapViewOfFile(fm, desiredAccess, dwFileOffsetHigh, dwFileOffsetLow, len);
CloseHandle(fm);
if (map == NULL)
{
errno = __map_mman_error(GetLastError(), EPERM);
return MAP_FAILED;
}
return map;
}
int munmap(void *addr, size_t len)
{
if (UnmapViewOfFile(addr))
return 0;
errno = __map_mman_error(GetLastError(), EPERM);
return -1;
}
int mprotect(void *addr, size_t len, int prot)
{
DWORD newProtect = __map_mmap_prot_page(prot);
DWORD oldProtect = 0;
if (VirtualProtect(addr, len, newProtect, &oldProtect))
return 0;
errno = __map_mman_error(GetLastError(), EPERM);
return -1;
}
int msync(void *addr, size_t len, int flags)
{
if (FlushViewOfFile(addr, len))
return 0;
errno = __map_mman_error(GetLastError(), EPERM);
return -1;
}
int mlock(const void *addr, size_t len)
{
if (VirtualLock((LPVOID)addr, len))
return 0;
errno = __map_mman_error(GetLastError(), EPERM);
return -1;
}
int munlock(const void *addr, size_t len)
{
if (VirtualUnlock((LPVOID)addr, len))
return 0;
errno = __map_mman_error(GetLastError(), EPERM);
return -1;
}
Reason : Under any windows C++ compiler, we don't have mman.h
. But I have found the following windows alternative lib.
Second:
Build modified fastBPE using:
python setup.py build --compiler=mingw32
And then install it:
python setup.py install
Third, two errors will occur in the last step. The fix to the following issue is similar to this page.
We modify cygwinccompiler.py
under distutils
:
In around line 85 we change to:
### PATCH###############################
# INCLUDES NEWEST mscvcr VERSION
#########################################
elif msc_ver >= '1900':
# Visual Studio 2015 / Visual C++ 14.0
# "msvcr140.dll no longer exists" http://blogs.msdn.com/b/vcblog/archive/2014/06/03/visual-studio-14-ctp.aspx
return ['vcruntime140']
else:
raise ValueError("Unknown MS Compiler version %s " % msc_ver)
Notice, my compiler version is 1915( to chekc your version, msc_pos = sys.version.find('MSC v.')
and then msc_ver = sys.version[msc_pos+6:msc_pos+10]
). After testing, vcruntime140
still works for this 1915 versioned compiler.
And then you need to copy a vcruntime140.dll
to your gcc lib. Identical to this page link.
After this step, you should be able to compile successfully albeit with some warnings.
And then you can install fastBPE
:
and then
pip install fairseq
These solutions did not work for me sadly. At the end of the third stage when I run
python setup.py install
,
throws another error
d:\development\fastbpe\fastbpe\fastBPE.hpp(18): fatal error C1083: Cannot open include file: 'unistd.h': No such file or directory error: command 'C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\BIN\\x86_amd64\\cl.exe' failed with exit status 2
@aliyesilkanat Possible fix for you: https://stackoverflow.com/questions/2793413/unistd-h-related-difficulty-when-compiling-bison-flex-program-under-vc/2872995#2872995
@aliyesilkanat Looks like you are stuck at the fastBPE. Did you use that
python setup.py build --compiler=mingw32
command?
@aliyesilkanat I tried a few days, finally, I got it. This tutorial provided by @zjplab is TOTALLY ESSENTIAL to anyone persists in Windows 10 python environment
where is "#include
I cannot locate the mman.c line as well
where is "#include
" ? @HoiBunCa Copy and paste to create one in your local folder
@zjplab i followed your instructions 1.Downloaded fastBPE repo 2.added "if msc_ver >= '1900': return ['vcruntime140']" in distutils\cygwinccompiler.py
Im getting this error 'thread' was not declared in this scope 28 | const size_t kThreads = max(1, min(10, int(thread::hardware_concurrency())))
Full stack trace: Tired to debug and search for this issue in other portals but was not able to get any solutions could you please assist
` python setup.py build --compiler=mingw32 running build running build_py running build_ext building 'fastBPE' extension C:\MinGW\bin\gcc.exe -mdll -O -Wall -IfastBPE -IC:\Users\aa\AppData\Local\Programs\Python\Python37\include -IC:\Users\aa\AppData\Local\Programs\Python\Python37\include -c fastBPE/fastBPE.cpp -o build\temp.win-amd64-3.7\Release\fastbpe\fastbpe.o -std=c++11 -Ofast -pthread In file included from fastBPE/fastBPE.cpp:653: fastBPE/fastBPE.hpp:28:44: error: 'thread' was not declared in this scope 28 | const size_t kThreads = max(1, min(10, int(thread::hardware_concurrency()))); | ^ |
#include |
+#include |
#include |
fprintf(stderr, "Read %lu words (%lu unique) from text file.\n", total, | |
---|---|---|---|---|---|---|---|
long unsigned int uint64_t {aka long long unsigned int} | |||||||
%I64u |
fastBPE/fastBPE.hpp:83:38: warning: format '%lu' expects argument of type 'long unsigned int', but argument 4 has type 'std::unordered_map<std::__cxx11::basic_string |
fprintf(stderr, "Read %lu words (%lu unique) from text file.\n", total, | ~~^ |
---|---|---|
long unsigned int | ||
%u | ||
84 | word_count.size()); | |
std::unordered_map<std::__cxx11::basic_string |
fastBPE/fastBPE.hpp: In function 'void fastBPE::outputText(const char, const char, std::unordered_map<std::__cxx11::basic_string |
if (ftruncate(fdOut, out_size) < 0) { | ^ |
strncat fastBPE/fastBPE.hpp:137:65: warning: format '%lu' expects argument of type 'long unsigned int', but argument 4 has type 'size_t' {aka 'unsigned int'} [-Wformat=] 137 | fprintf(stderr, "Couldn't truncate output file %s to size %lu\n", fpo, | ~~^ |
---|---|---|---|---|---|
long unsigned int | |||||
%u | |||||
138 | out_size); | ||||
size_t {aka unsigned int} |
fastBPE/fastBPE.hpp:149:31: warning: format '%lu' expects argument of type 'long unsigned int', but argument 3 has type 'long long unsigned int' [-Wformat=] 149 | fprintf(stderr, "Modified %lu words from text file.\n", p.second); | |
---|---|---|
long unsigned int long long unsigned int | ||
%I64u |
fastBPE/fastBPE.hpp: In function 'void fastBPE::readVocab(const char*, std::unordered_map<std::__cxx11::basic_string |
fprintf(stderr, "Read %lu words (%lu unique) from vocabulary file.\n", total, | |
---|---|---|
long unsigned int uint64_t {aka long long unsigned int} | ||
%I64u |
fastBPE/fastBPE.hpp:438:38: warning: format '%lu' expects argument of type 'long unsigned int', but argument 4 has type 'std::unordered_map<std::__cxx11::basic_string |
fprintf(stderr, "Read %lu words (%lu unique) from vocabulary file.\n", total, | ~~^ |
---|---|---|
long unsigned int | ||
%u | ||
439 | vocab.size()); | |
std::unordered_map<std::__cxx11::basic_string |
fastBPE/fastBPE.hpp: In function 'void fastBPE::readCodes(const char*, std::unordered_map<std::pair<std::cxx11::basic_string |
fprintf(stderr, "Read %lu codes from the codes file.\n", codes.size()); | |
---|---|---|
long unsigned int std::unordered_map<std::pair<std::__cxx11::basic_string |
||
%u |
fastBPE/fastBPE.hpp: In function 'void fastBPE::applybpe(const char, const char, const char, const char)':
fastBPE/fastBPE.hpp:607:37: error: size of array 'bpe' is not an integral constant-expression
607 | unordered_map<string, string> bpe[kThreads];
| ^~~~
fastBPE/fastBPE.hpp:608:10: error: 'thread' was not declared in this scope
608 | vector~
fastBPE/fastBPE.hpp:608:10: note: 'std::thread' is defined in header '~~~
fastBPE/fastBPE.hpp:623:14: error: invalid types 'int[size_t {aka unsigned int}]' for array subscript
623 | threads[i].join();
| ^
error: command 'C:\MinGW\bin\gcc.exe' failed with exit status 1`
Just adding comments of own experiences to help others with this. For reference I'm trying to get fastBPE for use for the salesforce CTRL model here: https://github.com/salesforce/ctrl
Problem 1:
Problem 2:
Problem 3:
UPDATE:
UPDATE 2:
when I use "pip install fastBPE" it error. Can you help me fix error?
Microsoft Windows [Version 10.0.17763.1282] (c) 2018 Microsoft Corporation. All rights reserved.
ERROR: Command errored out with exit status 1: 'c:\users\dorae\appdata\local\programs\python\python38\python.exe' -u -c 'import sys, setuptools, tokenize; sys.argv[0] = '"'"'C:\Users\dorae\AppData\Local\Temp\pip-install-k3niu8wz\fastBPE\setup.py'"'"'; file='"'"'C:\Users\dorae\AppData\Local\Temp\pip-install-k3niu8wz\fastBPE\setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(file);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, file, '"'"'exec'"'"'))' install --record 'C:\Users\dorae\AppData\Local\Temp\pip-record-brnadaju\install-record.txt' --single-version-externally-managed --compile --install-headers 'c:\users\dorae\appdata\local\programs\python\python38\Include\fastBPE' Check the logs for full command output. WARNING: You are using pip version 20.0.1; however, version 20.3.3 is available. You should consider upgrading via the 'c:\users\dorae\appdata\local\programs\python\python38\python.exe -m pip install --upgrade pip' command.
@ajbot2019 I followed your instruction but it was error.
These solutions did not work for me sadly. At the end of the third stage when I run
python setup.py install
, throws another errord:\development\fastbpe\fastbpe\fastBPE.hpp(18): fatal error C1083: Cannot open include file: 'unistd.h': No such file or directory error: command 'C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\BIN\\x86_amd64\\cl.exe' failed with exit status 2
so how did you solve it finally?
This is how I solved it, second to zjplab but slightly different:
git clone https://github.com/glample/fastBPE.git cd fastBPE notepad fastBPE.hpp
* BE CAREFULL from here ---> (1) remove
---> (2) add
--->(3) notepad mman.c
static int __map_mman_error(const DWORD err, const int deferr) { if (err == 0) return 0; //TODO: implement return err; }
static DWORD __map_mmap_prot_page(const int prot) { DWORD protect = 0;
if (prot == PROT_NONE)
return protect;
if ((prot & PROT_EXEC) != 0)
{
protect = ((prot & PROT_WRITE) != 0) ?
PAGE_EXECUTE_READWRITE : PAGE_EXECUTE_READ;
}
else
{
protect = ((prot & PROT_WRITE) != 0) ?
PAGE_READWRITE : PAGE_READONLY;
}
return protect;
}
static DWORD __map_mmap_prot_file(const int prot) { DWORD desiredAccess = 0;
if (prot == PROT_NONE)
return desiredAccess;
if ((prot & PROT_READ) != 0)
desiredAccess |= FILE_MAP_READ;
if ((prot & PROT_WRITE) != 0)
desiredAccess |= FILE_MAP_WRITE;
if ((prot & PROT_EXEC) != 0)
desiredAccess |= FILE_MAP_EXECUTE;
return desiredAccess;
}
void mmap(void addr, size_t len, int prot, int flags, int fildes, off_t off) { HANDLE fm, h;
void * map = MAP_FAILED;
const DWORD dwFileOffsetLow = (sizeof(off_t) <= sizeof(DWORD)) ?
(DWORD)off : (DWORD)(off & 0xFFFFFFFFL);
const DWORD dwFileOffsetHigh = (sizeof(off_t) <= sizeof(DWORD)) ?
(DWORD)0 : (DWORD)((off >> 32) & 0xFFFFFFFFL);
const DWORD protect = __map_mmap_prot_page(prot);
const DWORD desiredAccess = __map_mmap_prot_file(prot);
const off_t maxSize = off + (off_t)len;
const DWORD dwMaxSizeLow = (sizeof(off_t) <= sizeof(DWORD)) ?
(DWORD)maxSize : (DWORD)(maxSize & 0xFFFFFFFFL);
const DWORD dwMaxSizeHigh = (sizeof(off_t) <= sizeof(DWORD)) ?
(DWORD)0 : (DWORD)((maxSize >> 32) & 0xFFFFFFFFL);
errno = 0;
if (len == 0
/* Unsupported flag combinations */
|| (flags & MAP_FIXED) != 0
/* Usupported protection combinations */
|| prot == PROT_EXEC)
{
errno = EINVAL;
return MAP_FAILED;
}
h = ((flags & MAP_ANONYMOUS) == 0) ?
(HANDLE)_get_osfhandle(fildes) : INVALID_HANDLE_VALUE;
if ((flags & MAP_ANONYMOUS) == 0 && h == INVALID_HANDLE_VALUE)
{
errno = EBADF;
return MAP_FAILED;
}
fm = CreateFileMapping(h, NULL, protect, dwMaxSizeHigh, dwMaxSizeLow, NULL);
if (fm == NULL)
{
errno = __map_mman_error(GetLastError(), EPERM);
return MAP_FAILED;
}
map = MapViewOfFile(fm, desiredAccess, dwFileOffsetHigh, dwFileOffsetLow, len);
CloseHandle(fm);
if (map == NULL)
{
errno = __map_mman_error(GetLastError(), EPERM);
return MAP_FAILED;
}
return map;
}
int munmap(void *addr, size_t len) { if (UnmapViewOfFile(addr)) return 0;
errno = __map_mman_error(GetLastError(), EPERM);
return -1;
}
int mprotect(void *addr, size_t len, int prot) { DWORD newProtect = __map_mmap_prot_page(prot); DWORD oldProtect = 0;
if (VirtualProtect(addr, len, newProtect, &oldProtect))
return 0;
errno = __map_mman_error(GetLastError(), EPERM);
return -1;
}
int msync(void *addr, size_t len, int flags) { if (FlushViewOfFile(addr, len)) return 0;
errno = __map_mman_error(GetLastError(), EPERM);
return -1;
}
int mlock(const void *addr, size_t len) { if (VirtualLock((LPVOID)addr, len)) return 0;
errno = __map_mman_error(GetLastError(), EPERM);
return -1;
}
int munlock(const void *addr, size_t len) { if (VirtualUnlock((LPVOID)addr, len)) return 0;
errno = __map_mman_error(GetLastError(), EPERM);
return -1;
}
-->(4) notepad mman.h
/*
/ All the headers include this file. /
extern "C" {
/ Flags for msync. /
void mmap(void addr, size_t len, int prot, int flags, int fildes, off_t off); int munmap(void addr, size_t len); int mprotect(void addr, size_t len, int prot); int msync(void addr, size_t len, int flags); int mlock(const void addr, size_t len); int munlock(const void *addr, size_t len);
};
-->(5) go back "cd.." python setup.py install --user
YES that is it ! Annoying really, a globally used package is just so fully untested ! Steve
This is how I solved it, second to zjplab but slightly different:
git clone https://github.com/glample/fastBPE.git cd fastBPE notepad fastBPE.hpp
* BE CAREFULL from here ---> (1) remove #include <sys/mman.h ---> (2) add #include "mman.c" #include "mman.h"
--->(3) notepad mman.c
include
#include #include include "mman.h"
ifndef FILE_MAP_EXECUTE #define FILE_MAP_EXECUTE 0x0020 #endif / FILE_MAP_EXECUTE /
static int __map_mman_error(const DWORD err, const int deferr) { if (err == 0) return 0; //TODO: implement return err; }
static DWORD __map_mmap_prot_page(const int prot) { DWORD protect = 0;
if (prot == PROT_NONE) return protect; if ((prot & PROT_EXEC) != 0) { protect = ((prot & PROT_WRITE) != 0) ? PAGE_EXECUTE_READWRITE : PAGE_EXECUTE_READ; } else { protect = ((prot & PROT_WRITE) != 0) ? PAGE_READWRITE : PAGE_READONLY; } return protect;
}
static DWORD __map_mmap_prot_file(const int prot) { DWORD desiredAccess = 0;
if (prot == PROT_NONE) return desiredAccess; if ((prot & PROT_READ) != 0) desiredAccess |= FILE_MAP_READ; if ((prot & PROT_WRITE) != 0) desiredAccess |= FILE_MAP_WRITE; if ((prot & PROT_EXEC) != 0) desiredAccess |= FILE_MAP_EXECUTE; return desiredAccess;
}
void mmap(void addr, size_t len, int prot, int flags, int fildes, off_t off) { HANDLE fm, h;
void * map = MAP_FAILED;
ifdef _MSC_VER #pragma warning(push) #pragma warning(disable: 4293) #endif
const DWORD dwFileOffsetLow = (sizeof(off_t) <= sizeof(DWORD)) ? (DWORD)off : (DWORD)(off & 0xFFFFFFFFL); const DWORD dwFileOffsetHigh = (sizeof(off_t) <= sizeof(DWORD)) ? (DWORD)0 : (DWORD)((off >> 32) & 0xFFFFFFFFL); const DWORD protect = __map_mmap_prot_page(prot); const DWORD desiredAccess = __map_mmap_prot_file(prot); const off_t maxSize = off + (off_t)len; const DWORD dwMaxSizeLow = (sizeof(off_t) <= sizeof(DWORD)) ? (DWORD)maxSize : (DWORD)(maxSize & 0xFFFFFFFFL); const DWORD dwMaxSizeHigh = (sizeof(off_t) <= sizeof(DWORD)) ? (DWORD)0 : (DWORD)((maxSize >> 32) & 0xFFFFFFFFL);
ifdef _MSC_VER #pragma warning(pop) #endif
errno = 0; if (len == 0 /* Unsupported flag combinations */ || (flags & MAP_FIXED) != 0 /* Usupported protection combinations */ || prot == PROT_EXEC) { errno = EINVAL; return MAP_FAILED; } h = ((flags & MAP_ANONYMOUS) == 0) ? (HANDLE)_get_osfhandle(fildes) : INVALID_HANDLE_VALUE; if ((flags & MAP_ANONYMOUS) == 0 && h == INVALID_HANDLE_VALUE) { errno = EBADF; return MAP_FAILED; } fm = CreateFileMapping(h, NULL, protect, dwMaxSizeHigh, dwMaxSizeLow, NULL); if (fm == NULL) { errno = __map_mman_error(GetLastError(), EPERM); return MAP_FAILED; } map = MapViewOfFile(fm, desiredAccess, dwFileOffsetHigh, dwFileOffsetLow, len); CloseHandle(fm); if (map == NULL) { errno = __map_mman_error(GetLastError(), EPERM); return MAP_FAILED; } return map;
}
int munmap(void *addr, size_t len) { if (UnmapViewOfFile(addr)) return 0;
errno = __map_mman_error(GetLastError(), EPERM); return -1;
}
int mprotect(void *addr, size_t len, int prot) { DWORD newProtect = __map_mmap_prot_page(prot); DWORD oldProtect = 0;
if (VirtualProtect(addr, len, newProtect, &oldProtect)) return 0; errno = __map_mman_error(GetLastError(), EPERM); return -1;
}
int msync(void *addr, size_t len, int flags) { if (FlushViewOfFile(addr, len)) return 0;
errno = __map_mman_error(GetLastError(), EPERM); return -1;
}
int mlock(const void *addr, size_t len) { if (VirtualLock((LPVOID)addr, len)) return 0;
errno = __map_mman_error(GetLastError(), EPERM); return -1;
}
int munlock(const void *addr, size_t len) { if (VirtualUnlock((LPVOID)addr, len)) return 0;
errno = __map_mman_error(GetLastError(), EPERM); return -1;
}
-->(4) notepad mman.h
/*
- sys/mman.h
- mman-win32 */
ifndef _SYS_MMANH #define _SYS_MMANH
ifndef _WIN32_WINNT // Allow use of features specific to Windows XP or later. #define _WIN32_WINNT 0x0501 // Change this to the appropriate value to target other versions of Windows. #endif
/ All the headers include this file. / #ifndef _MSC_VER #include <_mingw.h> #endif
include <sys/types.h>
ifdef __cplusplus extern "C" { #endif
define PROT_NONE 0 #define PROT_READ 1 #define PROT_WRITE 2 #define PROT_EXEC 4
define MAP_FILE 0 #define MAP_SHARED 1 #define MAP_PRIVATE 2 #define MAP_TYPE 0xf #define MAP_FIXED 0x10 #define MAP_ANONYMOUS 0x20 #define MAP_ANON MAP_ANONYMOUS
define MAP_FAILED ((void *)-1)
/ Flags for msync. / #define MS_ASYNC 1 #define MS_SYNC 2 #define MS_INVALIDATE 4
void mmap(void addr, size_t len, int prot, int flags, int fildes, off_t off); int munmap(void addr, size_t len); int mprotect(void addr, size_t len, int prot); int msync(void addr, size_t len, int flags); int mlock(const void addr, size_t len); int munlock(const void *addr, size_t len);
ifdef __cplusplus }; #endif
endif / _SYS_MMANH /
-->(5) go back "cd.." python setup.py install --user
YES that is it ! Annoying really, a globally used package is just so fully untested ! Steve
Hello đồng hương, I followed your solution but still the error of unistd.h:
C:\Users\Hp\fastBPE\fastBPE\fastBPE.hpp(19): fatal error C1083: Cannot open include file: 'unistd.h': No such file or directory
error: command 'C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\BuildTools\\VC\\Tools\\MSVC\\14.29.30133\\bin\\HostX86\\x64\\cl.exe' failed with exit code 2
Can you suggest any action to do please ?
Hi, @calibretaliation Did you come up with any fix regarding the error of unistd.h ?
I have followed how to install fastBPE in Windows by @zjplab. However, when I run "python setup.py build --compiler=mingw32", it produces an error:
running build
running build_py
running build_ext
building 'fastBPE' extension
gcc -mdll -O -Wall -IfastBPE -IC:\Users\ASUS\anaconda3\envs\Testing_The_RS_AIP491\include -IC:\Users\ASUS\anaconda3\envs\Testing_The_RS_AIP491\Include -c fastBPE/fastBPE.cpp -o build\temp
.win-amd64-cpython-38\Release\fastbpe\fastbpe.o -std=c++11 -Ofast -pthread
In file included from fastBPE/fastBPE.cpp:789:
fastBPE/fastBPE.hpp: In function 'void fastBPE::readText(const char*, std::unordered_map<std::__cxx11::basic_string |
fprintf(stderr, "Read %lu words (%lu unique) from text file.\n", total, | |
---|---|---|
long unsigned int uint64_t {aka long long unsigned int} | ||
%llu |
fastBPE/fastBPE.hpp:84:38: warning: format '%lu' expects argument of type 'long unsigned int', but argument 4 has type 'std::unordered_map<std::__cxx11::basic_string |
fprintf(stderr, "Read %lu words (%lu unique) from text file.\n", total, | ~~^ |
---|---|---|
long unsigned int | ||
%u | ||
85 | word_count.size()); | |
std::unordered_map<std::__cxx11::basic_string |
fastBPE/fastBPE.hpp: In function 'void fastBPE::outputText(const char, const char, std::unordered_map<std::__cxx11::basic_string |
fprintf(stderr, "Couldn't truncate output file %s to size %lu\n", fpo, | ~~^ |
---|---|---|
long unsigned int | ||
%u | ||
139 | out_size); | |
size_t {aka unsigned int} |
fastBPE/fastBPE.hpp:150:31: warning: format '%lu' expects argument of type 'long unsigned int', but argument 3 has type 'long long unsigned int' [-Wformat=] 150 | fprintf(stderr, "Modified %lu words from text file.\n", p.second); | |
---|---|---|
long unsigned int long long unsigned int | ||
%llu |
fastBPE/fastBPE.hpp: In function 'void fastBPE::readVocab(const char*, std::unordered_map<std::__cxx11::basic_string |
fprintf(stderr, "Read %lu words (%lu unique) from vocabulary file.\n", total, | |
---|---|---|
long unsigned int uint64_t {aka long long unsigned int} | ||
%llu |
fastBPE/fastBPE.hpp:439:38: warning: format '%lu' expects argument of type 'long unsigned int', but argument 4 has type 'std::unordered_map<std::__cxx11::basic_string |
fprintf(stderr, "Read %lu words (%lu unique) from vocabulary file.\n", total, | ~~^ |
---|---|---|
long unsigned int | ||
%u | ||
440 | vocab.size()); | |
std::unordered_map<std::__cxx11::basic_string |
fastBPE/fastBPE.hpp: In function 'void fastBPE::readCodes(const char*, std::unordered_map<std::pair<std::cxx11::basic_string |
fprintf(stderr, "Read %lu codes from the codes file.\n", codes.size()); | |
---|---|---|
long unsigned int std::unordered_map<std::pair<std::__cxx11::basic_string |
t, fastBPE::pair_hash>::size_type {aka unsigned int} | %u writing build\temp.win-amd64-cpython-38\Release\fastbpe\fastBPE.cp38-win_amd64.def g++ -O -Wall -shared -s build\temp.win-amd64-cpython-38\Release\fastbpe\fastbpe.o build\temp.win-amd64-cpython-38\Release\fastbpe\fastBPE.cp38-win_amd64.def -LC:\Users\ASUS\anaconda3\envs \Testing_The_RS_AIP491\libs -LC:\Users\ASUS\anaconda3\envs\Testing_The_RS_AIP491 -LC:\Users\ASUS\anaconda3\envs\Testing_The_RS_AIP491\PCbuild\amd64 -lpython38 -lucrt -lvcruntime140 -o bui ld\lib.win-amd64-cpython-38\fastBPE.cp38-win_amd64.pyd C:/mingw32/bin/../lib/gcc/i686-w64-mingw32/12.2.0/../../../../i686-w64-mingw32/bin/ld.exe: C:\Users\ASUS\anaconda3\envs\Testing_The_RS_AIP491\libs/python38.lib(python38.dll): recognised b ut unhandled machine type (0x8664) in Import Library Format archive C:/mingw32/bin/../lib/gcc/i686-w64-mingw32/12.2.0/../../../../i686-w64-mingw32/bin/ld.exe: C:\Users\ASUS\anaconda3\envs\Testing_The_RS_AIP491\libs/python38.lib: error adding symbols: file format not recognized collect2.exe: error: ld returned 1 exit status error: command 'C:\mingw32\bin\g++.exe' failed with exit code 1
How can I fix it ?
Copy the file fastBPE.cpython-38-x86_64-linux-gnu.so and the folder fastBPE-0.1.0.dist-info in Ubuntu to your Windows envs
For example:
my environments is:
Ubuntu envs: anaconda3/envs/env_name/lib/python3.8/site-packages
Windows envs: miniconda3\envs\env_name\Lib\site-packages
It works in my win11! WoW
Hello, @liugj101. May you share the file to the public?
I am windows 10 user. Anaconda version torch-cpu 1.1.0 I installed MSVC v142 VS2019 C++ x64/x86 building tools (v14.23) when I tried to
pip install fairseq
, I got the following errors: