Windows installation failure: fastBPE(setup.py) error

I am windows 10 user. Anaconda version torch-cpu 1.1.0 I installed MSVC v142 VS2019 C++ x64/x86 building tools (v14.23) when I tried to pip install fairseq , I got the following errors:

 Building wheel for fastBPE (setup.py) ... error
  ERROR: Command errored out with exit status 1:
   command: 'C:\Users\zjplab\Miniconda3\envs\torch\python.exe' -u -c 'import sys, setuptools, tokenize; sys.argv[0] = '"'"'C:\\Users\\zjplab\\AppData\\Local\\Temp\\pip-install-dr_9ni_0\\fastBPE\\setup.py'"'"'; __file__='"'"'C:\\Users\\zjplab\\AppData\\Local\\Temp\\pip-install-dr_9ni_0\\fastBPE\\setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' bdist_wheel -d 'C:\Users\zjplab\AppData\Local\Temp\pip-wheel-2qy7fmwy' --python-tag cp37
       cwd: C:\Users\zjplab\AppData\Local\Temp\pip-install-dr_9ni_0\fastBPE\
  Complete output (19 lines):
  running bdist_wheel
  running build
  running build_py
  package init file 'fastBPE\__init__.py' not found (or not a regular file)
  running build_ext
  building 'fastBPE' extension
  creating build
  creating build\temp.win-amd64-3.7
  creating build\temp.win-amd64-3.7\Release
  creating build\temp.win-amd64-3.7\Release\fastBPE
  C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.23.28105\bin\HostX86\x64\cl.exe /c /nologo /Ox /W3 /GL /DNDEBUG /MT -IC:\Users\zjplab\Miniconda3\envs\torch\include -IC:\Users\zjplab\Miniconda3\envs\torch\include "-IC:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.23.28105\ATLMFC\include" "-IC:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.23.28105\include" "-IC:\Program Files (x86)\Windows Kits\NETFXSDK\4.8\include\um" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.18362.0\ucrt" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.18362.0\shared" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.18362.0\um" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.18362.0\winrt" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.18362.0\cppwinrt" /EHsc /TpfastBPE/fastBPE.cpp /Fobuild\temp.win-amd64-3.7\Release\fastBPE/fastBPE.obj -std=c++11 -Ofast -pthread
  cl : Command line warning D9025 : overriding '/Os' with '/Ot'
  cl : Command line warning D9002 : ignoring unknown option '-std=c++11'
  cl : Command line warning D9002 : ignoring unknown option '-Of'
  cl : Command line warning D9002 : ignoring unknown option '-Oa'
  cl : Command line warning D9002 : ignoring unknown option '-pthread'
  fastBPE.cpp
  C:\Users\zjplab\AppData\Local\Temp\pip-install-dr_9ni_0\fastBPE\fastBPE\fastBPE.hpp(15): fatal error C1083: Cannot open include file: 'sys/mman.h': No such file or directory
  error: command 'C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Community\\VC\\Tools\\MSVC\\14.23.28105\\bin\\HostX86\\x64\\cl.exe' failed with exit status 2
  ----------------------------------------
  ERROR: Failed building wheel for fastBPE
  Running setup.py clean for fastBPE
Failed to build fastBPE
Installing collected packages: regex, tqdm, pywin32, pypiwin32, portalocker, typing, sacrebleu, fastBPE, fairseq
  Running setup.py install for fastBPE ... error
    ERROR: Command errored out with exit status 1:
     command: 'C:\Users\zjplab\Miniconda3\envs\torch\python.exe' -u -c 'import sys, setuptools, tokenize; sys.argv[0] = '"'"'C:\\Users\\zjplab\\AppData\\Local\\Temp\\pip-install-dr_9ni_0\\fastBPE\\setup.py'"'"'; __file__='"'"'C:\\Users\\zjplab\\AppData\\Local\\Temp\\pip-install-dr_9ni_0\\fastBPE\\setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' install --record 'C:\Users\zjplab\AppData\Local\Temp\pip-record-uq6no_ej\install-record.txt' --single-version-externally-managed --compile
         cwd: C:\Users\zjplab\AppData\Local\Temp\pip-install-dr_9ni_0\fastBPE\
    Complete output (19 lines):
    running install
    running build
    running build_py
    package init file 'fastBPE\__init__.py' not found (or not a regular file)
    running build_ext
    building 'fastBPE' extension
    creating build
    creating build\temp.win-amd64-3.7
    creating build\temp.win-amd64-3.7\Release
    creating build\temp.win-amd64-3.7\Release\fastBPE
    C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.23.28105\bin\HostX86\x64\cl.exe /c /nologo /Ox /W3 /GL /DNDEBUG /MT -IC:\Users\zjplab\Miniconda3\envs\torch\include -IC:\Users\zjplab\Miniconda3\envs\torch\include "-IC:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.23.28105\ATLMFC\include" "-IC:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.23.28105\include" "-IC:\Program Files (x86)\Windows Kits\NETFXSDK\4.8\include\um" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.18362.0\ucrt" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.18362.0\shared" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.18362.0\um" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.18362.0\winrt" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.18362.0\cppwinrt" /EHsc /TpfastBPE/fastBPE.cpp /Fobuild\temp.win-amd64-3.7\Release\fastBPE/fastBPE.obj -std=c++11 -Ofast -pthread
    cl : Command line warning D9025 : overriding '/Os' with '/Ot'
    cl : Command line warning D9002 : ignoring unknown option '-std=c++11'
    cl : Command line warning D9002 : ignoring unknown option '-Of'
    cl : Command line warning D9002 : ignoring unknown option '-Oa'
    cl : Command line warning D9002 : ignoring unknown option '-pthread'
    fastBPE.cpp
    C:\Users\zjplab\AppData\Local\Temp\pip-install-dr_9ni_0\fastBPE\fastBPE\fastBPE.hpp(15): fatal error C1083: Cannot open include file: 'sys/mman.h': No such file or directory
    error: command 'C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Community\\VC\\Tools\\MSVC\\14.23.28105\\bin\\HostX86\\x64\\cl.exe' failed with exit status 2
    ----------------------------------------
ERROR: Command errored out with exit status 1: 'C:\Users\zjplab\Miniconda3\envs\torch\python.exe' -u -c 'import sys, setuptools, tokenize; sys.argv[0] = '"'"'C:\\Users\\zjplab\\AppData\\Local\\Temp\\pip-install-dr_9ni_0\\fastBPE\\setup.py'"'"'; __file__='"'"'C:\\Users\\zjplab\\AppData\\Local\\Temp\\pip-install-dr_9ni_0\\fastBPE\\setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' install --record 'C:\Users\zjplab\AppData\Local\Temp\pip-record-uq6no_ej\install-record.txt' --single-version-externally-managed --compile Check the logs for full command output.

[zjplab]

Ok. I have solved this problem myself. This error arises when you try to compile fastBPE( a dependency of fairseq) in windows. In order to solve this, you have to make modifications on:

• fastBPE source code
• python package distutils 's cygwinccompiler.py

First, git clone fastBPE source code to your local directory. Change line 15:

#include <mman.c>

to

#include "mman.c"

Add mman.h to fastBPE/fastBPE directory:

/*
 * sys/mman.h
 * mman-win32
 */

#ifndef _SYS_MMAN_H_
#define _SYS_MMAN_H_

#ifndef _WIN32_WINNT		// Allow use of features specific to Windows XP or later.                   
#define _WIN32_WINNT 0x0501	// Change this to the appropriate value to target other versions of Windows.
#endif						

/* All the headers include this file. */
#ifndef _MSC_VER
#include <_mingw.h>
#endif

#include <sys/types.h>

#ifdef __cplusplus
extern "C" {
#endif

#define PROT_NONE       0
#define PROT_READ       1
#define PROT_WRITE      2
#define PROT_EXEC       4

#define MAP_FILE        0
#define MAP_SHARED      1
#define MAP_PRIVATE     2
#define MAP_TYPE        0xf
#define MAP_FIXED       0x10
#define MAP_ANONYMOUS   0x20
#define MAP_ANON        MAP_ANONYMOUS

#define MAP_FAILED      ((void *)-1)

/* Flags for msync. */
#define MS_ASYNC        1
#define MS_SYNC         2
#define MS_INVALIDATE   4

void*   mmap(void *addr, size_t len, int prot, int flags, int fildes, off_t off);
int     munmap(void *addr, size_t len);
int     mprotect(void *addr, size_t len, int prot);
int     msync(void *addr, size_t len, int flags);
int     mlock(const void *addr, size_t len);
int     munlock(const void *addr, size_t len);

#ifdef __cplusplus
};
#endif

#endif /*  _SYS_MMAN_H_ */

and also mman.c

#include <windows.h>
#include <errno.h>
#include <io.h>

#include "mman.h"

#ifndef FILE_MAP_EXECUTE
#define FILE_MAP_EXECUTE    0x0020
#endif /* FILE_MAP_EXECUTE */

static int __map_mman_error(const DWORD err, const int deferr)
{
    if (err == 0)
        return 0;
    //TODO: implement
    return err;
}

static DWORD __map_mmap_prot_page(const int prot)
{
    DWORD protect = 0;
    
    if (prot == PROT_NONE)
        return protect;
        
    if ((prot & PROT_EXEC) != 0)
    {
        protect = ((prot & PROT_WRITE) != 0) ? 
                    PAGE_EXECUTE_READWRITE : PAGE_EXECUTE_READ;
    }
    else
    {
        protect = ((prot & PROT_WRITE) != 0) ?
                    PAGE_READWRITE : PAGE_READONLY;
    }
    
    return protect;
}

static DWORD __map_mmap_prot_file(const int prot)
{
    DWORD desiredAccess = 0;
    
    if (prot == PROT_NONE)
        return desiredAccess;
        
    if ((prot & PROT_READ) != 0)
        desiredAccess |= FILE_MAP_READ;
    if ((prot & PROT_WRITE) != 0)
        desiredAccess |= FILE_MAP_WRITE;
    if ((prot & PROT_EXEC) != 0)
        desiredAccess |= FILE_MAP_EXECUTE;
    
    return desiredAccess;
}

void* mmap(void *addr, size_t len, int prot, int flags, int fildes, off_t off)
{
    HANDLE fm, h;
    
    void * map = MAP_FAILED;
    
#ifdef _MSC_VER
#pragma warning(push)
#pragma warning(disable: 4293)
#endif

    const DWORD dwFileOffsetLow = (sizeof(off_t) <= sizeof(DWORD)) ? 
                    (DWORD)off : (DWORD)(off & 0xFFFFFFFFL);
    const DWORD dwFileOffsetHigh = (sizeof(off_t) <= sizeof(DWORD)) ?
                    (DWORD)0 : (DWORD)((off >> 32) & 0xFFFFFFFFL);
    const DWORD protect = __map_mmap_prot_page(prot);
    const DWORD desiredAccess = __map_mmap_prot_file(prot);

    const off_t maxSize = off + (off_t)len;

    const DWORD dwMaxSizeLow = (sizeof(off_t) <= sizeof(DWORD)) ? 
                    (DWORD)maxSize : (DWORD)(maxSize & 0xFFFFFFFFL);
    const DWORD dwMaxSizeHigh = (sizeof(off_t) <= sizeof(DWORD)) ?
                    (DWORD)0 : (DWORD)((maxSize >> 32) & 0xFFFFFFFFL);

#ifdef _MSC_VER
#pragma warning(pop)
#endif

    errno = 0;
    
    if (len == 0 
        /* Unsupported flag combinations */
        || (flags & MAP_FIXED) != 0
        /* Usupported protection combinations */
        || prot == PROT_EXEC)
    {
        errno = EINVAL;
        return MAP_FAILED;
    }
    
    h = ((flags & MAP_ANONYMOUS) == 0) ? 
                    (HANDLE)_get_osfhandle(fildes) : INVALID_HANDLE_VALUE;

    if ((flags & MAP_ANONYMOUS) == 0 && h == INVALID_HANDLE_VALUE)
    {
        errno = EBADF;
        return MAP_FAILED;
    }

    fm = CreateFileMapping(h, NULL, protect, dwMaxSizeHigh, dwMaxSizeLow, NULL);

    if (fm == NULL)
    {
        errno = __map_mman_error(GetLastError(), EPERM);
        return MAP_FAILED;
    }
  
    map = MapViewOfFile(fm, desiredAccess, dwFileOffsetHigh, dwFileOffsetLow, len);

    CloseHandle(fm);
  
    if (map == NULL)
    {
        errno = __map_mman_error(GetLastError(), EPERM);
        return MAP_FAILED;
    }

    return map;
}

int munmap(void *addr, size_t len)
{
    if (UnmapViewOfFile(addr))
        return 0;
        
    errno =  __map_mman_error(GetLastError(), EPERM);
    
    return -1;
}

int mprotect(void *addr, size_t len, int prot)
{
    DWORD newProtect = __map_mmap_prot_page(prot);
    DWORD oldProtect = 0;
    
    if (VirtualProtect(addr, len, newProtect, &oldProtect))
        return 0;
    
    errno =  __map_mman_error(GetLastError(), EPERM);
    
    return -1;
}

int msync(void *addr, size_t len, int flags)
{
    if (FlushViewOfFile(addr, len))
        return 0;
    
    errno =  __map_mman_error(GetLastError(), EPERM);
    
    return -1;
}

int mlock(const void *addr, size_t len)
{
    if (VirtualLock((LPVOID)addr, len))
        return 0;
        
    errno =  __map_mman_error(GetLastError(), EPERM);
    
    return -1;
}

int munlock(const void *addr, size_t len)
{
    if (VirtualUnlock((LPVOID)addr, len))
        return 0;
        
    errno =  __map_mman_error(GetLastError(), EPERM);
    
    return -1;
}

Reason : Under any windows C++ compiler, we don’t have mman.h . But I have found the following windows alternative lib.

[HoiBunCa]

where is “#include <mman.c>” ?