c3lang / c3c

Compiler for the C3 language
https://c3-lang.org
GNU Lesser General Public License v3.0
2.92k stars 180 forks source link

Add memory map module? #519

Open Andersama opened 2 years ago

Andersama commented 2 years ago

Digging though some example c++ memory map headers in https://github.com/mandreyel/mio/blob/master/include/mio we can find some function calls for memory mapping and unmapping.

struct mmap_context
{
    char* data;
    int64_t length;
    int64_t mapped_length;
#ifdef _WIN32
    file_handle_type file_mapping_handle;
#endif
};

inline mmap_context memory_map(const file_handle_type file_handle, const int64_t offset,
    const int64_t length, const access_mode mode, std::error_code& error)
{
    const int64_t aligned_offset = make_offset_page_aligned(offset);
    const int64_t length_to_map = offset - aligned_offset + length;
#ifdef _WIN32
    const int64_t max_file_size = offset + length;
    const auto file_mapping_handle = ::CreateFileMapping(
            file_handle,
            0,
            mode == access_mode::read ? PAGE_READONLY : PAGE_READWRITE,
            win::int64_high(max_file_size),
            win::int64_low(max_file_size),
            0);
    if(file_mapping_handle == invalid_handle)
    {
        error = detail::last_error();
        return {};
    }
    char* mapping_start = static_cast<char*>(::MapViewOfFile(
            file_mapping_handle,
            mode == access_mode::read ? FILE_MAP_READ : FILE_MAP_WRITE,
            win::int64_high(aligned_offset),
            win::int64_low(aligned_offset),
            length_to_map));
    if(mapping_start == nullptr)
    {
        // Close file handle if mapping it failed.
        ::CloseHandle(file_mapping_handle);
        error = detail::last_error();
        return {};
    }
#else // POSIX
    char* mapping_start = static_cast<char*>(::mmap(
            0, // Don't give hint as to where to map.
            length_to_map,
            mode == access_mode::read ? PROT_READ : PROT_WRITE,
            MAP_SHARED,
            file_handle,
            aligned_offset));
    if(mapping_start == MAP_FAILED)
    {
        error = detail::last_error();
        return {};
    }
#endif
    mmap_context ctx;
    ctx.data = mapping_start + offset - aligned_offset;
    ctx.length = length;
    ctx.mapped_length = length_to_map;
#ifdef _WIN32
    ctx.file_mapping_handle = file_mapping_handle;
#endif
    return ctx;
}

//this one is made part of a class, but it could likely just be it's own function with it's own parameters

template<access_mode AccessMode, typename ByteT>
void basic_mmap<AccessMode, ByteT>::unmap()
{
    if(!is_open()) { return; }
    // TODO do we care about errors here?
#ifdef _WIN32
    if(is_mapped())
    {
        ::UnmapViewOfFile(get_mapping_start());
        ::CloseHandle(file_mapping_handle_);
    }
#else // POSIX
    if(data_) { ::munmap(const_cast<pointer>(get_mapping_start()), mapped_length_); }
#endif

    // If `file_handle_` was obtained by our opening it (when map is called with
    // a path, rather than an existing file handle), we need to close it,
    // otherwise it must not be closed as it may still be used outside this
    // instance.
    if(is_handle_internal_)
    {
#ifdef _WIN32
        ::CloseHandle(file_handle_);
#else // POSIX
        ::close(file_handle_);
#endif
    }

    // Reset fields to their default values.
    data_ = nullptr;
    length_ = mapped_length_ = 0;
    file_handle_ = invalid_handle;
#ifdef _WIN32
    file_mapping_handle_ = invalid_handle;
#endif
}

And there's sync:

template<access_mode AccessMode, typename ByteT>
template<access_mode A>
typename std::enable_if<A == access_mode::write, void>::type
basic_mmap<AccessMode, ByteT>::sync(std::error_code& error)
{
    error.clear();
    if(!is_open())
    {
        error = std::make_error_code(std::errc::bad_file_descriptor);
        return;
    }

    if(data())
    {
#ifdef _WIN32
        if(::FlushViewOfFile(get_mapping_start(), mapped_length_) == 0
           || ::FlushFileBuffers(file_handle_) == 0)
#else // POSIX
        if(::msync(get_mapping_start(), mapped_length_, MS_SYNC) != 0)
#endif
        {
            error = detail::last_error();
            return;
        }
    }
#ifdef _WIN32
    if(::FlushFileBuffers(file_handle_) == 0)
    {
        error = detail::last_error();
    }
#endif
}

Translating the header to c3 may take some time, but should work.

Andersama commented 2 years ago

Notably the mio header split memory maps by templated classes to distinguish memory maps that have write access to disable the sync function. EG: trying to sync (write) to a readonly memory map (which is it's own class distict from a writeonly memory map) doesn't give an error...instead it does absolutely nothing.

The c3 module then might look like:

fn RMemoryMap readonly_memory_map(...);
fn WMemoryMap writable_memory_map(...);
fn MemoryMap memory_map(...) {
    if (readonly) {
        return readonly_memory_map(); //bitcast to MemoryMap
    } else {
        return writable_memory_map(); //bitcast to MemoryMap
    }
}
//MemoryMap calling sync actually checks the permission member of the structs to tell if it was a writable file.

Here's the c3 code I had written so far, which failed because windows needs additional libraries:

module mmap;
import libc;
//import std::core::os::linux;
//import std::core::os::macos;
//import std::core::os::windows;

struct Map {
    usize size;
    usize capacity;
    char *ptr;
}

$if (env::OS_TYPE == OsType.WIN32):
struct System_Info {
    union {
        uint dwOemId;          // Obsolete field...do not use
        struct {
            ushort wProcessorArchitecture;
            ushort wReserved;
        }  //DUMMYSTRUCTNAME
    }  //DUMMYUNIONNAME
    uint dwPageSize;
    void* lpMinimumApplicationAddress;
    void* lpMaximumApplicationAddress;
    uint* dwActiveProcessorMask;
    uint dwNumberOfProcessors;
    uint dwProcessorType;
    uint dwAllocationGranularity;
    ushort wProcessorLevel;
    ushort wProcessorRevision;
}
/*
union Large_Integer {
    struct {
        DWORD LowPart;
        LONG HighPart;
    } DUMMYSTRUCTNAME;
    struct {
        DWORD LowPart;
        LONG HighPart;
    } u;
    isize QuadPart;
}
*/
extern fn void getSystemInfo(System_Info*) @extname("GetSystemInfo"); //@stdcall

define Handle = void*;
define File_handle_type = Handle;

struct SecurityAttributes {
    uint nLength;
    void* lpSecurityDescriptor;
    int bInheritHandle;
}

extern fn int getFileSizeEx(File_handle_type, isize*) @stdcall @extname("GetFileSizeEx");
extern fn char* createFileMapping(File_handle_type, SecurityAttributes*, uint, uint, uint, char*) @stdcall @extname("CreateFileMappingA");
extern fn char* mapViewOfFile(File_handle_type, uint, uint, uint, usize) @stdcall @extname("MapViewOfFile");
extern fn int closeHandle(File_handle_type) @stdcall @extname("CloseHandle");

$else:
define File_handle_type = int;
$endif;

const File_handle_type INVALID_HANDLE = (File_handle_type)(isize)(0xffffffffffffffff);

fn usize page_size() {
    $if (env::OS_TYPE == OsType.WIN32):
    System_Info info;
    getSystemInfo(&info);
    return info.dwAllocationGranularity;
    $else:
    return sysconf(_SC_PAGE_SIZE);
    $endif;
}

fn libc::Errno last_error() @inline {
    return libc::errno();
}

enum MapStatus : int {
    OK,
    BAD_FILE_DESCRIPTOR,
    INVALID_ARGUMENT
    //INVALID_OFFSET,
    //INVALID_LENGTH,
}

enum AccessMode : int {
    READ,
    WRITE
}

struct MMap_CTX {
    char* data;
    isize length;
    isize mapped_length;
//$if (env::OS_TYPE == OsType.WIN32):
    File_handle_type file_mapping_handle;
//$endif;
}

fn usize query_file_size(File_handle_type handle, int *error) {
    *error = 0;
$if (env::OS_TYPE == OsType.WIN32):
    isize file_size = 0;
    if (getFileSizeEx(handle, &file_size) == 0) {
        *error = (int)last_error();
        return 0;
    }
    return file_size;
$else:
/*
    Stat sbuf;
    if (fstat(handle, &sbuf) == -1) {
        *error = last_error();
        return 0;
    }
    return sbuf.st_size;
*/
    return 0;
$endif;
}

fn int high(isize n) @inline {
    return (int)(n >> 32);
}

fn int low(isize n) @inline {
    return (int)(n & 0xffffffff);
}

fn usize make_offset_page_aligned(usize offset) @inline {
    usize page_sz = page_size();
    return (offset / page_sz) * page_sz;
}

fn MMap_CTX memory_map(File_handle_type handle, usize offset, usize len, int mode, int* error) {
    isize aligned_offset = make_offset_page_aligned(offset);
    isize length_to_map = offset - aligned_offset + len;

    File_handle_type file_mapping_handle = INVALID_HANDLE;
    char *mapping_start = null;
$if (env::OS_TYPE == OsType.WIN32):
    const int PAGE_READONLY = 0x02;
    const int PAGE_READWRITE = 0x04;
    //flipped around*...cause why not windows?
    const int FILE_MAP_READ = 0x04;
    const int FILE_MAP_WRITE = 0x02;

    isize max_file_size = offset+len;
    file_mapping_handle = createFileMapping(
        handle,
        null,
        mode == AccessMode.READ ? PAGE_READONLY : PAGE_READWRITE,
        high(max_file_size),
        low(max_file_size),
        null
        );

    mapping_start = mapViewOfFile(
        file_mapping_handle,
        mode == AccessMode.READ ? FILE_MAP_READ : FILE_MAP_WRITE,
        high(aligned_offset),
        low(aligned_offset),
        length_to_map
    );
    if (mapping_start == null) {
        closeHandle(file_mapping_handle);
        *error = (int)last_error();
        return MMap_CTX{};
    }
$else:
    mapping_start = mmap(
                0, // Don't give hint as to where to map.
                length_to_map,
                mode == access_mode::read ? PROT_READ : PROT_WRITE,
                MAP_SHARED,
                handle,
                aligned_offset);
    if (mapping_start == MAP_FAILED) {
        *error = last_error();
        return MMap_CTX{};
    }

$endif;

    MMap_CTX ctx;
    ctx.data = mapping_start + offset - aligned_offset;
    ctx.length = len;
    ctx.mapped_length = length_to_map;

$if (env::OS_TYPE == OsType.WIN32):
    ctx.file_mapping_handle = file_mapping_handle;
$else:
    ctx.file_mapping_handle = handle;
$endif;

    return ctx;
}

fn MMap_CTX map(File_handle_type handle, usize offset, usize len, int mode, int* error) {
    *error = 0;

    if (handle == INVALID_HANDLE) {
        *error = MapStatus.BAD_FILE_DESCRIPTOR;
        return MMap_CTX{};
    }

    usize file_size = query_file_size(handle, error);
    if (*error != MapStatus.OK) {
        return MMap_CTX{};
    }
    /*
    if (offset > file_size) {
        *error = MapStatus.INVALID_OFFSET;
        return;
    }
    */
    if ((offset + len) > file_size) {
        *error = MapStatus.INVALID_ARGUMENT;
        return MMap_CTX{};
    }

    return memory_map(handle, offset, len, mode, error);
}
lerno commented 2 years ago
  1. Memory mapping and so on are welcome additions. If you need something that works with the standard library you can add the required linking by adding arguments in the file linker.c look at static void linker_setup_windows(const char ***args_ref, LinkerType linker_type). Once it's in place we can continue improving on it. Ideally there's some use cases we can work against.
  2. I realize I ended up skipping through this because I was going to "think about it" and then I just forgot. I'll try to get better at responding promptly to these issues. But in the future I think there are two good ways to suggest a feature to get more eyes on it: (a) discuss it on the Discord (https://discord.gg/qN76R87) or (b) if you really don't like Discord, start a github discussion on the topic. Or do both. You can start a discussion and then reference this issue.
Andersama commented 2 years ago

No problem.

So I take it that in the language at the moment there's no feature to include a library if needed, they're all just hardcoded at the moment? In C you could use #pragma directives but that would mean a hardcoded path.

lerno commented 2 years ago

For external "C3 libraries" it's possible (see Raylib for example), but for the standard library, which is included in a different way (at the moment) it doesn't work (yet)