Incorrect memory segment limit checking

UniPCemu handles limits in a simple way: any access that's past the limit(or reversed with top-down descriptors) will AND the validity(1 being valid) with a combination of the case whether or not it's past the address width.

In UniPCemu's case, it uses some simple precalcs that are loaded whenever a segment descriptor is loaded:

typedef struct
{
    byte mask;
    byte nonequals;
    byte comparision;
} checkrights_cond;

checkrights_cond checkrights_conditions[0x10] = {
    { ~0x10,0,0 }, //0 Data, read-only
    { 0,0,0 }, //1 unused
    { 0,1,0 }, //2 Data, read/write! Allow always!
    { 0,0,0 }, //3 unused
    { ~0x10,0,0 }, //4 Data(expand down), read-only
    { 0,0,0 }, //5 unused
    { 0,1,0 }, //6 Data(expand down), read/write! Allow always!
    { 0,0,0 }, //7 unused
    { ~0x10,1,3 }, //8 Code, non-conforming, execute-only
    { 0,0,0 }, //9 unused
    { ~0x10,0,0 }, //10 Code, non-conforming, execute/read
    { 0,0,0 }, //11 unused
    { ~0x10,1,3 }, //12 Code, conforming, execute-only
    { 0,0,0 }, //13 unused
    { ~0x10,0,0 }, //14 Code, conforming, execute/read
    { 0,0,0 } //15 unused
};

void CPU_calcSegmentPrecalcs(SEGMENT_DESCRIPTOR *descriptor)
{
    word n;
    //Calculate the precalculations for execution for this descriptor!
    checkrights_cond *rights;
    uint_32 limits[2]; //What limit to apply?
    limits[0] = ((SEGDESCPTR_NONCALLGATE_LIMIT_HIGH(descriptor) << 16) | descriptor->desc.limit_low); //Base limit!
    limits[1] = ((limits[0] << 12) | 0xFFF); //4KB for a limit of 4GB, fill lower 12 bits with 1!
    descriptor->PRECALCS.limit = limits[SEGDESCPTR_GRANULARITY(descriptor)]; //Use the appropriate granularity to produce the limit!
    descriptor->PRECALCS.topdown = ((descriptor->desc.AccessRights & 0x1C) == 0x14); //Topdown segment?
    descriptor->PRECALCS.roof = (0xFFFF | (0xFFFF << (SEGDESCPTR_NONCALLGATE_D_B(descriptor) << 4))); //The roof of the descriptor!
    descriptor->PRECALCS.base = ((descriptor->desc.base_high << 24) | (descriptor->desc.base_mid << 16) | descriptor->desc.base_low); //Update the base address!
    rights = &checkrights_conditions[(descriptor->desc.AccessRights & 0xE)]; //What type do we check for(take it all, except the dirty bit)!
    for (n = 0; n < 0x100; ++n) //Calculate all conditions that error out or not!
    {
        descriptor->PRECALCS.rwe_errorout[n] = (((((n&rights->mask) == rights->comparision) == (rights->nonequals == 0)))&1); //Are we to error out on this condition?
    }
}

And the active processing of it is:

OPTINLINE byte verifyLimit(SEGMENT_DESCRIPTOR *descriptor, uint_64 offset)
{
    //Execute address test?
    INLINEREGISTER byte isvalid,topdown;
    topdown = descriptor->PRECALCS.topdown; //Are we a top-down segment?
    isvalid = (offset<=(uint_64)descriptor->PRECALCS.limit); //Valid address range!
    isvalid ^= topdown; //Apply expand-down data segment, if required, which reverses valid!
    isvalid &= (((offset>descriptor->PRECALCS.roof) & topdown)^1); //Limit to 16-bit/32-bit address space using topdown descriptors!
    isvalid &= 1; //Only 1-bit testing!
    return isvalid; //Are we valid?
}

byte CPU_MMU_checkrights_cause = 0; //What cause?
//Used by the CPU(VERR/VERW)&MMU I/O! forreading=0: Write, 1=Read normal, 3=Read opcode

byte CPU_MMU_checkrights(int segment, word segmentval, uint_64 offset, byte forreading, SEGMENT_DESCRIPTOR *descriptor, byte addrtest, byte is_offset16)
{
    //First: type checking!

    if (unlikely(GENERALSEGMENTPTR_P(descriptor)==0)) //Not present(invalid in the cache)? This also applies to NULL descriptors!
    {
        CPU_MMU_checkrights_cause = 1; //What cause?
        return 1; //#GP fault: not present in descriptor cache mean invalid, thus #GP!
    }

    //Basic access rights are always checked!
    if (GENERALSEGMENTPTR_S(descriptor)) //System segment? Check for additional type information!
    {
        //Entries 0,4,10,14: On writing, Entries 2,6: Never match, Entries 8,12: Writing or reading normally(!=3).
        //To ignore an entry for errors, specify mask 0, non-equals nonzero, comparison 0(a.k.a. ((forreading&0)!=0)
        if (unlikely(descriptor->PRECALCS.rwe_errorout[forreading])) //Are we to error out on this read/write/execute operation?
        {
            CPU_MMU_checkrights_cause = 3; //What cause?
            return 1; //Error!
        }
    }

    //Next: limit checking!
    if (likely(addrtest)) //Address test is to be performed?
    {
        if (likely(verifyLimit(descriptor,offset))) return 0; //OK? We're finished!
        //Not valid?
        {
            CPU_MMU_checkrights_cause = 6; //What cause?
            if (segment==CPU_SEGMENT_SS) //Stack fault?
            {
                return 3; //Error!
            }
            else //Normal #GP?
            {
                return 1; //Error!
            }
        }
    }

    //Don't perform rights checks: This is done when loading the segment register only!
    return 0; //OK!
}

//Used by the MMU! forreading: 0=Writes, 1=Read normal, 3=Read opcode fetch.
int CPU_MMU_checklimit(int segment, word segmentval, uint_64 offset, byte forreading, byte is_offset16) //Determines the limit of the segment, forreading=2 when reading an opcode!
{
    byte rights;
    //Determine the Limit!
    if (likely(EMULATED_CPU >= CPU_80286)) //Handle like a 80286+?
    {
        if (unlikely(segment==-1))
        {
            CPU_MMU_checkrights_cause = 0x80; //What cause?
            return 0; //Enable: we're an emulator call!
        }

        //Use segment descriptors, even when in real mode on 286+ processors!
        rights = CPU_MMU_checkrights(segment,segmentval, offset, forreading, &CPU[activeCPU].SEG_DESCRIPTOR[segment],1,is_offset16); //What rights resulting? Test the address itself too!
        if (unlikely(rights)) //Error?
        {
            switch (rights)
            {
            default: //Unknown status? Count #GP by default!
            case 1: //#GP(0) or pseudo protection fault(Real/V86 mode(V86 mode only during limit range exceptions, otherwise error code 0))?
                if (unlikely((forreading&0x10)==0)) CPU_GP(((getcpumode()==CPU_MODE_PROTECTED) || (!(((CPU_MMU_checkrights_cause==6) && (getcpumode()==CPU_MODE_8086)) || (getcpumode()==CPU_MODE_REAL))))?0:-2); //Throw (pseudo) fault when not prefetching!
                return 1; //Error out!
                break;
            case 2: //#NP?
                if (unlikely((forreading&0x10)==0)) THROWDESCNP(segmentval,0,(segmentval&4)?EXCEPTION_TABLE_LDT:EXCEPTION_TABLE_GDT); //Throw error: accessing non-present segment descriptor when not prefetching!
                return 1; //Error out!
                break;
            case 3: //#SS(0) or pseudo protection fault(Real/V86 mode)?
                if (unlikely((forreading&0x10)==0)) CPU_StackFault(((getcpumode()==CPU_MODE_PROTECTED) || (!(((CPU_MMU_checkrights_cause==6) && (getcpumode()==CPU_MODE_8086)) || (getcpumode()==CPU_MODE_REAL))))?0:-2); //Throw (pseudo) fault when not prefetching!
                return 1; //Error out!
                break;
            }
        }
        return 0; //OK!
    }
    return 0; //Don't give errors: handle like a 80(1)86!
}

The result of verifylimit is 1 when it's valid. When it returns 0, throw a #GP(for code and non-SS segment registers) or #SS(for stacks) fault, depending on the segment register used.

barotto / IBMulator

Incorrect memory segment limit checking #35