eddyleo / doubango

Automatically exported from code.google.com/p/doubango
0 stars 0 forks source link

Not clear if a torture test passes or fails #179

Closed GoogleCodeExporter closed 8 years ago

GoogleCodeExporter commented 8 years ago
What steps will reproduce the problem?
1.
2.
3.

What is the expected output? What do you see instead?

Please use labels and text to provide additional information.
Not clear if a torture test passes or fails.  In my wrapper tests I’ve added 
a PASS/FAIL for each test and an overall PASS/FAIL indication.

Original issue reported on code.google.com by boss...@yahoo.fr on 8 Jan 2013 at 11:43

GoogleCodeExporter commented 8 years ago
I believe a fix to this defect would be...

NOTE: I think for 17 (A.1.15. State Creation_4) & 18 (A.1.15. State Creation_5) 
you need to set XFAIL_YES rather than XFAIL_NO; based on RFC4465

//static void checkResult(const struct_torture_test* test, const tcomp_result_t 
*result, const void* output_ptr, tsk_size_t output_size)
//{
//  if(tsk_striequals(test->xoutput.ptr, "decompression_memory_size")){
//      printf("xoutput (decompression_memory_size): %s\n", 
(HostToNetworkShort(DMS) == *((uint16_t*)output_ptr)) ? "YES" : "NO");
//  }
//  else{
//      printf("xoutput: %s\n", startsWith(output_ptr, output_size, 
test->xoutput.ptr, test->xoutput.size) ? "YES" : "NO");
//  }
//  printf("xcycles: %s\n", (result->consumed_cycles == test->xcycles) ? "YES" : 
"NO");
//}

static int checkResult(const struct_torture_test* test, const tcomp_result_t 
*result, const void* output_ptr, tsk_size_t output_size)
{
    int test_status = 0;

    // if not expecting a failure
    if (!test->xfail)
    {
        if (result->isNack)
        {
            printf("FAIL: UNEXPECTED NACK\n");
            test_status += 1;
        }
        else
        {
            if(tsk_striequals(test->xoutput.ptr, "decompression_memory_size")){
                printf("xoutput (decompression_memory_size): %s\n", (HostToNetworkShort(DMS) == *((uint16_t*)output_ptr)) ? "YES" : "NO");
                if (HostToNetworkShort(DMS) != *((uint16_t*)output_ptr))
                    test_status += 1;
            }
            else{
                printf("xoutput: %s\n", startsWith(output_ptr, output_size, test->xoutput.ptr, test->xoutput.size) ? "YES" : "NO");
                if (!startsWith(output_ptr, output_size, test->xoutput.ptr, test->xoutput.size))
                    test_status += 1;
            }
            if (result->consumed_cycles == test->xcycles)
            {
                printf("xcycles: YES\n");
            }
            else
            {
                printf("xcycles: NO (exp: %d, got: %d)\n", test->xcycles, result->consumed_cycles);
                test_status +=1;
            }
        }
    }
    else // bit expecting a failure
    {
        printf("Expected failure: %s\n", test->xoutput.ptr);

        // fail: expecting a NACK and did not get one
        if (!(result->isNack))
        {
            printf("FAIL: MISSING NACK\n");
            test_status += 1;
        }
    }

    return test_status;
}

FROM test_tortures() lines I’ve added/modified have >>> above and <<< below 
them

#if RUN_TEST_LOOP
    for(;;)
#endif
    {
        int test_result = 0;
#if RUN_TEST_ALL
        start = 0, end = sizeof(tests)/sizeof(tests[0]);
#else
        start = RUN_TEST_NO, end = RUN_TEST_NO + 1;
#endif

        for(i = start; i<end; i++){
>>>
            test_result = 0;
<<<
            printf("=== Testing %s ===\n\n", tests[i].section_name);

            if(tests[i].stream){
                tcomp_result_setOutputTCPBuffer(result, buffer, OUTPUT_BUFFER_SIZE, STREAM_ID);
            }
            else{
                tcomp_result_setOutputUDPBuffer(result, buffer, OUTPUT_BUFFER_SIZE);
            }

            /* Performs decompression */
            res_size = tcomp_manager_decompress(manager, tests[i].bytecode, tests[i].bytecode_size, result);
            if(result->isNack){
                printf("\n==WE GOT A NACK\n\n");
                //sendto(tcomp_buffer_getBuffer(result->nack_info), tcomp_buffer_getSize(result->nack_info));
            }
            else{
                tcomp_result_setCompartmentId(result, COMPARTMENT_ID, strlen(COMPARTMENT_ID));
                tcomp_manager_provideCompartmentId(manager, result);
            }

            /* Checks result */
>>>
            test_result += checkResult(&tests[i], result, buffer, sizeof(buffer));
<<<

            /* Get sub-sequent stream messages */
            if(tests[i].stream && (res_size || result->isNack)){
                uint64_t consumed_cycles = result->consumed_cycles; // save "consumed_cycles" (shared by all sub-sequent messages)
                for(;;){
                    res_size = tcomp_manager_getNextStreamMessage(manager, result);
                    if(!res_size && !result->isNack){
                        break;
                    }
                    else if(res_size){
                        tcomp_result_setCompartmentId(result, COMPARTMENT_ID, strlen(COMPARTMENT_ID));
                        tcomp_manager_provideCompartmentId(manager, result);
                    }
                    else{
                        printf("\n==WE GOT A NACK\n\n");
                        //sendto(tcomp_buffer_getBuffer(result->nack_info), tcomp_buffer_getSize(result->nack_info));
                    }
                    result->consumed_cycles = consumed_cycles; // restore cycles just for checking. Had been cleared by "tcomp_manager_getNextStreamMessage()"
>>>
                    test_result += checkResult(&tests[i], result, buffer, sizeof(buffer));
<<<
                }
            }
>>>
        printf("=== %s STATUS: %s ===\n\n", tests[i].section_name, test_result ? "FAIL" : "PASS");
<<<
        }

    }/* LOOP */

Original comment by alistair...@gmail.com on 11 Jan 2013 at 3:14

GoogleCodeExporter commented 8 years ago
State Creation_4 & State Creation_5: we're doing complete check and haven't 
checked this yet. Opened: http://code.google.com/p/doubango/issues/detail?id=193
For the changes, could you please create a patch/diff? It's easier to check.

Original comment by boss...@yahoo.fr on 11 Jan 2013 at 3:59

GoogleCodeExporter commented 8 years ago
Patch file attached

Original comment by alistair...@gmail.com on 14 Jan 2013 at 10:32

Attachments:

GoogleCodeExporter commented 8 years ago
Will use assert()ions.

Original comment by boss...@yahoo.fr on 21 Jan 2013 at 7:57

GoogleCodeExporter commented 8 years ago
Fixed by r809. The application will crash if the result is not expected.

Original comment by boss...@yahoo.fr on 21 Jan 2013 at 8:50