Automatic Test Discovery

[christophercrouzet]

Having to write tests and then register them as part of a suite feels a bite redundant, error-prone since it would be easy to forget, and also quite verbose.

While reading CTest's source code, I noticed that data sections defined through compilers attributes could be accessed within the same translation unit, as seen below:

#include <stdio.h>

// Method seen on CTest <https://github.com/bvdberg/ctest>.

// TODO: add alternatives for non-GNU compliant compilers.
#define RXP_SECTION __attribute__ ((used, section (".rexo"), aligned(1)))

#define RX_TEST_CASE(name)                                                     \
    static void name##Func();                                                  \
    static struct RxTestCase name##Struct RXP_SECTION = {                      \
        .pfn = name##Func,                                                     \
    };                                                                         \
    static void name##Func()

typedef void (*RxPfnTestCase)();

static struct RxTestCase rxpRoot RXP_SECTION;

struct RxTestCase {
    const char *pName;
    RxPfnTestCase pfn;
};

RX_TEST_CASE(testFoo) {
    printf("foo\n");
}

RX_TEST_CASE(testBar) {
    printf("bar\n");
}

int
main()
{
    struct RxTestCase *pCase;

    pCase = &rxpRoot;

    ++pCase;
    pCase->pfn();

    ++pCase;
    pCase->pfn();

    return 0;
}

I'm planning to experiment with this approach and will look on how to expend it to the suites. Ideally, we'd end up with this kind of code on the user-side:

#include <rexo/rexo.h>

RX_TEST_CASE(myTestSuite, testFoo) {
    RX_CHECK_INT_EQUAL(1, 1);
}

RX_TEST_CASE(myTestSuite, testBar) {
    RX_CHECK_INT_EQUAL(2, 2);
}

int
main(int argc, const char **ppArgv)
{
    rxRun(argc, ppArgv);
    return 0;
}

Where both test cases would be registered within a same (default initialized) test suite.

In the case where a more specific test suite is required, such as one providing a fixture, then we would be able to explicitely define it as such:

#include <rexo/rexo.h>

enum RxStatus
mySetUp()
{
    // ...
    return RX_SUCCESS;
}

void
myTearDown()
{
    // ...
}

RX_TEST_SUITE(myTestSuite, .setUp = mySetUp, .tearDown = myTearDown);

RX_TEST_CASE(myTestSuite, testFoo) {
    // access some fixture data here.
    RX_CHECK_INT_EQUAL(1, 1);
}

int
main(int argc, const char **ppArgv)
{
    rxRun(argc, ppArgv);
    return 0;
}

[christophercrouzet]

For info, I managed to get a prototype working on gcc, clang, icc, and msvc. It's incomplete and needs much more work before being integrated into Rexo but it's looking promising.

#include <stdio.h>

#define RXP_TEST_CASE_PARAMETERS
typedef void (*RxPfnTestCase)(RXP_TEST_CASE_PARAMETERS);

struct RxpTestCaseDefinition {
    const char *pName;
    const char *pSuiteName;
    RxPfnTestCase pfnRun;
};

#if defined(__GNUC__)
extern const struct RxpTestCaseDefinition *__start_rxcase;
extern const struct RxpTestCaseDefinition *__stop_rxcase;
#define RXP_ADD_TO_CASE_SECTION __attribute__((used, section("rxcase")))
#define RXP_CASE_SECTION_BEGIN __start_rxcase
#define RXP_CASE_SECTION_END __stop_rxcase
#elif defined(_MSC_VER)
#pragma section("rxcase$a", read)
#pragma section("rxcase$s", read)
#pragma section("rxcase$z", read)
__declspec(allocate("rxcase$a")) static const
    struct RxpTestCaseDefinition *rxpCaseSectionBegin;
__declspec(allocate("rxcase$z")) static const
    struct RxpTestCaseDefinition *rxpCaseSectionEnd;
#define RXP_ADD_TO_CASE_SECTION __declspec(allocate("rxcase$s"))
#define RXP_CASE_SECTION_BEGIN (*(&rxpCaseSectionBegin + 1))
#define RXP_CASE_SECTION_END rxpCaseSectionEnd
#endif

#define RX_TEST_CASE(suiteName, name)                                          \
    static void name(RXP_TEST_CASE_PARAMETERS);                                \
    static const struct RxpTestCaseDefinition testCaseDefinition_##name        \
        = {#name, #suiteName, name};                                           \
    RXP_ADD_TO_CASE_SECTION                                                    \
    static const struct RxpTestCaseDefinition *pTestCaseDefinition_##name      \
        = &testCaseDefinition_##name;                                          \
    static void name(RXP_TEST_CASE_PARAMETERS)

RX_TEST_CASE(mySuite, foo)
{
    printf("foo!\n");
}

RX_TEST_CASE(mySuite, bar)
{
    printf("bar!\n");
}

int
main(void)
{
    const struct RxpTestCaseDefinition **ppCursor = &RXP_CASE_SECTION_BEGIN;

    for (; ppCursor < &RXP_CASE_SECTION_END; ++ppCursor) {
        if (*ppCursor == NULL) {
            continue;
        }

        (*ppCursor)->pfnRun();
    }

    return 0;
}

[christophercrouzet]

Quick heads up: it's 80% done and I'm pretty happy on the implementation so far.

The 20% remaining is about having the possibility to define metadata (e.g. set up and tear down functions) both at the suite and at the case levels, e.g.:

RX_TEST_SUITE(my_suite, .set_up=my_set_up_fn, .tear_down=my_tear_down_fn);

RX_TEST_CASE(my_suite, my_case, .set_up=another_set_up_fn)
{
    ...
}

I still have to figure out a nice approach to have the cases inherit the metadata from their suite while being able to override some specific fields (e.g.: the set_up function in the snippet above).

[christophercrouzet]

Sorry it took me a while but it's finally here and I'm really happy with the results!

A minimal version now looks like this:

#include <rexo.h>

RX_TEST_CASE(my_test_suite, my_test_case)
{
    RX_CHECK_STR_EQUAL("Hello", "world!");
}

int
main(int argc, const char **argv)
{
    return rx_run(argc, argv, 0, NULL) == RX_SUCCESS ? 0 : 1;
}

Let me know what you think @peterleegolang! :blush: