search

ThrowTheSwitch / Ceedling

Ruby-based unit testing and build system for C projects
http://throwtheswitch.org
Other
568 stars 241 forks source link

Is parameterised testing possible in Ceedling? #241

Open TAGC opened 6 years ago

TAGC commented 6 years ago

I've seen that Unity has support for parameterised testing but there's very little documentation available.

I'm trying to create a parametised test like this:

#define TEST_CASE(...)
// ...
TEST_CASE(0)
TEST_CASE(10)
void test_Device_Controller_Enables_PWM_When_Requested_By_PC(uint32_t _)
{
  // ...
}

However, when I run ceedling, I get this error:

Test 'test_basicPwmControl.c'
-----------------------------
Creating mock for pwmHandler...
Creating mock for pwmHardware...
Creating mock for commsHandler...
Creating mock for versionHandler...
Generating runner for test_basicPwmControl.c...
Compiling test_basicPwmControl_runner.c...
build/test/runners/test_basicPwmControl_runner.c: In function ‘main’:
build/test/runners/test_basicPwmControl_runner.c:87:12: error: too few arguments to function ‘test_Device_Controller_Enables_PWM_When_Requested_By_PC’
   RUN_TEST(test_Device_Controller_Enables_PWM_When_Requested_By_PC, 44);
            ^
build/test/runners/test_basicPwmControl_runner.c:14:7: note: in definition of macro ‘RUN_TEST’
       TestFunc(); \
       ^~~~~~~~
build/test/runners/test_basicPwmControl_runner.c:42:13: note: declared here
 extern void test_Device_Controller_Enables_PWM_When_Requested_By_PC(uint32_t _);
             ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
ERROR: Shell command failed.
> Shell executed command:
'gcc.exe -std=c99 -I"test" -I"test/support" -I"src" -I"src/c" -I"src/include" -I"lib" -I"lib/c" -I"lib/include" -I"C:/Users/<me>/Documents/IAR Embedded Workspace/testcube-firmware-v2/vendor/ceedling/vendor/unity/src" -I"C:/Users/<me>/Documents/IAR Embedded Workspace/testcube-firmware-v2/vendor/ceedling/vendor/cmock/src" -I"build/test/mocks" -DTEST -DUNITY_SUPPORT_TEST_CASES -DGNU_COMPILER -g -c "build/test/runners/test_basicPwmControl_runner.c" -o "build/test/out/test_basicPwmControl_runner.o"'
> And exited with status: [1].

This is the output from ceedling version:

Ceedling:: 0.28.2
CException:: 1.3.1.18
     CMock:: 2.4.4.215
     Unity:: 2.4.1.120

This is my project.yml file:

:project:
  :use_exceptions: FALSE
  :use_test_preprocessor: TRUE
  :use_auxiliary_dependencies: TRUE
  :build_root: build
  :release_build: FALSE
  :test_file_prefix: test_

:extension:
  :executable: .out

:flags:
  :test:
    :compile:
      :*:
        - -std=c99

:paths:
  :test:
    - +:test/**
    - -:test/support
  :source:
    - src/**
    - lib/**
  :support:
    - test/support

:defines:
  :commmon: &common_defines []
  :test:
    - *common_defines
    - TEST
    - UNITY_SUPPORT_TEST_CASES
  :test_preprocess:
    - *common_defines
    - TEST
    - UNITY_SUPPORT_TEST_CASES

:cmock:
  :mock_prefix: "mock_"
  :when_no_prototypes: :warn
  :enforce_strict_ordering: TRUE
  :plugins:
    - :ignore
    - :callback
    - :expect_any_args
    - :array
    - :return_thru_ptr
  :treat_as:
    uint8:    HEX8
    uint16:   HEX16
    uint32:   UINT32
    int8:     INT8
    bool:     UINT8

# LIBRARIES
# These libraries are automatically injected into the build process. Those specified as
# common will be used in all types of builds. Otherwise, libraries can be injected in just
# tests or releases. These options are MERGED with the options in supplemental yaml files.
:libraries:
  :placement: :end
  :flag: "${1}"  # or "-L ${1}" for example
  :common: &common_libraries []
  :test:
    - *common_libraries
  :release:
    - *common_libraries

:plugins:
  :load_paths:
    - vendor/ceedling/plugins
  :enabled:
    - stdout_pretty_tests_report
    - module_generator
TAGC commented 6 years ago

I've tried adding the following to my project.yml

:unity:
  :use_param_tests: true

Now the error is a little different:

Compiling test_basicPwmControl_runner.c...
build/test/runners/test_basicPwmControl_runner.c: In function ‘main’:
build/test/runners/test_basicPwmControl_runner.c:88:12: error: too few arguments to function ‘test_Device_Controller_Enables_PWM_When_Requested_By_PC’
   RUN_TEST(test_Device_Controller_Enables_PWM_When_Requested_By_PC, 44, RUN_TEST_NO_ARGS);
            ^
build/test/runners/test_basicPwmControl_runner.c:15:7: note: in definition of macro ‘RUN_TEST’
       TestFunc(__VA_ARGS__); \
       ^~~~~~~~
build/test/runners/test_basicPwmControl_runner.c:43:13: note: declared here
 extern void test_Device_Controller_Enables_PWM_When_Requested_By_PC(uint32_t _);
mvandervoord commented 6 years ago

Try adding that same line to the :test_runner: section as well. :)

TAGC commented 6 years ago

You mean like this?

...
:unity:
  :use_param_tests: true

:test_runner:
  :use_param_tests: true
...

I still get the same error.

mvandervoord commented 6 years ago

Hm. I'm surprised.

(1) I'd force everything to rebuild by issuing a rake clobber. (2) If there is still a problem, can you post what the RUN_TEST macro looks like in your build/test/runners/test_basicPwmControl_runner.c file?

TAGC commented 6 years ago

ceedling clobber / rake clobber doesn't have any effect.

Given this project.yml:

:unity:
  :use_param_tests: true

:test_runner:
  :use_param_tests: true

...

:defines:
  :commmon: &common_defines []
  :test:
    - *common_defines
    - TEST
    - UNITY_SUPPORT_TEST_CASES
  :test_preprocess:
    - *common_defines
    - TEST
    - UNITY_SUPPORT_TEST_CASES

And this test file:

#include "unity.h"

#define TEST_CASE(...)

void setUp(void)
{
}

void tearDown(void)
{
}

TEST_CASE(1)
TEST_CASE(2)
void test_Foo(uint32_t _)
{
}

Ceedling generates this runner:

/* AUTOGENERATED FILE. DO NOT EDIT. */

/*=======Test Runner Used To Run Each Test Below=====*/
#define RUN_TEST_NO_ARGS
#define RUN_TEST(TestFunc, TestLineNum, ...) \
{ \
  Unity.CurrentTestName = #TestFunc "(" #__VA_ARGS__ ")"; \
  Unity.CurrentTestLineNumber = TestLineNum; \
  Unity.NumberOfTests++; \
  if (TEST_PROTECT()) \
  { \
      setUp(); \
      TestFunc(__VA_ARGS__); \
  } \
  if (TEST_PROTECT()) \
  { \
    tearDown(); \
  } \
  UnityConcludeTest(); \
}

/*=======Automagically Detected Files To Include=====*/
#include "unity.h"
#include <setjmp.h>
#include <stdio.h>

int GlobalExpectCount;
int GlobalVerifyOrder;
char* GlobalOrderError;

/*=======External Functions This Runner Calls=====*/
extern void setUp(void);
extern void tearDown(void);
extern void test_Foo(uint32_t _);

/*=======Test Reset Option=====*/
void resetTest(void);
void resetTest(void)
{
  tearDown();
  setUp();
}

/*=======MAIN=====*/
int main(void)
{
  UnityBegin("test_temp.c");
  RUN_TEST(test_Foo, 15, RUN_TEST_NO_ARGS);

  return (UnityEnd());
}

And outputs this error:

Compiling test_temp_runner.c...
build/test/runners/test_temp_runner.c: In function ‘main’:
build/test/runners/test_temp_runner.c:50:12: error: too few arguments to function ‘test_Foo’
   RUN_TEST(test_Foo, 15, RUN_TEST_NO_ARGS);
            ^
build/test/runners/test_temp_runner.c:13:7: note: in definition of macro ‘RUN_TEST’
       TestFunc(__VA_ARGS__); \
       ^~~~~~~~
build/test/runners/test_temp_runner.c:34:13: note: declared here
 extern void test_Foo(uint32_t _);
mvandervoord commented 6 years ago

This definitely seems like a bug in the runner generator. Thanks for building a minimal example to reproduce it.

austinglaser commented 6 years ago

I believe I've run into this issue before.

A workaround

The problem I encountered is when setting

:project:
  # ...
  :use_test_preprocessor: TRUE

In this case, the preprocessor expands the TEST_CASE() macro away before the runner generator gets ahold of it. I made the following definition in a test-support header file:

#if defined(TEST_PP)
#   define TEST_VALUE(...) TEST_CASE(__VA_ARGS__)
#else
#   define TEST_VALUE(...)
#endif

and then added to my project.yml:

:defines:
  # ...
  :test_preprocess:
    # ...
    - TEST_PP

The reason to do things this way (rather than just disabling preprocessing) is that

#define A_LOCAL_CONSTANT 5

TEST_CASE(A_LOCAL_CONSTANT)
void test_works(uint32_t v)
{
    /* ... */
}

will fail due to A_LOCAL_CONSTANT being undefined in the runner.

An opinionated rant

With all that said, I still feel that Unity's parametrized testing story is extremely weak. Consider, for instance, this case (assuming the availability of the definition above):

#include <stdint.h>

TEST_VALUE(UINT32_MAX)
void test_works(uint32_t v)
{
    /* ... */
}

This won't match, because the TEST_VALUE definition expands to something like

TEST_CASE(
(4294967295U)
)

which the runner generator won't match. This can cause serious issues if, for instance, you define a test like

#include <stdint.h>

TEST_VALUE(2)
TEST_VALUE(3)
TEST_VALUE(UINT32_MAX)
TEST_VALUE(0)
void test_works(uint32_t v)
{
    /* ... */
}

where the preprocessor will silently ignore everything before the TEST_VALUE(UINT32_MAX), and only run the test with the value 0.

Of course, the current approach has ergonomic problems as well:

What to do about it

I'm interested in helping to build better parametrized testing into Unity and Ceedling, but it probably makes sense to discuss first what the best approach to take would be, and if it has a chance to be merged in (given considerations like backwards compatibility, etc).

My first idea is to take a fixture-like approach, which defines an array to be passed in element-by-element. It might look something like

/*
 * TEST_FIXTURE causes the runner to put an extern definition in the runner.
 * Alternatively, to match the current convention for tests, this could just be
 * a naming convention like uint32_t fixture_invalid_values[] = { ... };
 */
TEST_FIXTURE(uint32_t invalid_values[] = {
    0,
    UINT32_MAX,
    5,
    100,
});

/*
 * WITH_FIXTURE works similarly to the current TEST_CASE, but:
 *
 * -    Sidesteps the preprocessing question, since it takes an actual language
 *      symbol rather than a constant
 * -    Brings along a whole set of test-case values, improving re-usability
 *  -   Would allow a trailing semicolon
 */
WITH_FIXTURE(invalid_values);
void test_catches_bad_values(uint32_t val)
{
}

/*
 * WITH_FIXTURE would allow multiple fixtures, and give all possible
 * combinations (the cross product) of all values.
 */
WITH_FIXTURE(invalid_vals_1, invalid_names);
void test_handles_all_bad_things(uint32_t val, char const * name)
{
}

I can open an issue for further discussion if you're at all open to something like this, Mark

Miv13 commented 5 years ago

I get the same error as @TAGC even with @austinglaser 's workaround. Any news on this issue?

joel-felcana commented 4 years ago

Hi,

I'm trying to use this in my tests, but I can't make it work, and I can't find any documentation about it.

I created a new test with the latest ceedling:

$ ceedling version
   Ceedling:: 0.29.1
      Unity:: 2.5.0
      CMock:: 2.5.1
 CException:: 1.3.2

My test is:

#include <unity.h>

#include <stdio.h>

#define TEST_CASE(...)

/*
 */

void setUp(void) {
}

void tearDown(void) {
}

TEST_CASE(1)
TEST_CASE(100)
void test_parameterised(int num) {
    printf("Num is %d\n", num);
    TEST_ASSERT_EQUAL(1, num);
}

What I expect to get is:

What I get? One of the two options:

1) Either only one test is run, and num contains dirty memory:

Test 'test_parameterised.c'
---------------------------
Running test_parameterised.out...

-----------
TEST OUTPUT
-----------
[test_parameterised.c]
  - "Num is 4265064"

-------------------
FAILED TEST SUMMARY
-------------------
[test_parameterised.c]
  Test: test_parameterised
  At line (20): "Expected 1 Was 4265064"

--------------------
OVERALL TEST SUMMARY
--------------------
TESTED:  1
PASSED:  0
FAILED:  1
IGNORED: 0

---------------------
BUILD FAILURE SUMMARY
---------------------
Unit test failures.

2) Ceedling complainign that my test function is not a void func(void) type of function

$ ceedling test

Test 'test_parameterised.c'
---------------------------
Generating runner for test_parameterised.c...
Compiling test_parameterised_runner.c...
build/test/runners/test_parameterised_runner.c: In function 'main':
build/test/runners/test_parameterised_runner.c:82:12: warning: passing argument 1 of 'run_test' from incompatible pointer type [-Wincompatible-pointer-types]
   82 |   run_test(test_parameterised, "test_parameterised", 18);
      |            ^~~~~~~~~~~~~~~~~~
      |            |
      |            void (*)(int)
build/test/runners/test_parameterised_runner.c:46:40: note: expected 'UnityTestFunction' {aka 'void (*)(void)'} but argument is of type 'void (*)(int)'
   46 | static void run_test(UnityTestFunction func, const char* name, int line_num)
      |                      ~~~~~~~~~~~~~~~~~~^~~~
Compiling test_parameterised.c...
Compiling unity.c...
Compiling cmock.c...
Linking test_parameterised.out...
Running test_parameterised.out...

-----------
TEST OUTPUT
-----------
[test_parameterised.c]
  - "Num is 4265064"

-------------------
FAILED TEST SUMMARY
-------------------
[test_parameterised.c]
  Test: test_parameterised
  At line (20): "Expected 1 Was 4265064"

--------------------
OVERALL TEST SUMMARY
--------------------
TESTED:  1
PASSED:  0
FAILED:  1
IGNORED: 0

---------------------
BUILD FAILURE SUMMARY
---------------------
Unit test failures.

My project file is the standard, as generated by ceedling. The funny thing is that the result seems to be totally random. Sometimes I get the complaint about the func type, sometimes it runs. I tried adding

:unity:
  :use_param_tests: true
:cmock:
  :use_param_tests: true
:test_runner:
  :use_param_tests: true

And 

:defines:
  # in order to add common defines:
  #  1) remove the trailing [] from the :common: section
  #  2) add entries to the :common: section (e.g. :test: has TEST defined)
  :common: &common_defines []
  :test:
    - *common_defines
    - TEST
    - UNITY_SUPPORT_TEST_CASES
  :test_preprocess:
    - *common_defines
    - TEST
    - UNITY_SUPPORT_TEST_CASES

But nothing makes the parameterised test run. Sometimes adding something makes it run, sometimes breaks it... it's not like a config works 100% all the time, and changing the project makes it work or not.... For example, the first time I run it with the default config it failed, then adding the unity option made it work, then adding something else broke it, and then deleting everything and coming back to the default, which didn't work in the first place, then worked. I run clean after each config change.

fj-sanchez commented 3 years ago

I'm not sure if this is still relevant after https://github.com/ThrowTheSwitch/Ceedling/pull/497, I have managed to get it working by setting this in my project.yml:

:project:
  :use_test_preprocessor: TRUE
  :use_preprocessor_directives: TRUE

:unity:
  :use_param_tests: true

And then, in my test files I had to add this:

#define TEST_CASE(...)
#define TEST_RANGE(...)
pez3 commented 2 years ago

hi there... i'm trying to write some parameterized unit tests with unity. is there the possiblility to hand over arrays via TEST_CASE ?

thank you for your help greets peter

lapo4719 commented 2 months ago

I got TEST_CASE() TEST_RANGE() and TEST_MATRIX() working with this setup!

  1. I installed Ceedling 0.31.0 (latest as of today) and did a local install
    1. ceedling new your_project --local
  2. I did local install so I could swap the Unity vendor/ folder (both the auto/ and src/ code subfolders) with the latest Unity source v1.6.0 code which has support for these macros. You'll need the new Unity ruby scripts and the new Unity C code from that git repo.
    • Then to my project.yml I added many of the things in these posts above and flag/#defines I found in the latest Unity documentation: 
      
:project:
:use_exceptions: FALSE
:use_test_preprocessor: TRUE
:use_preprocessor_directives: TRUE # ADD THIS
:use_auxiliary_dependencies: TRUE # ADD THIS
:build_root: build
#  :release_build: TRUE
:test_file_prefix: test_
:which_ceedling: vendor/ceedling
:ceedling_version: 0.31.1
:default_tasks:
    • test:all

:unity: # ADD THIS :use_param_tests: true :includes:

:test_runner: # ADD THIS :use_param_tests: true

:test_build:

:use_assembly: TRUE

:release_build:

:output: MyApp.out

:use_assembly: FALSE

:environment:

:extension: :executable: .out

:paths: :test:

:defines:

in order to add common defines:

1) remove the trailing [] from the :common: section

2) add entries to the :common: section (e.g. :test: has TEST defined)

:common: &common_defines [] :test:

:cmock: :mockprefix: mock :when_no_prototypes: :warn :enforce_strict_ordering: TRUE :plugins:

Add -gcov to the plugins list to make sure of the gcov plugin

You will need to have gcov and gcovr both installed to make it work.

For more information on these options, see docs in plugins/gcov

:gcov: :reports:

:tools:

Ceedling defaults to using gcc for compiling, linking, etc.

As [:tools] is blank, gcc will be used (so long as it's in your system path)

See documentation to configure a given toolchain for use

LIBRARIES

These libraries are automatically injected into the build process. Those specified as

common will be used in all types of builds. Otherwise, libraries can be injected in just

tests or releases. These options are MERGED with the options in supplemental yaml files.

:libraries: :placement: :end :flag: "-l${1}" :path_flag: "-L ${1}" :system: [] # for example, you might list 'm' to grab the math library :test: [] :release: []

:plugins: :load_paths:


Maybe there are too many flags used here, but together, and with this parametrics_macro.h file, it lets me do use parametric features:

// parametric_macros.h

ifndef PARAMETRIC_MACROS_H

define PARAMETRIC_MACROS_H

// import this to use TEST_CASE() AND TEST_RANGE() -- these are unity features that ceedling supports // unity documentation for more details: https://github.com/ThrowTheSwitch/Unity/blob/master/docs/UnityHelperScriptsGuide.md

ifndef TEST_CASE

define TEST_CASE(...)

endif // TEST_CASE

// THESE ONLY WORK W/ UNITY > V1.6.0

ifndef TEST_RANGE

define TEST_RANGE(...)

endif // TEST_RANGE

ifndef TEST_MATRIX

define TEST_MATRIX(...)

endif // TEST_MATRIX

endif // PARAMETRIC_MACROS_H


You Must have the setUp() and tearDown() function definitions, for some reason they were getting screwed up in the generated runner if I didn't add them myself. But no big deal just leave them empty.

// /**** // File Name: Tests // *** */

include

include "unity.h"

include "parametric_macros.h"

include "stdbool.h"

void setUp(void) {

}

void tearDown(void) {

}

void test_hello_world(void) { bool myvar = true; TEST_ASSERT_EQUAL(myvar, true); }

TEST_CASE(1) TEST_CASE(3) void test_parametric_arguments(uint8_t my_param) { printf("testing parametric arguments: %d", my_param); TEST_ASSERT_TRUE_MESSAGE(my_param, "I print if assert is false"); }

TEST_RANGE([0, 0, 1]) void test_parametric_arguments_inclusive_range(uint8_t my_param) { printf("testing parametric arguments: %d", my_param);

// make a little debug message:
char *debug_msg = (char*)malloc(128 * sizeof(char));
sprintf(debug_msg, "my_param: %d", my_param);

// assert:
TEST_ASSERT_TRUE_MESSAGE(my_param == 0, debug_msg);

} TEST_RANGE(<0,1,1>) void test_parametric_arguments_exclusive_range(uint8_t my_param) { // make a little debug message: char debug_msg = (char)malloc(128 * sizeof(char)); sprintf(debug_msg, "my_param: %d", my_param);

TEST_ASSERT_TRUE_MESSAGE(my_param < 1, debug_msg);

}

TEST_MATRIX([3, 4, 7]) void test_parametric_arguments_with_test_matrix(uint8_t my_param) { // make a little debug message: char debug_msg = (char)malloc(128 * sizeof(char)); sprintf(debug_msg, "my_param: %d", my_param);

printf("I AM HERE! Running parametric MATRIX tests");

TEST_ASSERT_TRUE_MESSAGE(my_param < 8, debug_msg);

}


Output on command line:

ceedling verbosity[3] test:all

Test 'test_hello_world.c'

Generating runner for test_hello_world.c... Compiling test_hello_world_runner.c... Compiling test_hello_world.c... Linking test_hello_world.out... Running test_hello_world.out...

Test 'test_hello_world_cmock.c'

Generating include list for hello_world.h... Creating mock for hello_world... Generating runner for test_hello_world_cmock.c... Compiling test_hello_world_cmock_runner.c... Compiling test_hello_world_cmock.c... Compiling mock_hello_world.c... Linking test_hello_world_cmock.out... Running test_hello_world_cmock.out...

Test 'test_hello_world_parametric.c'

Generating runner for test_hello_world_parametric.c... Compiling test_hello_world_parametric_runner.c... Compiling test_hello_world_parametric.c... Linking test_hello_world_parametric.out... Running test_hello_world_parametric.out...

TEST OUTPUT

[test_hello_world_parametric.c]

OVERALL TEST SUMMARY

TESTED: 11 PASSED: 10 FAILED: 0 IGNORED: 1