spekframework / spek

A specification framework for Kotlin
Other
2.23k stars 180 forks source link

Table/data driven tests #638

Open raniejade opened 5 years ago

raniejade commented 5 years ago

The current recommended approach is by using listOf and nested listOfs for multiple rows.

listOf(
  listOf(1, 2, 3),
  listOf(1, 2, 3)
).forEach { data -> {
  ...
}

It has some major caveats:

Spek 1 has a data driven extension but it is tightly coupled into the DSL and is very opinionated to particular usage of Spek (on).

I'd like to bring some form of the Spek 1 plugin into 2.x:

withData(
  r(1, 2, 3),
  r(2, -1, 1)
) { (a, b, c) ->
  test("$a + $b == $c") { ... }
}

The whole implementation is the following:

interface RowN

data class Row1<A>(val a: A) : RowN
data class Row2<A, B>(val a: A, val b: B) : RowN
data class Row3<A, B, C>(val a: A, val b: B, val c: C) : RowN
data class Row4<A, B, C, D>(val a: A, val b: B, val c: C, val d: D) : RowN
data class Row5<A, B, C, D, E>(val a: A, val b: B, val c: C, val d: D, val e: E) : RowN
data class Row6<A, B, C, D, E, F>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F) : RowN
data class Row7<A, B, C, D, E, F, G>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G) : RowN
data class Row8<A, B, C, D, E, F, G, H>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H) : RowN
data class Row9<A, B, C, D, E, F, G, H, I>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I) : RowN
data class Row10<A, B, C, D, E, F, G, H, I, J>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J) : RowN
data class Row11<A, B, C, D, E, F, G, H, I, J, K>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K) : RowN
data class Row12<A, B, C, D, E, F, G, H, I, J, K, L>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L) : RowN
data class Row13<A, B, C, D, E, F, G, H, I, J, K, L, M>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M) : RowN
data class Row14<A, B, C, D, E, F, G, H, I, J, K, L, M, N>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N) : RowN
data class Row15<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N, val o: O) : RowN
data class Row16<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N, val o: O, val p: P) : RowN
data class Row17<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N, val o: O, val p: P, val q: Q) : RowN
data class Row18<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N, val o: O, val p: P, val q: Q, val r: R) : RowN
data class Row19<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N, val o: O, val p: P, val q: Q, val r: R, val s: S) : RowN
data class Row20<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N, val o: O, val p: P, val q: Q, val r: R, val s: S, val t: T) : RowN
data class Row21<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N, val o: O, val p: P, val q: Q, val r: R, val s: S, val t: T, val u: U) : RowN
data class Row22<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N, val o: O, val p: P, val q: Q, val r: R, val s: S, val t: T, val u: U, val v: V) : RowN
data class Row23<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N, val o: O, val p: P, val q: Q, val r: R, val s: S, val t: T, val u: U, val v: V, val w: W) : RowN
data class Row24<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N, val o: O, val p: P, val q: Q, val r: R, val s: S, val t: T, val u: U, val v: V, val w: W, val x: X) : RowN
data class Row25<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N, val o: O, val p: P, val q: Q, val r: R, val s: S, val t: T, val u: U, val v: V, val w: W, val x: X, val y: Y) : RowN
data class Row26<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y, Z>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N, val o: O, val p: P, val q: Q, val r: R, val s: S, val t: T, val u: U, val v: V, val w: W, val x: X, val y: Y, val z: Z) : RowN

fun<A> r(a: A) = Row1(a)
fun<A, B> r(a: A, b: B) = Row2(a, b)
fun<A, B, C> r(a: A, b: B, c: C) = Row3(a, b, c)
fun<A, B, C, D> r(a: A, b: B, c: C, d: D) = Row4(a, b, c, d)
fun<A, B, C, D, E> r(a: A, b: B, c: C, d: D, e: E) = Row5(a, b, c, d, e)
fun<A, B, C, D, E, F> r(a: A, b: B, c: C, d: D, e: E, f: F) = Row6(a, b, c, d, e, f)
fun<A, B, C, D, E, F, G> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G) = Row7(a, b, c, d, e, f, g)
fun<A, B, C, D, E, F, G, H> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H) = Row8(a, b, c, d, e, f, g, h)
fun<A, B, C, D, E, F, G, H, I> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I) = Row9(a, b, c, d, e, f, g, h, i)
fun<A, B, C, D, E, F, G, H, I, J> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J) = Row10(a, b, c, d, e, f, g, h, i, j)
fun<A, B, C, D, E, F, G, H, I, J, K> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K) = Row11(a, b, c, d, e, f, g, h, i, j, k)
fun<A, B, C, D, E, F, G, H, I, J, K, L> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L) = Row12(a, b, c, d, e, f, g, h, i, j, k, l)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M) = Row13(a, b, c, d, e, f, g, h, i, j, k, l, m)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N) = Row14(a, b, c, d, e, f, g, h, i, j, k, l, m, n)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N, o: O) = Row15(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N, o: O, p: P) = Row16(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N, o: O, p: P, q: Q) = Row17(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N, o: O, p: P, q: Q, r: R) = Row18(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N, o: O, p: P, q: Q, r: R, s: S) = Row19(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N, o: O, p: P, q: Q, r: R, s: S, t: T) = Row20(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N, o: O, p: P, q: Q, r: R, s: S, t: T, u: U) = Row21(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N, o: O, p: P, q: Q, r: R, s: S, t: T, u: U, v: V) = Row22(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N, o: O, p: P, q: Q, r: R, s: S, t: T, u: U, v: V, w: W) = Row23(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N, o: O, p: P, q: Q, r: R, s: S, t: T, u: U, v: V, w: W, x: X) = Row24(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N, o: O, p: P, q: Q, r: R, s: S, t: T, u: U, v: V, w: W, x: X, y: Y) = Row25(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y, Z> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N, o: O, p: P, q: Q, r: R, s: S, t: T, u: U, v: V, w: W, x: X, y: Y, z: Z) = Row26(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z)

fun<T: RowN> withData(vararg rows: T, block: (T) -> Unit) {
    rows.forEach(block)
}
raniejade commented 5 years ago

Another option which is somewhat dynamic:

val x by memoized<Int>()
val y by memoized<Int>()
val expected by memoized<Int>()

it("$x + $y = $expected") {
    assertEqual(expected, x + y)
}

where(
    c("x",  "y", "expected"),
    r( 1 ,   2 ,         3 ),
    r( 2 ,   2 ,         4 ),
    r( 1 ,  -1 ,         0 ),
)
raniejade commented 5 years ago

It might be also possible to do:

with { (a, b, c) ->

} where (
    r(1, 2, 3),
    r(1, 2, 3)
)

// or
with { (a, b, c) ->

} where {
    r(1, 2, 3)
    r(1, 2, 3)
}
schleifenkauz commented 4 years ago

Which options do you consider the best at the moment? I would really like to contribute by implementing it.

raniejade commented 4 years ago

@NKb03 I have a local branch that I'm experimenting on. The approach I'm leaning on is the same as what we have in Spek 1.

withData(
  r(1, 2, 3),
  r(2, -1, 1)
) { (a, b, c) ->
  test("$a + $b == $c") { ... }
}
schleifenkauz commented 4 years ago

I actually like the approach, which has the lambda expression after the where the most, because with this approach the data would not have to be actually stored. The where function could be inline too, eliminating any performance overhead. The memory and performance improvement may be neglectable although.

raniejade commented 4 years ago

@NKb03 the where approach won't work unless you switch the ordering (do where first - then the with) as the compiler won't have enough information about what the actual parameter types are.

ravenblackdusk commented 3 years ago

I've been implementing data driven tests with spek for my project and I'd like to share my experience, so that the problems that I faced could perhaps be better handled in your design.

class FooIT : Spek({
    setup()
    Feature("f") {
        Scenario("s") {
            val setup by memoized<Setup>()
            equalize(
                listOf(1, 2, 3, 4, 5),
                listOf("a", "b", "c"),
                listOf(true, false),
                Iterable { iterator { yieldAll(setup.users) } }, // setup cannot be accessed here so it must be evaluated lazily
            ).forEach {
                lateinit var first: Serializable // lots of extra lateinit lines
                lateinit var second: Serializable
                lateinit var third: Serializable
                lateinit var fourth: Serializable
                Given("g") {
                    val iterator = it.iterator()
                    first = iterator.next()
                    second = iterator.next()
                    third = iterator.next()
                    fourth = iterator.next()
                    // setup cannot be accessed in the description so you have to do something like this, which is not so good
                    println("given first $first second $second third $third fourth $fourth")
                }
                When("w") {
                }
                Then("t") {
                }
            }
        }
    }
})

in short not being able to access memoized in scenario scope causes a lot of problems. this is equalize in case you are intrested:

fun <T> equalize(vararg iterables: Iterable<T>): Iterable<Sequence<T>> {
    val max = iterables.filterIsInstance<Collection<T>>().maxOf { it.size }
    return Iterable {
        object : Iterator<Sequence<T>> {
            var index = 0
            val iterators = iterables.map { it.iterator() }.toMutableList()
            override fun hasNext() = index < max
            override fun next() = sequence {
                val listIterator = iterators.listIterator()
                while (listIterator.hasNext()) {
                    val iterable = iterables[listIterator.nextIndex()]
                    val iterator = listIterator.next()
                    yield(
                        when {
                            iterable is List<T> -> iterable[index % iterable.size]
                            iterator.hasNext() -> iterator.next()
                            else -> {
                                val nextIterator = iterable.iterator()
                                listIterator.set(nextIterator)
                                nextIterator.next()
                            }
                        }
                    )
                }
            }.also { index++ }
        }
    }
}