tests are flagged as passing, even when they don't

This is a new issue that I think just cropped up with your changes over the weekend. Not sure what happened.
Look at testResults.passed compared to all the test steps failing:
{
    "timeStarted": "2014-08-17T20:26:03.741Z",
    "timeEnded": "2014-08-17T20:26:04.032Z",
    "testsExecuted": 1,
    "testsFailed": 1,
    "testsSkipped": 0,
    "suiteResults": [{
        "suiteId": "test/integration/client.tests.json",
        "suiteName": "client.tests",
        "timeStarted": "2014-08-17T20:26:03.802Z",
        "timeEnded": "2014-08-17T20:26:04.032Z",
        "testsExecuted": 1,
        "testsFailed": 1,
        "testsSkipped": 0,
        "validationsPerformed": 0,
        "validationsFailed": 0,
        "testResults": {
            "passed": true,
            "suiteStepResults": [{
                    "id": "getAdminToken",
                    "testPhase": "suiteSetup",
                    "passed": false,
                    "timeSent": "2014-08-17T20:26:03.888Z",
                    "repeated": null,
                    "responseTime": 72.255132,
                    "rawRequest": "POST http://localhost:8080/token HTTP 1.1\ncontent-type: application/x-www-form-urlencoded\n\n\ngrant_type=password&username=redacted&password=redacted",
                    "rawResponse": "HTTP/1.1 403\ncontent-type: application/json; charset=utf-8\ncontent-length: 104\naccess-control-allow-origin: *\naccess-control-max-age: 86400\naccess-control-allow-methods: GET, HEAD, POST, PUT, PATCH, DELETE, OPTIONS\naccess-control-allow-headers: Authorization, Content-Type, If-None-Match, X-Requested-With\naccess-control-expose-headers: WWW-Authenticate, Server-Authorization, Location, Content-Range\ncache-control: no-cache\ndate: Sun, 17 Aug 2014 20:26:03 GMT\nconnection: keep-alive\n\n\n{\"statusCode\":403,\"error\":\"Forbidden\",\"message\":\"Unable to determine authenticated client from request\"}",
                    "validationResults": [{
                        "id": "statusCode",
                        "valid": false,
                        "expectedValue": 200,
                        "actualValue": 403,
                        "diffs": [{
                            "actual": 403,
                            "expected": 200
                        }],
                        "description": "Unexpected value"
                    }, {
                        "id": "header",
                        "valid": true
                    }],
                    "error": null
                }, {
                    "id": "createUnprivilegedClient",
                    "testPhase": "suiteSetup",
                    "passed": false,
                    "timeSent": "2014-08-17T20:26:03.984Z",
                    "repeated": null,
                    "responseTime": null,
                    "rawRequest": null,
                    "rawResponse": null,
                    "validationResults": [],
                    "error": "Error: Attempting to use a variable that has not been defined: 'adminAccessToken'"
                }, {
                    "id": "getUnprivilegedToken",
                    "testPhase": "suiteSetup",
                    "passed": false,
                    "timeSent": "2014-08-17T20:26:04.000Z",
                    "repeated": null,
                    "responseTime": 15.864803,
                    "rawRequest": "POST http://localhost:8080/token HTTP 1.1\ncontent-type: application/x-www-form-urlencoded\n\n\ngrant_type=password&username=at40ayau3x&password=test",
                    "rawResponse": "HTTP/1.1 403\ncontent-type: application/json; charset=utf-8\ncontent-length: 104\naccess-control-allow-origin: *\naccess-control-max-age: 86400\naccess-control-allow-methods: GET, HEAD, POST, PUT, PATCH, DELETE, OPTIONS\naccess-control-allow-headers: Authorization, Content-Type, If-None-Match, X-Requested-With\naccess-control-expose-headers: WWW-Authenticate, Server-Authorization, Location, Content-Range\ncache-control: no-cache\ndate: Sun, 17 Aug 2014 20:26:04 GMT\nconnection: keep-alive\n\n\n{\"statusCode\":403,\"error\":\"Forbidden\",\"message\":\"Unable to determine authenticated client from request\"}",
                    "validationResults": [{
                        "id": "statusCode",
                        "valid": false,
                        "expectedValue": 200,
                        "actualValue": 403,
                        "diffs": [{
                            "actual": 403,
                            "expected": 200
                        }],
                        "description": "Unexpected value"
                    }, {
                        "id": "header",
                        "valid": true
                    }],
                    "error": null
                },
                [{
                    "id": "createClient_Success",
                    "passed": false,
                    "repeated": null,
                    "testStepResults": [{
                        "id": "getRandomClientId",
                        "testPhase": "testSetup",
                        "passed": true,
                        "timeSent": "2014-08-17T20:26:04.022Z",
                        "repeated": null,
                        "responseTime": null,
                        "rawRequest": null,
                        "rawResponse": null,
                        "validationResults": [],
                        "error": null
                    }, {
                        "id": "createClient_Success",
                        "testPhase": "test",
                        "passed": false,
                        "timeSent": "2014-08-17T20:26:04.023Z",
                        "repeated": null,
                        "responseTime": null,
                        "rawRequest": null,
                        "rawResponse": null,
                        "validationResults": [],
                        "error": "Error: Attempting to use a variable that has not been defined: 'adminAccessToken'"
                    }, {
                        "id": "verifyClientCreated",
                        "testPhase": "testTeardown",
                        "passed": false,
                        "timeSent": "2014-08-17T20:26:04.023Z",
                        "repeated": null,
                        "responseTime": null,
                        "rawRequest": null,
                        "rawResponse": null,
                        "validationResults": [],
                        "error": "Error: Attempting to use a variable that has not been defined: 'adminAccessToken'"
                    }, {
                        "id": "removeClient",
                        "testPhase": "testTeardown",
                        "passed": false,
                        "timeSent": "2014-08-17T20:26:04.023Z",
                        "repeated": null,
                        "responseTime": null,
                        "rawRequest": null,
                        "rawResponse": null,
                        "validationResults": [],
                        "error": "Error: Attempting to use a variable that has not been defined: 'adminAccessToken'"
                    }],
                    "variables": {
                        "protocol": "http",
                        "host": "localhost:8080",
                        "adminClientId": "redacted",
                        "adminSecret": "redacted",
                        "testDataPrefix": "harvey-test---",
                        "unprivilegedClientSecret": "test",
                        "randomUnprivilegedClientId": "at40ayau3x",
                        "randomClientId": "akqaa5jaaa"
                    }
                }], {
                    "id": "removeUnprivilegedClient",
                    "testPhase": "suiteTeardown",
                    "passed": false,
                    "timeSent": "2014-08-17T20:26:04.032Z",
                    "repeated": null,
                    "responseTime": null,
                    "rawRequest": null,
                    "rawResponse": null,
                    "validationResults": [],
                    "error": "Error: Attempting to use a variable that has not been defined: 'adminAccessToken'"
                }
            ]
        }
    }]
}
tschwecke / harvey

tests are flagged as passing, even when they don't #57
