search

uqbar-project / wollok

Wollok Programming Language
GNU General Public License v3.0
60 stars 16 forks source link

Generate JSON report as wollok test execution output #434

Closed javierfernandes closed 8 years ago

javierfernandes commented 8 years ago

This is needed for mumuki integration

javierfernandes commented 8 years ago

@flbulgarelli @faloi can you give me an example of other langs ?

faloi commented 8 years ago

This is how RSpec does it:

describe 'numbers' do 
  it 'can be added' do
    expect(2 + 2).to eq 4
  end

  it 'can be sustracted' do
    expect(8 - 3).to eq 4
  end
end

rspec test.rb --format j

{
    "version": "3.3.0",
    "examples": [
        {
            "description": "can be added",
            "full_description": "numbers can be added",
            "status": "passed",
            "file_path": "./test.rb",
            "line_number": 2,
            "run_time": 0.000520181,
            "pending_message": null
        },
        {
            "description": "can be sustracted",
            "full_description": "numbers can be sustracted",
            "status": "failed",
            "file_path": "./test.rb",
            "line_number": 6,
            "run_time": 0.013283054,
            "pending_message": null,
            "exception": {
                "class": "RSpec::Expectations::ExpectationNotMetError",
                "message": "\nexpected: 4\n     got: 5\n\n(compared using ==)\n",
                "backtrace": [
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-support-3.3.0/lib/rspec/support.rb:86:in `block in <module:Support>'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-support-3.3.0/lib/rspec/support.rb:93:in `call'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-support-3.3.0/lib/rspec/support.rb:93:in `notify_failure'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-expectations-3.3.0/lib/rspec/expectations/fail_with.rb:27:in `fail_with'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-expectations-3.3.0/lib/rspec/expectations/handler.rb:38:in `handle_failure'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-expectations-3.3.0/lib/rspec/expectations/handler.rb:50:in `block in handle_matcher'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-expectations-3.3.0/lib/rspec/expectations/handler.rb:27:in `with_matcher'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-expectations-3.3.0/lib/rspec/expectations/handler.rb:48:in `handle_matcher'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-expectations-3.3.0/lib/rspec/expectations/expectation_target.rb:54:in `to'",
                    "/home/faloi/test.rb:7:in `block (2 levels) in <top (required)>'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-core-3.3.0/lib/rspec/core/example.rb:206:in `instance_exec'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-core-3.3.0/lib/rspec/core/example.rb:206:in `block in run'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-core-3.3.0/lib/rspec/core/example.rb:430:in `block in with_around_and_singleton_context_hooks'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-core-3.3.0/lib/rspec/core/example.rb:388:in `block in with_around_example_hooks'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-core-3.3.0/lib/rspec/core/hooks.rb:478:in `block in run'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-core-3.3.0/lib/rspec/core/hooks.rb:616:in `run_around_example_hooks_for'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-core-3.3.0/lib/rspec/core/hooks.rb:478:in `run'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-core-3.3.0/lib/rspec/core/example.rb:388:in `with_around_example_hooks'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-core-3.3.0/lib/rspec/core/example.rb:430:in `with_around_and_singleton_context_hooks'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-core-3.3.0/lib/rspec/core/example.rb:203:in `run'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-core-3.3.0/lib/rspec/core/example_group.rb:559:in `block in run_examples'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-core-3.3.0/lib/rspec/core/example_group.rb:555:in `map'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-core-3.3.0/lib/rspec/core/example_group.rb:555:in `run_examples'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-core-3.3.0/lib/rspec/core/example_group.rb:521:in `run'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-core-3.3.0/lib/rspec/core/runner.rb:115:in `block (3 levels) in run_specs'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-core-3.3.0/lib/rspec/core/runner.rb:115:in `map'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-core-3.3.0/lib/rspec/core/runner.rb:115:in `block (2 levels) in run_specs'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-core-3.3.0/lib/rspec/core/configuration.rb:1627:in `with_suite_hooks'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-core-3.3.0/lib/rspec/core/runner.rb:114:in `block in run_specs'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-core-3.3.0/lib/rspec/core/reporter.rb:77:in `report'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-core-3.3.0/lib/rspec/core/runner.rb:113:in `run_specs'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-core-3.3.0/lib/rspec/core/runner.rb:89:in `run'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-core-3.3.0/lib/rspec/core/runner.rb:73:in `run'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-core-3.3.0/lib/rspec/core/runner.rb:41:in `invoke'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/lib/ruby/gems/2.0.0/gems/rspec-core-3.3.0/exe/rspec:4:in `<top (required)>'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/bin/rspec:23:in `load'",
                    "/home/faloi/.rbenv/versions/2.0.0-p481/bin/rspec:23:in `<main>'"
                ]
            }
        }
    ],
    "summary": {
        "duration": 0.014512529,
        "example_count": 2,
        "failure_count": 1,
        "pending_count": 0
    },
    "summary_line": "2 examples, 1 failure"
}
javierfernandes commented 8 years ago

@faloi Cool. Give me 2 days, I have a sushi course to attend :P

faloi commented 8 years ago

Of those properties, I think only description, status and exception.message would be useful, at least for now. @flbulgarelli, do you agree?

faloi commented 8 years ago

It would also be nice if you can distinguish between failed (i.e. expectation not met) and errored (i.e. something unexpected happened) examples.

javierfernandes commented 8 years ago

Yes, I'm already doing that

Sample draft

{
    "version" : "1.3.0",
    "tests" : [
{
    "name" : "exported test from REPL session",
    "state" : "passed"
}
        ]
}

(sorry not right indentation)

Sample failure (expectation failed)

{
    "version" : "1.3.0",
    "tests" : [
{
        "name" : "exported test from REPL session",
        "state" : "failed"
        "error" : {
            "message" : "Expected [13] but found [10]",
            "file" : "file:/Users/jfernandes/dev/data/repo/wollok-dev/wollok-tests/src/wollok/examples/guia1/ej1_pepita.wtest",
            "lineNumber" : "15",
            "stackTrace" : "wollok.lib.ValueWasNotEqualsException: Expected [13] but found [10]
                at wollok.lib.AssertionException.valueNotWasEquals(AssertionException.java:67)
                at wollok.lib.AssertObject.assertEquals(AssertObject.java:64)
                at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
                at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
                at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
                at java.lang.reflect.Method.invoke(Method.java:606)
                at org.uqbar.project.wollok.ui.utils.XTendUtilExtensions.invokeConvertingArgs(XTendUtilExtensions.java:350)
                at org.uqbar.project.wollok.interpreter.nativeobj.AbstractWollokDeclarativeNativeObject.call(AbstractWollokDeclarativeNativeObject.java:77)
                at org.uqbar.project.wollok.interpreter.AbstractWollokCallable._invokeNative(AbstractWollokCallable.java:129)
                at org.uqbar.project.wollok.interpreter.AbstractWollokCallable.invokeNative(AbstractWollokCallable.java:187)
                at org.uqbar.project.wollok.interpreter.AbstractWollokCallable$1.apply(AbstractWollokCallable.java:73)
                at org.uqbar.project.wollok.interpreter.WollokInterpreter.performOnStack(WollokInterpreter.java:184)
                at org.uqbar.project.wollok.interpreter.AbstractWollokCallable.call(AbstractWollokCallable.java:95)
                at org.uqbar.project.wollok.interpreter.core.WollokObject.call(WollokObject.java:112)
                at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator._call(WollokInterpreterEvaluator.java:791)
                at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator.call(WollokInterpreterEvaluator.java:898)
                at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator.perform(WollokInterpreterEvaluator.java:787)
                at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator._evaluate(WollokInterpreterEvaluator.java:767)
                at org.uqbar.project.wollok.launch.WollokLauncherInterpreterEvaluator.evaluate(WollokLauncherInterpreterEvaluator.java:189)
                at org.uqbar.project.wollok.interpreter.WollokInterpreter.eval(WollokInterpreter.java:225)
                at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator.eval(WollokInterpreterEvaluator.java:132)
                at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator$1.apply(WollokInterpreterEvaluator.java:138)
                at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator$1.apply(WollokInterpreterEvaluator.java:1)
                at org.eclipse.xtext.xbase.lib.IteratorExtensions.fold(IteratorExtensions.java:626)
                at org.eclipse.xtext.xbase.lib.IterableExtensions.fold(IterableExtensions.java:534)
                at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator.evalAll(WollokInterpreterEvaluator.java:141)
                at org.uqbar.project.wollok.launch.WollokLauncherInterpreterEvaluator._evaluate(WollokLauncherInterpreterEvaluator.java:91)
                at org.uqbar.project.wollok.launch.WollokLauncherInterpreterEvaluator.evaluate(WollokLauncherInterpreterEvaluator.java:195)
                at org.uqbar.project.wollok.interpreter.WollokInterpreter.eval(WollokInterpreter.java:225)
                at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator.eval(WollokInterpreterEvaluator.java:132)
                at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator$1.apply(WollokInterpreterEvaluator.java:138)
                at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator$1.apply(WollokInterpreterEvaluator.java:1)
                at org.eclipse.xtext.xbase.lib.IteratorExtensions.fold(IteratorExtensions.java:626)
                at org.eclipse.xtext.xbase.lib.IterableExtensions.fold(IterableExtensions.java:534)
                at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator.evalAll(WollokInterpreterEvaluator.java:141)
                at org.uqbar.project.wollok.launch.WollokLauncherInterpreterEvaluator._evaluate(WollokLauncherInterpreterEvaluator.java:72)
                at org.uqbar.project.wollok.launch.WollokLauncherInterpreterEvaluator.evaluate(WollokLauncherInterpreterEvaluator.java:191)
                at org.uqbar.project.wollok.interpreter.WollokInterpreter.interpret(WollokInterpreter.java:141)
                at org.uqbar.project.wollok.interpreter.WollokInterpreter.interpret(WollokInterpreter.java:117)
                at org.uqbar.project.wollok.launch.WollokLauncher.doSomething(WollokLauncher.java:39)
                at org.uqbar.project.wollok.launch.WollokChecker.doMain(WollokChecker.java:82)
                at org.uqbar.project.wollok.launch.WollokLauncher.main(WollokLauncher.java:29)
            ""
        }
    }
        ]
}

Sample error

{
    "version" : "1.3.0",
    "tests" : [
{
    "name" : "exported test from REPL session",
    "state" : "error"
    "error" : {
        "message" : "",
        "stacktrace" : "org.uqbar.project.wollok.interpreter.core.WollokProgramExceptionWrapper
            at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator._evaluate(WollokInterpreterEvaluator.java:337)
            at org.uqbar.project.wollok.launch.WollokLauncherInterpreterEvaluator.evaluate(WollokLauncherInterpreterEvaluator.java:175)
            at org.uqbar.project.wollok.interpreter.WollokInterpreter.eval(WollokInterpreter.java:225)
            at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator.eval(WollokInterpreterEvaluator.java:132)
            at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator$1.apply(WollokInterpreterEvaluator.java:138)
            at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator$1.apply(WollokInterpreterEvaluator.java:1)
            at org.eclipse.xtext.xbase.lib.IteratorExtensions.fold(IteratorExtensions.java:626)
            at org.eclipse.xtext.xbase.lib.IterableExtensions.fold(IterableExtensions.java:534)
            at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator.evalAll(WollokInterpreterEvaluator.java:141)
            at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator._evaluate(WollokInterpreterEvaluator.java:659)
            at org.uqbar.project.wollok.launch.WollokLauncherInterpreterEvaluator.evaluate(WollokLauncherInterpreterEvaluator.java:149)
            at org.uqbar.project.wollok.interpreter.WollokInterpreter.eval(WollokInterpreter.java:225)
            at org.uqbar.project.wollok.interpreter.AbstractWollokCallable.eval(AbstractWollokCallable.java:133)
            at org.uqbar.project.wollok.interpreter.AbstractWollokCallable$1.apply(AbstractWollokCallable.java:83)
            at org.uqbar.project.wollok.interpreter.WollokInterpreter.performOnStack(WollokInterpreter.java:184)
            at org.uqbar.project.wollok.interpreter.AbstractWollokCallable.call(AbstractWollokCallable.java:95)
            at org.uqbar.project.wollok.interpreter.core.WollokObject.call(WollokObject.java:112)
            at org.uqbar.project.wollok.interpreter.core.WollokObject.throwMessageNotUnderstood(WollokObject.java:156)
            at org.uqbar.project.wollok.interpreter.core.WollokObject.call(WollokObject.java:128)
            at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator._call(WollokInterpreterEvaluator.java:791)
            at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator.call(WollokInterpreterEvaluator.java:898)
            at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator.perform(WollokInterpreterEvaluator.java:787)
            at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator._evaluate(WollokInterpreterEvaluator.java:767)
            at org.uqbar.project.wollok.launch.WollokLauncherInterpreterEvaluator.evaluate(WollokLauncherInterpreterEvaluator.java:189)
            at org.uqbar.project.wollok.interpreter.WollokInterpreter.eval(WollokInterpreter.java:225)
            at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator.eval(WollokInterpreterEvaluator.java:132)
            at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator$1.apply(WollokInterpreterEvaluator.java:138)
            at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator$1.apply(WollokInterpreterEvaluator.java:1)
            at org.eclipse.xtext.xbase.lib.IteratorExtensions.fold(IteratorExtensions.java:626)
            at org.eclipse.xtext.xbase.lib.IterableExtensions.fold(IterableExtensions.java:534)
            at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator.evalAll(WollokInterpreterEvaluator.java:141)
            at org.uqbar.project.wollok.launch.WollokLauncherInterpreterEvaluator._evaluate(WollokLauncherInterpreterEvaluator.java:91)
            at org.uqbar.project.wollok.launch.WollokLauncherInterpreterEvaluator.evaluate(WollokLauncherInterpreterEvaluator.java:195)
            at org.uqbar.project.wollok.interpreter.WollokInterpreter.eval(WollokInterpreter.java:225)
            at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator.eval(WollokInterpreterEvaluator.java:132)
            at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator$1.apply(WollokInterpreterEvaluator.java:138)
            at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator$1.apply(WollokInterpreterEvaluator.java:1)
            at org.eclipse.xtext.xbase.lib.IteratorExtensions.fold(IteratorExtensions.java:626)
            at org.eclipse.xtext.xbase.lib.IterableExtensions.fold(IterableExtensions.java:534)
            at org.uqbar.project.wollok.interpreter.WollokInterpreterEvaluator.evalAll(WollokInterpreterEvaluator.java:141)
            at org.uqbar.project.wollok.launch.WollokLauncherInterpreterEvaluator._evaluate(WollokLauncherInterpreterEvaluator.java:72)
            at org.uqbar.project.wollok.launch.WollokLauncherInterpreterEvaluator.evaluate(WollokLauncherInterpreterEvaluator.java:191)
            at org.uqbar.project.wollok.interpreter.WollokInterpreter.interpret(WollokInterpreter.java:141)
            at org.uqbar.project.wollok.interpreter.WollokInterpreter.interpret(WollokInterpreter.java:117)
            at org.uqbar.project.wollok.launch.WollokLauncher.doSomething(WollokLauncher.java:39)
            at org.uqbar.project.wollok.launch.WollokChecker.doMain(WollokChecker.java:82)
            at org.uqbar.project.wollok.launch.WollokLauncher.main(WollokLauncher.java:29)
        "
        "file" : "classpath:/wollok.wlk",
        "lineNumber" : "151"
    }
}
        ]
}

I still need to work more on it to

javierfernandes commented 8 years ago

Update, we will include a summary section with counters. Example


{
    "version" : "1.3.0",
    "tests" : [
            {
                "name" : "exported test from REPL session",
                "state" : "passed"
            }
    ],
    "summary" :  {
        "passed" : "1",
        "failures" : "0",
        "errors" : "0"
    }
}

@faloi what do you think ?

faloi commented 8 years ago

Seems ok, but the counters should be numbers, not strings.

El mar., nov. 17, 2015 19:58, javierfernandes notifications@github.com escribió:

Update, we will include a summary section with counters. Example

{ "version" : "1.3.0", "tests" : [ { "name" : "exported test from REPL session", "state" : "passed"

        }
],
"summary" :  {
    "passed" : "1",
    "failures" : "0",
    "errors" : "0"
}

}

@faloi https://github.com/faloi what do you think ?

— Reply to this email directly or view it on GitHub https://github.com/uqbar-project/wollok/issues/434#issuecomment-157537042 .

javierfernandes commented 8 years ago

Right.

Well here is the almost final result

screen shot 2015-11-17 at 22 58 43

Notices that within the "stackTrace" field we are escaping tabs and enters like c languages with \n \t Is that ok ?

Then I have another thing to discuss that I'm not sure how to fit into your proposal or the current JSON we are generating.

Before executing the test Wollok performs static checks (a lot). For example unexistent variable references, or "cannot instantiate abstract class", "must override method", etc. etc..

Those checks don't belong to a particular test, but the whole file. So I cannot render those (easily) as part of the result of a particular test.

For example in this case

import ej1_pepita.*

class A {}

test "Make pepita volar" {
    pepita.energia()
    pepita.volar(22)

    a.blah()

    assert.equals(10, alpiste.energia())
}

test "Test with an assert that fails" {
    pepita.energia()
    pepita.volar(22)

    assert.equals(19, alpiste.energia())
}

The test runner fails with this message (instead of json). It doesn't event start to run the tests at all.

[ERROR] /Users/jfernandes/dev/data/repo/wollok-dev/wollok-tests/src/wollok/examples/guia1/ej1_pepita.wtest:9:99 ERROR Couldn't resolve reference to WReferenciable 'a'.

So, can I render in this case something like this

{
    "version" : "1.3.0",
    "checks" : [
           { 
               "severity" : "ERROR",
               "description" : "Couldn't resolve reference to WReferenciable 'a'",
               "file" : "/Users/jfernandes/dev/data/repo/wollok-dev/wollok-tests/src/wollok/examples/guia1/ej1_pepita.wtest",
               "lineNumber" :  9,
               "offset" : 99
           } 
    ]
}

Would that work for you ??

faloi commented 8 years ago

In those exceptional cases, I think just returning the error string is ok (so does RSpec and stones-spec).

However, your approach is also fine. Perhaps you can include an error code, because the error message is in English and it would be our task to translate it. Or even better, i18n could be handled by Wollok, alongside with more newbie-friendly messages (like "You're trying to send "blah" message to an object referenced by "a", but "a" doesn't exist in this context. Maybe you misspelled it?").

Obviously, we could help on that. :)

El mar., nov. 17, 2015 23:08, javierfernandes notifications@github.com escribió:

Right.

Well here is the almost final result [image: screen shot 2015-11-17 at 22 58 43] https://cloud.githubusercontent.com/assets/4428120/11230610/e365bc98-8d7e-11e5-8476-8eaec26f2970.png

Notices that within the "stackTrace" field we are escaping tabs and enters like c languages with \n \t Is that ok ?

Then I have another thing to discuss that I'm not sure how to fit into your proposal or the current JSON we are generating.

Before executing the test Wollok performs static checks (a lot). For example unexistent variable references, or "cannot instantiate abstract class", "must override method", etc. etc..

Those checks don't belong to a particular test, but the whole file. So I cannot render those (easily) as part of the result of a particular test.

For example in this case

import ej1_pepita.* class A {}

test "Make pepita volar" { pepita.energia() pepita.volar(22)

a.blah()

assert.equals(10, alpiste.energia())

}

test "Test with an assert that fails" { pepita.energia() pepita.volar(22)

assert.equals(19, alpiste.energia())

}

The test runner fails with this message (instead of json). It doesn't event start to run the tests at all.

[ERROR] /Users/jfernandes/dev/data/repo/wollok-dev/wollok-tests/src/wollok/examples/guia1/ej1_pepita.wtest:9:99 ERROR Couldn't resolve reference to WReferenciable 'a'.

So, can I render in this case something like this

{ "version" : "1.3.0", "checks" : [ { "severity" : "ERROR", "description" : "Couldn't resolve reference to WReferenciable 'a'", "file" : "/Users/jfernandes/dev/data/repo/wollok-dev/wollok-tests/src/wollok/examples/guia1/ej1_pepita.wtest", "lineNumber" : 9, "offset" : 99 } ] }

Would that work for you ??

— Reply to this email directly or view it on GitHub https://github.com/uqbar-project/wollok/issues/434#issuecomment-157573939 .

javierfernandes commented 8 years ago

Yeah, actually it is already i18n, but my default language is "en" Anyway there are some parser/linker - level errors that we still need to customize in xtext for i18n, like that one (wrong reference).

I'll render it as json and include all data that I can take from xtext. And that should be enough to close this issue

faloi commented 8 years ago

I want you in my team, forever :punch:

javierfernandes commented 8 years ago

Ok. Done. Example output for a test which has a couple of errors (unused variables, / never assigned)

test "Make pepita volar" {
    pepita.energia()
    pepita.volar(22)

    a.blah()

    var b

    var c = 2

    assert.equals(10, alpiste.energia())
}

And the output goes like this

{  
   "version":"1.3.0",
   "checks":[  
      {  
         "severity":"WARNING",
         "code":"VARIABLE_NEVER_ASSIGNED",
         "message":"Variable is never assigned",
         "lineNumber":11,
         "offset":115,
         "length":1,
         "uri":"file:/Users/jfernandes/dev/data/repo/wollok-dev/wollok-tests/src/wollok/examples/guia1/ej1_pepita.wtest#//@tests.0/@elements.3",
         "syntaxError":false,
         "data":[  

         ]
      },
      {  
         "severity":"WARNING",
         "code":"WARNING_UNUSED_VARIABLE",
         "message":"Unused variable",
         "lineNumber":11,
         "offset":115,
         "length":1,
         "uri":"file:/Users/jfernandes/dev/data/repo/wollok-dev/wollok-tests/src/wollok/examples/guia1/ej1_pepita.wtest#//@tests.0/@elements.3",
         "syntaxError":false,
         "data":[  

         ]
      },
      {  
         "severity":"WARNING",
         "code":"WARNING_UNUSED_VARIABLE",
         "message":"Unused variable",
         "lineNumber":13,
         "offset":124,
         "length":1,
         "uri":"file:/Users/jfernandes/dev/data/repo/wollok-dev/wollok-tests/src/wollok/examples/guia1/ej1_pepita.wtest#//@tests.0/@elements.4",
         "syntaxError":false,
         "data":[  

         ]
      },
      {  
         "severity":"ERROR",
         "code":"org.eclipse.xtext.diagnostics.Diagnostic.Linking",
         "message":"Couldn't resolve reference to WReferenciable 'a'.",
         "lineNumber":9,
         "offset":99,
         "length":1,
         "uri":"file:/Users/jfernandes/dev/data/repo/wollok-dev/wollok-tests/src/wollok/examples/guia1/ej1_pepita.wtest#//@tests.0/@elements.2/@memberCallTarget",
         "syntaxError":false,
         "data":[  

         ]
      }
   ]
}

I just need to document how to use the launcher to output json in the wiki I think it is something like this (sorry been executing it directly from eclipse)

./winterpreter.sh -t -jsonOutput  myfile.wtest
javierfernandes commented 8 years ago

Was faster than 2 days :P

flbulgarelli commented 8 years ago

Awesome!