Jinja2 issue

[kinow]

A.py

import jinja2
import sys
import inspect
import dis

TEMPLATE = '''
[scheduling]
    initial cycle point = 20171129T20

    [[dependencies]]
        [[[PT5M]]]
            graph = """
                foo => bar
            """

[runtime]
    [[root]]
        [[[environment]]]
            watch_directory = {{ watch_directory }}

    [[CheckRequests]]
        [[[environment]]]
            processing_suite = {{ processing_suite }}'''

jinja_env = jinja2.Environment(undefined=jinja2.StrictUndefined)

###

#from __future__ import division, generator_stop
from jinja2.runtime import LoopContext, TemplateReference, Macro, Markup, TemplateRuntimeError, missing, concat, escape, markup_join, unicode_join, to_string, identity, TemplateNotFound, Namespace
name = None

def root(context, missing=missing, environment=jinja_env):
    resolve = context.resolve_or_missing
    undefined = environment.undefined
    if 0: yield None
    l_0_watch_directory = resolve('watch_directory')
    l_0_processing_suite = resolve('processing_suite')
    pass
    yield '\n[scheduling]\n    initial cycle point = 20171129T20\n\n    [[dependencies]]\n        [[[PT5M]]]\n            graph = """\n                foo => bar\n            """\n\n[runtime]\n    [[root]]\n        [[[environment]]]\n            watch_directory = %s\n\n    [[CheckRequests]]\n        [[[environment]]]\n            processing_suite = %s' % (
         (undefined(name='watch_directory') if l_0_watch_directory is missing else l_0_watch_directory),
         (undefined(name='processing_suite') if l_0_processing_suite is missing else l_0_processing_suite),
    )

blocks = {}
debug_info = '14=13&18=14'

### 

if __name__ == '__main__':
    #main()
    # jinja_env.from_string(TEMPLATE).render(exists='foo')
    # t = jinja_env.parse(source=TEMPLATE)

    # from jinja2.lexer import Lexer
    # l = Lexer(jinja_env)
    # t = l.tokeniter(source=TEMPLATE, name="Bla")
    # print(list(t))

    #template = jinja2.Template(source=TEMPLATE)
    template = jinja_env.from_string(source=TEMPLATE)
    vars = {'exists': 'foo'}
    context = template.new_context(vars)
    root_render_func = root
    try:
        # r = root_render_func(context)
        #template.root_render_func = root_render_func
        # print(inspect.getsource(template.root_render_func))
        # print(dis.dis(template.root_render_func))
        try:
            template.render(vars)
        except:
            ...
        template.root_render_func = root_render_func
        template.render(vars)
    except:
        exc_info = sys.exc_info()
        jinja_env.handle_exception(exc_info, True)

B.py

import sys
import jinja2

undefined=jinja2.StrictUndefined

def thro():
    yield "Values: %s %s %s" %\
          (undefined('bruno') if 1 > 0 else 'BRUNO',
           undefined('maria') if 1 > 2 else 'MARIA',
           'Terco')

def main():
    exc_info = None
    try:
        print(list(thro()))
        print("OK!")
    except:
        exc_info = sys.exc_info()
    print(exc_info)
    tb = exc_info[2]
    from pprint import pprint
    pprint(tb)

if __name__ == '__main__':
    main()

C.py

import sys

class Thro3():

    def te(self):
        raise IOError()

    def __str__(self):
        self.te()

def thro2():
    raise ValueError()

def thro():
    yield "Values: %s %s %s" %\
          ('bruno',
           Thro3() if 1 > 0 else 'MARIA',
           'Terco')

def main():
    exc_info = None
    try:
        print(list(thro()))
        print("OK!")
    except:
        exc_info = sys.exc_info()
    print(exc_info)
    tb = exc_info[2]
    from pprint import pprint
    pprint(tb)

if __name__ == '__main__':
    main()

[kinow]

jinja2.txt

0.
ORIGINAL:

1. \n
[scheduling]
    initial cycle point = 20171129T20

    [[dependencies]]
        [[[PT5M]]]
            graph = """
                foo => bar
            """

[runtime]
    [[root]]
        [[[environment]]]
14.            watch_directory = {{ watch_directory }}

    [[CheckRequests]]
        [[[environment]]]
18.            processing_suite = {{ processing_suite }}

1. from __future__ import division, generator_stop
from jinja2.runtime import LoopContext, TemplateReference, Macro, Markup, TemplateRuntimeError, missing, concat, escape, markup_join, unicode_join, to_string, identity, TemplateNotFound, Namespace
name = None

def root(context, missing=missing, environment=environment):
    resolve = context.resolve_or_missing
    undefined = environment.undefined
    if 0: yield None
    l_0_watch_directory = resolve('watch_directory')
    l_0_processing_suite = resolve('processing_suite')
    pass
    yield '\n[scheduling]\n    initial cycle point = 20171129T20\n\n    [[dependencies]]\n        [[[PT5M]]]\n            graph = """\n                foo => bar\n            """\n\n[runtime]\n    [[root]]\n        [[[environment]]]\n            watch_directory = %s\n\n    [[CheckRequests]]\n        [[[environment]]]\n            processing_suite = %s' % (
13.         (undefined(name='watch_directory') if l_0_watch_directory is missing else l_0_watch_directory), 
14.         (undefined(name='processing_suite') if l_0_processing_suite is missing else l_0_processing_suite), 
    )

blocks = {}
debug_info = '14=13&18=14'

1.
environment.undefined is jinja2.StrictUndefined

2.
Lexer:
(lineno, type, value)
[

(
    1,
    'data',
    '\n[scheduling]\n    initial cycle point = 20171129T20\n\n    [[dependencies]]\n        [[[PT5M]]]\n            graph = """\n                foo => bar\n            """\n\n[runtime]\n    [[root]]\n        [[[environment]]]\n            watch_directory = '
),
(
    14,
    'variable_begin',
    '{{'), (14, 'whitespace', ' '), (14, 'name', 'watch_directory'), (14, 'whitespace', ' '), (14, 'variable_end', '}}'
),
(
    14,
    'data',
    '\n\n    [[CheckRequests]]\n        [[[environment]]]\n            processing_suite = '
),
(
    18,
    'variable_begin',
    '{{'), (18, 'whitespace', ' '), (18, 'name', 'processing_suite'), (18, 'whitespace', ' '), (18, 'variable_end', '}}'
)

]

3. TEMPLATE obj
Template(
    body=
    [
        Output(
            nodes=
            [
                TemplateData(data='\n
                [scheduling]\n    initial cycle point = 20171129T20\n\n    
                [[dependencies]]\n        
                [
                [
                [PT5M]]]\n            graph = """\n                foo => bar\n            """\n\n
                [runtime]\n    
                [
                [root]]\n        
                [
                [
                [environment]]]\n            watch_directory = '), Name(name='watch_directory', ctx='load'), TemplateData(data='\n\n    
                [
                [CheckRequests]]\n        
                [
                [
                [environment]]]\n            processing_suite = '), Name(name='processing_suite', ctx='load')
            ]
        )
    ]
)

4. DEBUG INFO

(DEBUG_INFO, CODE_LINE)
<class 'list'>: [(14, 13), (18, 14)]

[kinow]

test_jinja2.py

from unittest import main, TestCase

import jinja2

TEMPLATE = '''
[scheduling]
    initial cycle point = 20171129T20

    [[dependencies]]
        [[[PT5M]]]
            graph = """
                foo => bar
            """

[runtime]
    [[root]]
        [[[environment]]]
            watch_directory = {{ watch_directory }}

    [[CheckRequests]]
        [[[environment]]]
            processing_suite = {{ processing_suite }}'''

class TestJinja2(TestCase):

    def test_error_variable_line_number(self):
        # jinja_env = jinja2.Environment(undefined=jinja2.StrictUndefined)
        jinja_env = jinja2.Environment(undefined=jinja2.StrictUndefined)
        jinja_env.from_string(TEMPLATE).render(exists='foo')

if __name__ == '__main__':
    main()

[kinow]

Links:

• https://bugs.python.org/issue16806
• https://bugs.python.org/issue33211
• https://bugs.python.org/issue30465

[kinow]

Z.py

class Z(object):
    def __eq__(self, _):
        raise ValueError('Z error')

def raise_generator():
    yield 'three values are: %s %s %s' % (
        'primeiro',
        Z(),  # traceback must point to this lineno
        'terceiro'  # but points to this lineno (__str__ only, __eq__ is OK)
    )

list(raise_generator())

Z.py (with tracer)

import sys
import trace

# create a Trace object, telling it what to ignore, and whether to
# do tracing or line-counting or both.
tracer = trace.Trace(
    ignoredirs=[sys.prefix, sys.exec_prefix],
    trace=0,
    count=1)

class Z(object):
    def __str__(self):
        #raise ValueError('Z error')
        return "Z"

def raise_generator():
    yield 'three values are: %s %s %s' % (
        'primeiro',
        Z().__str__(),  # traceback must point to this lineno
        'terceiro'  # but points to this lineno (__str__ only, __eq__ is OK)
    )

def main():
    print(list(raise_generator()))

if __name__ == '__main__':
    # run the new command using the given tracer
    tracer.run('main()')

    # make a report, placing output in the current directory
    r = tracer.results()
    r.write_results(show_missing=True, coverdir="/tmp/test")

Output:

>>>>>> import sys
>>>>>> import trace

       # create a Trace object, telling it what to ignore, and whether to
       # do tracing or line-counting or both.
>>>>>> tracer = trace.Trace(
>>>>>>     ignoredirs=[sys.prefix, sys.exec_prefix],
>>>>>>     trace=0,
>>>>>>     count=1)

>>>>>> class Z(object):
>>>>>>     def __str__(self):
               #raise ValueError('Z error')
    1:         return "Z"

>>>>>> def raise_generator():
    1:     yield 'three values are: %s %s %s' % (
    1:         'primeiro',
    1:         Z().__str__(),  # traceback must point to this lineno
    1:         'terceiro'  # but points to this lineno (__str__ only, __eq__ is OK)
           )

>>>>>> def main():
    1:     print(list(raise_generator()))

>>>>>> if __name__ == '__main__':
           # run the new command using the given tracer
>>>>>>     tracer.run('main()')

           # make a report, placing output in the current directory
>>>>>>     r = tracer.results()
>>>>>>     r.write_results(show_missing=True, coverdir="/tmp/test")

[kinow]

Z.py (string "")

import dis

class Z(object):
    def __str__(self):
        raise ValueError('Z error')

def raise_generator():
    yield 'three values are: %s %s %s' % (
        'primeiro',
        Z(),  # traceback must point to this lineno
        'terceiro'  # but points to this lineno (__str__ only, __eq__ is OK)
    )

dis.dis(raise_generator)

print(list(raise_generator()))

Output:

  9           0 LOAD_CONST               1 ('three values are: %s %s %s')

 10           2 LOAD_CONST               2 ('primeiro')

 11           4 LOAD_GLOBAL              0 (Z)
              6 CALL_FUNCTION            0

 12           8 LOAD_CONST               3 ('terceiro')
             10 BUILD_TUPLE              3
             12 BINARY_MODULO
             14 YIELD_VALUE
             16 POP_TOP
             18 LOAD_CONST               0 (None)
             20 RETURN_VALUE
Traceback (most recent call last):
  File "/home/kinow/Development/python/workspace/cylc-flow/cylc/flow/tests/Z.py", line 18, in <module>
    print(list(raise_generator()))
  File "/home/kinow/Development/python/workspace/cylc-flow/cylc/flow/tests/Z.py", line 12, in raise_generator
    'terceiro'  # but points to this lineno (__str__ only, __eq__ is OK)
  File "/home/kinow/Development/python/workspace/cylc-flow/cylc/flow/tests/Z.py", line 5, in __str__
    raise ValueError('Z error')
ValueError: Z error

Z.py (string .str())

import dis

class Z(object):
    def __str__(self):
        raise ValueError('Z error')

def raise_generator():
    yield 'three values are: %s %s %s' % (
        'primeiro',
        Z().__str__(),  # traceback must point to this lineno
        'terceiro'  # but points to this lineno (__str__ only, __eq__ is OK)
    )

dis.dis(raise_generator)

print(list(raise_generator()))

Output:

  9           0 LOAD_CONST               1 ('three values are: %s %s %s')

 10           2 LOAD_CONST               2 ('primeiro')

 11           4 LOAD_GLOBAL              0 (Z)
              6 CALL_FUNCTION            0
              8 LOAD_METHOD              1 (__str__)
             10 CALL_METHOD              0

 12          12 LOAD_CONST               3 ('terceiro')
             14 BUILD_TUPLE              3
             16 BINARY_MODULO
             18 YIELD_VALUE
             20 POP_TOP
             22 LOAD_CONST               0 (None)
             24 RETURN_VALUE
Traceback (most recent call last):
  File "/home/kinow/Development/python/workspace/cylc-flow/cylc/flow/tests/Z.py", line 18, in <module>
    print(list(raise_generator()))
  File "/home/kinow/Development/python/workspace/cylc-flow/cylc/flow/tests/Z.py", line 11, in raise_generator
    Z().__str__(),  # traceback must point to this lineno
  File "/home/kinow/Development/python/workspace/cylc-flow/cylc/flow/tests/Z.py", line 5, in __str__
    raise ValueError('Z error')
ValueError: Z error

[kinow]

https://bugs.python.org/issue37647

[kinow]

Example Python code created from a template by Jinja:

from __future__ import division, generator_stop
from jinja2.runtime import LoopContext, TemplateReference, Macro, Markup, TemplateRuntimeError, missing, concat, escape, markup_join, unicode_join, to_string, identity, TemplateNotFound, Namespace
name = 'undefined.html'

def root(context, missing=missing, environment=environment):
    resolve = context.resolve_or_missing
    undefined = environment.undefined
    if 0: yield None
    l_0_two = resolve('two')
    l_0_four = resolve('four')
    l_0_six = resolve('six')
    pass
    yield 'This is a text with\none\n%s (error happens here first!)\nthree\n%s (error here as well)\nfive\n%s (no error)\nand more lines.\n\nThe line using the "two" variable must be the one to appear in the\ntraceback with a matching \'two\' is missing message.' % (
        (undefined(name='two') if l_0_two is missing else l_0_two), 
        (undefined(name='four') if l_0_four is missing else l_0_four), 
        (undefined(name='six') if l_0_six is missing else l_0_six),