how to shrink effectively with hypothesis stateful testing?

[zhoucheng361]

I write a stateful test to test the ACL feature, when I run the test with shrink enabled , I got a failure example including 56 steps. I wonder how I can optimize my testing code to let it shrink more effectively? Another question is , when I got a long steps failure example, can I do shrink based on it with hypothesis framework?

cat .github/scripts/fsrand2.py

import os
import random
from string import ascii_lowercase
import subprocess
from hypothesis import given, settings, Verbosity, Phase, HealthCheck, seed, strategies as st
from hypothesis.stateful import RuleBasedStateMachine, Bundle, initialize, rule, multiple
from numpy import multiply
SEED=int(os.environ.get('SEED', random.randint(0, 1000000000)))
MAX_EXAMPLE=int(os.environ.get('MAX_EXAMPLE', '100'))
st_entry_name = st.text(alphabet=ascii_lowercase, min_size=4, max_size=4)
st_open_mode = st.sampled_from(['w', 'x', 'a'])
st_content = st.binary(min_size=0, max_size=100)
USERS=['root']
ACL_USERS=['root', 'user1', 'user2', 'user3', '']
ACL_GROUPS=['root', 'group1', 'group2', 'group3', '']
ROOT_DIR1='/tmp/fsrand1'
ROOT_DIR2='/tmp/fsrand2'

def run_cmd(command: str) -> str:
    print('run_cmd:'+command)
    try:
        output = subprocess.run(command.split(), check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as e:
        raise e
    return output.stdout.decode()

@seed(SEED)
@settings(verbosity=Verbosity.debug, 
    max_examples=MAX_EXAMPLE, 
    stateful_step_count=50, 
    derandomize = False,
    deadline=None, 
    report_multiple_bugs=False, 
    phases=[Phase.explicit, Phase.reuse, Phase.generate, Phase.target, Phase.shrink, Phase.explain ], 
    suppress_health_check=[HealthCheck.too_slow])
class JuicefsMachine(RuleBasedStateMachine):
    Files = Bundle('files')
    Folders = Bundle('folders')
    Entries = Files | Folders
    EntryWithACL = Bundle('entry_with_acl')

    @initialize(target=Folders)
    def init_folders(self):
        return ""

    def __init__(self):
        super(JuicefsMachine, self).__init__()

    @rule(target=Files, 
          parent = Folders.filter(lambda x: x != multiple()), 
          file_name = st_entry_name, 
          mode = st_open_mode, 
          content = st_content, 
          user = st.sampled_from(USERS))
    def create_file(self, parent, file_name, mode, content, user):
        result1 = self.do_create_file(ROOT_DIR1, parent, file_name, mode, content, user)
        result2 = self.do_create_file(ROOT_DIR2, parent, file_name, mode, content, user)
        assert self.equal(result1, result2), f'create_file:\nresult1 is {result1}\nresult2 is {result2}'
        if isinstance(result1, tuple):
            return os.path.join(parent, file_name)
        else:
            return multiple()

    def do_create_file(self, root_dir, parent, file_name, mode, content, user):
        relpath = os.path.join(parent, file_name)
        abspath = os.path.join(root_dir, relpath)
        try:
            with open(abspath, mode) as file:
                file.write(str(content))
        except Exception as e :
            return str(e)
        return os.stat(abspath)

    @rule(target = Files , 
          dest_file = Files.filter(lambda x: x != multiple()), 
          parent = Folders.filter(lambda x: x != multiple()),
          link_file_name = st_entry_name, 
          user = st.sampled_from(USERS) )
    def symlink(self, dest_file, parent, link_file_name, user):
        # assume(not os.path.exists(os.path.join(ROOT_DIR1, parent_dir, link_file_name)))
        result1 = self.do_symlink(ROOT_DIR1, dest_file, parent, link_file_name, user)
        result2 = self.do_symlink(ROOT_DIR2, dest_file, parent, link_file_name, user)
        assert self.equal(result1, result2), f'symlink:\nresult1 is {result1}\nresult2 is {result2}'
        if isinstance(result1, tuple):
            return os.path.join(parent, link_file_name)
        else:
            return multiple()

    def do_symlink(self, root_dir, dest_file, parent, link_file_name, user):
        dest_abs_path = os.path.join(root_dir, dest_file)
        link_rel_path = os.path.join(parent, link_file_name)
        link_abs_path = os.path.join(root_dir, link_rel_path)
        relative_path = os.path.relpath(dest_abs_path, os.path.dirname(link_abs_path))
        try:
            os.symlink(relative_path, link_abs_path)
        except Exception as e:
            return str(e)
        return os.stat(link_abs_path)

    @rule(
          target=EntryWithACL,
          sudo_user = st.sampled_from(USERS),
          entry = Entries.filter(lambda x: x != multiple()), 
          user=st.sampled_from(ACL_USERS),
          user_perm = st.sets(st.sampled_from(['r', 'w', 'x', ''])),
          group=st.sampled_from(ACL_GROUPS),
          group_perm = st.sets(st.sampled_from(['r', 'w', 'x', ''])),
          other_perm = st.sets(st.sampled_from(['r', 'w', 'x', ''])),
          set_mask = st.just(False),
          mask = st.sets(st.sampled_from(['r', 'w', 'x', ''])),
          default = st.just(False),
          recursive = st.booleans(),
          recalc_mask = st.booleans(),
          not_recalc_mask = st.booleans(),
          logical = st.booleans(),
          physical = st.booleans(),
          )
    def set_acl(self, sudo_user, entry, user, user_perm, group, group_perm, other_perm, set_mask, mask, default, recursive, recalc_mask, not_recalc_mask, logical, physical):
        result1 = self.do_set_acl(ROOT_DIR1, sudo_user, entry, user, user_perm, group, group_perm, other_perm, set_mask, mask, default, recursive, recalc_mask, not_recalc_mask, logical, physical)
        result2 = self.do_set_acl(ROOT_DIR2, sudo_user, entry, user, user_perm, group, group_perm, other_perm, set_mask, mask, default, recursive, recalc_mask, not_recalc_mask, logical, physical)
        assert result1==result2, f'set_acl:\nresult1 is {result1}\nresult2 is {result2}'
        if isinstance(result1, tuple):
            return entry
        else:
            return multiple()

    def do_set_acl(self, root_dir, sudo_user, entry, user, user_perm, group, group_perm, other_perm, set_mask, mask, default, recursive, recalc_mask, not_recalc_mask, logical, physical):
        abspath = os.path.join(root_dir, entry)
        user_perm = ''.join(user_perm) == '' and '-' or ''.join(user_perm)
        group_perm = ''.join(group_perm) == '' and '-' or ''.join(group_perm)
        other_perm = ''.join(other_perm) == '' and '-' or ''.join(other_perm)
        mask = ''.join(mask) == '' and '-' or ''.join(mask)
        default = default and '-d' or ''
        recursive = recursive and '-R' or ''
        recalc_mask = recalc_mask and '--mask' or ''
        not_recalc_mask = not_recalc_mask and '--no-mask' or ''
        logical = (recursive and logical) and '-L' or ''
        physical = (recursive and physical) and '-P' or ''
        try:
            text = f'u:{user}:{user_perm},g:{group}:{group_perm},o::{other_perm}'
            if set_mask:
                text += f',m::{mask}'
            run_cmd(f'sudo -u {sudo_user} setfacl {default} {recursive} {recalc_mask} {not_recalc_mask} {logical} {physical} -m {text} {abspath}')
            acl = run_cmd(f'getfacl {abspath}')
        except subprocess.CalledProcessError as e:
            return str(e)
        return (acl,)    

if __name__ == '__main__':
    juicefs_machine = JuicefsMachine.TestCase()
    juicefs_machine.runTest()

Run the test:

MAX_EXAMPLE=100 STEP_COUNT=100 python3 .github/scripts/fsrand2.py

Check the fail example

here is what I got after the test , the failure example include 56 steps, which is not shrunk effectively.

Traceback (most recent call last):
  File ".github/scripts/fsrand2.py", line 1474, in <module>
    example3()
  File ".github/scripts/fsrand2.py", line 1468, in example3
    state.symlink(dest_file=v56, link_file_name='tabm', parent=v1, user='root')
  File ".github/scripts/fsrand2.py", line 864, in symlink
    dest_file = Files.filter(lambda x: x != multiple()),
  File "/usr/local/lib/python3.8/dist-packages/hypothesis/stateful.py", line 675, in rule_wrapper
    return f(*args, **kwargs)
  File ".github/scripts/fsrand2.py", line 864, in symlink
    dest_file = Files.filter(lambda x: x != multiple()),
  File "/usr/local/lib/python3.8/dist-packages/hypothesis/stateful.py", line 796, in precondition_wrapper
    return f(*args, **kwargs)
  File ".github/scripts/fsrand2.py", line 873, in symlink
    assert self.equal(result1, result2), f'symlink:\nresult1 is {result1}\nresult2 is {result2}'
AssertionError: symlink:
result1 is (0, 0, 4, '0o100773', 1)
result2 is (0, 0, 4, '0o100772', 1)

state = JuicefsMachine()
v1 = state.init_folders()
v2 = state.create_file(content=b'C', file_name='vhgb', mode='w', parent=v1, user='root')
v3 = state.symlink(dest_file=v2, link_file_name='qslq', parent=v1, user='root')
v4 = state.symlink(dest_file=v3, link_file_name='ciwg', parent=v1, user='root')
v5 = state.set_acl(default=False, entry=v1, group='group2', group_perm={v1, 'r', 'w', 'x'}, logical=True, mask={'w', 'x'}, not_recalc_mask=False, other_perm={v1}, physical=False, recalc_mask=False, recursive=False, set_mask=False, sudo_user='root', user=v1, user_perm={v1, 'x'})
v6 = state.set_acl(default=False, entry=v1, group='root', group_perm=set(), logical=True, mask=set(), not_recalc_mask=False, other_perm=set(), physical=True, recalc_mask=False, recursive=False, set_mask=False, sudo_user='root', user=v1, user_perm={v1, 'r', 'w', 'x'})
v7 = state.symlink(dest_file=v3, link_file_name='zzka', parent=v1, user='root')
v8 = state.set_acl(default=False, entry=v1, group='group3', group_perm=set(), logical=False, mask={v1, 'r', 'w', 'x'}, not_recalc_mask=False, other_perm={v1, 'r', 'w', 'x'}, physical=False, recalc_mask=False, recursive=False, set_mask=False, sudo_user='root', user='root', user_perm=set())
v9 = state.set_acl(default=False, entry=v1, group='root', group_perm={'r', 'w'}, logical=False, mask=set(), not_recalc_mask=True, other_perm=set(), physical=True, recalc_mask=False, recursive=True, set_mask=False, sudo_user='root', user='root', user_perm={v1, 'r', 'w', 'x'})
v10 = state.set_acl(default=False, entry=v1, group='group1', group_perm=set(), logical=False, mask=set(), not_recalc_mask=True, other_perm={'r', 'x'}, physical=True, recalc_mask=True, recursive=False, set_mask=False, sudo_user='root', user=v1, user_perm={'x'})
v11 = state.symlink(dest_file=v3, link_file_name='bape', parent=v1, user='root')
v12 = state.symlink(dest_file=v11, link_file_name='dbua', parent=v1, user='root')
v13 = state.create_file(content=b'm\xb9\xb7@', file_name='sznu', mode='x', parent=v1, user='root')
v14 = state.set_acl(default=False, entry=v1, group='group3', group_perm={'w', 'x'}, logical=False, mask=set(), not_recalc_mask=False, other_perm={'w'}, physical=True, recalc_mask=True, recursive=False, set_mask=False, sudo_user='root', user=v1, user_perm={v1, 'r', 'w', 'x'})
v15 = state.create_file(content=b'', file_name='jpri', mode='w', parent=v1, user='root')
v16 = state.symlink(dest_file=v4, link_file_name='gnud', parent=v1, user='root')
v17 = state.set_acl(default=False, entry=v1, group='root', group_perm={v1, 'r', 'w', 'x'}, logical=True, mask={'w'}, not_recalc_mask=True, other_perm=set(), physical=False, recalc_mask=True, recursive=True, set_mask=False, sudo_user='root', user='root', user_perm=set())
v18 = state.set_acl(default=False, entry=v15, group='group1', group_perm=set(), logical=True, mask={'r', 'w'}, not_recalc_mask=True, other_perm=set(), physical=True, recalc_mask=False, recursive=False, set_mask=False, sudo_user='root', user=v1, user_perm=set())
v19 = state.set_acl(default=False, entry=v1, group='group2', group_perm={v1, 'r', 'w', 'x'}, logical=True, mask=set(), not_recalc_mask=False, other_perm={v1, 'r', 'w', 'x'}, physical=True, recalc_mask=True, recursive=True, set_mask=False, sudo_user='root', user=v1, user_perm=set())
v20 = state.symlink(dest_file=v12, link_file_name='daka', parent=v1, user='root')
v21 = state.set_acl(default=False, entry=v16, group='group1', group_perm={v1}, logical=True, mask={v1, 'r', 'w', 'x'}, not_recalc_mask=True, other_perm={v1, 'r', 'w'}, physical=False, recalc_mask=False, recursive=False, set_mask=False, sudo_user='root', user=v1, user_perm={'r'})
v22 = state.symlink(dest_file=v2, link_file_name='jkej', parent=v1, user='root')
v23 = state.set_acl(default=False, entry=v1, group='group1', group_perm={v1, 'r', 'w', 'x'}, logical=True, mask={v1, 'r', 'w', 'x'}, not_recalc_mask=True, other_perm={v1, 'r', 'w', 'x'}, physical=True, recalc_mask=True, recursive=True, set_mask=False, sudo_user='root', user='root', user_perm=set())
v24 = state.create_file(content=b'\xe9', file_name='mdaa', mode='w', parent=v1, user='root')
v25 = state.set_acl(default=False, entry=v15, group='group3', group_perm={v1, 'w', 'x'}, logical=False, mask={'r', 'w', 'x'}, not_recalc_mask=True, other_perm={v1, 'r', 'w', 'x'}, physical=True, recalc_mask=False, recursive=True, set_mask=False, sudo_user='root', user='root', user_perm={v1, 'r', 'w', 'x'})
v26 = state.set_acl(default=False, entry=v1, group='group2', group_perm={v1, 'r', 'w'}, logical=True, mask={v1, 'r', 'w', 'x'}, not_recalc_mask=True, other_perm=set(), physical=True, recalc_mask=True, recursive=False, set_mask=False, sudo_user='root', user='root', user_perm={v1, 'w', 'x'})
v27 = state.create_file(content=b'\xfe\x97', file_name='ctin', mode='x', parent=v1, user='root')
v28 = state.set_acl(default=False, entry=v20, group='group2', group_perm={v1, 'r', 'x'}, logical=True, mask=set(), not_recalc_mask=True, other_perm={'w'}, physical=True, recalc_mask=False, recursive=False, set_mask=False, sudo_user='root', user=v1, user_perm={'r', 'x'})
v29 = state.set_acl(default=False, entry=v1, group='group2', group_perm={v1, 'r', 'x'}, logical=False, mask={'r'}, not_recalc_mask=False, other_perm=set(), physical=True, recalc_mask=False, recursive=True, set_mask=False, sudo_user='root', user=v1, user_perm={v1, 'w'})
v30 = state.create_file(content=b'q\xa4|\xd4\x0cR', file_name='aocx', mode='w', parent=v1, user='root')
v31 = state.set_acl(default=False, entry=v27, group='group3', group_perm={'r'}, logical=True, mask={'r'}, not_recalc_mask=True, other_perm={v1, 'r', 'w', 'x'}, physical=False, recalc_mask=False, recursive=True, set_mask=False, sudo_user='root', user='root', user_perm=set())
v32 = state.create_file(content=b'K', file_name='tahn', mode='w', parent=v1, user='root')
v33 = state.set_acl(default=False, entry=v32, group=v1, group_perm=set(), logical=True, mask=set(), not_recalc_mask=True, other_perm={v1, 'r', 'x'}, physical=True, recalc_mask=True, recursive=True, set_mask=False, sudo_user='root', user='root', user_perm=set())
v34 = state.create_file(content=b'm!\x9b\xc7', file_name='tdvh', mode='x', parent=v1, user='root')
v35 = state.create_file(content=b'\xf8\xb8\xfc\x00\xcd\xc9', file_name='ahwm', mode='w', parent=v1, user='root')
v36 = state.create_file(content=b'\xff', file_name='rawe', mode='a', parent=v1, user='root')
v37 = state.create_file(content=b'g\xe6\x83R', file_name='nflb', mode='x', parent=v1, user='root')
v38 = state.set_acl(default=False, entry=v1, group='group1', group_perm={v1}, logical=False, mask=set(), not_recalc_mask=False, other_perm={v1, 'w'}, physical=False, recalc_mask=False, recursive=True, set_mask=False, sudo_user='root', user=v1, user_perm={v1, 'r', 'w', 'x'})
v39 = state.symlink(dest_file=v3, link_file_name='ctpf', parent=v1, user='root')
v40 = state.create_file(content=b't~', file_name='sdss', mode='x', parent=v1, user='root')
v41 = state.set_acl(default=False, entry=v22, group='group1', group_perm=set(), logical=False, mask=set(), not_recalc_mask=True, other_perm={v1, 'r', 'w', 'x'}, physical=True, recalc_mask=False, recursive=False, set_mask=False, sudo_user='root', user='root', user_perm={v1})
v42 = state.set_acl(default=False, entry=v1, group='group3', group_perm=set(), logical=True, mask={v1, 'r', 'w', 'x'}, not_recalc_mask=False, other_perm={v1, 'x'}, physical=False, recalc_mask=False, recursive=False, set_mask=False, sudo_user='root', user='root', user_perm={'r'})
v43 = state.symlink(dest_file=v24, link_file_name='xhyi', parent=v1, user='root')
v44 = state.symlink(dest_file=v12, link_file_name='rffh', parent=v1, user='root')
v45 = state.symlink(dest_file=v35, link_file_name='vojz', parent=v1, user='root')
v46 = state.symlink(dest_file=v30, link_file_name='tqry', parent=v1, user='root')
v47 = state.symlink(dest_file=v4, link_file_name='nuub', parent=v1, user='root')
v48 = state.create_file(content=b'\x05\x7f', file_name='dsod', mode='w', parent=v1, user='root')
v49 = state.symlink(dest_file=v47, link_file_name='ikmu', parent=v1, user='root')
v50 = state.set_acl(default=False, entry=v49, group='root', group_perm=set(), logical=True, mask={'r'}, not_recalc_mask=False, other_perm={'r'}, physical=True, recalc_mask=True, recursive=True, set_mask=False, sudo_user='root', user=v1, user_perm={'w'})
v51 = state.create_file(content=b'\x00\x01\x00\x03\x00\x01\x01\x03', file_name='acba', mode='x', parent=v1, user='root')
v52 = state.set_acl(default=False, entry=v47, group='group3', group_perm={'w'}, logical=False, mask=set(), not_recalc_mask=False, other_perm={v1, 'r', 'w'}, physical=False, recalc_mask=True, recursive=True, set_mask=False, sudo_user='root', user='root', user_perm=set())
v53 = state.symlink(dest_file=v4, link_file_name='cbai', parent=v1, user='root')
v54 = state.set_acl(default=False, entry=v51, group=v1, group_perm=set(), logical=True, mask={'r'}, not_recalc_mask=False, other_perm={'w', 'x'}, physical=False, recalc_mask=False, recursive=False, set_mask=False, sudo_user='root', user='root', user_perm=set())
v55 = state.create_file(content=b'', file_name='abab', mode='a', parent=v1, user='root')
v56 = state.set_acl(default=False, entry=v1, group='group3', group_perm=set(), logical=True, mask={'w'}, not_recalc_mask=True, other_perm={v1, 'r'}, physical=False, recalc_mask=False, recursive=True, set_mask=False, sudo_user='root', user=v1, user_perm=set())
state.symlink(dest_file=v49, link_file_name='cawa', parent=v1, user='root')
state.teardown()

[Zac-HD]

We have general advice at https://github.com/HypothesisWorks/hypothesis/blob/master/guides/strategies-that-shrink.rst, and the same principles apply for stateful testing.

There's no way to control shrinking separately from how you implement your strategies and test code; the main tip is to ensure that (1) a small example is possible, and (2) making local changes will produce similar valid/failing inputs.

For any further questions, I'd recommend Stack Overflow.