search

mozilla / experimenter

A web application for managing user experiments for Mozilla Firefox.
https://experimenter.services.mozilla.com
Mozilla Public License 2.0
114 stars 179 forks source link

Intermittent test failure: TestGenerateNimbusChangeLog.test_generate_nimbus_changelog_with_prior_change #6119

Closed jaredlockhart closed 3 years ago

jaredlockhart commented 3 years ago

Seen on circle: https://app.circleci.com/pipelines/github/mozilla/experimenter/10622/workflows/64d4471e-de9c-4aca-9076-ceaf8fe0011c/jobs/31575

=================================== FAILURES ===================================
_ TestGenerateNimbusChangeLog.test_generate_nimbus_changelog_with_prior_change _
[gw1] linux -- Python 3.9.6 /usr/local/bin/python3

self = <experimenter.experiments.tests.test_changelog_utils.test_changelog_utils_nimbus.TestGenerateNimbusChangeLog testMethod=test_generate_nimbus_changelog_with_prior_change>

    def test_generate_nimbus_changelog_with_prior_change(self):
        experiment = NimbusExperimentFactory.create_with_lifecycle(
            NimbusExperimentFactory.Lifecycles.CREATED
        )

        self.assertEqual(experiment.changes.count(), 1)

        experiment.status = NimbusExperiment.Status.DRAFT
        experiment.status_next = NimbusExperiment.Status.LIVE
        experiment.publish_status = NimbusExperiment.PublishStatus.REVIEW
        experiment.save()

        generate_nimbus_changelog(experiment, self.user, "test message")

        self.assertEqual(experiment.changes.count(), 2)

        change = experiment.changes.latest_change()

        self.assertEqual(change.experiment, experiment)
        self.assertEqual(change.message, "test message")
        self.assertEqual(change.changed_by, self.user)
        self.assertEqual(change.old_status, NimbusExperiment.Status.DRAFT)
        self.assertEqual(change.old_status_next, None)
        self.assertEqual(change.old_publish_status, NimbusExperiment.PublishStatus.IDLE)
        self.assertEqual(change.new_status, NimbusExperiment.Status.DRAFT)
        self.assertEqual(change.new_status_next, NimbusExperiment.Status.LIVE)
        self.assertEqual(change.new_publish_status, NimbusExperiment.PublishStatus.REVIEW)
>       self.assertEqual(
            change.experiment_data,
            dict(NimbusExperimentChangeLogSerializer(experiment).data),
        )
E       AssertionError: {'name': 'Programmable value-added workforc[3792 chars]4000} != {'reference_branch': OrderedDict([('name', [3892 chars]: []}
E         {'application': 'firefox-desktop',
E       +  'branches': [OrderedDict([('name', 'Profit-focused intangible policy'),
E       +                            ('slug', 'profit-focused-intangible-policy'),
E       +                            ('description',
E       -  'branches': [{'description': 'Gas shake section soldier. Well table draw '
E       -                               'perform far control standard.\n'
E       -                               'Take by center among eight throughout baby '
E       -                               'plan. Admit sell dinner summer activity yard '
E       -                               'owner father.',
E       -                'feature_enabled': True,
E       -                'feature_value': '{"peace-organization": "soon-produce"}',
E       -                'name': 'Open-architected high-level synergy',
E       -                'ratio': 1,
E       -                'slug': 'open-architected-high-level-synergy'},
E       -               {'description': 'Strategy fish skin alone learn ball. Many '
E       ?               ^^^^^^^^^^^^^^^
E       
E       +                             'Strategy fish skin alone learn ball. Many '
E       ?               ^^^^^^^^^^^^^
E       
E       -                               'especially many goal color. Family it live '
E       ? --
E       
E       +                             'especially many goal color. Family it live '
E       -                               'challenge society pretty whose.',
E       ? --
E       
E       +                             'challenge society pretty whose.'),
E       ?                                                              +
E       
E       +                            ('ratio', 1),
E       -                'feature_enabled': True,
E       ?                                 ^
E       
E       +                            ('feature_enabled', True),
E       ?                +++++++++++++                 ^     +
E       
E       +                            ('feature_value',
E       -                'feature_value': '{"must-entire": "allow-prepare-wind"}',
E       ?                ^^^^^^^^^^^^^^^^
E       
E       +                             '{"must-entire": "allow-prepare-wind"}')]),
E       ?                ^^^^^^^^^^^^                                        +++
E       
E       -                'name': 'Profit-focused intangible policy',
E       +               OrderedDict([('name', 'Open-architected high-level synergy'),
E       +                            ('slug', 'open-architected-high-level-synergy'),
E       +                            ('description',
E       +                             'Gas shake section soldier. Well table draw '
E       +                             'perform far control standard.\n'
E       +                             'Take by center among eight throughout baby plan. '
E       +                             'Admit sell dinner summer activity yard owner '
E       +                             'father.'),
E       -                'ratio': 1,
E       ?                       ^
E       
E       +                            ('ratio', 1),
E       ?                +++++++++++++       ^  +
E       
E       -                'slug': 'profit-focused-intangible-policy'}],
E       +                            ('feature_enabled', True),
E       +                            ('feature_value',
E       +                             '{"peace-organization": "soon-produce"}')])],
E          'channel': '',
E          'countries': [],
E       -  'feature_config': {'application': 'focus-android',
E       +  'feature_config': OrderedDict([('name', 'Optimized didactic groupware'),
E       +                                 ('slug', 'optimized-didactic-groupware'),
E       +                                 ('description',
E       -                     'description': 'Support section American. It must risk '
E       ?                     ^^^^^^^^^^^^^^
E       
E       +                                  'Support section American. It must risk feel '
E       ?                     ^^^^^^^^^^^^                                         +++++
E       
E       -                                    'feel indeed.\n'
E       ? --                                  -----
E       
E       +                                  'indeed.\n'
E       -                                    'Use quality forward move in. Into food '
E       ? --
E       
E       +                                  'Use quality forward move in. Into food '
E       -                                    'suggest recent establish still soon.',
E       ? --
E       
E       +                                  'suggest recent establish still soon.'),
E       ?                                                                        +
E       
E       -                     'name': 'Optimized didactic groupware',
E       +                                 ('application', 'focus-android'),
E       -                     'owner_email': 'katherinejordan@kennedy.com',
E       ?                                  ^
E       
E       +                                 ('owner_email', 'katherinejordan@kennedy.com'),
E       ?                     +++++++++++++             ^                              +
E       
E       -                     'schema': '{\n'
E       +                                 ('schema',
E       +                                  '{\n'
E       -                               '    "$schema": '
E       +                                  '    "$schema": '
E       ? +++
E       
E       -                               '"http://json-schema.org/draft-07/schema#",\n'
E       +                                  '"http://json-schema.org/draft-07/schema#",\n'
E       ? +++
E       
E       -                               '    "description": "Fake schema that matches '
E       +                                  '    "description": "Fake schema that matches '
E       ? +++
E       
E       -                               'NimbusBranchFactory feature_value factory",\n'
E       +                                  'NimbusBranchFactory feature_value factory",\n'
E       ? +++
E       
E       -                               '    "type": "object",\n'
E       +                                  '    "type": "object",\n'
E       ? +++
E       
E       -                               '    "patternProperties": {\n'
E       +                                  '    "patternProperties": {\n'
E       ? +++
E       
E       -                               '        "^.*$": { "type": "string" }\n'
E       +                                  '        "^.*$": { "type": "string" }\n'
E       ? +++
E       
E       -                               '    },\n'
E       +                                  '    },\n'
E       ? +++
E       
E       -                               '    "additionalProperties": false\n'
E       +                                  '    "additionalProperties": false\n'
E       ? +++
E       
E       -                               '}\n',
E       +                                  '}\n')]),
E       ? +++                                   +++
E       
E       -                     'slug': 'optimized-didactic-groupware'},
E          'firefox_min_version': '89.!',
E          'hypothesis': 'Hope kid method senior film nothing. Beat nor tough many '
E                        'medical ok successful. Field hand house I person.\n'
E                        'Probably old whom ask north at lawyer. Resource pretty card.\n'
E                        'Tree take only foot worker model. Congress must space alone '
E                        'week agency. Information foot just stuff growth.\n'
E                        'Force coach often. Former because list often down.\n'
E                        'Success build product economy risk. No college right card '
E                        'later voice act.\n'
E                        'Base attorney industry pressure free middle. In current '
E                        'sometimes also civil miss pattern power. Response occur '
E                        'finally realize.\n'
E                        'Then herself plan million ten brother. Message political '
E                        'instead when. Firm high art within.\n'
E                        'Talk thousand charge of only. Daughter happy score deep '
E                        'respond.\n'
E                        'Try person trip information. Democrat whole threat network.\n'
E                        'It career since then. Their finish study staff lose high '
E                        'executive.\n'
E                        'Keep front western continue debate. Together else whole last '
E                        'never. System billion local might include either public.',
E          'is_paused': False,
E          'locales': [],
E          'name': 'Programmable value-added workforce',
E          'owner': 'keith34@lopez-salinas.org',
E          'population_percent': '50.0000',
E          'primary_outcomes': ['fenix_outcome', 'desktop_outcome_2'],
E          'projects': ['distributed-systematic-artificial-intelligence_',
E                       'function-based-interactive-throughput_',
E                       'object-based-bifurcated-archive_'],
E          'proposed_duration': 57,
E          'proposed_enrollment': 7,
E          'public_description': 'Effort maintain yet letter. Study body unit '
E                                'relationship sometimes.\n'
E                                'Modern describe design shoulder purpose challenge. '
E                                'Attorney quickly medical evidence reveal discover '
E                                'beyond.',
E          'publish_status': 'Review',
E          'published_dto': None,
E       -  'reference_branch': {'description': 'Gas shake section soldier. Well table '
E       +  'reference_branch': OrderedDict([('name',
E       +                                    'Open-architected high-level synergy'),
E       +                                   ('slug',
E       +                                    'open-architected-high-level-synergy'),
E       +                                   ('description',
E       +                                    'Gas shake section soldier. Well table draw '
E       -                                      'draw perform far control standard.\n'
E       ? --                                    -----
E       
E       +                                    'perform far control standard.\n'
E       -                                      'Take by center among eight throughout '
E       ? --
E       
E       +                                    'Take by center among eight throughout baby '
E       ?                                                                           +++++
E       
E       -                                      'baby plan. Admit sell dinner summer '
E       ? --                                    -----
E       
E       +                                    'plan. Admit sell dinner summer activity '
E       ?                                                                    +++++++++
E       
E       -                                      'activity yard owner father.',
E       ? --                                    ---------
E       
E       +                                    'yard owner father.'),
E       ?                                                        +
E       
E       +                                   ('ratio', 1),
E       -                       'feature_enabled': True,
E       ?                                        ^
E       
E       +                                   ('feature_enabled', True),
E       ?                       +++++++++++++                 ^     +
E       
E       +                                   ('feature_value',
E       -                       'feature_value': '{"peace-organization": "soon-produce"}',
E       ?                       ^^^^^^^^^^^^^^^^
E       
E       +                                    '{"peace-organization": "soon-produce"}')]),
E       ?                       ^^^^^^^^^^^^                                         +++
E       
E       -                       'name': 'Open-architected high-level synergy',
E       -                       'ratio': 1,
E       -                       'slug': 'open-architected-high-level-synergy'},
E          'results_data': None,
E          'risk_brand': False,
E          'risk_mitigation_link': 'https://www.reid.com/search/',
E          'risk_partner_related': False,
E          'risk_revenue': True,
E          'secondary_outcomes': ['desktop_outcome_3', 'desktop_outcome_1'],
E          'slug': 'programmable-value-added-workforce',
E          'status': 'Draft',
E          'status_next': 'Live',
E          'targeting_config_slug': '',
E          'total_enrolled_clients': 24000}

experimenter/experiments/tests/test_changelog_utils/test_changelog_utils_nimbus.py:226: AssertionError

┆Issue is synchronized with this Jira Task

jaredlockhart commented 3 years ago

Okay well I just ran this a few thousand times and didn't see it again. The other intermittent I worked on was easily reproducible within a 1000x run, so maybe this was a cosmic ray fluke. I'll close this and if we see it again we can revisit it.