Closed andresriancho closed 9 years ago
from difflib import SequenceMatcher
from tests.vulndb_test import VulnDBTest
class TestSimilarTexts(VulnDBTest):
MAX_RATE = 0.8
def get_rate(self, a, b):
return SequenceMatcher(None, a, b).ratio()
def test_similar_texts(self):
invalid = []
for _file_1, db_data_1 in self.get_all_json():
for _file_2, db_data_2 in self.get_all_json():
if _file_1 == _file_2:
continue
description_1 = self.to_string(db_data_1['description'])
description_2 = self.to_string(db_data_2['description'])
if self.get_rate(description_1, description_2) > self.MAX_RATE:
invalid.append((_file_1, _file_2, 'description'))
fix_1 = self.to_string(db_data_1['fix']['guidance'])
fix_2 = self.to_string(db_data_2['fix']['guidance'])
if self.get_rate(fix_1, fix_2) > self.MAX_RATE:
invalid.append((_file_1, _file_2, 'fix_guidance'))
self.assertEqual(invalid, [])
Well... that outputs A LOT of similarities...
Create a unittest that will find descriptions that are duplicated/very similar between two files. I'm worried about some of the data we imported from arachni, namely all the
xss_*
we have at https://github.com/vulndb/data/tree/master/db . If they are duplicated we should remove them, and the unittest will also help is avoid similar issues in the future.