rhyolight
commented
10 years ago I saw this test fail on July 25th as well: https://github.com/numenta/nupic/wiki/Intermittently-Failing-Tests
Open subutai opened 10 years ago
rhyolight
commented
10 years ago I saw this test fail on July 25th as well: https://github.com/numenta/nupic/wiki/Intermittently-Failing-Tests
rhyolight
commented
10 years ago Just saw this again in one of my builds: https://travis-ci.org/rhyolight/nupic/jobs/31440561
baroobob
commented
10 years ago When I was tracking down numenta/nupic.core#159, I noticed that the permanences of the Python SP and the C++ SP would differ in the 5th decimal place.
cogmission
commented
10 years ago I've noticed this to be a function of the Random Number Generator. The algorithm used is the Mersenne Twister on both Python and Numpy - but somehow differs in the 5th or 6th decimal place. I mentioned this exact occurrence to Subutai in a related issue regarding uniformity across Languages - this was in regards to writing a "Litmus Test" to validate externally written ports which want qualify/adhere to standards.
On Fri, Sep 5, 2014 at 4:41 PM, James Bridgewater notifications@github.com wrote:
When I was tracking down numenta/nupic.core#159 https://github.com/numenta/nupic.core/issues/159, I noticed that the permanences of the Python SP and the C++ SP would differ in the 5th decimal place.
— Reply to this email directly or view it on GitHub https://github.com/numenta/nupic/issues/1155#issuecomment-54685244.
subutai
commented
10 years ago Occurred again in another pipeline:
tests/unit/py2/nupic/research/spatial_pooler_compatability_test.py:293: SpatialPoolerCompatabilityTest.testCompatability2 FAILED
=================================== FAILURES ===================================
______________ SpatialPoolerCompatabilityTest.testCompatability2 _______________
self = <tests.unit.py2.nupic.research.spatial_pooler_compatability_test.SpatialPoolerCompatabilityTest testMethod=testCompatability2>
def testCompatability2(self):
params = {
"inputDimensions": [12,7],
"columnDimensions": [4,15],
"potentialRadius": 22,
"potentialPct": 0.3,
"globalInhibition": False,
"localAreaDensity": 0,
"numActiveColumnsPerInhArea": 5,
"stimulusThreshold": 2,
"synPermInactiveDec": 0.04,
"synPermActiveInc": 0.14,
"synPermConnected": 0.178,
"minPctOverlapDutyCycle": 0.021,
"minPctActiveDutyCycle": 0.0012,
"dutyCyclePeriod": 20,
"maxBoost": 11.0,
"seed": 6,
"spVerbosity": 0
}
> self.runSideBySide(params, convertEveryIteration = True)
tests/unit/py2/nupic/research/spatial_pooler_compatability_test.py:313:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <tests.unit.py2.nupic.research.spatial_pooler_compatability_test.SpatialPoolerCompatabilityTest testMethod=testCompatability2>
params = {'columnDimensions': [4, 15], 'dutyCyclePeriod': 20, 'globalInhibition': False, 'inputDimensions': [12, 7], ...}
seed = None, learnMode = None, convertEveryIteration = True
def runSideBySide(self, params, seed = None,
learnMode = None,
convertEveryIteration = False):
"""
Run the PY and CPP implementations side by side on random inputs.
If seed is None a random seed will be chosen based on time, otherwise
the fixed seed will be used.
If learnMode is None learning will be randomly turned on and off.
If it is False or True then set it accordingly.
If convertEveryIteration is True, the CPP will be copied from the PY
instance on every iteration just before each compute.
"""
randomState = getNumpyRandomGenerator(seed)
cppSp = CreateSP("cpp", params)
pySp = CreateSP("py", params)
self.compare(pySp, cppSp)
numColumns = pySp.getNumColumns()
numInputs = pySp.getNumInputs()
threshold = 0.8
inputMatrix = (
randomState.rand(numRecords,numInputs) > threshold).astype(uintType)
# Run side by side for numRecords iterations
for i in xrange(numRecords):
if learnMode is None:
learn = (randomState.rand() > 0.5)
else:
learn = learnMode
if self.verbosity > 1:
print "Iteration:",i,"learn=",learn
PyActiveArray = numpy.zeros(numColumns).astype(uintType)
CppActiveArray = numpy.zeros(numColumns).astype(uintType)
inputVector = inputMatrix[i,:]
pySp.compute(inputVector, learn, PyActiveArray)
cppSp.compute(inputVector, learn, CppActiveArray)
self.assertListEqual(list(PyActiveArray), list(CppActiveArray))
> self.compare(pySp,cppSp)
tests/unit/py2/nupic/research/spatial_pooler_compatability_test.py:203:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <tests.unit.py2.nupic.research.spatial_pooler_compatability_test.SpatialPoolerCompatabilityTest testMethod=testCompatability2>
pySp = <nupic.research.spatial_pooler.SpatialPooler object at 0x479d790>
cppSp = <nupic.bindings.algorithms.SpatialPooler; proxy of <Swig Object of type 'nta::algorithms::spatial_pooler::SpatialPooler *' at 0x49603e0> >
def compare(self, pySp, cppSp):
self.assertAlmostEqual(pySp.getNumColumns(),
cppSp.getNumColumns())
self.assertAlmostEqual(pySp.getNumInputs(),
cppSp.getNumInputs())
self.assertAlmostEqual(pySp.getPotentialRadius(),
cppSp.getPotentialRadius())
self.assertAlmostEqual(pySp.getPotentialPct(),
cppSp.getPotentialPct())
self.assertAlmostEqual(pySp.getGlobalInhibition(),
cppSp.getGlobalInhibition())
self.assertAlmostEqual(pySp.getNumActiveColumnsPerInhArea(),
cppSp.getNumActiveColumnsPerInhArea())
self.assertAlmostEqual(pySp.getLocalAreaDensity(),
cppSp.getLocalAreaDensity())
self.assertAlmostEqual(pySp.getStimulusThreshold(),
cppSp.getStimulusThreshold())
self.assertAlmostEqual(pySp.getInhibitionRadius(),
cppSp.getInhibitionRadius())
self.assertAlmostEqual(pySp.getDutyCyclePeriod(),
cppSp.getDutyCyclePeriod())
self.assertAlmostEqual(pySp.getMaxBoost(),
cppSp.getMaxBoost())
self.assertAlmostEqual(pySp.getIterationNum(),
cppSp.getIterationNum())
self.assertAlmostEqual(pySp.getIterationLearnNum(),
cppSp.getIterationLearnNum())
self.assertAlmostEqual(pySp.getSpVerbosity(),
cppSp.getSpVerbosity())
self.assertAlmostEqual(pySp.getUpdatePeriod(),
cppSp.getUpdatePeriod())
self.assertAlmostEqual(pySp.getSynPermTrimThreshold(),
cppSp.getSynPermTrimThreshold())
self.assertAlmostEqual(pySp.getSynPermActiveInc(),
cppSp.getSynPermActiveInc())
self.assertAlmostEqual(pySp.getSynPermInactiveDec(),
cppSp.getSynPermInactiveDec())
self.assertAlmostEqual(pySp.getSynPermBelowStimulusInc(),
cppSp.getSynPermBelowStimulusInc())
self.assertAlmostEqual(pySp.getSynPermConnected(),
cppSp.getSynPermConnected())
self.assertAlmostEqual(pySp.getMinPctOverlapDutyCycles(),
cppSp.getMinPctOverlapDutyCycles())
self.assertAlmostEqual(pySp.getMinPctActiveDutyCycles(),
cppSp.getMinPctActiveDutyCycles())
numColumns = pySp.getNumColumns()
numInputs = pySp.getNumInputs()
pyBoost = numpy.zeros(numColumns).astype(realType)
cppBoost = numpy.zeros(numColumns).astype(realType)
pySp.getBoostFactors(pyBoost)
cppSp.getBoostFactors(cppBoost)
self.assertListAlmostEqual(list(pyBoost), list(cppBoost))
pyOverlap = numpy.zeros(numColumns).astype(realType)
cppOverlap = numpy.zeros(numColumns).astype(realType)
pySp.getOverlapDutyCycles(pyOverlap)
cppSp.getOverlapDutyCycles(cppOverlap)
self.assertListAlmostEqual(list(pyOverlap), list(cppOverlap))
pyActive = numpy.zeros(numColumns).astype(realType)
cppActive = numpy.zeros(numColumns).astype(realType)
pySp.getActiveDutyCycles(pyActive)
cppSp.getActiveDutyCycles(cppActive)
self.assertListAlmostEqual(list(pyActive), list(cppActive))
pyMinOverlap = numpy.zeros(numColumns).astype(realType)
cppMinOverlap = numpy.zeros(numColumns).astype(realType)
pySp.getMinOverlapDutyCycles(pyMinOverlap)
cppSp.getMinOverlapDutyCycles(cppMinOverlap)
self.assertListAlmostEqual(list(pyMinOverlap), list(cppMinOverlap))
pyMinActive = numpy.zeros(numColumns).astype(realType)
cppMinActive = numpy.zeros(numColumns).astype(realType)
pySp.getMinActiveDutyCycles(pyMinActive)
cppSp.getMinActiveDutyCycles(cppMinActive)
self.assertListAlmostEqual(list(pyMinActive), list(cppMinActive))
for i in xrange(pySp.getNumColumns()):
if self.verbosity > 2: print "Column:",i
pyPot = numpy.zeros(numInputs).astype(uintType)
cppPot = numpy.zeros(numInputs).astype(uintType)
pySp.getPotential(i, pyPot)
cppSp.getPotential(i, cppPot)
self.assertListEqual(list(pyPot),list(cppPot))
pyPerm = numpy.zeros(numInputs).astype(realType)
cppPerm = numpy.zeros(numInputs).astype(realType)
pySp.getPermanence(i, pyPerm)
cppSp.getPermanence(i, cppPerm)
> self.assertListAlmostEqual(list(pyPerm),list(cppPerm))
tests/unit/py2/nupic/research/spatial_pooler_compatability_test.py:149:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <tests.unit.py2.nupic.research.spatial_pooler_compatability_test.SpatialPoolerCompatabilityTest testMethod=testCompatability2>
alist = [0.0, 0.0, 0.0, 0.16262999, 0.3285, 0.0, ...]
blist = [0.0, 0.0, 0.0, 0.16262999, 0.3285, 0.0, ...]
def assertListAlmostEqual(self, alist, blist):
self.assertEqual(len(alist), len(blist))
for a, b in zip(alist, blist):
diff = abs(a - b)
> self.assertLess(diff, 1e-5)
E AssertionError: 1.001358e-05 not less than 1e-05
tests/unit/py2/nupic/research/spatial_pooler_compatability_test.py:55: AssertionError
vitaly-krugl
commented
10 years ago See https://github.com/numenta/nupic.core/pull/197. @scottpurdy said that the use of https://github.com/numenta/nupic.core/pull/197 might mitigate this issue.
scottpurdy
commented
10 years ago @vitaly-krugl - you sort of put the words in my mouth there - my actual response was "No" that it won't solve this issue and then I gave the caveat that there is a chance you could resolve this issue by using the new code from numenta/nupic.core#197
rhyolight
commented
10 years ago I keep track of intermittently failing tests here: https://github.com/numenta/nupic/wiki/Intermittently-Failing-Tests. I wonder if https://github.com/numenta/nupic.core/pull/197 will help any other of these tests?
Matt Taylor OS Community Flag-Bearer Numenta
On Tue, Sep 30, 2014 at 2:52 PM, Scott Purdy notifications@github.com wrote:
@vitaly-krugl https://github.com/vitaly-krugl - you sort of put the words in my mouth there - my actual response was "No" that it won't solve this issue and then I gave the caveat that there is a chance you could resolve this issue by using the new code from numenta/nupic.core#197 https://github.com/numenta/nupic.core/pull/197
— Reply to this email directly or view it on GitHub https://github.com/numenta/nupic/issues/1155#issuecomment-57389077.
vitaly-krugl
commented
10 years ago @vitaly-krugl - you sort of put the words in my mouth there - my actual response was "No" that it won't solve this issue and then I gave the caveat that there is a chance you could resolve this issue by using the new code from numenta/nupic.core#197
@scottpurdy: sorry, didn't mean to put the words in your mouth. I simply wanted to communicate that there is a possibility that the code in the Random class "might mitigate this issue"
cogmission
commented
10 years ago I think the min error should be set to whatever increment/decrement amount is being used in the relevant test, otherwise all you're testing is the RNG which is not really part of the test. Otherwise we are chasing our own tails.
On Tue, Sep 30, 2014 at 4:28 PM, Subutai Ahmad notifications@github.com wrote:
Occurred again in another pipeline:
tests/unit/py2/nupic/research/spatial_pooler_compatability_test.py:293: SpatialPoolerCompatabilityTest.testCompatability2 FAILED
=================================== FAILURES =================================== __ SpatialPoolerCompatabilityTest.testCompatability2 ___ self =
def testCompatability2(self): params = { "inputDimensions": [12,7], "columnDimensions": [4,15], "potentialRadius": 22, "potentialPct": 0.3, "globalInhibition": False, "localAreaDensity": 0, "numActiveColumnsPerInhArea": 5, "stimulusThreshold": 2, "synPermInactiveDec": 0.04, "synPermActiveInc": 0.14, "synPermConnected": 0.178, "minPctOverlapDutyCycle": 0.021, "minPctActiveDutyCycle": 0.0012, "dutyCyclePeriod": 20, "maxBoost": 11.0, "seed": 6, "spVerbosity": 0 }self.runSideBySide(params, convertEveryIteration = True)tests/unit/py2/nupic/research/spatial_pooler_compatability_test.py:313:
self =
params = {'columnDimensions': [4, 15], 'dutyCyclePeriod': 20, 'globalInhibition': False, 'inputDimensions': [12, 7], ...} seed = None, learnMode = None, convertEveryIteration = True def runSideBySide(self, params, seed = None, learnMode = None, convertEveryIteration = False): """ Run the PY and CPP implementations side by side on random inputs. If seed is None a random seed will be chosen based on time, otherwise the fixed seed will be used. If learnMode is None learning will be randomly turned on and off. If it is False or True then set it accordingly. If convertEveryIteration is True, the CPP will be copied from the PY instance on every iteration just before each compute. """ randomState = getNumpyRandomGenerator(seed) cppSp = CreateSP("cpp", params) pySp = CreateSP("py", params) self.compare(pySp, cppSp) numColumns = pySp.getNumColumns() numInputs = pySp.getNumInputs() threshold = 0.8 inputMatrix = ( randomState.rand(numRecords,numInputs) > threshold).astype(uintType) # Run side by side for numRecords iterations for i in xrange(numRecords): if learnMode is None: learn = (randomState.rand() > 0.5) else: learn = learnMode if self.verbosity > 1: print "Iteration:",i,"learn=",learn PyActiveArray = numpy.zeros(numColumns).astype(uintType) CppActiveArray = numpy.zeros(numColumns).astype(uintType) inputVector = inputMatrix[i,:] pySp.compute(inputVector, learn, PyActiveArray) cppSp.compute(inputVector, learn, CppActiveArray) self.assertListEqual(list(PyActiveArray), list(CppActiveArray))self.compare(pySp,cppSp)tests/unit/py2/nupic/research/spatial_pooler_compatability_test.py:203:
self =
pySp = <nupic.research.spatial_pooler.SpatialPooler object at 0x479d790> cppSp = <nupic.bindings.algorithms.SpatialPooler; proxy of <Swig Object of type 'nta::algorithms::spatial_pooler::SpatialPooler *' at 0x49603e0> > def compare(self, pySp, cppSp): self.assertAlmostEqual(pySp.getNumColumns(), cppSp.getNumColumns()) self.assertAlmostEqual(pySp.getNumInputs(), cppSp.getNumInputs()) self.assertAlmostEqual(pySp.getPotentialRadius(), cppSp.getPotentialRadius()) self.assertAlmostEqual(pySp.getPotentialPct(), cppSp.getPotentialPct()) self.assertAlmostEqual(pySp.getGlobalInhibition(), cppSp.getGlobalInhibition()) self.assertAlmostEqual(pySp.getNumActiveColumnsPerInhArea(), cppSp.getNumActiveColumnsPerInhArea()) self.assertAlmostEqual(pySp.getLocalAreaDensity(), cppSp.getLocalAreaDensity()) self.assertAlmostEqual(pySp.getStimulusThreshold(), cppSp.getStimulusThreshold()) self.assertAlmostEqual(pySp.getInhibitionRadius(), cppSp.getInhibitionRadius()) self.assertAlmostEqual(pySp.getDutyCyclePeriod(), cppSp.getDutyCyclePeriod()) self.assertAlmostEqual(pySp.getMaxBoost(), cppSp.getMaxBoost()) self.assertAlmostEqual(pySp.getIterationNum(), cppSp.getIterationNum()) self.assertAlmostEqual(pySp.getIterationLearnNum(), cppSp.getIterationLearnNum()) self.assertAlmostEqual(pySp.getSpVerbosity(), cppSp.getSpVerbosity()) self.assertAlmostEqual(pySp.getUpdatePeriod(), cppSp.getUpdatePeriod()) self.assertAlmostEqual(pySp.getSynPermTrimThreshold(), cppSp.getSynPermTrimThreshold()) self.assertAlmostEqual(pySp.getSynPermActiveInc(), cppSp.getSynPermActiveInc()) self.assertAlmostEqual(pySp.getSynPermInactiveDec(), cppSp.getSynPermInactiveDec()) self.assertAlmostEqual(pySp.getSynPermBelowStimulusInc(), cppSp.getSynPermBelowStimulusInc()) self.assertAlmostEqual(pySp.getSynPermConnected(), cppSp.getSynPermConnected()) self.assertAlmostEqual(pySp.getMinPctOverlapDutyCycles(), cppSp.getMinPctOverlapDutyCycles()) self.assertAlmostEqual(pySp.getMinPctActiveDutyCycles(), cppSp.getMinPctActiveDutyCycles()) numColumns = pySp.getNumColumns() numInputs = pySp.getNumInputs() pyBoost = numpy.zeros(numColumns).astype(realType) cppBoost = numpy.zeros(numColumns).astype(realType) pySp.getBoostFactors(pyBoost) cppSp.getBoostFactors(cppBoost) self.assertListAlmostEqual(list(pyBoost), list(cppBoost)) pyOverlap = numpy.zeros(numColumns).astype(realType) cppOverlap = numpy.zeros(numColumns).astype(realType) pySp.getOverlapDutyCycles(pyOverlap) cppSp.getOverlapDutyCycles(cppOverlap) self.assertListAlmostEqual(list(pyOverlap), list(cppOverlap)) pyActive = numpy.zeros(numColumns).astype(realType) cppActive = numpy.zeros(numColumns).astype(realType) pySp.getActiveDutyCycles(pyActive) cppSp.getActiveDutyCycles(cppActive) self.assertListAlmostEqual(list(pyActive), list(cppActive)) pyMinOverlap = numpy.zeros(numColumns).astype(realType) cppMinOverlap = numpy.zeros(numColumns).astype(realType) pySp.getMinOverlapDutyCycles(pyMinOverlap) cppSp.getMinOverlapDutyCycles(cppMinOverlap) self.assertListAlmostEqual(list(pyMinOverlap), list(cppMinOverlap)) pyMinActive = numpy.zeros(numColumns).astype(realType) cppMinActive = numpy.zeros(numColumns).astype(realType) pySp.getMinActiveDutyCycles(pyMinActive) cppSp.getMinActiveDutyCycles(cppMinActive) self.assertListAlmostEqual(list(pyMinActive), list(cppMinActive)) for i in xrange(pySp.getNumColumns()): if self.verbosity > 2: print "Column:",i pyPot = numpy.zeros(numInputs).astype(uintType) cppPot = numpy.zeros(numInputs).astype(uintType) pySp.getPotential(i, pyPot) cppSp.getPotential(i, cppPot) self.assertListEqual(list(pyPot),list(cppPot)) pyPerm = numpy.zeros(numInputs).astype(realType) cppPerm = numpy.zeros(numInputs).astype(realType) pySp.getPermanence(i, pyPerm) cppSp.getPermanence(i, cppPerm)self.assertListAlmostEqual(list(pyPerm),list(cppPerm))tests/unit/py2/nupic/research/spatial_pooler_compatability_test.py:149:
self =
alist = [0.0, 0.0, 0.0, 0.16262999, 0.3285, 0.0, ...] blist = [0.0, 0.0, 0.0, 0.16262999, 0.3285, 0.0, ...] def assertListAlmostEqual(self, alist, blist): self.assertEqual(len(alist), len(blist)) for a, b in zip(alist, blist): diff = abs(a - b)self.assertLess(diff, 1e-5)E AssertionError: 1.001358e-05 not less than 1e-05
tests/unit/py2/nupic/research/spatial_pooler_compatability_test.py:55: AssertionError
— Reply to this email directly or view it on GitHub https://github.com/numenta/nupic/issues/1155#issuecomment-57386017.
We find it hard to hear what another is saying because of how loudly "who one is", speaks...
This test fails occasionally when run with certain random seeds. Example output (including the seed that failed) is here: