missed cache - Githubissues

edgarcosta commented 7 months ago

from amortizedHGM import *

H = AmortizingHypergeometricData(cyclotomic=[[5],[1,1,1,1]])

#%time traces = H.amortized_padic_H_values(2, 2**13)
traces = H.amortized_padic_H_values(2, 2**23)

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
File /Applications/sage-dev/src/sage/misc/cachefunc.pyx:1962, in sage.misc.cachefunc.CachedMethodCaller.__call__()
   1961 try:
-> 1962     return cache[k]
   1963 except TypeError:  # k is not hashable

KeyError: ((8388608, 0, 0), ())

During handling of the above exception, another exception occurred:

KeyError                                  Traceback (most recent call last)
File amortizedHGM/hgm_misc.pyx:230, in amortizedHGM.hgm_misc.gamma_expansion_product()

File amortizedHGM/hgm_misc.pyx:203, in amortizedHGM.hgm_misc.expansion_from_cache()

KeyError: (4, 5, 1048583)

During handling of the above exception, another exception occurred:

KeyError                                  Traceback (most recent call last)
File /Applications/sage-dev/local/var/lib/sage/venv-python3.11/lib/python3.11/site-packages/amortizedHGM/gamma_expansions.py:198, in pAdicLogGammaCache.expansion(self, abp)
    197 try:
--> 198     return expansion_from_cache(self.cache, *abp, self.e)
    199 except KeyError:

File amortizedHGM/hgm_misc.pyx:195, in amortizedHGM.hgm_misc.expansion_from_cache()

File amortizedHGM/hgm_misc.pyx:203, in amortizedHGM.hgm_misc.expansion_from_cache()

KeyError: (4, 5, 1048583)

During handling of the above exception, another exception occurred:

AssertionError                            Traceback (most recent call last)
File <timed exec>:1

File /Applications/sage-dev/local/var/lib/sage/venv-python3.11/lib/python3.11/site-packages/amortizedHGM/hgm_modpe.py:948, in AmortizingHypergeometricData.amortized_padic_H_values(self, t, N, chained, debug)
    945 else:
    946     # Precompute Gamma values and zero offsets.
    947     self.gammas_cache.increase_N(N)
--> 948     self.displacements(N, ZZ(0), ZZ(0))
    949     tmp = ZZ(0)
    950 vectors = {p: tmp for p in self._prime_range(t, N)[1][0]}

File /Applications/sage-dev/src/sage/misc/cachefunc.pyx:1967, in sage.misc.cachefunc.CachedMethodCaller.__call__()
   1965         return cache[k]
   1966 except KeyError:
-> 1967     w = self._instance_call(*args, **kwds)
   1968     cache[k] = w
   1969     return w

File /Applications/sage-dev/src/sage/misc/cachefunc.pyx:1842, in sage.misc.cachefunc.CachedMethodCaller._instance_call()
   1840         True
   1841     """
-> 1842     return self.f(self._instance, *args, **kwds)
   1843 
   1844 cdef fix_args_kwds(self, tuple args, dict kwds):

File /Applications/sage-dev/local/var/lib/sage/venv-python3.11/lib/python3.11/site-packages/amortizedHGM/hgm_modpe.py:562, in AmortizingHypergeometricData.displacements(self, N, start, pclass)
    557 tmp = tuple(t.as_integer_ratio() for t in tmp)
    558 l = None if max(ei1, ei) <= 1 else (tmp2, r, d, 
    559     1 if ei1 < 1 else factorial(ZZ(ei1-1)), 
    560     1 if ei<1 else factorial(ZZ(ei-1))**3, inter_polys, R1.gen())
--> 562 ans = {p: gammas_to_displacements(p, ei1, ei,
    563        *gamma_expansion_product(l0, p, eimax), tmp, l)
    564        for p in self._prime_range(ZZ(-1), N)[d][pclass]} #inner loop
    565 # If start==0, we need to extract a normalization factor.
    566 if start == 0:

File /Applications/sage-dev/local/var/lib/sage/venv-python3.11/lib/python3.11/site-packages/amortizedHGM/hgm_modpe.py:563, in <dictcomp>(.0)
    557 tmp = tuple(t.as_integer_ratio() for t in tmp)
    558 l = None if max(ei1, ei) <= 1 else (tmp2, r, d, 
    559     1 if ei1 < 1 else factorial(ZZ(ei1-1)), 
    560     1 if ei<1 else factorial(ZZ(ei-1))**3, inter_polys, R1.gen())
    562 ans = {p: gammas_to_displacements(p, ei1, ei,
--> 563        *gamma_expansion_product(l0, p, eimax), tmp, l)
    564        for p in self._prime_range(ZZ(-1), N)[d][pclass]} #inner loop
    565 # If start==0, we need to extract a normalization factor.
    566 if start == 0:

File amortizedHGM/hgm_misc.pyx:207, in amortizedHGM.hgm_misc.gamma_expansion_product()

File amortizedHGM/hgm_misc.pyx:232, in amortizedHGM.hgm_misc.gamma_expansion_product()

File /Applications/sage-dev/local/var/lib/sage/venv-python3.11/lib/python3.11/site-packages/amortizedHGM/gamma_expansions.py:207, in pAdicLogGammaCache.expansion(self, abp)
    205     self.increase_N(N)
    206 self._set_expansion_at_offset(b)
--> 207 assert abp in self.cache or (b-a, b, p) in self.cache
    208 return self.expansion(abp)

AssertionError:

edgarcosta commented 7 months ago

N = 2^20 works, but N = 2^21 also fails

kedlaya commented 7 months ago

On my end I'm seeing some path dependence. In a fresh session,

sage: from amortizedHGM import *
sage: H = AmortizingHypergeometricData(cyclotomic=[[5],[1,1,1,]])
sage: traces = H.amortized_padic_H_values(2, 2**14)

works, but in another fresh session,

sage: from amortizedHGM import *
sage: H = AmortizingHypergeometricData(cyclotomic=[[5],[1,1,1,]])
sage: traces = H.amortized_padic_H_values(2, 2**13)
sage: traces = H.amortized_padic_H_values(2, 2**14)

fails in a similar way; it is missing a value for p=8209 which is indeed the first prime beyond the end of the old range.

I'm guessing this means the code to extend the range isn't getting triggered correctly in pAdicGammaLogCache.increase_N. To wit:

sage: from amortizedHGM.gamma_expansions import pAdicLogGammaCache
sage: cache = pAdicLogGammaCache(5)
sage: cache.clear_cache()
sage: cache.increase_N(20)
[7, 11, 13, 17, 19]
sage: cache.increase_N(40)
[7, 11, 13, 17, 19]

One solution would be to add self.clear_cache() before calling self._expansion_at_0, to force the lazy attribute to be regenerated.

kedlaya commented 7 months ago

Correction: need to clear self.clear_cache() before resetting self.N.

Latest two commits to main should address this.

edgarcosta commented 5 months ago

I still observe this with

H = AmortizingHypergeometricData(alpha_beta=[[1/2,1/3,2/3,1/6,5/6],
    [0,0,0,0,0]])
_ = H.amortized_padic_H_values(-80^3, 10)
_ = H.amortized_padic_H_values(-80^3, 200)

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
File /usr/local/sage/src/sage/misc/cachefunc.pyx:1934, in sage.misc.cachefunc.CachedMethodCaller.__call__()
   1933 try:
-> 1934     return cache[k]
   1935 except TypeError:  # k is not hashable
KeyError: ((-512000, 2048, None, False), ())

During handling of the above exception, another exception occurred:
KeyError                                  Traceback (most recent call last)
File /usr/local/sage/src/sage/misc/cachefunc.pyx:1934, in sage.misc.cachefunc.CachedMethodCaller.__call__()
   1933 try:
-> 1934     return cache[k]
   1935 except TypeError:  # k is not hashable
KeyError: ((2048, 0, 0), ())

During handling of the above exception, another exception occurred:
KeyError                                  Traceback (most recent call last)
File amortizedHGM/hgm_misc.pyx:230, in amortizedHGM.hgm_misc.gamma_expansion_product()
File amortizedHGM/hgm_misc.pyx:203, in amortizedHGM.hgm_misc.expansion_from_cache()
KeyError: (5, 6, 1031)

During handling of the above exception, another exception occurred:
KeyError                                  Traceback (most recent call last)
File ~/.sage/local/lib/python3.11/site-packages/amortizedHGM/gamma_expansions.py:198, in pAdicLogGammaCache.expansion(self, abp)
    197 try:
--> 198     return expansion_from_cache(self.cache, *abp, self.e)
    199 except KeyError:
File amortizedHGM/hgm_misc.pyx:195, in amortizedHGM.hgm_misc.expansion_from_cache()
File amortizedHGM/hgm_misc.pyx:203, in amortizedHGM.hgm_misc.expansion_from_cache()
KeyError: (5, 6, 1031)

During handling of the above exception, another exception occurred:
AssertionError                            Traceback (most recent call last)
Cell In [5], line 9
      7 N = Integer(2)**e
      8 s = time()
----> 9 euler_factors = HJV.euler_factors(-Integer(80)**Integer(3), N)
     10 w = time() -s
     11 print(f"{e} {w:.2f} s\n")
File /usr/local/sage/src/sage/misc/cachefunc.pyx:1939, in sage.misc.cachefunc.CachedMethodCaller.__call__()
   1937         return cache[k]
   1938 except KeyError:
-> 1939     w = self._instance_call(*args, **kwds)
   1940     cache[k] = w
   1941     return w
File /usr/local/sage/src/sage/misc/cachefunc.pyx:1815, in sage.misc.cachefunc.CachedMethodCaller._instance_call()
   1813         True
   1814     """
-> 1815     return self.f(self._instance, *args, **kwds)
   1816
   1817 cdef fix_args_kwds(self, tuple args, dict kwds):
Cell In [3], line 279, in LAmortizingHypergeometricData.euler_factors(self, t, N, chained, verbose)
    271 """
    272 Returns the conductor exponent and Euler factor at wild primes of self
    273
   (...)
    276 * "N" -- integer, the number of Dirichlet coefficients needed in the L-series
    277 """
    278 # Collect prime Frobenius traces.
--> 279 prime_traces = self.amortized_padic_H_values(t, N, chained=chained)
    280 if verbose:
    281     print("Computed prime Frobenius traces")
File ~/.sage/local/lib/python3.11/site-packages/amortizedHGM/hgm_modpe.py:949, in AmortizingHypergeometricData.amortized_padic_H_values(self, t, N, chained, debug)
    946 else:
    947     # Precompute Gamma values and zero offsets.
    948     self.gammas_cache.increase_N(N)
--> 949     self.displacements(N, ZZ(0), ZZ(0))
    950     tmp = ZZ(0)
    951 vectors = {p: tmp for p in self._prime_range(t, N)[1][0]}
File /usr/local/sage/src/sage/misc/cachefunc.pyx:1939, in sage.misc.cachefunc.CachedMethodCaller.__call__()
   1937         return cache[k]
   1938 except KeyError:
-> 1939     w = self._instance_call(*args, **kwds)
   1940     cache[k] = w
   1941     return w
File /usr/local/sage/src/sage/misc/cachefunc.pyx:1815, in sage.misc.cachefunc.CachedMethodCaller._instance_call()
   1813         True
   1814     """
-> 1815     return self.f(self._instance, *args, **kwds)
   1816
   1817 cdef fix_args_kwds(self, tuple args, dict kwds):
File ~/.sage/local/lib/python3.11/site-packages/amortizedHGM/hgm_modpe.py:563, in AmortizingHypergeometricData.displacements(self, N, start, pclass)
    558 tmp = tuple(t.as_integer_ratio() for t in tmp)
    559 l = None if max(ei1, ei) <= 1 else (tmp2, r, d,
    560     1 if ei1 < 1 else factorial(ZZ(ei1-1)),
    561     1 if ei<1 else factorial(ZZ(ei-1))**3, inter_polys, R1.gen())
--> 563 ans = {p: gammas_to_displacements(p, ei1, ei,
    564        *gamma_expansion_product(l0, p, eimax), tmp, l)
    565        for p in self._prime_range(ZZ(-1), N)[d][pclass]} #inner loop
    566 # If start==0, we need to extract a normalization factor.
    567 if start == 0:
File ~/.sage/local/lib/python3.11/site-packages/amortizedHGM/hgm_modpe.py:564, in <dictcomp>(.0)
    558 tmp = tuple(t.as_integer_ratio() for t in tmp)
    559 l = None if max(ei1, ei) <= 1 else (tmp2, r, d,
    560     1 if ei1 < 1 else factorial(ZZ(ei1-1)),
    561     1 if ei<1 else factorial(ZZ(ei-1))**3, inter_polys, R1.gen())
    563 ans = {p: gammas_to_displacements(p, ei1, ei,
--> 564        *gamma_expansion_product(l0, p, eimax), tmp, l)
    565        for p in self._prime_range(ZZ(-1), N)[d][pclass]} #inner loop
    566 # If start==0, we need to extract a normalization factor.
    567 if start == 0:
File amortizedHGM/hgm_misc.pyx:207, in amortizedHGM.hgm_misc.gamma_expansion_product()
File amortizedHGM/hgm_misc.pyx:232, in amortizedHGM.hgm_misc.gamma_expansion_product()
File ~/.sage/local/lib/python3.11/site-packages/amortizedHGM/gamma_expansions.py:207, in pAdicLogGammaCache.expansion(self, abp)
    205     self.increase_N(N)
    206 self._set_expansion_at_offset(b)
--> 207 assert abp in self.cache or (b-a, b, p) in self.cache
    208 return self.expansion(abp)
AssertionError:

edgarcosta commented 5 months ago

i was usng the wrong version

edgarcosta / amortizedHGM

missed cache #4