search

stac-utils / stac-validator

Validator for the stac-spec
Apache License 2.0
46 stars 27 forks source link

Validator times out with very large catalogs #31

Closed jbants closed 5 years ago

jbants commented 5 years ago

stac_validator.py https://cbers-stac-0-6.s3.amazonaws.com/catalog.json results in:

  Traceback (most recent call last):
    File "/home/anik/anaconda3/envs/stac/lib/python3.6/site-packages/trio/_timeouts.py", line 116, in fail_at
      yield scope
    File "/home/anik/anaconda3/envs/stac/lib/python3.6/site-packages/asks/utils.py", line 15, in timeout_manager
      return (await coro(*args))
    File "/home/anik/anaconda3/envs/stac/lib/python3.6/site-packages/asks/sessions.py", line 361, in _grab_connection
      sock = await self._make_connection(host_loc)
    File "/home/anik/anaconda3/envs/stac/lib/python3.6/site-packages/asks/sessions.py", line 335, in _make_connection
      sock, port = await self._connect(host_loc)
    File "/home/anik/anaconda3/envs/stac/lib/python3.6/site-packages/asks/sessions.py", line 103, in _connect
      (host, int(port))), port
    File "/home/anik/anaconda3/envs/stac/lib/python3.6/site-packages/asks/sessions.py", line 81, in _open_connection_https
      source_addr=self.source_address)
    File "/home/anik/anaconda3/envs/stac/lib/python3.6/site-packages/multio/_event_loop_wrappers.py", line 29, in trio_open_connection
      sock = await trio.open_ssl_over_tcp_stream(host, port)
    File "/home/anik/anaconda3/envs/stac/lib/python3.6/site-packages/trio/_highlevel_ssl_helpers.py", line 62, in open_ssl_over_tcp_stream
      happy_eyeballs_delay=happy_eyeballs_delay,
    File "/home/anik/anaconda3/envs/stac/lib/python3.6/site-packages/trio/_highlevel_open_tcp_stream.py", line 232, in open_tcp_stream
      targets = await getaddrinfo(host, port, type=SOCK_STREAM)
    File "/home/anik/anaconda3/envs/stac/lib/python3.6/site-packages/trio/_socket.py", line 188, in getaddrinfo
      cancellable=True
    File "/home/anik/anaconda3/envs/stac/lib/python3.6/site-packages/trio/_threads.py", line 400, in run_sync_in_worker_thread
      return await _core.wait_task_rescheduled(abort)
    File "/home/anik/anaconda3/envs/stac/lib/python3.6/site-packages/trio/_core/_traps.py", line 165, in wait_task_rescheduled
      return (await _async_yield(WaitTaskRescheduled(abort_func))).unwrap()
    File "/home/anik/anaconda3/envs/stac/lib/python3.6/site-packages/outcome/_sync.py", line 111, in unwrap
      raise captured_error
    File "/home/anik/anaconda3/envs/stac/lib/python3.6/site-packages/trio/_core/_run.py", line 632, in raise_cancel
      raise exc
  trio.Cancelled

  During handling of the above exception, another exception occurred:

  Traceback (most recent call last):
    File "/home/anik/anaconda3/envs/stac/lib/python3.6/site-packages/asks/utils.py", line 15, in timeout_manager
      return (await coro(*args))
    File "/home/anik/anaconda3/envs/stac/lib/python3.6/site-packages/multio/__init__.py", line 73, in __aexit__
      return self.__exit__(exc_type, exc_val, exc_tb)
    File "/home/anik/anaconda3/envs/stac/lib/python3.6/site-packages/multio/__init__.py", line 67, in __exit__
      return self.manager.__exit__(exc_type, exc_val, exc_tb)
    File "/home/anik/anaconda3/envs/stac/lib/python3.6/contextlib.py", line 100, in __exit__
      self.gen.throw(type, value, traceback)
    File "/home/anik/anaconda3/envs/stac/lib/python3.6/site-packages/trio/_timeouts.py", line 118, in fail_at
      raise TooSlowError
  trio.TooSlowError

  The above exception was the direct cause of the following exception:

  Traceback (most recent call last):
    File "/home/anik/PycharmProjects/STAC/stac-validator/stac_validator.py", line 164, in _validate_child
      _ = await stac.run()
    File "/home/anik/PycharmProjects/STAC/stac-validator/stac_validator.py", line 219, in run
      resp = await asks.get(self.stac_file)
    File "/home/anik/anaconda3/envs/stac/lib/python3.6/site-packages/asks/base_funcs.py", line 30, in request
      r = await s.request(method, url=uri, **kwargs)
    File "/home/anik/anaconda3/envs/stac/lib/python3.6/site-packages/asks/sessions.py", line 170, in request
      connection_timeout, self._grab_connection, url)
    File "/home/anik/anaconda3/envs/stac/lib/python3.6/site-packages/asks/utils.py", line 17, in timeout_manager
      raise RequestTimeout from e
  asks.errors.RequestTimeout
jbants commented 5 years ago

removing trio and asks solves this