Continuous Test runs haven't succeeded since 30 September

[crwilcox]

https://github.com/googleapis/python-firestore/commit/6e73cbdb6fc788732133ea847d0dc6ae54c966d2

I think this commit caused our continuous run to fail.

nox > * system_emulated-3.7: failed

__________________ ERROR at teardown of test_create_document ___________________

args = (database: "projects/google-cloud-firestore-emulator/databases/(default)"
writes {
  delete: "projects/google-cloud-firestore-emulator/databases/(default)/documents/doc-create-1633031445418/doc-1633031445418"
}
,)
kwargs = {'metadata': [('google-cloud-resource-prefix', 'projects/google-cloud-firestore-emulator/databases/(default)'), ('auth...estore-emulator/databases/%28default%29'), ('x-goog-api-client', 'gl-python/3.7.9 grpc/1.41.0 gax/2.0.1 gapic/2.3.4')]}

    @functools.wraps(callable_)
    def error_remapped_callable(*args, **kwargs):
        try:
>           return callable_(*args, **kwargs)

.nox/system_emulated-3-7/lib/python3.7/site-packages/google/api_core/grpc_helpers.py:66: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <grpc._channel._UnaryUnaryMultiCallable object at 0x7fcf4d2d4710>
request = database: "projects/google-cloud-firestore-emulator/databases/(default)"
writes {
  delete: "projects/google-cloud-firestore-emulator/databases/(default)/documents/doc-create-1633031445418/doc-1633031445418"
}

timeout = None
metadata = [('google-cloud-resource-prefix', 'projects/google-cloud-firestore-emulator/databases/(default)'), ('authorization', '...restore-emulator/databases/%28default%29'), ('x-goog-api-client', 'gl-python/3.7.9 grpc/1.41.0 gax/2.0.1 gapic/2.3.4')]
credentials = None, wait_for_ready = None, compression = None

    def __call__(self,
                 request,
                 timeout=None,
                 metadata=None,
                 credentials=None,
                 wait_for_ready=None,
                 compression=None):
        state, call, = self._blocking(request, timeout, metadata, credentials,
                                      wait_for_ready, compression)
>       return _end_unary_response_blocking(state, call, False, None)

.nox/system_emulated-3-7/lib/python3.7/site-packages/grpc/_channel.py:946: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

state = <grpc._channel._RPCState object at 0x7fcf4d0d0050>
call = <grpc._cython.cygrpc.SegregatedCall object at 0x7fcf4cec3aa0>
with_call = False, deadline = None

    def _end_unary_response_blocking(state, call, with_call, deadline):
        if state.code is grpc.StatusCode.OK:
            if with_call:
                rendezvous = _MultiThreadedRendezvous(state, call, None, deadline)
                return state.response, rendezvous
            else:
                return state.response
        else:
>           raise _InactiveRpcError(state)
E           grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
E               status = StatusCode.UNAVAILABLE
E               details = "failed to connect to all addresses"
E               debug_error_string = "{"created":"@1633031695.025817925","description":"Failed to pick subchannel","file":"src/core/ext/filters/client_channel/client_channel.cc","file_line":3158,"referenced_errors":[{"created":"@1633031695.025816594","description":"failed to connect to all addresses","file":"src/core/lib/transport/error_utils.cc","file_line":147,"grpc_status":14}]}"
E           >

.nox/system_emulated-3-7/lib/python3.7/site-packages/grpc/_channel.py:849: _InactiveRpcError

The above exception was the direct cause of the following exception:

target = functools.partial(<function _wrap_unary_errors.<locals>.error_remapped_callable at 0x7fcf4d29db00>, database: "project...estore-emulator/databases/%28default%29'), ('x-goog-api-client', 'gl-python/3.7.9 grpc/1.41.0 gax/2.0.1 gapic/2.3.4')])
predicate = <function if_exception_type.<locals>.if_exception_type_predicate at 0x7fcf4d29da70>
sleep_generator = <generator object exponential_sleep_generator at 0x7fcf4d0a7150>
deadline = 60.0, on_error = None

    def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
        """Call a function and retry if it fails.

        This is the lowest-level retry helper. Generally, you'll use the
        higher-level retry helper :class:`Retry`.

        Args:
            target(Callable): The function to call and retry. This must be a
                nullary function - apply arguments with `functools.partial`.
            predicate (Callable[Exception]): A callable used to determine if an
                exception raised by the target should be considered retryable.
                It should return True to retry or False otherwise.
            sleep_generator (Iterable[float]): An infinite iterator that determines
                how long to sleep between retries.
            deadline (float): How long to keep retrying the target. The last sleep
                period is shortened as necessary, so that the last retry runs at
                ``deadline`` (and not considerably beyond it).
            on_error (Callable[Exception]): A function to call while processing a
                retryable exception.  Any error raised by this function will *not*
                be caught.

        Returns:
            Any: the return value of the target function.

        Raises:
            google.api_core.RetryError: If the deadline is exceeded while retrying.
            ValueError: If the sleep generator stops yielding values.
            Exception: If the target raises a method that isn't retryable.
        """
        if deadline is not None:
            deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
                seconds=deadline
            )
        else:
            deadline_datetime = None

        last_exc = None

        for sleep in sleep_generator:
            try:
>               return target()

.nox/system_emulated-3-7/lib/python3.7/site-packages/google/api_core/retry.py:190: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

args = (database: "projects/google-cloud-firestore-emulator/databases/(default)"
writes {
  delete: "projects/google-cloud-firestore-emulator/databases/(default)/documents/doc-create-1633031445418/doc-1633031445418"
}
,)
kwargs = {'metadata': [('google-cloud-resource-prefix', 'projects/google-cloud-firestore-emulator/databases/(default)'), ('auth...estore-emulator/databases/%28default%29'), ('x-goog-api-client', 'gl-python/3.7.9 grpc/1.41.0 gax/2.0.1 gapic/2.3.4')]}

    @functools.wraps(callable_)
    def error_remapped_callable(*args, **kwargs):
        try:
            return callable_(*args, **kwargs)
        except grpc.RpcError as exc:
>           raise exceptions.from_grpc_error(exc) from exc
E           google.api_core.exceptions.ServiceUnavailable: 503 failed to connect to all addresses

.nox/system_emulated-3-7/lib/python3.7/site-packages/google/api_core/grpc_helpers.py:68: ServiceUnavailable

The above exception was the direct cause of the following exception:

    @pytest.fixture
    def cleanup():
        operations = []
        yield operations.append

        for operation in operations:
>           operation()

tests/system/test_system.py:69: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
google/cloud/firestore_v1/document.py:355: in delete
    request=request, metadata=self._client._rpc_metadata, **kwargs,
google/cloud/firestore_v1/services/firestore/client.py:815: in commit
    response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,)
.nox/system_emulated-3-7/lib/python3.7/site-packages/google/api_core/gapic_v1/method.py:142: in __call__
    return wrapped_func(*args, **kwargs)
.nox/system_emulated-3-7/lib/python3.7/site-packages/google/api_core/retry.py:288: in retry_wrapped_func
    on_error=on_error,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

target = functools.partial(<function _wrap_unary_errors.<locals>.error_remapped_callable at 0x7fcf4d29db00>, database: "project...estore-emulator/databases/%28default%29'), ('x-goog-api-client', 'gl-python/3.7.9 grpc/1.41.0 gax/2.0.1 gapic/2.3.4')])
predicate = <function if_exception_type.<locals>.if_exception_type_predicate at 0x7fcf4d29da70>
sleep_generator = <generator object exponential_sleep_generator at 0x7fcf4d0a7150>
deadline = 60.0, on_error = None

    def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
        """Call a function and retry if it fails.

        This is the lowest-level retry helper. Generally, you'll use the
        higher-level retry helper :class:`Retry`.

        Args:
            target(Callable): The function to call and retry. This must be a
                nullary function - apply arguments with `functools.partial`.
            predicate (Callable[Exception]): A callable used to determine if an
                exception raised by the target should be considered retryable.
                It should return True to retry or False otherwise.
            sleep_generator (Iterable[float]): An infinite iterator that determines
                how long to sleep between retries.
            deadline (float): How long to keep retrying the target. The last sleep
                period is shortened as necessary, so that the last retry runs at
                ``deadline`` (and not considerably beyond it).
            on_error (Callable[Exception]): A function to call while processing a
                retryable exception.  Any error raised by this function will *not*
                be caught.

        Returns:
            Any: the return value of the target function.

        Raises:
            google.api_core.RetryError: If the deadline is exceeded while retrying.
            ValueError: If the sleep generator stops yielding values.
            Exception: If the target raises a method that isn't retryable.
        """
        if deadline is not None:
            deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
                seconds=deadline
            )
        else:
            deadline_datetime = None

        last_exc = None

        for sleep in sleep_generator:
            try:
                return target()

            # pylint: disable=broad-except
            # This function explicitly must deal with broad exceptions.
            except Exception as exc:
                if not predicate(exc):
                    raise
                last_exc = exc
                if on_error is not None:
                    on_error(exc)

            now = datetime_helpers.utcnow()

            if deadline_datetime is not None:
                if deadline_datetime <= now:
                    raise exceptions.RetryError(
                        "Deadline of {:.1f}s exceeded while calling {}".format(
                            deadline, target
                        ),
                        last_exc,
>                   ) from last_exc
E                   google.api_core.exceptions.RetryError: Deadline of 60.0s exceeded while calling functools.partial(<function _wrap_unary_errors.<locals>.error_remapped_callable at 0x7fcf4d29db00>, database: "projects/google-cloud-firestore-emulator/databases/(default)"
E                   writes {
E                     delete: "projects/google-cloud-firestore-emulator/databases/(default)/documents/doc-create-1633031445418/doc-1633031445418"
E                   }
E                   , metadata=[('google-cloud-resource-prefix', 'projects/google-cloud-firestore-emulator/databases/(default)'), ('authorization', 'Bearer owner'), ('x-goog-request-params', 'database=projects/google-cloud-firestore-emulator/databases/%28default%29'), ('x-goog-api-client', 'gl-python/3.7.9 grpc/1.41.0 gax/2.0.1 gapic/2.3.4')]), last exception: 503 failed to connect to all addresses

.nox/system_emulated-3-7/lib/python3.7/site-packages/google/api_core/retry.py:210: RetryError

[crwilcox]

ERROR: (gcloud.beta.emulators.firestore.start) To use the Google Cloud Firestore emulator, a Java 8+ JRE must be installed and on your system PATH

gcr.io/cloud-devrel-kokoro-resources/python-multi doesn't have a java runtime installed. We may be able to add it.

[crwilcox]

I am going to consider this fixed as we run the emulator on an action, so not running it on continuous is okay.