Seems like a stop-gap fix would be can_connect guard + defer.
Logs
unit-prom-0: 16:13:12.848 DEBUG unit.prom/0.juju-log certificates:71: Emitting custom event <CertChanged via PrometheusCharm/CertHandler[prometheus-server-cert]/on/cert_changed[43]>.
unit-prom-0: 16:13:12.881 ERROR unit.prom/0.juju-log certificates:71: Uncaught exception while in charm code:
Traceback (most recent call last):
File "/usr/lib/python3.8/urllib/request.py", line 1354, in do_open
h.request(req.get_method(), req.selector, req.data, headers,
File "/usr/lib/python3.8/http/client.py", line 1256, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.8/http/client.py", line 1302, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.8/http/client.py", line 1251, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.8/http/client.py", line 1011, in _send_output
self.send(msg)
File "/usr/lib/python3.8/http/client.py", line 951, in send
self.connect()
File "/var/lib/juju/agents/unit-prom-0/charm/venv/ops/pebble.py", line 256, in connect
self.sock.connect(self.socket_path)
FileNotFoundError: [Errno 2] No such file or directory
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/var/lib/juju/agents/unit-prom-0/charm/venv/ops/pebble.py", line 1571, in _request_raw
response = self.opener.open(request, timeout=self.timeout)
File "/usr/lib/python3.8/urllib/request.py", line 525, in open
response = self._open(req, data)
File "/usr/lib/python3.8/urllib/request.py", line 542, in _open
result = self._call_chain(self.handle_open, protocol, protocol +
File "/usr/lib/python3.8/urllib/request.py", line 502, in _call_chain
result = func(*args)
File "/var/lib/juju/agents/unit-prom-0/charm/venv/ops/pebble.py", line 270, in http_open
return self.do_open(_UnixSocketConnection, req, # type:ignore
File "/usr/lib/python3.8/urllib/request.py", line 1357, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error [Errno 2] No such file or directory>
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "./src/charm.py", line 939, in <module>
main(PrometheusCharm)
File "/var/lib/juju/agents/unit-prom-0/charm/venv/ops/main.py", line 441, in main
_emit_charm_event(charm, dispatcher.event_name)
File "/var/lib/juju/agents/unit-prom-0/charm/venv/ops/main.py", line 149, in _emit_charm_event
event_to_emit.emit(*args, **kwargs)
File "/var/lib/juju/agents/unit-prom-0/charm/venv/ops/framework.py", line 344, in emit
framework._emit(event)
File "/var/lib/juju/agents/unit-prom-0/charm/venv/ops/framework.py", line 833, in _emit
self._reemit(event_path)
File "/var/lib/juju/agents/unit-prom-0/charm/venv/ops/framework.py", line 922, in _reemit
custom_handler(event)
File "/var/lib/juju/agents/unit-prom-0/charm/lib/charms/tls_certificates_interface/v2/tls_certificates.py", line 1401, in _on_relation_changed
self.on.certificate_available.emit(
File "/var/lib/juju/agents/unit-prom-0/charm/venv/ops/framework.py", line 344, in emit
framework._emit(event)
File "/var/lib/juju/agents/unit-prom-0/charm/venv/ops/framework.py", line 833, in _emit
self._reemit(event_path)
File "/var/lib/juju/agents/unit-prom-0/charm/venv/ops/framework.py", line 922, in _reemit
custom_handler(event)
File "/var/lib/juju/agents/unit-prom-0/charm/lib/charms/observability_libs/v0/cert_handler.py", line 273, in _on_certificate_available
self.on.cert_changed.emit() # pyright: ignore
File "/var/lib/juju/agents/unit-prom-0/charm/venv/ops/framework.py", line 344, in emit
framework._emit(event)
File "/var/lib/juju/agents/unit-prom-0/charm/venv/ops/framework.py", line 833, in _emit
self._reemit(event_path)
File "/var/lib/juju/agents/unit-prom-0/charm/venv/ops/framework.py", line 922, in _reemit
custom_handler(event)
File "/var/lib/juju/agents/unit-prom-0/charm/lib/charms/tempo_k8s/v0/charm_instrumentation.py", line 455, in wrapped_function
return callable(*args, **kwargs) # type: ignore
File "./src/charm.py", line 382, in _on_server_cert_changed
self.container.remove_path(path, recursive=True)
File "/var/lib/juju/agents/unit-prom-0/charm/venv/ops/model.py", line 2361, in remove_path
self._pebble.remove_path(str(path), recursive=recursive)
File "/var/lib/juju/agents/unit-prom-0/charm/venv/ops/pebble.py", line 2130, in remove_path
resp = self._request('POST', '/v1/files', None, body)
File "/var/lib/juju/agents/unit-prom-0/charm/venv/ops/pebble.py", line 1538, in _request
response = self._request_raw(method, path, query, headers, data)
File "/var/lib/juju/agents/unit-prom-0/charm/venv/ops/pebble.py", line 1584, in _request_raw
raise ConnectionError(e.reason)
ops.pebble.ConnectionError: [Errno 2] No such file or directory
unit-prom-0: 16:13:13.130 ERROR juju.worker.uniter.operation hook "certificates-relation-changed" (via hook dispatching script: dispatch) failed: exit status 1
Issue
File operations are attempted before pebble-ready:
https://github.com/canonical/prometheus-k8s-operator/blob/d33f51f39d990de3b8dcb9436bf69291a8e8b891/src/charm.py#L380-L382
Seems like a stop-gap fix would be
can_connect
guard + defer.Logs