search

UMNLibraries / experts_etl

Moves data from UMN to Pure (Experts@Minnesota), and vice versa.
3 stars 0 forks source link

Ensure that /swadm/var/log/experts_etl/experts_etl.log exists before attempting to instantiate the logger? #6

Open nihiliad opened 5 years ago

nihiliad commented 2 months ago

Possibly related error, revealed by running systemctl status -l -n 200 experts_etl on the remote machines as root:

--- Logging error ---
--- Logging error ---
Traceback (most recent call last):
File "/opt/python-3.9.10/lib/python3.9/logging/handlers.py", line 74, in emit
self.doRollover()
File "/opt/python-3.9.10/lib/python3.9/logging/handlers.py", line 428, in doRollover
self.rotate(self.baseFilename, dfn)
File "/opt/python-3.9.10/lib/python3.9/logging/handlers.py", line 117, in rotate
self.rotator(source, dest)
File "/swadm/opt/experts_etl/releases/20240313090436/experts_etl/loggers.py", line 42, in rotator
os.remove(source)
FileNotFoundError: [Errno 2] No such file or directory: '/swadm/var/log/experts_etl/experts_etl.log'
Call stack:
Traceback (most recent call last):
File "/opt/python-3.9.10/lib/python3.9/logging/handlers.py", line 74, in emit
self.doRollover()
File "/opt/python-3.9.10/lib/python3.9/logging/handlers.py", line 428, in doRollover
self.rotate(self.baseFilename, dfn)
File "/opt/python-3.9.10/lib/python3.9/logging/handlers.py", line 117, in rotate
self.rotator(source, dest)
File "/swadm/opt/experts_etl/releases/20240313090436/experts_etl/loggers.py", line 42, in rotator
os.remove(source)
FileNotFoundError: [Errno 2] No such file or directory: '/swadm/var/log/experts_etl/experts_etl.log'
Call stack:
File "/swadm/opt/experts_etl/current/runner.py", line 177, in <module>
daemon()
File "/swadm/opt/experts_etl/current/runner.py", line 173, in daemon
run()
File "/swadm/opt/experts_etl/current/runner.py", line 142, in run
with mp.Pool() as pool:
File "/opt/python-3.9.10/lib/python3.9/multiprocessing/context.py", line 119, in Pool
return Pool(processes, initializer, initargs, maxtasksperchild,
File "/opt/python-3.9.10/lib/python3.9/multiprocessing/pool.py", line 212, in __init__
self._repopulate_pool()
File "/opt/python-3.9.10/lib/python3.9/multiprocessing/pool.py", line 303, in _repopulate_pool
return self._repopulate_pool_static(self._ctx, self.Process,
File "/opt/python-3.9.10/lib/python3.9/multiprocessing/pool.py", line 326, in _repopulate_pool_static
w.start()
File "/opt/python-3.9.10/lib/python3.9/multiprocessing/process.py", line 121, in start
self._popen = self._Popen(self)
File "/opt/python-3.9.10/lib/python3.9/multiprocessing/context.py", line 277, in _Popen
return Popen(process_obj)
File "/opt/python-3.9.10/lib/python3.9/multiprocessing/popen_fork.py", line 19, in __init__
self._launch(process_obj)
File "/opt/python-3.9.10/lib/python3.9/multiprocessing/popen_fork.py", line 71, in _launch
code = process_obj._bootstrap(parent_sentinel=child_r)
File "/opt/python-3.9.10/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
self.run()
File "/opt/python-3.9.10/lib/python3.9/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/opt/python-3.9.10/lib/python3.9/multiprocessing/pool.py", line 125, in worker
result = (True, func(*args, **kwds))
File "/swadm/opt/experts_etl/current/runner.py", line 86, in extract_load_collection
collection.run(collection_api_name, **params)
File "/swadm/opt/experts_etl/releases/20240313090436/experts_etl/pure_to_edw/collection.py", line 26, in run
experts_etl_logger.info('starting: extracting/loading raw json', extra={'pure_api_record_type': collection_api_name})
Message: 'starting: extracting/loading raw json'
Arguments: ()
File "/swadm/opt/experts_etl/current/runner.py", line 177, in <module>
daemon()
File "/swadm/opt/experts_etl/current/runner.py", line 173, in daemon
run()
File "/swadm/opt/experts_etl/current/runner.py", line 142, in run
with mp.Pool() as pool:
File "/opt/python-3.9.10/lib/python3.9/multiprocessing/context.py", line 119, in Pool
return Pool(processes, initializer, initargs, maxtasksperchild,
File "/opt/python-3.9.10/lib/python3.9/multiprocessing/pool.py", line 212, in __init__
self._repopulate_pool()
File "/opt/python-3.9.10/lib/python3.9/multiprocessing/pool.py", line 303, in _repopulate_pool
return self._repopulate_pool_static(self._ctx, self.Process,
File "/opt/python-3.9.10/lib/python3.9/multiprocessing/pool.py", line 326, in _repopulate_pool_static
w.start()
File "/opt/python-3.9.10/lib/python3.9/multiprocessing/process.py", line 121, in start
self._popen = self._Popen(self)
File "/opt/python-3.9.10/lib/python3.9/multiprocessing/context.py", line 277, in _Popen
return Popen(process_obj)
File "/opt/python-3.9.10/lib/python3.9/multiprocessing/popen_fork.py", line 19, in __init__
self._launch(process_obj)
File "/opt/python-3.9.10/lib/python3.9/multiprocessing/popen_fork.py", line 71, in _launch
code = process_obj._bootstrap(parent_sentinel=child_r)
File "/opt/python-3.9.10/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
self.run()
File "/opt/python-3.9.10/lib/python3.9/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/opt/python-3.9.10/lib/python3.9/multiprocessing/pool.py", line 125, in worker
result = (True, func(*args, **kwds))
File "/swadm/opt/experts_etl/current/runner.py", line 86, in extract_load_collection
collection.run(collection_api_name, **params)
File "/swadm/opt/experts_etl/releases/20240313090436/experts_etl/pure_to_edw/collection.py", line 26, in run
experts_etl_logger.info('starting: extracting/loading raw json', extra={'pure_api_record_type': collection_api_name})
Message: 'starting: extracting/loading raw json'
Arguments: ()