Python scripts for ETL (extract, transform and load) jobs for Ethereum blocks, transactions, ERC20 / ERC721 tokens, transfers, receipts, logs, contracts, internal transactions. Data is available in Google BigQuery https://goo.gl/oY5BCQ
This is regarding the ethereum-etl[streaming] to Postgres over SSL:
I tried using this format 'postgresql+pg8000://username:password@host/db_name?sslmode=require' as --output and I am getting the below error. What is the correct way to stream to Postgres over SSL.
2022-10-25 09:40:48,416 - root [INFO] - Exporting with MultiItemExporter 2022-10-25 09:40:48,416 - root [ERROR] - An exception occurred while syncing block data. Traceback (most recent call last): File "/usr/local/lib/python3.10/site-packages/blockchainetl/streaming/streamer.py", line 77, in _do_stream synced_blocks = self._sync_cycle() File "/usr/local/lib/python3.10/site-packages/blockchainetl/streaming/streamer.py", line 98, in _sync_cycle self.blockchain_streamer_adapter.export_all(self.last_synced_block + 1, target_block) File "/usr/local/lib/python3.10/site-packages/ethereumetl/streaming/eth_streamer_adapter.py", line 103, in export_all self.item_exporter.export_items(all_items) File "/usr/local/lib/python3.10/site-packages/blockchainetl/jobs/exporters/multi_item_exporter.py", line 34, in export_items exporter.export_items(items) File "/usr/local/lib/python3.10/site-packages/blockchainetl/jobs/exporters/postgres_item_exporter.py", line 49, in export_items connection = self.engine.connect() File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 3066, in connect return self._connection_cls(self, close_with_result=close_with_result) File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 91, in __init__ else engine.raw_connection() File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 3145, in raw_connection return self._wrap_pool_connect(self.pool.connect, _connection) File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 3112, in _wrap_pool_connect return fn() File "/usr/local/lib/python3.10/site-packages/sqlalchemy/pool/base.py", line 301, in connect return _ConnectionFairy._checkout(self) File "/usr/local/lib/python3.10/site-packages/sqlalchemy/pool/base.py", line 748, in _checkout fairy = _ConnectionRecord.checkout(pool) File "/usr/local/lib/python3.10/site-packages/sqlalchemy/pool/base.py", line 419, in checkout rec = pool._do_get() File "/usr/local/lib/python3.10/site-packages/sqlalchemy/pool/impl.py", line 144, in _do_get with util.safe_reraise(): File "/usr/local/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__ compat.raise_( File "/usr/local/lib/python3.10/site-packages/sqlalchemy/util/compat.py", line 198, in raise_ raise exception File "/usr/local/lib/python3.10/site-packages/sqlalchemy/pool/impl.py", line 142, in _do_get return self._create_connection() File "/usr/local/lib/python3.10/site-packages/sqlalchemy/pool/base.py", line 247, in _create_connection return _ConnectionRecord(self) File "/usr/local/lib/python3.10/site-packages/sqlalchemy/pool/base.py", line 362, in __init__ self.__connect(first_connect_check=True) File "/usr/local/lib/python3.10/site-packages/sqlalchemy/pool/base.py", line 597, in __connect with util.safe_reraise(): File "/usr/local/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__ compat.raise_( File "/usr/local/lib/python3.10/site-packages/sqlalchemy/util/compat.py", line 198, in raise_ raise exception File "/usr/local/lib/python3.10/site-packages/sqlalchemy/pool/base.py", line 592, in __connect connection = pool._invoke_creator(self) File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/create.py", line 576, in connect return dialect.connect(*cargs, **cparams) File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/default.py", line 548, in connect return self.dbapi.connect(*cargs, **cparams) TypeError: connect() got an unexpected keyword argument 'sslmode'
This is regarding the ethereum-etl[streaming] to Postgres over SSL: I tried using this format 'postgresql+pg8000://username:password@host/db_name?sslmode=require' as --output and I am getting the below error. What is the correct way to stream to Postgres over SSL.
2022-10-25 09:40:48,416 - root [INFO] - Exporting with MultiItemExporter 2022-10-25 09:40:48,416 - root [ERROR] - An exception occurred while syncing block data. Traceback (most recent call last): File "/usr/local/lib/python3.10/site-packages/blockchainetl/streaming/streamer.py", line 77, in _do_stream synced_blocks = self._sync_cycle() File "/usr/local/lib/python3.10/site-packages/blockchainetl/streaming/streamer.py", line 98, in _sync_cycle self.blockchain_streamer_adapter.export_all(self.last_synced_block + 1, target_block) File "/usr/local/lib/python3.10/site-packages/ethereumetl/streaming/eth_streamer_adapter.py", line 103, in export_all self.item_exporter.export_items(all_items) File "/usr/local/lib/python3.10/site-packages/blockchainetl/jobs/exporters/multi_item_exporter.py", line 34, in export_items exporter.export_items(items) File "/usr/local/lib/python3.10/site-packages/blockchainetl/jobs/exporters/postgres_item_exporter.py", line 49, in export_items connection = self.engine.connect() File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 3066, in connect return self._connection_cls(self, close_with_result=close_with_result) File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 91, in __init__ else engine.raw_connection() File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 3145, in raw_connection return self._wrap_pool_connect(self.pool.connect, _connection) File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 3112, in _wrap_pool_connect return fn() File "/usr/local/lib/python3.10/site-packages/sqlalchemy/pool/base.py", line 301, in connect return _ConnectionFairy._checkout(self) File "/usr/local/lib/python3.10/site-packages/sqlalchemy/pool/base.py", line 748, in _checkout fairy = _ConnectionRecord.checkout(pool) File "/usr/local/lib/python3.10/site-packages/sqlalchemy/pool/base.py", line 419, in checkout rec = pool._do_get() File "/usr/local/lib/python3.10/site-packages/sqlalchemy/pool/impl.py", line 144, in _do_get with util.safe_reraise(): File "/usr/local/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__ compat.raise_( File "/usr/local/lib/python3.10/site-packages/sqlalchemy/util/compat.py", line 198, in raise_ raise exception File "/usr/local/lib/python3.10/site-packages/sqlalchemy/pool/impl.py", line 142, in _do_get return self._create_connection() File "/usr/local/lib/python3.10/site-packages/sqlalchemy/pool/base.py", line 247, in _create_connection return _ConnectionRecord(self) File "/usr/local/lib/python3.10/site-packages/sqlalchemy/pool/base.py", line 362, in __init__ self.__connect(first_connect_check=True) File "/usr/local/lib/python3.10/site-packages/sqlalchemy/pool/base.py", line 597, in __connect with util.safe_reraise(): File "/usr/local/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__ compat.raise_( File "/usr/local/lib/python3.10/site-packages/sqlalchemy/util/compat.py", line 198, in raise_ raise exception File "/usr/local/lib/python3.10/site-packages/sqlalchemy/pool/base.py", line 592, in __connect connection = pool._invoke_creator(self) File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/create.py", line 576, in connect return dialect.connect(*cargs, **cparams) File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/default.py", line 548, in connect return self.dbapi.connect(*cargs, **cparams) TypeError: connect() got an unexpected keyword argument 'sslmode'