data loading error in 2019 at Balancing inter-provincial trade

[rehananis]

2023-09-07 17:36:15,195 - openIO-Canada - INFO - Reading all the Excel files... 2023-09-07 17:36:44,026 - openIO-Canada - INFO - Formatting the Supply and Use tables... 2023-09-07 17:38:40,612 - openIO-Canada - INFO - Modifying names of duplicated sectors... 2023-09-07 17:38:41,344 - openIO-Canada - INFO - Organizing final demand sectors... 2023-09-07 17:38:43,674 - openIO-Canada - INFO - Removing IOIC codes from index... 2023-09-07 17:38:43,846 - openIO-Canada - INFO - Endogenizing capitals of OpenIO-Canada... 2023-09-07 17:39:56,995 - openIO-Canada - INFO - Balancing inter-provincial trade...

KeyError Traceback (most recent call last) Input In [4], in <cell line: 1>() ----> 1 tables = openIO.IOTables(folder_path='C://Users/ranis/Downloads/carbon/2019/Detail level/', 2 exiobase_folder='C://Users/ranis/Downloads/carbon/IOT_2019_pxp/', 3 endogenizing_capitals=True)

File C:\/Users/ranis/Downloads/carbon\openIO.py:165, in IOTables.init(self, folder_path, exiobase_folder, endogenizing_capitals) 162 self.endogenizing_capitals() 164 logger.info("Balancing inter-provincial trade...") --> 165 self.province_import_export( 166 pd.read_excel( 167 folder_path+[i for i in [j for j in os.walk(folder_path)][0][2] if 'Provincial_trade_flow' in i][0], 168 'Data')) 170 if self.exiobase_folder: 171 logger.info('Pre-treatment of international trade data...')

File C:\/Users/ranis/Downloads/carbon\openIO.py:724, in IOTables.province_import_export(self, province_trade_file) 719 scaled_imports_K = ((import_distribution_K.T import_markets.fillna(0).loc[ 720 importing_province, exporting_province]).T).reindex(import_distribution_K.index).fillna(0) 721 scaled_imports_Y = ((import_distribution_Y.T import_markets.fillna(0).loc[ 722 importing_province, exporting_province]).T).reindex(import_distribution_Y.index).fillna(0) --> 724 self.assert_order(exporting_province, importing_province, scaled_imports_U, scaled_imports_Y, 725 scaled_imports_K) 727 # assign new values into self.U 728 self.U.loc[exporting_province, importing_province] = ( 729 scaled_imports_U.loc[:, self.U.columns.levels[1]].reindex( 730 self.U.loc[exporting_province, importing_province].columns, axis=1).values)

File C:\/Users/ranis/Downloads/carbon\openIO.py:2490, in IOTables.assert_order(self, exporting_province, importing_province, scaled_imports_U, scaled_imports_Y, scaled_imports_K) 2482 assert all(self.K.loc[exporting_province, importing_province].index == 2483 scaled_imports_K.loc[:, self.K.columns.levels[1]].reindex( 2484 self.K.loc[exporting_province, importing_province].columns, axis=1).index) 2485 assert all(self.K.loc[exporting_province, importing_province].columns == 2486 scaled_imports_K.loc[:, self.K.columns.levels[1]].reindex( 2487 self.K.loc[exporting_province, importing_province].columns, axis=1).columns) 2489 assert all(self.Y.loc[exporting_province, importing_province].index == -> 2490 scaled_imports_Y.loc[:, self.Y.columns.levels[1]].reindex( 2491 self.Y.loc[exporting_province, importing_province].columns, axis=1).index) 2492 assert all(self.Y.loc[exporting_province, importing_province].columns == 2493 scaled_imports_Y.loc[:, self.Y.columns.levels[1]].reindex( 2494 self.Y.loc[exporting_province, importing_province].columns, axis=1).columns)

File ~\AppData\Roaming\Python\Python39\site-packages\pandas\core\indexing.py:1147, in _LocationIndexer.getitem(self, key) 1145 if self._is_scalar_access(key): 1146 return self.obj._get_value(*key, takeable=self._takeable) -> 1147 return self._getitem_tuple(key) 1148 else: 1149 # we by definition only have the 0th axis 1150 axis = self.axis or 0

File ~\AppData\Roaming\Python\Python39\site-packages\pandas\core\indexing.py:1330, in _LocIndexer._getitem_tuple(self, tup) 1328 with suppress(IndexingError): 1329 tup = self._expand_ellipsis(tup) -> 1330 return self._getitem_lowerdim(tup) 1332 # no multi-index, so validate all of the indexers 1333 tup = self._validate_tuple_indexer(tup)

File ~\AppData\Roaming\Python\Python39\site-packages\pandas\core\indexing.py:1015, in _LocationIndexer._getitem_lowerdim(self, tup) 1013 # we may have a nested tuples indexer here 1014 if self._is_nested_tuple_indexer(tup): -> 1015 return self._getitem_nested_tuple(tup) 1017 # we maybe be using a tuple to represent multiple dimensions here 1018 ax0 = self.obj._get_axis(0)

File ~\AppData\Roaming\Python\Python39\site-packages\pandas\core\indexing.py:1127, in _LocationIndexer._getitem_nested_tuple(self, tup) 1124 axis -= 1 1125 continue -> 1127 obj = getattr(obj, self.name)._getitem_axis(key, axis=axis) 1128 axis -= 1 1130 # if we have a scalar, we are done

File ~\AppData\Roaming\Python\Python39\site-packages\pandas\core\indexing.py:1382, in _LocIndexer._getitem_axis(self, key, axis) 1379 if hasattr(key, "ndim") and key.ndim > 1: 1380 raise ValueError("Cannot index with multidimensional key") -> 1382 return self._getitem_iterable(key, axis=axis) 1384 # nested tuple slicing 1385 if is_nested_tuple(key, labels):

File ~\AppData\Roaming\Python\Python39\site-packages\pandas\core\indexing.py:1322, in _LocIndexer._getitem_iterable(self, key, axis) 1319 self._validate_key(key, axis) 1321 # A collection of keys -> 1322 keyarr, indexer = self._get_listlike_indexer(key, axis) 1323 return self.obj._reindex_with_indexers( 1324 {axis: [keyarr, indexer]}, copy=True, allow_dups=True 1325 )

File ~\AppData\Roaming\Python\Python39\site-packages\pandas\core\indexing.py:1520, in _LocIndexer._get_listlike_indexer(self, key, axis) 1517 ax = self.obj._get_axis(axis) 1518 axis_name = self.obj._get_axis_name(axis) -> 1520 keyarr, indexer = ax._get_indexer_strict(key, axis_name) 1522 return keyarr, indexer

File ~\AppData\Roaming\Python\Python39\site-packages\pandas\core\indexes\multi.py:2648, in MultiIndex._get_indexer_strict(self, key, axis_name) 2645 if len(keyarr) and not isinstance(keyarr[0], tuple): 2646 indexer = self._get_indexer_level_0(keyarr) -> 2648 self._raise_if_missing(key, indexer, axis_name) 2649 return self[indexer], indexer 2651 return super()._get_indexer_strict(key, axis_name)

File ~\AppData\Roaming\Python\Python39\site-packages\pandas\core\indexes\multi.py:2669, in MultiIndex._raise_if_missing(self, key, indexer, axis_name) 2666 raise KeyError(f"{keyarr[cmask]} not in index") 2667 # We get here when levels still contain values which are not 2668 # actually in Index anymore -> 2669 raise KeyError(f"{keyarr} not in index") 2670 else: 2671 return super()._raise_if_missing(key, indexer, axis_name)

KeyError: "Index(['Changes in inventories', 'Governments final consumption expenditure',\n 'Gross fixed capital formation, Construction',\n 'Gross fixed capital formation, Intellectual property products',\n 'Gross fixed capital formation, Machinery and equipment',\n 'Household final consumption expenditure', 'International exports',\n 'Non-profit institutions serving households' final consumption expenditure'],\n dtype='object') not in index"

[MaximeAgez]

Hi @rehananis. Do you have the pandas package to the specified version in the requirements.txt file? I believe this error arises with newer version of pandas.

[MaximeAgez]

Was the issue resolved?

[rehananis]

Yes, but i stuck at some other point. I am trying to produce demo results, I used 2019 data. I tried to export tables using the export function from OpenIO file in jupyter notebook but the export file is empty. Later I start calculating the QC household consumption command but it said data doesn’t exist Best regards RehanOn Sep 10, 2023, at 1:36 AM, Maxime Agez @.***> wrote: Was the issue resolved?

—Reply to this email directly, view it on GitHub, or unsubscribe.You are receiving this because you were mentioned.Message ID: @.***>

[MaximeAgez]

Yeah the export function is outdated. You can use the pickle format to store all dataframes from the system and reload everything

import pickle
import gzip

with gzip.open("path_where_to_store_the_data","rb") as f:
     pickle.dump(self, f)

You can then reimport the saved system with:

with gzip.open("path_where_to_store_the_data","wb") as f:
     self = pickle.load(f)

[rehananis]

Hi, thanks for your help but I am still unable to export results. File "", line 4 with gzip.open("F:\carbon\","rb") as f: ^ SyntaxError: EOL while scanning string literal

[MaximeAgez]

When using the \ instead of the /, you need to add r in front of the path. Also you should create a .gz.pickle file so add .gz.pickle at the end of your file path such as: with gzip.open(r"F:\carbon\openio.gz.pickle","rb") as f:

[rehananis]

import pickle import gzip ​ with gzip.open("F:/carbon/openio.gz.pickle", 'rb') as f: pickle.dump(self, f)

FileNotFoundError Traceback (most recent call last)

in 2 import gzip 3 ----> 4 with gzip.open("F:/carbon/openio.gz.pickle", 'rb') as f: 5 pickle.dump(self, f) c:\users\rehan\appdata\local\programs\python\python39\lib\gzip.py in open(filename, mode, compresslevel, encoding, errors, newline) 56 gz_mode = mode.replace("t", "") 57 if isinstance(filename, (str, bytes, os.PathLike)): ---> 58 binary_file = GzipFile(filename, gz_mode, compresslevel) 59 elif hasattr(filename, "read") or hasattr(filename, "write"): 60 binary_file = GzipFile(None, gz_mode, compresslevel, filename) c:\users\rehan\appdata\local\programs\python\python39\lib\gzip.py in __init__(self, filename, mode, compresslevel, fileobj, mtime) 171 mode += 'b' 172 if fileobj is None: --> 173 fileobj = self.myfileobj = builtins.open(filename, mode or 'rb') 174 if filename is None: 175 filename = getattr(fileobj, 'name', '') FileNotFoundError: [Errno 2] No such file or directory: 'F:/carbon/openio.gz.pickle'

[MaximeAgez]

Try to first create the file on the C disk. That might be the reason

[rehananis]

I solve it to save use

import pickle import gzip

with gzip.open("F:/carbon/openio1.gz.pickle", 'wb') as f: pickle.dump(tables, f)

[rehananis]

I use this to reload in another file sucessfully import sys sys.path.append('F://carbon/') import openIO import pandas as pd import numpy as np

with gzip.open("F:/carbon/openio.gz.pickle", 'rb') as f: tables = pickle.load(f)

[rehananis]

thanks for your help. Your openIO is very useful for me to understand the IO Tables and do some analysis. thank you again for your quick response.