Running this one single test passes for SQLAlchemy==1.2:
(env36) hiebert@pcic-2004:~/code/modelmeta$ py.test -v -x -k test_find_or_insert_data_file_variable_dsg
============================================== test session starts ===============================================
platform linux -- Python 3.6.8, pytest-4.6.3, py-1.8.0, pluggy-0.12.0 -- /home/hiebert/code/modelmeta/env36/bin/python3.6
cachedir: .pytest_cache
rootdir: /home/hiebert/code/modelmeta, inifile: pytest.ini, testpaths: tests
plugins: alembic-verify-0.1.4
collected 515 items / 513 deselected / 2 selected
tests/mm_cataloguer/test_index.py::test_find_or_insert_data_file_variable_dsg[streamflow-False] PASSED [ 50%]
tests/mm_cataloguer/test_index.py::test_find_or_insert_data_file_variable_dsg[streamflow-True] PASSED [100%]
================================================ warnings summary ================================================
tests/mm_cataloguer/test_index.py::test_find_or_insert_data_file_variable_dsg[streamflow-False]
/home/hiebert/code/modelmeta/env36/lib/python3.6/site-packages/sqlalchemy/orm/relationships.py:1426: SAWarning: Got None for value of column data_files.data_file_id; this is unsupported for a relationship comparison and will not currently produce an IS comparison (but may in a future release)
"(but may in a future release)" % column)
-- Docs: https://docs.pytest.org/en/latest/warnings.html
============================== 2 passed, 513 deselected, 1 warnings in 3.56 seconds ==============================
But fails in 1.3. Guess we should have heeded the warning above!
(env36) hiebert@pcic-2004:~/code/modelmeta$ pip install SQLAlchemy==1.3
Collecting SQLAlchemy==1.3
Downloading https://files.pythonhosted.org/packages/35/9e/5eb467ed50cdd8e88b808a7e65045020fa12b3b9c2ab51de0f452d269d4d/SQLAlchemy-1.3.0.tar.gz (5.9MB)
|████████████████████████████████| 5.9MB 804kB/s
Installing collected packages: SQLAlchemy
Found existing installation: SQLAlchemy 1.2.0
Uninstalling SQLAlchemy-1.2.0:
Successfully uninstalled SQLAlchemy-1.2.0
Running setup.py install for SQLAlchemy ... done
Successfully installed SQLAlchemy-1.3.0
(env36) hiebert@pcic-2004:~/code/modelmeta$ py.test -v -x -k test_find_or_insert_data_file_variable_dsg
============================================== test session starts ===============================================
platform linux -- Python 3.6.8, pytest-4.6.3, py-1.8.0, pluggy-0.12.0 -- /home/hiebert/code/modelmeta/env36/bin/python3.6
cachedir: .pytest_cache
rootdir: /home/hiebert/code/modelmeta, inifile: pytest.ini, testpaths: tests
plugins: alembic-verify-0.1.4
collected 515 items / 513 deselected / 2 selected
tests/mm_cataloguer/test_index.py::test_find_or_insert_data_file_variable_dsg[streamflow-False] FAILED [ 50%]
==================================================== FAILURES ====================================================
__________________________ test_find_or_insert_data_file_variable_dsg[streamflow-False] __________________________
self = <sqlalchemy.engine.base.Connection object at 0x7fac5f2f0a58>
dialect = <sqlalchemy.dialects.postgresql.psycopg2.PGDialect_psycopg2 object at 0x7fac5fc8a080>
constructor = <bound method DefaultExecutionContext._init_compiled of <class 'sqlalchemy.dialects.postgresql.psycopg2.PGExecutionContext_psycopg2'>>
statement = <sqlalchemy.dialects.postgresql.psycopg2.PGCompiler_psycopg2 object at 0x7fac5f2f0ac8>
parameters = [immutabledict({})]
args = (<sqlalchemy.dialects.postgresql.psycopg2.PGCompiler_psycopg2 object at 0x7fac5f2f0ac8>, [immutabledict({})])
conn = <sqlalchemy.pool.base._ConnectionFairy object at 0x7fac5f2f0a20>
def _execute_context(
self, dialect, constructor, statement, parameters, *args
):
"""Create an :class:`.ExecutionContext` and execute, returning
a :class:`.ResultProxy`."""
try:
try:
conn = self.__connection
except AttributeError:
# escape "except AttributeError" before revalidating
# to prevent misleading stacktraces in Py3K
conn = None
if conn is None:
conn = self._revalidate_connection()
> context = constructor(dialect, self, conn, *args)
env36/lib/python3.6/site-packages/sqlalchemy/engine/base.py:1179:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cls = <class 'sqlalchemy.dialects.postgresql.psycopg2.PGExecutionContext_psycopg2'>
dialect = <sqlalchemy.dialects.postgresql.psycopg2.PGDialect_psycopg2 object at 0x7fac5fc8a080>
connection = <sqlalchemy.engine.base.Connection object at 0x7fac5f2f0a58>
dbapi_connection = <sqlalchemy.pool.base._ConnectionFairy object at 0x7fac5f2f0a20>
compiled = <sqlalchemy.dialects.postgresql.psycopg2.PGCompiler_psycopg2 object at 0x7fac5f2f0ac8>
parameters = [immutabledict({})]
@classmethod
def _init_compiled(
cls, dialect, connection, dbapi_connection, compiled, parameters
):
"""Initialize execution context for a Compiled construct."""
self = cls.__new__(cls)
self.root_connection = connection
self._dbapi_connection = dbapi_connection
self.dialect = connection.dialect
self.compiled = compiled
# this should be caught in the engine before
# we get here
assert compiled.can_execute
self.execution_options = compiled.execution_options.union(
connection._execution_options
)
self.result_column_struct = (
compiled._result_columns,
compiled._ordered_columns,
compiled._textual_ordered_columns,
)
self.unicode_statement = util.text_type(compiled)
if not dialect.supports_unicode_statements:
self.statement = self.unicode_statement.encode(
self.dialect.encoding
)
else:
self.statement = self.unicode_statement
self.isinsert = compiled.isinsert
self.isupdate = compiled.isupdate
self.isdelete = compiled.isdelete
self.is_text = compiled.isplaintext
if not parameters:
self.compiled_parameters = [compiled.construct_params()]
else:
self.compiled_parameters = [
compiled.construct_params(m, _group_number=grp)
> for grp, m in enumerate(parameters)
]
env36/lib/python3.6/site-packages/sqlalchemy/engine/default.py:683:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
.0 = <enumerate object at 0x7fac5f282708>
compiled.construct_params(m, _group_number=grp)
> for grp, m in enumerate(parameters)
]
env36/lib/python3.6/site-packages/sqlalchemy/engine/default.py:683:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <sqlalchemy.dialects.postgresql.psycopg2.PGCompiler_psycopg2 object at 0x7fac5f2f0ac8>
params = immutabledict({}), _group_number = 0, _check = True
def construct_params(self, params=None, _group_number=None, _check=True):
"""return a dictionary of bind parameter keys and values"""
if params:
pd = {}
for bindparam in self.bind_names:
name = self.bind_names[bindparam]
if bindparam.key in params:
pd[name] = params[bindparam.key]
elif name in params:
pd[name] = params[name]
elif _check and bindparam.required:
if _group_number:
raise exc.InvalidRequestError(
"A value is required for bind parameter %r, "
"in parameter group %d"
% (bindparam.key, _group_number),
code="cd3x",
)
else:
raise exc.InvalidRequestError(
"A value is required for bind parameter %r"
% bindparam.key,
code="cd3x",
)
elif bindparam.callable:
pd[name] = bindparam.effective_value
else:
pd[name] = bindparam.value
return pd
else:
pd = {}
for bindparam in self.bind_names:
if _check and bindparam.required:
if _group_number:
raise exc.InvalidRequestError(
"A value is required for bind parameter %r, "
"in parameter group %d"
% (bindparam.key, _group_number),
code="cd3x",
)
else:
raise exc.InvalidRequestError(
"A value is required for bind parameter %r"
% bindparam.key,
code="cd3x",
)
if bindparam.callable:
> pd[self.bind_names[bindparam]] = bindparam.effective_value
env36/lib/python3.6/site-packages/sqlalchemy/sql/compiler.py:692:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = BindParameter('%(140378308285384 param)s', None, type_=Integer())
@property
def effective_value(self):
"""Return the value of this bound parameter,
taking into account if the ``callable`` parameter
was set.
The ``callable`` value will be evaluated
and returned if present, else ``value``.
"""
if self.callable:
> return self.callable()
env36/lib/python3.6/site-packages/sqlalchemy/sql/elements.py:1213:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
def _go():
last_known = to_return = state._last_known_values[prop.key]
existing_is_available = last_known is not attributes.NO_VALUE
# we support that the value may have changed. so here we
# try to get the most recent value including re-fetching.
# only if we can't get a value now due to detachment do we return
# the last known value
current_value = mapper._get_state_attr_by_column(
state,
dict_,
column,
passive=attributes.PASSIVE_RETURN_NEVER_SET
if state.persistent
else attributes.PASSIVE_NO_FETCH ^ attributes.INIT_OK,
)
if current_value is attributes.NEVER_SET:
if not existing_is_available:
raise sa_exc.InvalidRequestError(
"Can't resolve value for column %s on object "
"%s; no value has been set for this column"
> % (column, state_str(state))
E sqlalchemy.exc.InvalidRequestError: Can't resolve value for column data_files.data_file_id on object <DataFile at 0x7fac5f418518>; no value has been set for this column
env36/lib/python3.6/site-packages/sqlalchemy/orm/relationships.py:1580: InvalidRequestError
The above exception was the direct cause of the following exception:
test_session_with_empty_db = <sqlalchemy.orm.session.Session object at 0x7fac5fcaef28>
tiny_dsg_dataset = <class 'nchelpers.CFDataset'>
root group (NETCDF4 data model, file format HDF5):
title: RVIC history file
comm...omp_ind(outlets), |S1 outlet_name(outlets,nc_chars), float32 streamflow(time,outlets)
groups:
insert = False, data_file_1 = <modelmeta.v2.DataFile object at 0x7fac5f418518>
def test_find_or_insert_data_file_variable_dsg(
test_session_with_empty_db, tiny_dsg_dataset, insert, data_file_1):
var_name = tiny_dsg_dataset.dependent_varnames()[0]
variable = tiny_dsg_dataset.variables[var_name]
# data_file = insert_data_file(test_session_with_empty_db, tiny_dsg_dataset)
dfv = check_find_or_insert(
find_or_insert_data_file_variable,
cond_insert_data_file_variable_dsg_plus,
test_session_with_empty_db,
tiny_dsg_dataset,
var_name,
data_file_1,
> invoke=insert
)
tests/mm_cataloguer/test_index.py:756:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
tests/mm_cataloguer/test_index.py:127: in check_find_or_insert
thing_found_or_inserted = find_or_insert_thing(*args)
mm_cataloguer/index_netcdf.py:964: in find_or_insert_data_file_variable
dfv = find_data_file_variable(sesh, cf, var_name, data_file)
mm_cataloguer/index_netcdf.py:770: in find_data_file_variable
return q.first()
env36/lib/python3.6/site-packages/sqlalchemy/orm/query.py:3214: in first
ret = list(self[0:1])
env36/lib/python3.6/site-packages/sqlalchemy/orm/query.py:3006: in __getitem__
return list(res)
env36/lib/python3.6/site-packages/sqlalchemy/orm/query.py:3316: in __iter__
return self._execute_and_instances(context)
env36/lib/python3.6/site-packages/sqlalchemy/orm/query.py:3341: in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
env36/lib/python3.6/site-packages/sqlalchemy/engine/base.py:988: in execute
return meth(self, multiparams, params)
env36/lib/python3.6/site-packages/sqlalchemy/sql/elements.py:287: in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
env36/lib/python3.6/site-packages/sqlalchemy/engine/base.py:1107: in _execute_clauseelement
distilled_params,
env36/lib/python3.6/site-packages/sqlalchemy/engine/base.py:1182: in _execute_context
e, util.text_type(statement), parameters, None, None
env36/lib/python3.6/site-packages/sqlalchemy/engine/base.py:1466: in _handle_dbapi_exception
util.raise_from_cause(sqlalchemy_exception, exc_info)
env36/lib/python3.6/site-packages/sqlalchemy/util/compat.py:383: in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
env36/lib/python3.6/site-packages/sqlalchemy/util/compat.py:128: in reraise
raise value.with_traceback(tb)
env36/lib/python3.6/site-packages/sqlalchemy/engine/base.py:1179: in _execute_context
context = constructor(dialect, self, conn, *args)
env36/lib/python3.6/site-packages/sqlalchemy/engine/default.py:683: in _init_compiled
for grp, m in enumerate(parameters)
env36/lib/python3.6/site-packages/sqlalchemy/engine/default.py:683: in <listcomp>
for grp, m in enumerate(parameters)
env36/lib/python3.6/site-packages/sqlalchemy/sql/compiler.py:692: in construct_params
pd[self.bind_names[bindparam]] = bindparam.effective_value
env36/lib/python3.6/site-packages/sqlalchemy/sql/elements.py:1213: in effective_value
return self.callable()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
def _go():
last_known = to_return = state._last_known_values[prop.key]
existing_is_available = last_known is not attributes.NO_VALUE
# we support that the value may have changed. so here we
# try to get the most recent value including re-fetching.
# only if we can't get a value now due to detachment do we return
# the last known value
current_value = mapper._get_state_attr_by_column(
state,
dict_,
column,
passive=attributes.PASSIVE_RETURN_NEVER_SET
if state.persistent
else attributes.PASSIVE_NO_FETCH ^ attributes.INIT_OK,
)
if current_value is attributes.NEVER_SET:
if not existing_is_available:
raise sa_exc.InvalidRequestError(
"Can't resolve value for column %s on object "
"%s; no value has been set for this column"
> % (column, state_str(state))
E sqlalchemy.exc.StatementError: (sqlalchemy.exc.InvalidRequestError) Can't resolve value for column data_files.data_file_id on object <DataFile at 0x7fac5f418518>; no value has been set for this column
E [SQL: SELECT data_file_variables_dsg_time_series.data_file_variable_dsg_ts_id AS data_file_variables_dsg_time_series_data_file_variable_ds_1, data_file_variables.data_file_variable_id AS data_file_variables_data_file_variable_id, data_file_variables.geometry_type AS data_file_variables_geometry_type, data_file_variables.derivation_method AS data_file_variables_derivation_method, data_file_variables.variable_cell_methods AS data_file_variables_variable_cell_methods, data_file_variables.netcdf_variable_name AS data_file_variables_netcdf_variable_name, data_file_variables.disabled AS data_file_variables_disabled, data_file_variables.range_min AS data_file_variables_range_min, data_file_variables.range_max AS data_file_variables_range_max, data_file_variables.data_file_id AS data_file_variables_data_file_id, data_file_variables.variable_alias_id AS data_file_variables_variable_alias_id, data_files_1.data_file_id AS data_files_1_data_file_id, data_files_1.filename AS data_files_1_filename, data_files_1.first_1mib_md5sum AS data_files_1_first_1mib_md5sum, data_files_1.unique_id AS data_files_1_unique_id, data_files_1.x_dim_name AS data_files_1_x_dim_name, data_files_1.y_dim_name AS data_files_1_y_dim_name, data_files_1.z_dim_name AS data_files_1_z_dim_name, data_files_1.t_dim_name AS data_files_1_t_dim_name, data_files_1.index_time AS data_files_1_index_time, data_files_1.run_id AS data_files_1_run_id, data_files_1.time_set_id AS data_files_1_time_set_id, runs_1.run_id AS runs_1_run_id, runs_1.run_name AS runs_1_run_name, runs_1.driving_run AS runs_1_driving_run, runs_1.initialized_from AS runs_1_initialized_from, runs_1.model_id AS runs_1_model_id, runs_1.emission_id AS runs_1_emission_id, runs_1.project AS runs_1_project, emissions_1.emission_id AS emissions_1_emission_id, emissions_1.emission_long_name AS emissions_1_emission_long_name, emissions_1.emission_short_name AS emissions_1_emission_short_name, models_1.model_id AS models_1_model_id, models_1.model_long_name AS models_1_model_long_name, models_1.model_short_name AS models_1_model_short_name, models_1.model_organization AS models_1_model_organization, models_1.type AS models_1_type
E FROM data_file_variables JOIN data_file_variables_dsg_time_series ON data_file_variables.data_file_variable_id = data_file_variables_dsg_time_series.data_file_variable_dsg_ts_id LEFT OUTER JOIN data_files AS data_files_1 ON data_files_1.data_file_id = data_file_variables.data_file_id LEFT OUTER JOIN runs AS runs_1 ON runs_1.run_id = data_files_1.run_id LEFT OUTER JOIN emissions AS emissions_1 ON emissions_1.emission_id = runs_1.emission_id LEFT OUTER JOIN models AS models_1 ON models_1.model_id = runs_1.model_id
E WHERE %(param_1)s = data_file_variables.data_file_id AND data_file_variables.netcdf_variable_name = %(netcdf_variable_name_1)s
E LIMIT %(param_2)s]
E [parameters: [{}]]
env36/lib/python3.6/site-packages/sqlalchemy/orm/relationships.py:1580: StatementError
==================================== 1 failed, 513 deselected in 3.46 seconds ====================================
Running this one single test passes for
SQLAlchemy==1.2:But fails in
1.3. Guess we should have heeded the warning above!