more helpful error message about run format?

[cmacdonald]

Can we improve the error message? What does the run contain/type, what are the expected formats?

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-219-30de4914907d> in <module>
----> 1 eval_rq1(dataset.get_topics("test-2019"), dataset.get_qrels("test-2019"), METRICS)

<ipython-input-218-6a00610862d6> in eval_rq1(topics, qrels, metrics, cutoff, batch_size)
     25         baseline=0,
     26         correction='b',
---> 27         names=['baseline'] + names,
     28     )
     29 

/opt/conda/lib/python3.7/site-packages/pyterrier/pipelines.py in Experiment(retr_systems, topics, qrels, eval_metrics, names, perquery, dataframe, batch_size, drop_unused, baseline, test, correction, correction_alpha, highlight, round)
    306     # run and evaluate each system
    307     for name,system in zip(names, retr_systems):
--> 308         time, evalMeasuresDict = _run_and_evaluate(system, topics, qrels, eval_metrics, perquery=perquery or baseline is not None, batch_size=batch_size)
    309 
    310         if perquery or baseline is not None:

/opt/conda/lib/python3.7/site-packages/pyterrier/pipelines.py in _run_and_evaluate(system, topics, qrels, metrics, perquery, batch_size)
    126             rev_mapping,
    127             num_q,
--> 128             perquery)
    129 
    130     elif batch_size is None:

/opt/conda/lib/python3.7/site-packages/pyterrier/pipelines.py in _ir_measures_to_dict(seq, metrics, rev_mapping, num_q, perquery)
     90         # qid -> measure -> value
     91         rtr=defaultdict(dict)
---> 92         for m in seq:
     93             metric = m.measure
     94             metric = rev_mapping.get(metric, str(metric))

/opt/conda/lib/python3.7/site-packages/ir_measures/providers/base.py in _iter_calc(self, measures, qrels, run)
     33         # Implementations must either provide _calc_ctxt or _iter_calc
     34         with self.calc_ctxt(measures, qrels) as _iter_calc:
---> 35             yield from _iter_calc(run)
     36 
     37     def calc_aggregate(self, measures, qrels, run):

/opt/conda/lib/python3.7/site-packages/ir_measures/providers/fallback_provider.py in _iter_calc(run)
     36                 runs_teed = RunConverter(run).tee(len(contexts))
     37                 for ctxt, run in zip(contexts, runs_teed):
---> 38                     yield from ctxt(run.run)
     39             yield _iter_calc
     40 

/opt/conda/lib/python3.7/site-packages/ir_measures/providers/pytrec_eval_provider.py in _iter_calc(run)
     55         def _iter_calc(run):
     56             # Convert qrels to dict_of_dict (input format used by pytrec_eval)
---> 57             run = ir_measures.util.RunConverter(run).as_dict_of_dict()
     58             for invoker in invokers:
     59                 yield from invoker.iter_calc(run)

/opt/conda/lib/python3.7/site-packages/ir_measures/util.py in as_dict_of_dict(self)
    132         else:
    133             result = {}
--> 134             for scored_doc in self.as_namedtuple_iter():
    135                 if scored_doc.query_id not in result:
    136                     result[scored_doc.query_id] = {}

/opt/conda/lib/python3.7/site-packages/ir_measures/util.py in as_namedtuple_iter(self)
    149             yield from self.run.itertuples()
    150         if t == 'UNKNOWN':
--> 151             raise ValueError('unknown run format')
    152 
    153     def as_pd_dataframe(self):

ValueError: unknown run format

[cmacdonald]

In this case, the problem is a missing docno column (which probably becomes doc_id)

There is a debate as to whether PyTerrier should have detected this.

[seanmacavaney]

Good call- @cmacdonald can you review #15 when you have a chance?

The error message I went with was:

unknown run format: DataFrame missing columns: ['doc_id'] (found ['query_id', 'docno', 'score'])

(similar message when qrels format is wrong)