editing the demo file

[abdoelsayed2016]

Hello

thank you for you sharing code but i have question becuase the results of Nq-open and squad have error when i tested them

i changed the code of run demo to make evaultion for the test dataset without using server and using this commad line

python run_demo_edit.py --run_mode eval_request --index_port 51997 --test_path $DATA_DIR/open-qa/squad/test_preprocessed.json --eval_batch_size 64 --save_pred --truecase --cuda --index_name start/1048576_flat_OPQ96 --dump_dir $SAVE_DIR/densephrases-multi_wiki-20181220/dump/

the code work fine but the result is very bad i dont know what the reason

03/07/2023 10:46:24 - INFO - eval_phrase_retrieval -   EM: 0.22, F1: 0.45
03/07/2023 10:46:24 - INFO - eval_phrase_retrieval -   1) Who got the first Nobel Prize in physics
03/07/2023 10:46:24 - INFO - eval_phrase_retrieval -   => groundtruths: ['Wilhelm Conrad Röntgen'], top 5 prediction: ['G', 'He', 'Chad', 'W', 'Davis']
03/07/2023 10:46:24 - INFO - eval_phrase_retrieval -   2) When is the next Deadpool movie being released
03/07/2023 10:46:24 - INFO - eval_phrase_retrieval -   => groundtruths: ['May 18 , 2018'], top 5 prediction: ['214', '[', '2099', 'Pet', '2010']
03/07/2023 10:46:24 - INFO - eval_phrase_retrieval -   3) Which mode is used for short wave broadcast service
03/07/2023 10:46:24 - INFO - eval_phrase_retrieval -   => groundtruths: ['MFSK', 'Olivia'], top 5 prediction: ['98', 'O', '136', "Schöbel's", '990",']
03/07/2023 10:46:26 - INFO - eval_phrase_retrieval -   {'exact_match_top1': 0.22160664819944598, 'f1_score_top1': 0.4542936288088642}
03/07/2023 10:46:26 - INFO - eval_phrase_retrieval -   {'exact_match_top10': 0.9418282548476454, 'f1_score_top10': 2.4090489381348115}
03/07/2023 10:46:26 - INFO - eval_phrase_retrieval -   {'redundancy of top10': 1.4556786703601108}

and i notice that the prediction is only one word here is an example of the predication and true label

Bond ['lament on various worldwide problems']
mon ['peptide bond']
1943 ['1952']
Shahi ['Akbar the Great', 'Babur']

some time give one character

here is my edit code

import os
os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"
import json
import argparse
import torch
import os
import random
import numpy as np
import requests
import logging

from tqdm import tqdm
from time import time
from flask import Flask, request, jsonify, render_template, redirect
from flask_cors import CORS
from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop
from requests_futures.sessions import FuturesSession

from eval_phrase_retrieval import evaluate_results, evaluate_results_kilt
from densephrases.utils.single_utils import load_encoder
from densephrases.utils.data_utils import TrueCaser
from densephrases.utils.open_utils import load_phrase_index, load_cross_encoder, load_qa_pairs, get_query2vec
from densephrases import Options
from concurrent.futures import Future

logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s', datefmt='%m/%d/%Y %H:%M:%S',
                    level=logging.INFO)
logger = logging.getLogger(__name__)

class DensePhrasesDemo(object):
    def __init__(self, args, inmemory=False, batch_size=64, query_encoder=None, tokenizer=None):
        self.args = args
        self.base_ip = args.base_ip
        self.query_port = args.query_port
        self.index_port = args.index_port
        self.truecase = TrueCaser(os.path.join(os.environ['DATA_DIR'], args.truecase_path))
        self.args.examples_path = os.path.join('densephrases/demo/static', args.examples_path)
        self.mips = load_phrase_index(args)
        self.device = 'cuda' if args.cuda else 'cpu'
        if query_encoder is None:
            self.query_encoder, self.tokenizer, _ = load_encoder(self.device, args, query_only=True)
        self.query2vec = get_query2vec(
            query_encoder=self.query_encoder, tokenizer=self.tokenizer, args=args, batch_size=batch_size
        )

    def query2vec_api(self,x):
            batch_query = json.loads(x['query'])
            #print(batch_query)
            # start_time = time()
            outs = list(self.query2vec(batch_query))
            #print(outs)
            # logger.info(f'query2vec {time()-start_time:.3f} for {len(batch_query)} queries: {batch_query[0]}')
            #outs = json.dumps(outs)
            return outs #json.loads(outs)#jsonify(outs)

    def api(self,x):
            query = x['query']
            query = query[:-1] if query.endswith('?') else query
            if args.truecase:
                if query[1:].lower() == query[1:]:
                    query = self.truecase.get_true_case(query)
            out = batch_search(
                [query],
                max_answer_length=args.max_answer_length,
                top_k=args.top_k,
                nprobe=args.nprobe,
            )
            out['ret'] = out['ret'][0]
            #outs = json.loads(json.dumps(outs))
            return outs
            #return jsonify(out)

    def get_examples():
            with open(args.examples_path, 'r') as fp:
                examples = [line.strip() for line in fp.readlines()]
            examples = json.loads(json.dumps(examples))
            return #jsonify(examples)

    def batch_search(self, batch_query, max_answer_length=20, top_k=10,
                         nprobe=64, return_vecs=False):
            t0 = time()
            outs, _ = self.embed_query(batch_query)
            start = np.concatenate([out[0] for out in outs], 0)
            end = np.concatenate([out[1] for out in outs], 0)
            query_vec = np.concatenate([start, end], 1)

            rets = self.mips.search(
                query_vec, q_texts=batch_query, nprobe=nprobe,
                top_k=top_k, max_answer_length=max_answer_length,
                return_vecs=return_vecs, aggregate=True,
            )
            for ret_idx, ret in enumerate(rets):
                for rr in ret:
                    rr['query_tokens'] = outs[ret_idx][2]
            t1 = time()
            out = {'ret': rets, 'time': int(1000 * (t1 - t0))}
            return out
    def batch_api(self, post_data):
            batch_query = json.loads(post_data['query'])
            max_answer_length = int(post_data['max_answer_length'])
            top_k = int(post_data['top_k'])
            nprobe = int(post_data['nprobe'])
            out = self.batch_search(
                batch_query,
                max_answer_length=max_answer_length,
                top_k=top_k,
                nprobe=nprobe,
            )
            return out #json.loads(json.dumps(out)) #jsonify(out)
    def get_address(self, port):
        assert self.base_ip is not None and len(port) > 0
        return self.base_ip + ':' + port

    """def embed_query(self, batch_query):
        #emb_session = FuturesSession()
        data={'query': json.dumps(batch_query)}
        r = self.query2vec_api(data) #emb_session.post(self.get_address(self.query_port) + '/query2vec_api',data={'query': json.dumps(batch_query)})
        print(type(r))
        def map_():
            result = r.result()
            emb = result.json()
            return emb, result.elapsed.total_seconds() * 1000
        return map_"""
    def embed_query(self, batch_query):
        data = {'query': json.dumps(batch_query)}
        r = self.query2vec_api(data)

        return r,0

    def query(self, query):
        params = {'query': query}
        res = requests.get(self.get_address(self.index_port) + '/api', params=params)

        try:
            outs = json.loads(res)
        except Exception as e:
            logger.info(f'no response or error for q {query}')
            logger.info(res)
        return outs

    def batch_query(self, batch_query, batch_context=None, max_answer_length=20, top_k=10, nprobe=64):
        post_data = {
            'query': json.dumps(batch_query),
            'context': json.dumps(batch_context) if batch_context is not None else json.dumps(batch_query),
            'max_answer_length': max_answer_length,
            'top_k': top_k,
            'nprobe': nprobe,
        }
        res = self.batch_api(post_data)#requests.post(self.get_address(self.index_port) + '/batch_api', data=post_data)
        """if res.status_code != 200:
            logger.info('Wrong behavior %d' % res.status_code)"""
        try:
            outs = json.loads(res)
        except Exception as e:
            logger.info(f'no response or error for q {batch_query}')
            logger.info(res)
        return res

    def eval_request(self, args):
        # Load dataset
        qids, questions, answers, _ = load_qa_pairs(args.test_path, args)

        # Run batch_query and evaluate
        step = args.eval_batch_size
        predictions = []
        evidences = []
        titles = []
        scores = []
        start_time = None
        num_q = 0
        for q_idx in tqdm(range(0, len(questions), step)):
            if q_idx >= 5*step: # exclude warmup
                if start_time is None:
                    start_time = time()
                num_q += len(questions[q_idx:q_idx+step])
            result = self.batch_query(
                questions[q_idx:q_idx+step],
                max_answer_length=args.max_answer_length,
                top_k=args.top_k,
                nprobe=args.nprobe,
            )
            prediction = [[ret['answer'] for ret in out] if len(out) > 0 else [''] for out in result['ret']]
            evidence = [[ret['context'] for ret in out] if len(out) > 0 else [''] for out in result['ret']]
            title = [[ret['title'] for ret in out] if len(out) > 0 else [''] for out in result['ret']]
            score = [[ret['score'] for ret in out] if len(out) > 0 else [-1e10] for out in result['ret']]
            predictions += prediction
            evidences += evidence
            titles += title
            scores += score
        logger.info(f'{time()-start_time:.3f} sec for {num_q} questions => {num_q/(time()-start_time):.1f} Q/Sec')
        eval_fn = evaluate_results if not args.is_kilt else evaluate_results_kilt
        eval_fn(
            predictions, qids, questions, answers, args, evidences=evidences, scores=scores, titles=titles,
        )

if __name__ == '__main__':
    # See options in densephrases.options
    options = Options()
    options.add_model_options()
    options.add_index_options()
    options.add_retrieval_options()
    options.add_data_options()
    options.add_demo_options()
    args = options.parse()

    # Seed for reproducibility
    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    if torch.cuda.is_available():
        torch.cuda.manual_seed_all(args.seed)

    server = DensePhrasesDemo(args)

    if args.run_mode == 'q_serve':
        logger.info(f'Query address: {server.get_address(server.query_port)}')
        server.serve_query_encoder(args.query_port, args)

    elif args.run_mode == 'p_serve':
        logger.info(f'Index address: {server.get_address(server.index_port)}')
        server.serve_phrase_index(args.index_port, args)

    elif args.run_mode == 'query':
        query = 'Name three famous writers'
        result = server.query(query)
        logger.info(f'Answers to a question: {query}')
        logger.info(f'{[r["answer"] for r in result["ret"]]}')

    elif args.run_mode == 'batch_query':
        queries = [
            'Where',
            'when did medicare begin in the united states',
            'who sings don\'t stand so close to me',
            'Name three famous writers',
            'Who was defeated by computer in chess game?'
        ]
        contexts = [
            'Uncle jesse\'s original full name was James Lee. And he was born in South Korea.',
            'Uncle jesse\'s original name was James Lee. And he was born in South Korea in 1333. US medicare started in 1222.',
            'Uncle jesse\'s original name was James Lee. And he sang this song.',
            'Uncle jesse\'s original name was James Lee. And Jens wrote this novel.',
            'Uncle jesse\'s original name was James Lee. The man was defeated by Alphago.'
        ]
        result = server.batch_query(
            queries,
            contexts, # feed context for cross encoders
            max_answer_length=args.max_answer_length,
            top_k=args.top_k,
            nprobe=args.nprobe,
        )
        for query, re in zip(queries, result['ret'].values()):
            logger.info(f'Answers to a question: {query}')
            logger.info(f'{re}')

    elif args.run_mode == 'eval_request':
        print("sadadadada")
        server.eval_request(args)

    else:
        raise NotImplementedError

[abdoelsayed2016]

i fixed it and it works i forgot to add load_dir but it worked only one time after i tired to run it again i got this error the new command is

python run_demo_edit.py --run_mode eval_request --index_port 51997 --test_path $DATA_DIR/open-qa/squad/test_preprocessed.json --eval_batch_size 64 --save_pred --truecase --cuda --index_name start/1048576_flat_OPQ96 --dump_dir $SAVE_DIR/densephrases-multi_wiki-20181220/dump/ --load_dir princeton-nlp/densephrases-multi-query-multi --max_answer_length 50

could anyone please help me to fixed this error

File "run_demo_edit.py", line 45, in __init__
    self.query_encoder, self.tokenizer, _ = load_encoder(self.device, args, query_only=True)
  File "..../....../.../DensePhrases/densephrases/utils/single_utils.py", line 95, in load_encoder
    model = load_class(
  File "...../.../..../.conda/envs/densephrases-v1.1.0/lib/python3.8/site-packages/transformers/modeling_utils.py", line 1294, in from_pretrained
    raise EnvironmentError(
OSError: Error no file named ['pytorch_model.bin', 'tf_model.h5', 'model.ckpt.index', 'flax_model.msgpack'] found in directory princeton-nlp/densephrases-multi-query-multi or `from_tf` and `from_flax` set to False.

[abdoelsayed2016]

i figure out the problem but i dont know how to solve it i tired different model from https://huggingface.co/princeton-nlp it is works for first time but when i run the code again it gave the error above

File "run_demo_edit.py", line 45, in __init__
    self.query_encoder, self.tokenizer, _ = load_encoder(self.device, args, query_only=True)
  File "..../....../.../DensePhrases/densephrases/utils/single_utils.py", line 95, in load_encoder
    model = load_class(
  File "...../.../..../.conda/envs/densephrases-v1.1.0/lib/python3.8/site-packages/transformers/modeling_utils.py", line 1294, in from_pretrained
    raise EnvironmentError(
OSError: Error no file named ['pytorch_model.bin', 'tf_model.h5', 'model.ckpt.index', 'flax_model.msgpack'] found in directory princeton-nlp/densephrases-multi-query-multi or `from_tf` and `from_flax` set to False.

[abdoelsayed2016]

i fixed it by downloading the models from hugging face