module 'd2l.torch' has no attribute 'count_corpus'

[wangze1219]

import math import os import random import torch from d2l import torch as d2l import os import matplotlib.pyplot as plt

os.environ["KMP_DUPLICATE_LIB_OK"]="TRUE"

@save

d2l.DATA_HUB['ptb'] = (d2l.DATA_URL + 'ptb.zip', '319d85e578af0cdc590547f26231e4e31cdf1e42')

@save

def read_ptb(): """将PTB数据集加载到文本行的列表中""" data_dir = d2l.download_extract('ptb')

Readthetrainingset.

with open(os.path.join(data_dir, 'ptb.train.txt')) as f:
    raw_text = f.read()
return [line.split() for line in raw_text.split('\n')]

sentences = read_ptb() f'# sentences数: {len(sentences)}' vocab = d2l.Vocab(sentences, min_freq=10) f'vocab size: {len(vocab)}'

@save

def subsample(sentences, vocab): """下采样高频词"""

排除未知词元''

sentences = [[token for token in line if vocab[token] != vocab.unk]
             for line in sentences]
counter = d2l.count_corpus(sentences)
num_tokens = sum(counter.values())

# 如果在下采样期间保留词元，则返回True
def keep(token):
    return(random.uniform(0, 1) <
           math.sqrt(1e-4 / counter[token] * num_tokens))

return ([[token for token in line if keep(token)] for line in sentences],
        counter)

subsampled, counter = subsample(sentences, vocab) d2l.show_list_len_pair_hist( ['origin', 'subsampled'], '# tokens per sentence', 'count', sentences, subsampled) plt.show()

[AnirudhDagar]

Please pip install d2l==0.17.6 to use older version of d2l which has these saved functions. In the latest version, we refactored the code and removed them.

[zaffnet]

What a nightmare. So much backward incompatibility! Could you mention which version to use at the beginning of each chapter?

[AnirudhDagar]

If you are using the latest version of the book then it should work with the latest d2l package.