如何提取音频特征

[tailangjun]

我刚试了一下实例代码

# fairseq 使用
import torch
import torch.nn.functional as F
import soundfile as sf
from fairseq import checkpoint_utils

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

model_path=""
wav_path=""

def postprocess(feats, normalize=False):
    if feats.dim() == 2:
        feats = feats.mean(-1)

    assert feats.dim() == 1, feats.dim()

    if normalize:
        with torch.no_grad():
            feats = F.layer_norm(feats, feats.shape)
    return feats

print("loading model(s) from {}".format(model_path))
models, saved_cfg, task = checkpoint_utils.load_model_ensemble_and_task(
    [model_path],
    suffix="",
)
print("loaded model(s) from {}".format(model_path))
print(f"normalize: {saved_cfg.task.normalize}")

model = models[0]
model = model.to(device)
model = model.half()
model.eval()

wav, sr = sf.read(wav_path)
feat = torch.from_numpy(wav).float()
feat = postprocess(feat, normalize=saved_cfg.task.normalize)
feats = feat.view(1, -1)
padding_mask = (
    torch.BoolTensor(feats.shape).fill_(False)
)
inputs = {
    "source": feats.half().to(device),
    "padding_mask": padding_mask.to(device),
}

with torch.no_grad():
    logits = model.extract_features(**inputs)
    print(logits)

返回是这样子的 不知道是不是把这块的数据提取出来做reshape就可以了

[tailangjun]

感觉就是这个了，不知道对不对

# numpy
[[[-0.1685  -0.00496  0.1835  ... -0.1613   0.0525  -0.2196 ]
  [-0.333    0.1344   0.06665 ... -0.3665   0.2031   0.1443 ]
  [-0.4343   0.07825  0.1136  ... -0.3713   0.2035   0.1287 ]
  ...
  [-0.2524  -0.1392   0.1398  ... -0.2316  -0.114   -0.08954]
  [-0.3652  -0.1733   0.3245  ... -0.11316 -0.01608  0.05212]
  [-0.2258  -0.188    0.4443  ... -0.05884  0.2161  -0.0806 ]]]

# shape
(1, 492, 1024)

[xiaofanku]

我想问你一下, model_path是指: chinese-wav2vec2-large-fairseq-ckpt.pt这个文件的路径吗