src/audio_models/model.py

import os
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import Wav2Vec2Config

from .torch_utils import get_mask_from_lengths
from .wav2vec2 import Wav2Vec2Model


class Audio2MeshModel(nn.Module):
    def __init__(
        self,
        config
    ):  
        super().__init__()
        out_dim = config['out_dim']
        latent_dim = config['latent_dim']
        model_path = config['model_path']
        only_last_fetures = config['only_last_fetures']
        from_pretrained = config['from_pretrained']

        self._only_last_features = only_last_fetures

        self.audio_encoder_config = Wav2Vec2Config.from_pretrained(model_path, local_files_only=True)
        if from_pretrained:
            self.audio_encoder = Wav2Vec2Model.from_pretrained(model_path, local_files_only=True)
        else:
            self.audio_encoder = Wav2Vec2Model(self.audio_encoder_config)
        self.audio_encoder.feature_extractor._freeze_parameters()

        hidden_size = self.audio_encoder_config.hidden_size

        self.in_fn = nn.Linear(hidden_size, latent_dim)
       
        self.out_fn = nn.Linear(latent_dim, out_dim)
        nn.init.constant_(self.out_fn.weight, 0)
        nn.init.constant_(self.out_fn.bias, 0)

    def forward(self, audio, label, audio_len=None):
        attention_mask = ~get_mask_from_lengths(audio_len) if audio_len else None
        
        seq_len = label.shape[1]

        embeddings = self.audio_encoder(audio, seq_len=seq_len, output_hidden_states=True,
                                        attention_mask=attention_mask)

        if self._only_last_features:
            hidden_states = embeddings.last_hidden_state
        else:
            hidden_states = sum(embeddings.hidden_states) / len(embeddings.hidden_states)

        layer_in = self.in_fn(hidden_states)
        out = self.out_fn(layer_in)

        return out, None

    def infer(self, input_value, seq_len):
        embeddings = self.audio_encoder(input_value, seq_len=seq_len, output_hidden_states=True)

        if self._only_last_features:
            hidden_states = embeddings.last_hidden_state
        else:
            hidden_states = sum(embeddings.hidden_states) / len(embeddings.hidden_states)

        layer_in = self.in_fn(hidden_states)
        out = self.out_fn(layer_in)

        return out