Upload codes

config.py

@@ -0,0 +1,34 @@
+import json
+import yaml
+
+def get_config_universal(dataset_name):
+    with open('./configs/' + dataset_name + '_config.json') as f:
+        config = json.load(f)
+    return config
+
+
+def get_sweep_config_universal(dataset_name):
+    with open('./configs/sweep_' + dataset_name + '_config.yaml') as f:
+        config = yaml.load(f,Loader=yaml.FullLoader)
+    return config
+
+
+def get_config():
+    with open('configs/camargo_config.json') as f:
+        config = json.load(f)
+    return config
+
+def get_kiha_config():
+    with open('./configs/kiha_config.json') as f:
+        config = json.load(f)
+    return config
+
+def get_model_config(model_config):
+    with open(f'./configs/{model_config}.json') as f:
+        config = json.load(f)
+    return config
+
+def get_sweep_config():
+    with open('configs/sweep_camargo_config.yaml') as f:
+        config = yaml.load(f,Loader=yaml.FullLoader)
+    return config

dataset.py

@@ -0,0 +1,261 @@
+import numpy as np
+import pandas as pd
+from loading.loadpickledataset import LoadPickleDataSet
+from preprocessing.augmentation.gaussiannoise import GaussianNoise
+from preprocessing.augmentation.imurotation import IMURotation
+from preprocessing.filter_imu import FilterIMU
+from preprocessing.filter_opensim import FilterOpenSim
+from preprocessing.remove_outlier import remove_outlier
+from preprocessing.resample import Resample
+from preprocessing.segmentation.fixwindowsegmentation import FixWindowSegmentation
+from preprocessing.segmentation.gaitcyclesegmentation import GaitCycleSegmentation
+from preprocessing.segmentation.zeropaddingsegmentation import ZeroPaddingSegmentation
+
+
+class DataSet:
+    def __init__(self, config, load_dataset=True):
+        self.config = config
+        self.x = []
+        self.y = []
+        self.labels = []
+        self.selected_trial_type = config['selected_trial_type']
+        self.selected_activity_label = config['selected_activity_label']
+        self.segmentation_method = config['segmentation_method']
+        if self.config['gc_dataset']:
+            self.segmentation_method = 'zeropadding'
+        self.resample = config['resample']
+        self.n_sample = len(self.y)
+        if load_dataset:
+            self.load_dataset()
+            self.train_subjects = config['train_subjects']
+            self.test_subjects = config['test_subjects']
+            self.train_activity = config['train_activity']
+            self.test_activity = config['test_activity']
+        # self.winsize = 128
+        self.train_dataset = {}
+        self.test_dataset = {}
+
+    def load_dataset(self):
+        getdata_handler = LoadPickleDataSet(self.config)
+        x, y, labels = getdata_handler.run_get_dataset()
+        self.x, self.y, self.labels = self.run_activity_based_filter(x, y, labels)
+        self._preprocess()
+
+    def _preprocess(self):
+        self.x, self.y, self.labels = remove_outlier(self.x, self.y, self.labels)
+        if self.resample:
+            self.x, self.y, self.labels = self.run_resample_signal(self.x, self.y, self.labels)
+        if self.config['opensim_filter']:
+            filteropensim_handler = FilterOpenSim(self.y, lowcut=6, fs=100, order=2)
+            self.y = filteropensim_handler.run_lowpass_filter()
+        if self.config['imu_filter']:
+            filterimu_handler = FilterIMU(self.x, lowcut=10, fs=100, order=2)
+            self.x = filterimu_handler.run_lowpass_filter()
+
+
+    def run_resample_signal(self, x, y, labels):
+        resample_handler = Resample(x, y, labels, 200, 100)
+        x, y, labels = resample_handler._run_resample()
+        return x, y, labels
+
+    def run_segmentation(self, x, y, labels):
+        if self.segmentation_method == 'fixedwindow':
+            segmentation_handler = FixWindowSegmentation(x, y, labels, winsize=self.config['target_padding_length'], overlap=0.5, start_over=True)
+            self.x, self.y, self.labels = segmentation_handler._run_segmentation()
+
+        elif self.segmentation_method == 'zeropadding':
+            segmentation_handler = ZeroPaddingSegmentation(x, y, labels, target_padding_length=self.config['target_padding_length'], start_over=True)
+            self.x, self.y, self.labels = segmentation_handler._run_segmentation()
+
+        elif self.segmentation_method == 'gaitcycle':
+            segmentation_handler = GaitCycleSegmentation(x, y, labels, winsize=128, overlap=0.5, start_over=True)
+            self.x, self.y, self.labels = segmentation_handler._run_segmentation()
+
+        if self.config['opensim_filter']:
+            filteropensim_handler = FilterOpenSim(self.y, lowcut=6, fs=100, order=2)
+            self.y = filteropensim_handler.run_lowpass_filter()
+
+        if self.config['rotation']:
+            imu_rotation_handler = IMURotation(knom=10)
+            self.x, self.y, self.labels = imu_rotation_handler.run_rotation(self.x.copy(), self.y.copy(), self.labels.copy())
+
+        if self.config['gaussian_noise']:
+            gaussian_noise_handler = GaussianNoise(0, .05)
+            self.x, self.y, self.labels = gaussian_noise_handler.run_add_noise(self.x, self.y, self.labels)
+        del x, y, labels
+        return self.x, self.y, self.labels
+
+    def run_activity_based_filter(self, x, y, label):
+        '''
+        :return: updated x, y, and labels which contains only the selected labels (activity section)
+        '''
+        updated_x = []
+        update_y = []
+        updated_label = []
+        s = 0
+        for ll, xx, yy, in zip(label, x, y):
+            # print(ll['subject'][0])
+            # print(ll['trialNum'][0])
+            if self.config['dataset_name']=='camargo' and ll['trialType'].isin(self.selected_trial_type).all() and self.selected_activity_label == ['all_idle']:
+                l_temp = ll[ll['trialType'].isin(self.selected_trial_type)]
+                l_temp_index = l_temp.index.values
+                xx_temp = xx[l_temp_index]
+                yy_temp = yy[l_temp_index]
+
+                updated_x.append(xx_temp)
+                update_y.append(yy_temp)
+                updated_label.append(l_temp)
+            elif self.config['dataset_name']=='camargo' and ll['trialType'].isin(self.selected_trial_type).all() and self.selected_activity_label == ['all']:
+                update_selected_activity_label = list(ll['Label'].unique())
+                update_selected_activity_label = [i for i in update_selected_activity_label if i not in ['idle', 'stand']]
+                l_temp = ll[(ll['trialType'].isin(self.selected_trial_type)) & (ll['Label'].isin(update_selected_activity_label))]
+                l_temp_index = l_temp.index.values
+                xx_temp = xx[l_temp_index]
+                yy_temp = yy[l_temp_index]
+                updated_x.append(xx_temp)
+                update_y.append(yy_temp)
+                updated_label.append(l_temp)
+
+            elif self.config['dataset_name'] == 'camargo' and ll['trialType'].isin(self.selected_trial_type).all() and self.selected_activity_label == ['all_split']:
+                ll_temp = ll.copy()
+                ll_temp['trialType2'] =ll_temp['Label']
+                if ll['trialType'][0] =='levelground':
+                    # get the turn index if it's there
+                    turn1_indx = ll_temp[ll_temp['Label'] == 'turn1'].index.values
+                    turn2_indx = ll_temp[ll_temp['Label'] == 'turn2'].index.values
+                    # check which turn is turn 1
+                    if turn1_indx[0]<turn2_indx[0]:
+                        pass
+                    else:
+                        turn2_indx_temp = turn1_indx
+                        turn1_indx = turn2_indx
+                        turn2_indx = turn2_indx_temp
+                    # devide into two segments
+                    seg1 = ll_temp.iloc[0:turn1_indx[-1]+1]
+                    seg2 = ll_temp.iloc[turn2_indx[0]:]
+                    seg1_trialType2 = seg1['trialType2'].replace({'idle': 'idle', 'stand': 'idle', 'turn1': 'idle', 'turn2': 'idle',
+                                                                           'stand-walk':'levelground1', 'walk':'levelground1',
+                                                                           'walk-stand': 'levelground1'})
+                    seg2_trialType2 = seg2['trialType2'].replace({'idle': 'idle', 'stand': 'idle', 'turn1': 'idle','turn2': 'idle',
+                                                                           'stand-walk':'levelground2', 'walk':'levelground2',
+                                                                           'walk-stand': 'levelground2'})
+                    ll_temp['trialType2'] = pd.concat([seg1_trialType2, seg2_trialType2])
+                    ll = ll_temp
+                elif ll['trialType'][0] =='ramp':
+                    ll_temp['trialType2'] = ll_temp['trialType2'].replace({'idle': 'idle',
+                              'walk-rampascent': 'rampascent', 'rampascent':'rampascent','rampascent-walk': 'rampascent',
+                              'walk-rampdescent': 'rampdescent', 'rampdescent':'rampdescent','rampdescent-walk': 'rampdescent'})
+                    ll = ll_temp
+                elif ll['trialType'][0] == 'stair':
+                    ll_temp['trialType2'] = ll_temp['trialType2'].replace({'idle': 'idle',
+                              'walk-stairascent': 'stairascent', 'stairascent':'stairascent','stairascent-walk': 'stairascent',
+                              'walk-stairdescent': 'stairdescent', 'stairdescent':'stairdescent','stairdescent-walk': 'stairdescent'})
+                    ll = ll_temp
+
+                update_selected_activity_label = list(ll['trialType2'].unique())
+                # remove stand, idle, turn1, turn2 samples
+                update_selected_activity_label = [i for i in update_selected_activity_label if
+                                                  i not in ['idle']]
+                for activity_label in update_selected_activity_label:
+                    # if trial type == levelground ->save stand-walk and walk into one trial and walk-stand into another trial. all samples would be continues
+                    # if ramp or stair--> save trial for ascent and descent individually
+                    if isinstance(activity_label, str):
+                        l_temp = ll[(ll['trialType'].isin(self.selected_trial_type)) & (ll['trialType2']==activity_label)]
+                        l_temp_index = l_temp.index.values
+                        xx_temp = xx[l_temp_index]
+                        yy_temp = yy[l_temp_index]
+                        updated_x.append(xx_temp)
+                        update_y.append(yy_temp)
+                        updated_label.append(l_temp)
+                    if len(xx_temp)==0:
+                        print(i)
+            elif self.config['dataset_name']=='camargo':
+                l_temp = ll[(ll['trialType'].isin(self.selected_trial_type)) & (ll['Label'].isin(self.selected_activity_label))]
+                l_temp_index = l_temp.index.values
+                xx_temp = xx[l_temp_index]
+                yy_temp = yy[l_temp_index]
+
+                updated_x.append(xx_temp)
+                update_y.append(yy_temp)
+                updated_label.append(l_temp)
+            elif self.config['dataset_name']=='kiha':
+                l_temp = ll[(ll['trialType'].isin(self.selected_trial_type))]
+                l_temp_index = l_temp.index.values
+                xx_temp = xx[l_temp_index]
+                yy_temp = yy[l_temp_index]
+
+
+                updated_x.append(xx_temp)
+                update_y.append(yy_temp)
+                updated_label.append(l_temp)
+            # else:
+            #     continue
+        return updated_x, update_y, updated_label
+
+    def concatenate_data(self):
+        self.labels = pd.concat(self.labels, axis=0, ignore_index = True)
+        self.x = np.concatenate(self.x, axis=0)
+        self.y = np.concatenate(self.y, axis=0)
+
+    def run_dataset_split_loop(self):
+        train_labels = []
+        test_labels = []
+        train_x = []
+        train_y = []
+        test_x = []
+        test_y = []
+        for t, trial in enumerate(self.labels):
+            if all(trial['subject'].isin(self.train_subjects)) and all(trial['trialType2'].isin(self.train_activity)):
+                train_labels.append(trial)
+                train_x.append(self.x[t])
+                train_y.append(self.y[t])
+
+            elif all(trial['subject'].isin(self.test_subjects)) and all(trial['trialType2'].isin(self.test_activity)):
+                test_labels.append(trial)
+                test_x.append(self.x[t])
+                test_y.append(self.y[t])
+
+        self.train_dataset['x'] = train_x
+        self.train_dataset['y'] = train_y
+        self.train_dataset['labels'] = train_labels
+
+        self.test_dataset['x'] = test_x
+        self.test_dataset['y'] = test_y
+        self.test_dataset['labels'] = test_labels
+        return self.train_dataset, self.test_dataset
+
+    def run_dataset_split(self):
+        if set(self.test_subjects).issubset(self.train_subjects):
+             train_labels = self.labels[~self.labels['subject'].isin(self.test_subjects)]
+             test_labels = self.labels[(self.labels['subjects'].isin(self.test_subjects))]
+        else:
+             train_labels = self.labels[self.labels['subject'].isin(self.train_subjects)]
+             test_labels = self.labels[(self.labels['subject'].isin(self.test_subjects))]
+        print(train_labels['subject'].unique())
+        print(test_labels['subject'].unique())
+
+
+        train_index = train_labels.index.values
+        test_index = test_labels.index.values
+        print('training length', len(train_index))
+        print('test length', len(test_index))
+
+        train_x = self.x[train_index]
+        train_y = self.y[train_index]
+        # self.train_dataset['x'] = train_x.reshape([int(train_x.shape[0]/self.config['target_padding_length']), self.config['target_padding_length'], train_x.shape[1]])
+        # self.train_dataset['y'] = train_y.reshape([int(train_y.shape[0]/self.config['target_padding_length']), self.config['target_padding_length'], train_y.shape[1]])
+        self.train_dataset['x'] = train_x
+        self.train_dataset['y'] = train_y
+        self.train_dataset['labels'] = train_labels.reset_index(drop=True)
+
+        test_x = self.x[test_index]
+        test_y = self.y[test_index]
+        # self.test_dataset['x'] = test_x.reshape([int(test_x.shape[0]/self.config['target_padding_length']), self.config['target_padding_length'], test_x.shape[1]])
+        # self.test_dataset['y'] = test_y.reshape([int(test_y.shape[0]/self.config['target_padding_length']), self.config['target_padding_length'], test_y.shape[1]])
+        self.test_dataset['x'] = test_x
+        self.test_dataset['y'] = test_y
+        self.test_dataset['labels'] = test_labels.reset_index(drop=True)
+        del train_labels, test_labels, train_x, train_y, test_x, test_y
+        return self.train_dataset,  self.test_dataset
+
+

datasetbuilder.py

@@ -0,0 +1,62 @@
+import numpy as np
+import torch
+from torch.utils.data import Dataset
+from preprocessing.augmentation.gaussiannoise import GaussianNoise
+from preprocessing.transformation.transformation import Transformation
+import torch.nn.functional as F
+from sklearn import preprocessing
+import torch
+
+
+class DataSetBuilder(Dataset):
+    def __init__(self, x, y, labels, transform_method=None, scaler=None, noise=None, classification=None):
+        self.x = x
+        self.y = y
+        self.labels = labels
+        self.y_label = []
+
+        self.transform_method = transform_method
+        self.scaler = scaler
+        self.noise = noise
+        self.classification = classification
+        self._preprocess()
+        if self.classification:
+            self._run_label_encoding()
+        self.n_sample = len(y)
+
+        # x = np.transpose(self.x, (0, 2, 1))
+        self.x = torch.from_numpy(x).double()
+        self.y = torch.from_numpy(self.y).double()
+
+    def _run_label_encoding(self):
+        le = preprocessing.LabelEncoder()
+        y_label = le.fit_transform(self.labels[:, 0, 3])
+        y_label = torch.as_tensor(y_label)
+        # self.y_label = F.one_hot(y_label.to(torch.int64))
+        self.y_label = y_label.to(torch.int64)
+
+    def _preprocess(self):
+        if self.transform_method['data_transformer_method'] is not None:
+            self._run_transform()
+        if self.noise is not None:
+            self._run_noise()
+
+    def _run_transform(self):
+        transform_handler = Transformation(method=self.transform_method['data_transformer_method'], by=self.transform_method['data_transformer_by'])
+        if self.scaler is None:
+            self.scaler, self.x = transform_handler.run_transform(train=self.x, scaler_fit=self.scaler)
+        else:
+            self.x = transform_handler.run_transform(val=self.x, scaler_fit=self.scaler)
+
+    def _run_noise(self, ):
+        gaussiannoise_handler = GaussianNoise(mean=0, std=1)
+        self.x, self.y, self.labels = gaussiannoise_handler.run_add_noise(self.x, self.y, self.labels)
+
+    def __len__(self):
+        return self.n_sample
+
+    def __getitem__(self, item):
+        if self.classification:
+            return self.x[item], self.y[item], self.y_label[item]
+        else:
+            return self.x[item], self.y[item]

modelbuilder.py

@@ -0,0 +1,127 @@
+import torch
+from torch import nn
+# from tsai.models.TST import TST
+from sklearn.neighbors import KNeighborsRegressor
+from config import get_model_config
+from loss.weightedmseloss import WeightedMSELoss
+from loss.weightedmultioutputsloss import WeightedMultiOutputLoss
+from loss.weightedrmseloss import WeightedRMSELoss
+from model.Hernandez2021cnnlstm import Hernandez2021CNNLSTM
+from model.bilstmmodel import BiLSTMModel
+from model.cnnlstm import CNNLSTM
+from model.dorschky2020cnn import Dorschky2020CNN
+from model.gholami2020cnn import Gholami2020CNN
+from model.lstmlstm import Seq2Seq
+from model.lstmlstmattention import Seq2SeqAtt
+from model.lstmlstmrec import Seq2SeqRec
+from model.lstmmodel import LSTMModel
+from model.tcnmodel import TCNModel
+from model.transformer import Transformer
+from model.transformer_seq2seq import Seq2SeqTransformer
+from model.transformer_tsai import TransformerTSAI
+from model.zrenner2018cnn import Zrenner2018CNN
+from utils.update_config import update_model_config
+
+
+class ModelBuilder:
+    def __init__(self, config):
+        self.config = config
+        self.n_input_channel = len(self.config['selected_sensors'])*6
+        self.n_output = len(self.config['selected_opensim_labels'])
+        self.model_name = self.config['model_name']
+        self.model_config = get_model_config(f'config_{self.model_name}')
+        self.model_config = update_model_config(self.config, self.model_config)
+        self.optimizer_name = self.config['optimizer_name']
+        self.learning_rate = self.config['learning_rate']
+        self.l2_weight_decay_status = self.config['l2_weight_decay_status']
+        self.l2_weight_decay = self.config['l2_weight_decay']
+        self.loss = self.config['loss']
+        self.weight = self.config['loss_weight']
+        self.device = self.config['device']
+        # self.device = torch.device('cuda:2' if torch.cuda.is_available() else 'cpu')
+        if not self.n_output == len(self.weight):
+            self.weight = None
+
+    def run_model_builder(self):
+        model = self.get_model_architecture()
+        criterion = self.get_criterion(self.weight)
+        optimizer = self.get_optimizer()
+        return model, optimizer, criterion
+
+    def get_model_architecture(self):
+        if self.model_name == 'lstm': # done
+            self.model = LSTMModel(self.model_config)
+        elif self.model_name == 'bilstm': # done
+            self.model = BiLSTMModel(self.model_config)
+        elif self.model_name == 'cnnlstm': # done
+            self.model = CNNLSTM(self.model_config)
+        elif self.model_name == 'hernandez2021cnnlstm': # done
+            self.model = Hernandez2021CNNLSTM(self.model_config)
+        elif self.model_name == 'seq2seq': # done
+            self.model = Seq2Seq(self.config)
+        elif self.model_name == 'seq2seqrec':
+            self.model = Seq2SeqRec(self.n_input_channel, self.n_output)
+        elif self.model_name == 'seq2seqatt':# done
+            self.model = Seq2SeqAtt(self.model_config)
+        elif self.model_name == 'transformer': #done
+            self.model = Transformer(d_input=self.n_input_channel, d_model=12, d_output=self.n_output, d_len=self.config['target_padding_length'], h=8, N=1, attention_size=None,
+                  dropout=0.5, chunk_mode=None, pe='original', multihead=True)
+        elif self.model_name == 'seq2seqtransformer':
+            self.model = Seq2SeqTransformer(d_input=self.n_input_channel, d_model=24, d_output=self.n_output, h=8, N=4, attention_size=None,
+                  dropout=0.1, chunk_mode=None, pe='original')
+        elif self.model_name == 'transformertsai':
+            c_in = self.n_input_channel  # aka channels, features, variables, dimensions
+            c_out = self.n_output
+            seq_len = self.config['target_padding_length']
+            y_range = self.config['target_padding_length']
+            max_seq_len = self.config['target_padding_length']
+            d_model = self.model_config['tsai_d_model']
+            n_heads = self.model_config['tsai_n_heads']
+            d_k = d_v = None  # if None --> d_model // n_heads
+            d_ff = self.model_config['tsai_d_ff']
+            res_dropout = self.model_config['tsai_res_dropout_p']
+            activation = "gelu"
+            n_layers = self.model_config['tsai_n_layers']
+            fc_dropout = self.model_config['tsai_fc_dropout_p']
+            classification = self.model_config['classification']
+            kwargs = {}
+            self.model = TransformerTSAI(c_in, c_out, seq_len, max_seq_len=max_seq_len, d_model=d_model, n_heads=n_heads,
+                            d_k=d_k, d_v=d_v, d_ff=d_ff, res_dropout=res_dropout, act=activation, n_layers=n_layers,
+                            fc_dropout=fc_dropout, classification=classification,  **kwargs)
+        elif self.model_name == 'Gholami2020CNN':
+            self.model = Gholami2020CNN(self.model_config)
+        elif self.model_name == 'Dorschky2020CNN':
+            self.model = Dorschky2020CNN(self.model_config)
+        elif self.model_name == 'Zrenner2018CNN':
+            self.model = Zrenner2018CNN(self.model_config)
+        elif self.model_name == 'tcn':
+            self.model = TCNModel(self.model_config)
+        elif self.model_name == 'knn':
+            self.model = KNeighborsRegressor()
+        return self.model
+
+    def get_optimizer(self):
+        if self.optimizer_name == 'Adam':
+            if self.l2_weight_decay_status:
+                self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.learning_rate, weight_decay=self.l2_weight_decay)
+            else:
+                self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.learning_rate)
+        return self.optimizer
+
+    def get_criterion(self, weight=None):
+        if self.loss == 'RMSE' and weight is not None:
+            weight = torch.tensor(weight).to(self.device)
+            self.criterion = WeightedRMSELoss(weight)
+        elif self.loss == 'RMSE' and weight is None:
+            self.criterion = torch.sqrt(nn.MSELoss())
+        elif self.loss == 'MSE' and weight is not None:
+            weight = torch.tensor(weight).to(self.device)
+            self.criterion = WeightedMSELoss(weight)
+        elif self.loss == 'MSE-CE' and weight is not None:
+            weight = torch.tensor(weight).to(self.device)
+            self.criterion = WeightedMultiOutputLoss(weight)
+        else:
+            self.criterion = nn.MSELoss()
+        return self.criterion
+
+

test.py

@@ -0,0 +1,164 @@
+import torch
+import wandb
+
+from model.lstmlstm import Seq2SeqTest
+from model.lstmlstmattention import Seq2SeqAttTest
+from model.transformer_seq2seq import Seq2SeqTransformerTest
+from modelbuilder import ModelBuilder
+
+
+class Test:
+    def run_testing(self, config, model, test_dataloader):
+        self.config = config
+        self.device = config['device']
+        self.loss = self.config['loss']
+        self.weight = self.config['loss_weight']
+        self.model_name = self.config['model_name']
+        self.classification = config['classification']
+        self.n_output = len(self.config['selected_opensim_labels'])
+        if not self.n_output == len(self.weight):
+            self.weight = None
+        modelbuilder_handler = ModelBuilder(self.config)
+        criterion = modelbuilder_handler.get_criterion(self.weight)
+        self.tester = self.setup_tester()
+        y_pred, y_true, loss = self.tester(model, test_dataloader, criterion, self.device)
+        return y_pred, y_true, loss
+
+    def setup_tester(self):
+        if self.model_name == 'seq2seqatt':
+            tester = self.testing_seq2seqatt
+        elif self.model_name == 'seq2seqtransformer':
+            tester = self.testing_transformer_seq2seq
+        elif (self.model_name == 'transformer' and not self.classification) or (self.model_name == 'transformertsai' and not self.classification):
+            tester = self.testing_transformer
+        elif self.classification:
+            tester = self.testing_w_classification
+        else:
+            tester = self.testing
+        return tester
+
+    def testing(self, model, test_dataloader, criterion, device):
+        model.eval()
+        with torch.no_grad():
+            test_loss = []
+            test_preds = []
+            test_trues = []
+            for x, y in test_dataloader:
+                x = x.to(device)
+                y = y.to(device)
+                y_pred = model(x.float())
+                loss = criterion(y, y_pred)
+                test_loss.append(loss.item())
+                test_preds.append(y_pred)
+                test_trues.append(y)
+            test_loss = torch.mean(torch.tensor(test_loss))
+            print('Test Accuracy of the model: {}'.format(test_loss))
+        # wandb.log({"Test Loss": test_loss})
+        return torch.cat(test_preds, 0), torch.cat(test_trues, 0), test_loss
+
+    def testing_w_classification(self, model, test_dataloader, criterion, device):
+        model.eval()
+        with torch.no_grad():
+            test_loss = []
+            test_preds = []
+            test_trues = []
+            for x, y, y_label in test_dataloader:
+                x = x.to(device).float()
+                y_label = y_label.type(torch.LongTensor).to(device)  # The targets passed to nn.CrossEntropyLoss() should be in torch.long format
+                y = y.to(device)
+                y_pred = model(x)
+                y_pred[0] = y_pred[0].double()
+                y_pred[1] = y_pred[1].double()
+                y_true = [y, y_label]
+                loss = criterion(y_pred, y_true)
+                test_loss.append(loss.item())
+                test_preds.append(y_pred)
+                test_trues.append(y_true)
+            test_loss = torch.mean(torch.tensor(test_loss))
+            print('Test Accuracy of the model: {}'.format(test_loss))
+        wandb.log({"Test Loss": test_loss})
+        test_preds_reg = []
+        test_trues_reg = []
+        for pred, true in zip(test_preds, test_trues):
+            test_preds_reg.append(pred[0])
+            test_trues_reg.append(true[0])
+        return torch.cat(test_preds_reg, 0), torch.cat(test_trues_reg, 0), test_loss
+
+    def testing_seq2seq(self, model, test_dataloader, criterion, device):
+        model.eval()
+        with torch.no_grad():
+            test_loss = []
+            test_preds = []
+            test_trues = []
+            for x, y in test_dataloader:
+                x = x.to(device)
+                y = y.to(device)
+                # y_pred = model(x.float(), y.float())  # just for seq 2 seq
+                y_pred = Seq2SeqTest(model, x.float())
+                loss = criterion(y_pred[:, 1:, :].to(device), y[:, 1:, :])
+                test_loss.append(loss.item())
+                test_preds.append(y_pred)
+                test_trues.append(y)
+            test_loss = torch.mean(torch.tensor(test_loss))
+            print('Test Accuracy of the model: {}'.format(test_loss))
+        wandb.log({"Test Loss": test_loss})
+        return torch.cat(test_preds, 0), torch.cat(test_trues, 0), test_loss
+
+    def testing_seq2seqatt(self, model, test_dataloader, criterion, device):
+        model.eval()
+        with torch.no_grad():
+            test_loss = []
+            test_preds = []
+            test_trues = []
+            for x, y in test_dataloader:
+                x = x.to(device)
+                y = y.to(device)
+                # y_pred = model(x.float(), y.float())  # just for seq 2 seq
+                y_pred = Seq2SeqAttTest(model, x.float())
+                loss = criterion(y_pred[:, 1:, :].to(device), y[:, 1:, :])
+                test_loss.append(loss.item())
+                test_preds.append(y_pred)
+                test_trues.append(y)
+            test_loss = torch.mean(torch.tensor(test_loss))
+            print('Test Accuracy of the model: {}'.format(test_loss))
+        wandb.log({"Test Loss": test_loss})
+        return torch.cat(test_preds, 0), torch.cat(test_trues, 0), test_loss
+
+    def testing_transformer(self, model, test_dataloader, criterion, device):
+        model.eval()
+        with torch.no_grad():
+            test_loss = []
+            test_preds = []
+            test_trues = []
+            for x, y in test_dataloader:
+                x = x.to(device)
+                y = y.to(device)
+                y_pred = model(x.float())  # just for transformer
+                loss = criterion(y, y_pred.to(device))
+                test_loss.append(loss.item())
+                test_preds.append(y_pred)
+                test_trues.append(y)
+            test_loss = torch.mean(torch.tensor(test_loss))
+            print('Test Accuracy of the model: {}'.format(test_loss))
+        wandb.log({"Test Loss": test_loss})
+        return torch.cat(test_preds, 0), torch.cat(test_trues, 0), test_loss
+
+    def testing_transformer_seq2seq(self, model, test_dataloader, criterion, device):
+        model.eval()
+        with torch.no_grad():
+            test_loss = []
+            test_preds = []
+            test_trues = []
+            for x, y in test_dataloader:
+                x = x.to(device)
+                y = y.to(device)
+                y_pred = Seq2SeqTransformerTest(model, x.float())
+                # y_pred = model(x.float(), y.float()[:, :-1, :])  # just for seq 2 seq transformer
+                loss = criterion(y_pred, y.to(device))
+                test_loss.append(loss.item())
+                test_preds.append(y_pred)
+                test_trues.append(y[:, 1:, :])
+            test_loss = torch.mean(torch.tensor(test_loss))
+            print('Test Accuracy of the model: {}'.format(test_loss))
+        # wandb.log({"Test Loss": test_loss})
+        return torch.cat(test_preds, 0), torch.cat(test_trues, 0), test_loss