enose_2025/TrainerClass.py

import numpy as np
import pandas as pd
import tensorflow as tf
import matplotlib.cm as cm
import matplotlib.pyplot as plt
import matplotlib
matplotlib.rcParams['text.usetex'] = True

from sklearn.preprocessing import KBinsDiscretizer
from sklearn.model_selection import StratifiedGroupKFold, StratifiedShuffleSplit, GridSearchCV
from sklearn.metrics import mean_squared_error, mean_absolute_error
from sklearn.preprocessing import MinMaxScaler

from xgboost import XGBRegressor

import keras
from keras import layers
from ray import tune
from ray.tune.schedulers import ASHAScheduler


# from ray import tune
# import ray
# from keras.callbacks import TensorBoard
# from keras.models import Sequential
# from keras.callbacks import EarlyStopping
# from keras.layers import Dense, BatchNormalization, Dropout
# from kerastuner.tuners import RandomSearch, Hyperband, GridSearch

from datetime import datetime
import enlighten
import logging
import zipfile
import random
import joblib
import pickle
import time
import json
import os


def get_seed():
  return random.randint(0, 2**32 - 1)

class eNoseTrainer:
    def __init__(self, loader, test_size=0.2, debug=False):
        self.ledger = pd.DataFrame(columns=["node", "ts", "Dataset", "Samples", "Target", "Train Size", "Train Ratio", "Model", "Params", "Ratio", "Train mse", "mse", "mae", "rmse"])
        self.loader = loader
        self.name = self.loader.label_file
        self.state = dict()

        os.makedirs(self.name, exist_ok=True)
        self.start = int(time.time())

        log_format = '%(asctime)s | %(levelname)-8s | %(name)-15s | %(message)s'
        date_format = '%Y-%m-%d %H:%M:%S'
        logging.basicConfig(format=log_format, datefmt=date_format)

        target_log = '{}/load-{}.log'.format(self.name, self.start)
        fh = logging.FileHandler(target_log)

        self.debug = debug

        self.logger = logging.getLogger("eNoseTrainer")
        if self.debug:
            self.logger.setLevel(logging.DEBUG)
            fh.setLevel(logging.DEBUG)
        else:
            self.logger.setLevel(logging.INFO)
            fh.setLevel(logging.INFO)
        self.logger.addHandler(fh)

        self.ratio = test_size

        self.loader.stats()
        self.loadCheckPoint()

    def loadCheckPoint(self):
        if not os.path.isfile('{}/Simulaciones.xlsx'.format(self.name)):
            self.saveCheckPoint()

        with pd.ExcelFile('{}/Simulaciones.xlsx'.format(self.name)) as xls:
            self.ledger = pd.read_excel(xls, sheet_name='Historial')
            self.trained = self.ledger.shape[0]

        # with open('{}/vars.pickle'.format(self.name), 'rb') as pfile:
        #     self.ratio, self.state  = pickle.load(pfile)

    def saveCheckPoint(self):
        with pd.ExcelWriter('{}/Simulaciones.xlsx'.format(self.name), engine='xlsxwriter') as xls:
            self.ledger.to_excel(xls, sheet_name='Historial', index=False)

        # with open('{}/vars.pickle'.format(self.name), 'wb') as pfile:
        #     pickle.dump((self.ratio, self.state), pfile, protocol=pickle.HIGHEST_PROTOCOL)

        self.trained = self.ledger.shape[0]

    def wrap_and_save(self):
        self.logger.info("{:=^60}".format(' Saving Summary and Wrap the output in a ZipFile '))


        with zipfile.ZipFile('{}-{}.zip'.format(self.name, self.start), 'w', zipfile.ZIP_DEFLATED) as zipf:
            for root, dirs, files in os.walk(self.name):
                for file in files:
                    zipf.write(os.path.join(root, file))

    def row_exists(self, dataset, model):
        search_result = self.ledger[(self.ledger["Dataset"]==dataset) & (self.ledger["Target"]==self.loader.target) & (self.ledger["Model"]==model) & (self.ledger["Ratio"]==self.ratio)].shape[0] > 0
        self.logger.debug(f'Looking for {dataset}, {model}, {self.loader.target}, {self.ratio} => {search_result} {self.ledger.shape}')
        return search_result

    def model_A(self, hp):

        model = Sequential()
        model.add(Dense(units=hp.Int('units_input', min_value=48, max_value=56, step=8), input_dim=self.nvars, activation='relu'))
        model.add(BatchNormalization())
        model.add(Dropout(rate=hp.Float('dropout_input', min_value=0.1, max_value=0.1, step=0.1)))

        model.add(Dense(units=hp.Int('units_hidden', min_value=32, max_value=48, step=8), activation='relu'))
        model.add(BatchNormalization())
        model.add(Dropout(rate=hp.Float('dropout_hidden', min_value=0.4, max_value=0.4, step=0.1)))

        model.add(Dense(1, activation='sigmoid'))

        model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy', AUC()])

        return model

    def train_and_score_model_keras(self, X_train, X_test, Y_train, Y_test, seed, label):
#        set_random_seed(seed)
        ntrials = 6
        tuner = RandomSearch(
            self.get_model_train_keras,
            objective='val_loss', #val_loss
#            seed=seed,
            max_trials=ntrials,
#            executions_per_trial=1,  # Número de ejecuciones por cada configuración
            directory=self.name,
            project_name='{}-{}'.format(label,seed))

        self.logger.info("{:~^60}".format(' {}-{} '.format(label,seed)))


        search_dir = "{}/keras-tuner-{}/".format(self.name,label)
        os.makedirs(search_dir, exist_ok=True)
        search_callback = TensorBoard(log_dir=search_dir)
        early_stopping_search = EarlyStopping(monitor='val_loss', patience=13, min_delta=0.005, start_from_epoch=7, restore_best_weights=True)
        tuner.search(X_train, Y_train, epochs=150, batch_size=10, validation_data=(X_test, Y_test), callbacks=[early_stopping_search, search_callback])
        best_hps = tuner.get_best_hyperparameters(num_trials=1)[0]

        self.trained += 1
        self.bar.update()
        return mse, mae, rmse, optimized_model, model_params


    def train_and_score_conv1D_v1(self, X_train, X_test, y_train, y_test, epochs=30, num_samples=25):
        def build_model(config, input_shape, output_dim):
            model = keras.Sequential([
                layers.Conv1D(filters=config['filters'], kernel_size=config['kernel_size'], activation='relu', input_shape=input_shape),
                layers.MaxPooling1D(pool_size=config['pool_size']),
                layers.Conv1D(filters=config['filters'] * 2, kernel_size=config['kernel_size'], activation='relu'),
                layers.MaxPooling1D(pool_size=config['pool_size']),
                layers.Flatten(),
                layers.Dense(config['dense_units'], activation='relu'),
                layers.Dropout(config['dropout']),
                layers.Dense(output_dim)
            ])
            model.compile(optimizer=keras.optimizers.Adam(learning_rate=config['lr']), loss='mse')
            return model

        def train_model(config):
            input_shape = X_train.shape[1:]
            output_dim = Y_train.shape[1]

            model = build_model(config, input_shape, output_dim)
            early_stopping = keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)

            model.fit(
                X_train, Y_train,
                validation_data=(X_test, Y_test),
                epochs=epochs,
                batch_size=config['batch_size'],
                verbose=0,
                callbacks=[early_stopping]
            )

            Y_pred = model.predict(X_test)
            mse = mean_squared_error(Y_test, Y_pred)
            tune.report(mse=mse)

        config_space = {
            'filters': tune.choice([32, 64, 128]),
            'kernel_size': tune.choice([3, 5]),
            'pool_size': tune.choice([2, 3]),
            'dense_units': tune.choice([32, 64, 128]),
            'dropout': tune.choice([0.1, 0.2, 0.3]),
            'lr': tune.choice([0.001, 0.0005, 0.0001]),
            'batch_size': tune.choice([16, 32, 64])
        }

        scheduler = ASHAScheduler(metric='mse', mode='min', max_t=epochs, grace_period=5, reduction_factor=2)
        analysis = tune.run(train_model, config=config_space, num_samples=num_samples, scheduler=scheduler)

        best_config = analysis.get_best_config(metric='mse', mode='min')
        best_model = build_model(best_config, X_train.shape[1:], Y_train.shape[1])
        best_model.fit(X_train, Y_train, epochs=epochs, batch_size=best_config['batch_size'], verbose=0)

        Y_train_pred = best_model.predict(X_train)
        Y_test_pred = best_model.predict(X_test)

        mse_train = mean_squared_error(Y_train, Y_train_pred)
        mae_test = mean_absolute_error(y_test, Y_test_pred)
        mse_test = mean_squared_error(Y_test, Y_test_pred)
        rmse_test = np.sqrt(mse_test)

#        # Calculate evaluation metrics
        mse = mean_squared_error(y_test, y_pred)
        rmse = np.sqrt(mse)

        return mse_train, mae_test, mse_test, rmse_test, best_model, best_config
    def get_model_train(self):
        return [
            XGBRegressor(objective='reg:squarederror'),
        ]

    def get_tunable_params(self, model):
        if isinstance(model, XGBRegressor):
            return {
                'tree_method': ["hist"],
                "n_estimators": [100, 128, 150],
                'max_depth': [6, 7, 8],
                'subsample': [0.5, 0.6, 0.7],
                'multi_strategy': ['one_output_per_tree', 'multi_output_tree']
            }
        elif isinstance(model, RandomForestClassifier):
            return {
                "n_estimators": [50, 100, 200],
                "max_depth": [5, 10, 15],
                "max_features": [2, 5, 10] #['n', 'max_depth', 'max_features', 'max_leaf_nodes', 'max_samples', 'min_impurity_decrease', 'min_samples_leaf', 'min_samples_split', 'min_weight_fraction_leaf', 'monotonic_cst', 'n_estimators', 'n_jobs', 'oob_score', 'random_state', 'verbose', 'warm_start']
            }
        else:
            return {}

    def train_and_score_model(self, model, X_train, X_test, Y_train, Y_test):
        param_dist = self.get_tunable_params(model)

        cv = StratifiedShuffleSplit(n_splits=int(1/(2*self.ratio))+1, test_size=self.ratio, random_state=get_seed())
        grid_search = GridSearchCV(estimator=model, param_grid=param_dist, scoring='neg_mean_squared_error', cv=cv, verbose=2, n_jobs=-1)

        grid_search.fit(X_train, Y_train)

        optimized_model = grid_search.best_estimator_
        model_params = grid_search.best_params_

        y_aux = optimized_model.predict(X_train)
        tmse = mean_squared_error(Y_train, y_aux)

        y_pred = optimized_model.predict(X_test)
        mse = mean_squared_error(Y_test, y_pred)
        mae = mean_absolute_error(Y_test, y_pred)
        rmse = np.sqrt(mse)

        return tmse, mse, mae, rmse, optimized_model, model_params

    def gen_plots(self, dataset, model_id, target=None):
        if isinstance(target, list):
            self.loader.target_list=target
        if isinstance(target, str):
            self.loader.target_list= list(target)

        if dataset.endswith("-conv3"):
            self.loader.smooth = 'conv3'
        else:
            self.loader.smooth = None

        self.loader.reset()
        if not self.row_exists(dataset, model_id):
            self.logger.error(f'No se encuentra la simulacion {dataset}, {model_id}')
            return

        model_file = '{}/{}/{}/{}'.format(self.name, self.loader.target, dataset, model_id )
        if not os.path.isfile(model_file):
            self.logger.error('No se encuentra el modelo')
            return

        trained_model = joblib.load(model_file)

        pics_folder = '{}/{}/{}/plots'.format(self.name, self.loader.target, dataset)
        os.makedirs(pics_folder, exist_ok=True)

        df = self.loader.scaled_data

        Y_samples = np.zeros((len(df), len(self.loader.target_list)))
        for i, sample in enumerate(df):
            Y_samples[i] = np.array([[df[sample]['label'][key] for key in self.loader.target_list]])

        self.logger.debug(f"Y_samples.shape: {Y_samples.shape}")

        target_scaler = MinMaxScaler()
        Y_samples = target_scaler.fit_transform(Y_samples)

        cmapx = cm.get_cmap('winter', len(self.loader.source_channels))
        cmapy = cm.get_cmap('prism', Y_samples.shape[1])

        for measurament, (r, l) in self.loader.dataset['range'].items():
            # df[measurament]['data'].plot(figsize=(12, 6), title=f"{measurament} Prediction")
            plt.figure(figsize=(12, 6))
            plt.title(f"[{dataset}] {model_id}. Sample {measurament}")
            plt.xlabel("Sensor Readings")

            plt.vlines(x=r, ymin=0, ymax=1, colors='blue', linestyle='dashed')
            plt.vlines(x=l, ymin=0, ymax=1, colors='blue', linestyle='dashed')


            Y_value = np.zeros((1, len(self.loader.target_list)))
            Y_value[0] = np.array([[df[measurament]['label'][key] for key in self.loader.target_list]])

            self.logger.debug(f"Y_value.shape: {Y_value.shape}")
            self.logger.debug(f"Y_value: {Y_value}")

            Y_scaled = target_scaler.transform(Y_value).reshape(1, -1)
            self.logger.debug(f"Y_scaled.shape: {Y_scaled.shape}")
            self.logger.debug(f"Y_scaled: {Y_scaled}")

            y_pred = trained_model.predict(df[measurament]['data'].to_numpy())
            self.logger.debug(f"y_pred.shape: {y_pred.shape}")
#            self.logger.debug(f"y_pred: {Y_scaled}")

            if y_pred.ndim == 2:
                plt.ylabel("Target dashed / Pred solid")
                for i, channel_name in enumerate(df[measurament]['data'].columns):
                    plt.plot(df[measurament]['data'][channel_name], linestyle = 'dotted', color=cmapx(i), alpha=0.2)
                for i in range(y_pred.shape[1]):
                    self.logger.debug(f"Y_scaled[0][i]: {Y_scaled[0][i]}")
                    plt.axhline(y=Y_scaled[0][i], xmin=0, xmax=df[measurament]['data'].shape[0], color=cmapy(i), linestyle='dashed')
                    plt.plot(y_pred[:, i], color=cmapy(i), linestyle='solid')
            else:
                plt.ylabel("Samples dotted / Target dashed / Pred solid")
                for i, channel_name in enumerate(df[measurament]['data'].columns):
                    plt.plot(df[measurament]['data'][channel_name], linestyle = 'dotted', color=cmapx(i))
                plt.plot(y_pred, color=cmapy(0), linestyle='solid')
                plt.axhline(y=Y_scaled, xmin=0, xmax=df[measurament]['data'].shape[0], color=cmapy(i), linestyle='dashed')

            filename = os.path.join(pics_folder, f"{measurament}_{model_id}.png")
            plt.savefig(filename, format='png')
            self.logger.info(f"Saved plot as {filename}")

            plt.close()

    def fit(self):
        windows = [16, 32, 64, 128]
        total_train_queue = 2*int(1/self.ratio)*(len(self.get_model_train())+1)
        self.logger.info("{:=^60}".format(f'Begin Fit {total_train_queue} Models'))
        self.trained = 0
        manager = enlighten.get_manager()
        self.bar = manager.counter(total=total_train_queue, count=self.trained, desc='Tunning', unit='Models',
                                format='{desc}{desc_pad}{percentage:3.0f}%|{bar}| {count:{len_total}d}/{total:d} [{elapsed}<{eta}, {rate:.2f}{unit_pad}{unit}/s]'
                                )

        discretizer = KBinsDiscretizer(n_bins=200, encode='ordinal', strategy='uniform')
        gss = StratifiedGroupKFold(n_splits=int(1/self.ratio), shuffle=True, random_state=get_seed())

        node = os.uname()[1]
        self.loader.smooth = None
        self.loader.reset()

        X_xboost, Y_xboost, G_xboost = self.loader.load_dataset_xboost()
        # self.logger.debug(f"X_xboost: {X_xboost.shape}")
        self.logger.debug(f"Y_xboost: {Y_xboost.shape}")
        # self.logger.debug(f"G_xboost: {G_xboost.shape}")

        Y_discrete = discretizer.fit_transform(Y_xboost)
        if Y_discrete.ndim == 2:
            Y_discrete = np.sum(Y_discrete, axis=1)
        # self.logger.debug(f"Y_discrete: {Y_discrete.shape}")

        dataset = 'Tabular'
        for i, (train_index, test_index) in enumerate(gss.split(X_xboost, Y_discrete, G_xboost)):
            self.logger.info("{:=^60}".format(f'CV {i}/{int(1/self.ratio)} {dataset}'))
            os.makedirs('{}/{}/{}'.format(self.name, self.loader.target, dataset), exist_ok=True)
            X_train, X_test = X_xboost[train_index], X_xboost[test_index]
            Y_train, Y_test = Y_xboost[train_index], Y_xboost[test_index]
            # self.logger.debug(f"X_train: {X_train.shape}")
            # self.logger.debug(f"X_test: {X_test.shape}")
            self.logger.debug(f"Y_train: {Y_train.shape}")
            self.logger.debug(f"Y_test: {Y_test.shape}")


            for model in self.get_model_train():
                model_id = "{}_{}".format(type(model).__name__, i)
                self.trained += 1

                if self.row_exists(dataset, model_id):
                    self.bar.update()
                    continue

                model_file = '{}/{}/{}/{}'.format(self.name, self.loader.target, dataset, model_id )

                tmse, mse, mae, rmse, optimized_model, model_params = self.train_and_score_model(model, X_train, X_test, Y_train, Y_test)

                ts = datetime.now().strftime("%d/%m/%Y %H:%M:%S")
                joblib.dump(optimized_model, model_file)

                newrow = pd.DataFrame( [{"node": node,
                                    "ts": ts,
                                    "Dataset": dataset,
                                    "Samples": Y_xboost.shape[0],
                                    "Target": self.loader.target,
                                    "Train Size": Y_train.shape[0],
                                    "Train Ratio": Y_train.shape[0]/Y_xboost.shape[0],
                                    "Ratio": self.ratio,
                                    "Model": model_id,
                                    "Params": json.dumps(model_params),
                                    "Train mse": tmse,
                                    "mse": mse,
                                    "mae": mae,
                                    "rmse": rmse
                                    }] )
                self.ledger = pd.concat([self.ledger, newrow], ignore_index=True)
                self.bar.update()

            self.saveCheckPoint()

        self.loader.smooth = 'conv3'
        self.loader.reset()
        X_xboost, Y_xboost, G_xboost = self.loader.load_dataset_xboost()
        # self.logger.debug(f"X_xboost: {X_xboost.shape}")
        self.logger.debug(f"Y_xboost: {Y_xboost.shape}")
        # self.logger.debug(f"G_xboost: {G_xboost.shape}")

        Y_discrete = discretizer.fit_transform(Y_xboost)
        if Y_discrete.ndim == 2:
            Y_discrete = np.sum(Y_discrete, axis=1)

        dataset = 'Tabular-conv3'
        for i, (train_index, test_index) in enumerate(gss.split(X_xboost, Y_discrete, G_xboost)):
            self.logger.info("{:=^60}".format(f'CV {i}/{int(1/self.ratio)} {dataset}'))
            os.makedirs('{}/{}/{}'.format(self.name, self.loader.target, dataset), exist_ok=True)
            X_train, X_test = X_xboost[train_index], X_xboost[test_index]
            Y_train, Y_test = Y_xboost[train_index], Y_xboost[test_index]
            # self.logger.debug(f"X_train: {X_train.shape}")
            # self.logger.debug(f"X_test: {X_test.shape}")
            self.logger.debug(f"Y_train: {Y_train.shape}")
            self.logger.debug(f"Y_test: {Y_test.shape}")

            for model in self.get_model_train():
                model_id = "{}_{}".format(type(model).__name__, i)
                self.trained += 1

                if self.row_exists(dataset, model_id):
                    self.bar.update()
                    continue

                model_file = '{}/{}/{}/{}'.format(self.name, self.loader.target, dataset, model_id )

                tmse, mse, mae, rmse, optimized_model, model_params = self.train_and_score_model(model, X_train, X_test, Y_train, Y_test)

                ts = datetime.now().strftime("%d/%m/%Y %H:%M:%S")
                joblib.dump(optimized_model, model_file)

                newrow = pd.DataFrame( [{"node": node,
                                    "ts": ts,
                                    "Dataset": dataset,
                                    "Samples": Y_xboost.shape[0],
                                    "Target": self.loader.target,
                                    "Train Size": Y_train.shape[0],
                                    "Train Ratio": Y_train.shape[0]/Y_xboost.shape[0],
                                    "Ratio": self.ratio,
                                    "Model": model_id,
                                    "Params": json.dumps(model_params),
                                    "Train mse": tmse,
                                    "mse": mse,
                                    "mae": mae,
                                    "rmse": rmse
                                    }] )
                self.ledger = pd.concat([self.ledger, newrow], ignore_index=True)
                self.bar.update()

            self.saveCheckPoint()


        self.loader.smooth = None
        self.loader.reset()
        for window in windows:
            X_conv1d, Y_conv1d, G_conv1d = self.loader.load_dataset_window(window)
            self.logger.debug(f"X_conv1d: {X_conv1d.shape}")
            self.logger.debug(f"Y_conv1d: {Y_conv1d.shape}")
            self.logger.debug(f"G_conv1d: {G_conv1d.shape}")

            Y_discrete = discretizer.fit_transform(Y_conv1d)
            if Y_discrete.ndim == 2:
                Y_discrete = np.sum(Y_discrete, axis=1)

            dataset = f'Conv1d-base-w{window}'
            for i, (train_index, test_index) in enumerate(gss.split(X_conv1d, Y_discrete, G_conv1d)):
                self.logger.info("{:=^60}".format(f'CV {i}/{int(1/self.ratio)} {dataset}'))

                os.makedirs('{}/{}/{}-w{}'.format(self.name, self.loader.target, dataset, window), exist_ok=True)
                X_train, X_test = X_conv1d[train_index], X_conv1d[test_index]
                Y_train, Y_test = Y_conv1d[train_index], Y_conv1d[test_index]
                # self.logger.debug(f"X_train: {X_train.shape}")
                # self.logger.debug(f"X_test: {X_test.shape}")
                self.logger.debug(f"Y_train: {Y_train.shape}")
                self.logger.debug(f"Y_test: {Y_test.shape}")

                model_id = "Conv1d-base_{}".format(i)
                self.trained += 1

                if self.row_exists(dataset, model_id):
                    self.bar.update()
                    continue

                model_file = '{}/{}/{}-w{}/{}'.format(self.name, self.loader.target, dataset, window, model_id )

                tmse, mse, mae, rmse, optimized_model, model_params = self.train_and_score_conv1D_v1(X_train, X_test, Y_train, Y_test)

                ts = datetime.now().strftime("%d/%m/%Y %H:%M:%S")
                optimized_model.save(model_file)
                optimized_model.save_weights(f"{model_file}.weights.h5")

                newrow = pd.DataFrame( [{"node": node,
                                    "ts": ts,
                                    "Dataset": dataset,
                                    "Samples": Y_xboost.shape[0],
                                    "Target": self.loader.target,
                                    "Train Size": Y_train.shape[0],
                                    "Train Ratio": Y_train.shape[0]/Y_xboost.shape[0],
                                    "Ratio": self.ratio,
                                    "Model": model_id,
                                    "Params": json.dumps(model_params),
                                    "Train mse": tmse,
                                    "mse": mse,
                                    "mae": mae,
                                    "rmse": rmse
                                    }] )
                self.ledger = pd.concat([self.ledger, newrow], ignore_index=True)
                self.bar.update()
            self.saveCheckPoint()

        self.loader.smooth = 'conv3'
        self.loader.reset()
        for window in windows:
            X_conv1d, Y_conv1d, G_conv1d = self.loader.load_dataset_window(window)
            self.logger.debug(f"X_conv1d: {X_conv1d.shape}")
            self.logger.debug(f"Y_conv1d: {Y_conv1d.shape}")
            self.logger.debug(f"G_conv1d: {G_conv1d.shape}")

            Y_discrete = discretizer.fit_transform(Y_conv1d)
            if Y_discrete.ndim == 2:
                Y_discrete = np.sum(Y_discrete, axis=1)

            dataset = f'Conv1d-base-w{window}-conv3'
            for i, (train_index, test_index) in enumerate(gss.split(X_conv1d, Y_discrete, G_conv1d)):
                self.logger.info("{:=^60}".format(f'CV {i}/{int(1/self.ratio)} {dataset}'))

                os.makedirs('{}/{}/{}-w{}'.format(self.name, self.loader.target, dataset, window), exist_ok=True)
                X_train, X_test = X_conv1d[train_index], X_conv1d[test_index]
                Y_train, Y_test = Y_conv1d[train_index], Y_conv1d[test_index]
                # self.logger.debug(f"X_train: {X_train.shape}")
                # self.logger.debug(f"X_test: {X_test.shape}")
                self.logger.debug(f"Y_train: {Y_train.shape}")
                self.logger.debug(f"Y_test: {Y_test.shape}")


                model_id = "Conv1d-base_{}".format(i)
                self.trained += 1

                if self.row_exists(dataset, model_id):
                    self.bar.update()
                    continue

                model_file = '{}/{}/{}-w{}/{}'.format(self.name, self.loader.target, dataset, window, model_id )

                tmse, mse, mae, rmse, optimized_model, model_params = self.train_and_score_conv1D_v1(X_train, X_test, Y_train, Y_test)

                ts = datetime.now().strftime("%d/%m/%Y %H:%M:%S")
                optimized_model.save(model_file)
                optimized_model.save_weights(f"{model_file}.weights.h5")

                newrow = pd.DataFrame( [{"node": node,
                                    "ts": ts,
                                    "Dataset": dataset,
                                    "Samples": Y_xboost.shape[0],
                                    "Target": self.loader.target,
                                    "Train Size": Y_train.shape[0],
                                    "Train Ratio": Y_train.shape[0]/Y_xboost.shape[0],
                                    "Ratio": self.ratio,
                                    "Model": model_id,
                                    "Params": json.dumps(model_params),
                                    "Train mse": tmse,
                                    "mse": mse,
                                    "mae": mae,
                                    "rmse": rmse
                                    }] )
                self.ledger = pd.concat([self.ledger, newrow], ignore_index=True)
                self.bar.update()
            self.saveCheckPoint()


        self.bar.close()