diff --git a/.gitignore b/.gitignore index 6bc968a..ed54378 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,9 @@ +**bz2 +**tar +**pdf +**png +paper* +prese* *xlsx pics *zip diff --git a/LoaderClass.py b/LoaderClass.py index 1017f6f..1ff8f1f 100644 --- a/LoaderClass.py +++ b/LoaderClass.py @@ -88,8 +88,10 @@ class GasSensorDataLoader: self.logger.info(f"Reset requested. Init for {len(self.target_list)} targets => {self.target}") - delattr(self, "delta_data") - delattr(self, "scaled_data") + if hasattr(self, "delta_data"): + delattr(self, "delta_data") + if hasattr(self, "scaled_data"): + delattr(self, "scaled_data") self.init_minmax() self.stats() @@ -176,7 +178,7 @@ class GasSensorDataLoader: plt.plot(data_instance[key]['data'][channel_name]) plt.xlabel("Time") - plt.ylabel("Sensor Reading") + plt.ylabel("Sensor Array Readings") plt.title(f"{title} Sensor Channel: {channel_name}") if save: @@ -192,8 +194,8 @@ class GasSensorDataLoader: self.logger.debug(f"{title}All sensor readings for measurament: {measurament_name}.") if measurament_name in data_instance: data_instance[measurament_name]['data'].plot(figsize=(12, 6), title=f"{title} measurament: {measurament_name}") - plt.xlabel("Time") - plt.ylabel("Sensor Readings") + plt.xlabel("sample") + plt.ylabel(f"{title} value") plt.legend(bbox_to_anchor=(0.95, 0.5), loc="center left") for xvalue, txtcolor in limits: plt.vlines(x=xvalue, ymin=0, ymax=1, colors=txtcolor) @@ -210,9 +212,9 @@ class GasSensorDataLoader: def plotRawdata(self, save=True): self.logger.debug("Plotting raw data for all measuraments and channels.") for measurament in self.data: - self.plot_measurament(self.data, measurament, save=save, tag="raw", title="[Original] ") + self.plot_measurament(self.data, measurament, save=save, tag="rawv", title="[V]") for channel in self.data[next(iter(self.data))]['data'].columns: - self.plot_channel(self.data, channel, save=save, tag="raw", title="[Original] ") + self.plot_channel(self.data, channel, save=save, tag="rawv", title="[V]") def plotDeltadata(self, save=True): if not hasattr(self, 'delta_data'): @@ -220,9 +222,9 @@ class GasSensorDataLoader: self.logger.debug("Plotting raw data for all measuraments and channels.") for measurament in self.delta_data: - self.plot_measurament(self.delta_data, measurament, save=save, tag="delta", title="[$\Delta V$] ") + self.plot_measurament(self.delta_data, measurament, save=save, tag="delta", title="[$\\Delta V$] ") for channel in self.delta_data[next(iter(self.delta_data))]['data'].columns: - self.plot_channel(self.delta_data, channel, save=save, tag="delta", title="[$\delta V$] ") + self.plot_channel(self.delta_data, channel, save=save, tag="delta", title="[$\\Delta V$] ") def plotScaleddata(self, save=True): if not hasattr(self, 'scaled_data'): @@ -230,18 +232,24 @@ class GasSensorDataLoader: self.logger.debug("Plotting raw data for all measuraments and channels.") for measurament in self.scaled_data: - self.plot_measurament(self.scaled_data, measurament, save=save, tag="scaled", title="[Scaled] ") + self.plot_measurament(self.scaled_data, measurament, save=save, tag="scaled", title="[$\\Delta \\bar{V}$] ") for channel in self.scaled_data[next(iter(self.scaled_data))]['data'].columns: - self.plot_channel(self.scaled_data, channel, save=save, tag="scaled", title="[Scaled] ") + self.plot_channel(self.scaled_data, channel, save=save, tag="scaled", title="[$\\Delta \\bar{V}$] ") def plotScaledBoundaries(self, save=True): if not hasattr(self, 'scaled_data'): self.init_minmax() + if self.smooth is None: + tag = "raw" + else: + tag = "denoise" +# tag = self.smooth + self.logger.debug("Plotting raw data for all measuraments and channels.") for measurament in self.scaled_data: r, l, m = self.findIndicesAbovethreshold(self.scaled_data[measurament]['data']) - self.plot_measurament(self.scaled_data, measurament, save=save, tag="train", title=f"[Interval {self.threshold} max] ", limits=[(r, 'blue'), (l, 'blue'), (m, 'red')]) + self.plot_measurament(self.scaled_data, measurament, save=save, tag="{}_bound".format(tag), title=f"[$\\Delta \\bar{{V}}_{{{tag}}}$] ", limits=[(r, 'blue'), (l, 'blue'), (m, 'red')]) def findIndicesAbovethreshold(self, df): if not isinstance(df, pd.DataFrame): diff --git a/TrainerClass.py b/TrainerClass.py index da2b153..51441b7 100644 --- a/TrainerClass.py +++ b/TrainerClass.py @@ -219,6 +219,7 @@ class eNoseTrainer: return tmse, mse, mae, rmse, optimized_model, model_params def gen_plots(self, dataset, model_id, target=None): + import re if isinstance(target, list): self.loader.target_list=target if isinstance(target, str): @@ -230,16 +231,38 @@ class eNoseTrainer: self.loader.smooth = None self.loader.reset() + + if dataset.startswith("Conv1D"): + width = int(m.group(1)) if (m := re.search(r'w(\d+)', dataset)) else 1 + self.logger.debug(f'Conv1D: {dataset} of width {width}') + y_padding = np.zeros((width-1, self.loader.target_len)) + elif dataset.startswith("Tabular"): + self.logger.debug(f'Tabular: {dataset}') + width = 1 + else: + self.logger.error(f'Tipo de dataset desconocido {dataset}') + return + if not self.row_exists(dataset, model_id): self.logger.error(f'No se encuentra la simulacion {dataset}, {model_id}') return model_file = '{}/{}/{}/{}'.format(self.name, self.loader.target, dataset, model_id ) - if not os.path.isfile(model_file): - self.logger.error('No se encuentra el modelo') - return - trained_model = joblib.load(model_file) + if width > 1: + if not os.path.isfile(f'{model_file}.keras'): + self.logger.debug(f'{model_file}') + self.logger.error('No se encuentra el modelo') + return + trained_model = keras.models.load_model(f"{model_file}.keras") + trained_model.load_weights(f"{model_file}.weights.h5") + else: + if not os.path.isfile(model_file): + self.logger.debug(f'{model_file}') + self.logger.error('No se encuentra el modelo') + return + trained_model = joblib.load(model_file) + pics_folder = '{}/{}/{}/plots'.format(self.name, self.loader.target, dataset) os.makedirs(pics_folder, exist_ok=True) @@ -261,8 +284,19 @@ class eNoseTrainer: for measurament, (r, l) in self.loader.dataset['range'].items(): # df[measurament]['data'].plot(figsize=(12, 6), title=f"{measurament} Prediction") plt.figure(figsize=(12, 6)) - plt.title(f"[{dataset}] {model_id}. Sample {measurament}") - plt.xlabel("Sensor Readings") + + if dataset.startswith('Conv1D'): + label = dataset.split('-') + model_label = '-'.join(label[:2]) + else: + model_label = "XGBRegressor" + + if self.loader.smooth is not None: + model_label += " + denoise" + + plt.title(f"[{model_label}] Sample {measurament}") + + plt.xlabel("Sensor Array Readings") plt.vlines(x=r, ymin=0, ymax=1, colors='blue', linestyle='dashed') plt.vlines(x=l, ymin=0, ymax=1, colors='blue', linestyle='dashed') @@ -278,8 +312,27 @@ class eNoseTrainer: self.logger.debug(f"Y_scaled.shape: {Y_scaled.shape}") self.logger.debug(f"Y_scaled: {Y_scaled}") - y_pred = trained_model.predict(df[measurament]['data'].to_numpy()) - self.logger.debug(f"y_pred.shape: {y_pred.shape}") + if width > 1: + plt.vlines(x=r+width, ymin=0, ymax=1, colors='cyan', linestyle='dashed') + + X_data = self.loader.scaled_data[measurament]['data'] + total_samples = X_data.shape[0] - width + 1 + x_samples = np.zeros((total_samples, width, self.loader.data_channels)) + + for i in range(total_samples): + x_samples[i] = X_data.iloc[i:i + width].values + + y_pred_w = trained_model.predict(x_samples) + self.logger.debug(f"y_pred_w.shape: {y_pred_w.shape}") + self.logger.debug(f"y_padding.shape: {y_padding.shape}") + self.logger.debug(f"X_data.shape: {X_data.shape}") + + y_pred = np.concatenate((y_padding, y_pred_w)) + self.logger.debug(f"y_pred.shape: {y_pred.shape}") + else: + y_pred = trained_model.predict(df[measurament]['data'].to_numpy()) + self.logger.debug(f"y_pred.shape: {y_pred.shape}") + # self.logger.debug(f"y_pred: {Y_scaled}") if y_pred.ndim == 2: diff --git a/train_sequence.py b/train_sequence.py index f1feb7d..d17f182 100644 --- a/train_sequence.py +++ b/train_sequence.py @@ -7,25 +7,43 @@ warnings.filterwarnings("ignore") source_channels=["MQ 8", "MQ 9", "MQ 135", "TGS 813", "TGS 821", "TGS 2600", "TGS 2602", "TGS 2611-0", "TGS 2612", "TGS 2620"] target_variables=['C2H2', 'CH4', 'C3H6', 'CO', 'C2H6', 'C3H8', 'C2H4', 'H2', 'O2'] -eNoseLoader = GasSensorDataLoader("enose_dataset", threshold=0.85, source_channels=source_channels, target_list=target_variables, debug=False) +eNoseLoader = GasSensorDataLoader("enose_dataset", threshold=0.85, source_channels=source_channels, target_list=target_variables, debug=True) + + +# Mostrar los dataset originales +eNoseLoader.smooth = None +eNoseLoader.reset() +eNoseLoader.plotRawdata() +eNoseLoader.plotScaledBoundaries() +eNoseLoader.smooth = 'conv3' +eNoseLoader.reset() +eNoseLoader.plotScaledBoundaries() + +# Carga el Entrenador eNose = eNoseTrainer(eNoseLoader, test_size=0.2, debug=False) -eNoseLoader.target_list=['H2', 'C2H2', 'CH4', 'C2H4', 'C2H6',] -eNose.fit() -eNoseLoader.target_list=['H2',] -eNose.fit() -eNoseLoader.target_list=['C2H2',] -eNose.fit() -eNoseLoader.target_list=['CH4',] -eNose.fit() -eNoseLoader.target_list=['C2H4',] -eNose.fit() -eNoseLoader.target_list=['C2H6',] -eNose.fit() -eNose.wrap_and_save() - +# Entrenar los modelos +# eNoseLoader.target_list=['H2', 'C2H2', 'CH4', 'C2H4', 'C2H6',] +# eNose.fit() # eNoseLoader.target_list=['H2',] -# eNose.gen_plots('Tabular-conv3','XGBRegressor_0') +# eNose.fit()| +# eNoseLoader.target_list=['C2H2',] +# eNose.fit() +# eNoseLoader.target_list=['CH4',] +# eNose.fit() +# eNoseLoader.target_list=['C2H4',] +# eNose.fit() +# eNoseLoader.target_list=['C2H6',] +# eNose.fit() +# eNose.wrap_and_save() + +# Grafica las predicciones +eNose.gen_plots('Tabular','XGBRegressor_3', target=['H2', 'C2H2', 'CH4', 'C2H4', 'C2H6',]) +#eNose.gen_plots('Tabular-conv3','XGBRegressor_4', target=['H2',]) + +eNose.gen_plots('Conv1D-w32-conv3','Conv1D_v1_1', target=['H2','C2H2', 'CH4', 'C2H4', 'C2H6',]) +# eNose.gen_plots('Conv1D-w32-conv3','Conv1D_v1_2', target=['H2',]) +#eNose.gen_plots('Conv1D-w32','Conv1D_v1_3', target=['H2', 'C2H2', 'CH4', 'C2H4', 'C2H6',]) # eNoseLoader.target_list=['H2', 'C2H2', 'CH4', 'C2H4', 'C2H6',] # eNose.gen_plots('Tabular','XGBRegressor_1')