last value

.gitignore

@@ -1,3 +1,9 @@
+**bz2
+**tar
+**pdf
+**png
+paper*
+prese*
 *xlsx
 pics
 *zip

LoaderClass.py

@@ -88,8 +88,10 @@ class GasSensorDataLoader:
 
         self.logger.info(f"Reset requested. Init for {len(self.target_list)} targets => {self.target}")
 
-        delattr(self, "delta_data")
+        if hasattr(self, "delta_data"):
-        delattr(self, "scaled_data")
+            delattr(self, "delta_data")
+        if hasattr(self, "scaled_data"):
+            delattr(self, "scaled_data")
         self.init_minmax()
         self.stats()
 
@@ -176,7 +178,7 @@ class GasSensorDataLoader:
                 plt.plot(data_instance[key]['data'][channel_name])
 
         plt.xlabel("Time")
-        plt.ylabel("Sensor Reading")
+        plt.ylabel("Sensor Array Readings")
         plt.title(f"{title} Sensor Channel: {channel_name}")
 
         if save:
@@ -192,8 +194,8 @@ class GasSensorDataLoader:
         self.logger.debug(f"{title}All sensor readings for measurament: {measurament_name}.")
         if measurament_name in data_instance:
             data_instance[measurament_name]['data'].plot(figsize=(12, 6), title=f"{title} measurament: {measurament_name}")
-            plt.xlabel("Time")
+            plt.xlabel("sample")
-            plt.ylabel("Sensor Readings")
+            plt.ylabel(f"{title} value")
             plt.legend(bbox_to_anchor=(0.95, 0.5), loc="center left")
             for xvalue, txtcolor in limits:
                 plt.vlines(x=xvalue, ymin=0, ymax=1, colors=txtcolor)
@@ -210,9 +212,9 @@ class GasSensorDataLoader:
     def plotRawdata(self, save=True):
         self.logger.debug("Plotting raw data for all measuraments and channels.")
         for measurament in self.data:
-            self.plot_measurament(self.data, measurament, save=save, tag="raw", title="[Original] ")
+            self.plot_measurament(self.data, measurament, save=save, tag="rawv", title="[V]")
         for channel in self.data[next(iter(self.data))]['data'].columns:
-            self.plot_channel(self.data, channel, save=save, tag="raw", title="[Original] ")
+            self.plot_channel(self.data, channel, save=save, tag="rawv", title="[V]")
 
     def plotDeltadata(self, save=True):
         if not hasattr(self, 'delta_data'):
@@ -220,9 +222,9 @@ class GasSensorDataLoader:
 
         self.logger.debug("Plotting raw data for all measuraments and channels.")
         for measurament in self.delta_data:
-            self.plot_measurament(self.delta_data, measurament, save=save, tag="delta", title="[$\Delta V$] ")
+            self.plot_measurament(self.delta_data, measurament, save=save, tag="delta", title="[$\\Delta V$] ")
         for channel in self.delta_data[next(iter(self.delta_data))]['data'].columns:
-            self.plot_channel(self.delta_data, channel, save=save, tag="delta", title="[$\delta V$] ")
+            self.plot_channel(self.delta_data, channel, save=save, tag="delta", title="[$\\Delta V$] ")
 
     def plotScaleddata(self, save=True):
         if not hasattr(self, 'scaled_data'):
@@ -230,18 +232,24 @@ class GasSensorDataLoader:
 
         self.logger.debug("Plotting raw data for all measuraments and channels.")
         for measurament in self.scaled_data:
-            self.plot_measurament(self.scaled_data, measurament, save=save, tag="scaled", title="[Scaled] ")
+            self.plot_measurament(self.scaled_data, measurament, save=save, tag="scaled", title="[$\\Delta \\bar{V}$] ")
         for channel in self.scaled_data[next(iter(self.scaled_data))]['data'].columns:
-            self.plot_channel(self.scaled_data, channel, save=save, tag="scaled", title="[Scaled] ")
+            self.plot_channel(self.scaled_data, channel, save=save, tag="scaled", title="[$\\Delta \\bar{V}$] ")
 
     def plotScaledBoundaries(self, save=True):
         if not hasattr(self, 'scaled_data'):
             self.init_minmax()
 
+        if self.smooth is None:
+            tag = "raw"
+        else:
+            tag = "denoise"
+#            tag = self.smooth
+
         self.logger.debug("Plotting raw data for all measuraments and channels.")
         for measurament in self.scaled_data:
             r, l, m = self.findIndicesAbovethreshold(self.scaled_data[measurament]['data'])
-            self.plot_measurament(self.scaled_data, measurament, save=save, tag="train", title=f"[Interval {self.threshold} max] ", limits=[(r, 'blue'), (l, 'blue'), (m, 'red')])
+            self.plot_measurament(self.scaled_data, measurament, save=save, tag="{}_bound".format(tag), title=f"[$\\Delta \\bar{{V}}_{{{tag}}}$] ", limits=[(r, 'blue'), (l, 'blue'), (m, 'red')])
 
     def findIndicesAbovethreshold(self, df):
         if not isinstance(df, pd.DataFrame):

TrainerClass.py

@@ -219,6 +219,7 @@ class eNoseTrainer:
         return tmse, mse, mae, rmse, optimized_model, model_params
 
     def gen_plots(self, dataset, model_id, target=None):
+        import re
         if isinstance(target, list):
             self.loader.target_list=target
         if isinstance(target, str):
@@ -230,16 +231,38 @@ class eNoseTrainer:
             self.loader.smooth = None
 
         self.loader.reset()
+
+        if dataset.startswith("Conv1D"):
+            width = int(m.group(1)) if (m := re.search(r'w(\d+)', dataset)) else 1
+            self.logger.debug(f'Conv1D: {dataset} of width {width}')
+            y_padding = np.zeros((width-1, self.loader.target_len))
+        elif dataset.startswith("Tabular"):
+            self.logger.debug(f'Tabular: {dataset}')
+            width = 1
+        else:
+            self.logger.error(f'Tipo de dataset desconocido {dataset}')
+            return
+
         if not self.row_exists(dataset, model_id):
             self.logger.error(f'No se encuentra la simulacion {dataset}, {model_id}')
             return
 
         model_file = '{}/{}/{}/{}'.format(self.name, self.loader.target, dataset, model_id )
-        if not os.path.isfile(model_file):
-            self.logger.error('No se encuentra el modelo')
-            return
 
-        trained_model = joblib.load(model_file)
+        if width > 1:
+            if not os.path.isfile(f'{model_file}.keras'):
+                self.logger.debug(f'{model_file}')
+                self.logger.error('No se encuentra el modelo')
+                return
+            trained_model = keras.models.load_model(f"{model_file}.keras")
+            trained_model.load_weights(f"{model_file}.weights.h5")
+        else:
+            if not os.path.isfile(model_file):
+                self.logger.debug(f'{model_file}')
+                self.logger.error('No se encuentra el modelo')
+                return
+            trained_model = joblib.load(model_file)
+
 
         pics_folder = '{}/{}/{}/plots'.format(self.name, self.loader.target, dataset)
         os.makedirs(pics_folder, exist_ok=True)
@@ -261,8 +284,19 @@ class eNoseTrainer:
         for measurament, (r, l) in self.loader.dataset['range'].items():
             # df[measurament]['data'].plot(figsize=(12, 6), title=f"{measurament} Prediction")
             plt.figure(figsize=(12, 6))
-            plt.title(f"[{dataset}] {model_id}. Sample {measurament}")
+
-            plt.xlabel("Sensor Readings")
+            if dataset.startswith('Conv1D'):
+                label = dataset.split('-')
+                model_label = '-'.join(label[:2])
+            else:
+                model_label = "XGBRegressor"
+
+            if self.loader.smooth is not None:
+                model_label += " + denoise"
+
+            plt.title(f"[{model_label}] Sample {measurament}")
+
+            plt.xlabel("Sensor Array Readings")
 
             plt.vlines(x=r, ymin=0, ymax=1, colors='blue', linestyle='dashed')
             plt.vlines(x=l, ymin=0, ymax=1, colors='blue', linestyle='dashed')
@@ -278,8 +312,27 @@ class eNoseTrainer:
             self.logger.debug(f"Y_scaled.shape: {Y_scaled.shape}")
             self.logger.debug(f"Y_scaled: {Y_scaled}")
 
-            y_pred = trained_model.predict(df[measurament]['data'].to_numpy())
+            if width > 1:
-            self.logger.debug(f"y_pred.shape: {y_pred.shape}")
+                plt.vlines(x=r+width, ymin=0, ymax=1, colors='cyan', linestyle='dashed')
+
+                X_data = self.loader.scaled_data[measurament]['data']
+                total_samples = X_data.shape[0] - width + 1
+                x_samples = np.zeros((total_samples, width, self.loader.data_channels))
+
+                for i in range(total_samples):
+                    x_samples[i] = X_data.iloc[i:i + width].values
+
+                y_pred_w = trained_model.predict(x_samples)
+                self.logger.debug(f"y_pred_w.shape: {y_pred_w.shape}")
+                self.logger.debug(f"y_padding.shape: {y_padding.shape}")
+                self.logger.debug(f"X_data.shape: {X_data.shape}")
+
+                y_pred = np.concatenate((y_padding, y_pred_w))
+                self.logger.debug(f"y_pred.shape: {y_pred.shape}")
+            else:
+                y_pred = trained_model.predict(df[measurament]['data'].to_numpy())
+                self.logger.debug(f"y_pred.shape: {y_pred.shape}")
+
 #            self.logger.debug(f"y_pred: {Y_scaled}")
 
             if y_pred.ndim == 2:

train_sequence.py

@@ -7,25 +7,43 @@ warnings.filterwarnings("ignore")
 source_channels=["MQ 8", "MQ 9", "MQ 135", "TGS 813", "TGS 821", "TGS 2600", "TGS 2602", "TGS 2611-0", "TGS 2612", "TGS 2620"]
 target_variables=['C2H2', 'CH4', 'C3H6', 'CO', 'C2H6', 'C3H8', 'C2H4', 'H2', 'O2']
 
-eNoseLoader = GasSensorDataLoader("enose_dataset", threshold=0.85, source_channels=source_channels, target_list=target_variables, debug=False)
+eNoseLoader = GasSensorDataLoader("enose_dataset", threshold=0.85, source_channels=source_channels, target_list=target_variables, debug=True)
+
+
+# Mostrar los dataset originales
+eNoseLoader.smooth = None
+eNoseLoader.reset()
+eNoseLoader.plotRawdata()
+eNoseLoader.plotScaledBoundaries()
+eNoseLoader.smooth = 'conv3'
+eNoseLoader.reset()
+eNoseLoader.plotScaledBoundaries()
+
+# Carga el Entrenador
 eNose = eNoseTrainer(eNoseLoader, test_size=0.2, debug=False)
 
-eNoseLoader.target_list=['H2', 'C2H2', 'CH4', 'C2H4', 'C2H6',]
+# Entrenar los modelos
-eNose.fit()
+# eNoseLoader.target_list=['H2', 'C2H2', 'CH4', 'C2H4', 'C2H6',]
-eNoseLoader.target_list=['H2',]
+# eNose.fit()
-eNose.fit()
-eNoseLoader.target_list=['C2H2',]
-eNose.fit()
-eNoseLoader.target_list=['CH4',]
-eNose.fit()
-eNoseLoader.target_list=['C2H4',]
-eNose.fit()
-eNoseLoader.target_list=['C2H6',]
-eNose.fit()
-eNose.wrap_and_save()
-
 # eNoseLoader.target_list=['H2',]
-# eNose.gen_plots('Tabular-conv3','XGBRegressor_0')
+# eNose.fit()|
+# eNoseLoader.target_list=['C2H2',]
+# eNose.fit()
+# eNoseLoader.target_list=['CH4',]
+# eNose.fit()
+# eNoseLoader.target_list=['C2H4',]
+# eNose.fit()
+# eNoseLoader.target_list=['C2H6',]
+# eNose.fit()
+# eNose.wrap_and_save()
+
+# Grafica las predicciones
+eNose.gen_plots('Tabular','XGBRegressor_3', target=['H2', 'C2H2', 'CH4', 'C2H4', 'C2H6',])
+#eNose.gen_plots('Tabular-conv3','XGBRegressor_4', target=['H2',])
+
+eNose.gen_plots('Conv1D-w32-conv3','Conv1D_v1_1', target=['H2','C2H2', 'CH4', 'C2H4', 'C2H6',])
+# eNose.gen_plots('Conv1D-w32-conv3','Conv1D_v1_2', target=['H2',])
+#eNose.gen_plots('Conv1D-w32','Conv1D_v1_3', target=['H2', 'C2H2', 'CH4', 'C2H4', 'C2H6',])
 
 # eNoseLoader.target_list=['H2', 'C2H2', 'CH4', 'C2H4', 'C2H6',]
 # eNose.gen_plots('Tabular','XGBRegressor_1')