trainer
parent
065de4c5e8
commit
df2c1f3775
177
TrainerClass.py
177
TrainerClass.py
|
|
@ -15,9 +15,7 @@ from xgboost import XGBRegressor
|
|||
|
||||
import keras
|
||||
from keras import layers
|
||||
import ray
|
||||
from ray import tune
|
||||
from ray.tune.schedulers import ASHAScheduler
|
||||
import optuna
|
||||
|
||||
|
||||
# from ray import tune
|
||||
|
|
@ -177,82 +175,70 @@ class eNoseTrainer:
|
|||
else:
|
||||
return {}
|
||||
|
||||
def search_best_conv1D_v1(self, X_train_orig, X_test_orig, Y_train_orig, Y_test_orig, epochs=30, nsamples=0.1):
|
||||
ray.init(ignore_reinit_error=True, configure_logging=True, logging_level=logging.INFO)
|
||||
X_train_ref = ray.put(X_train_orig)
|
||||
Y_train_ref = ray.put(Y_train_orig)
|
||||
X_test_ref = ray.put(X_test_orig)
|
||||
Y_test_ref = ray.put(Y_test_orig)
|
||||
def search_best_conv1D_v1(self, X_train, X_test, Y_train, Y_test, epochs=50, num_trials=100):
|
||||
# Stratified sampling
|
||||
|
||||
# input channels 80 window
|
||||
# l1 minimo 2, maximo 5 => 40 | 16
|
||||
# maxpool 2 o 3 => 20 |
|
||||
# l2 minimo 2 maximo 5 => 4 | 2/5
|
||||
# maxpool 2 o 3 => 2 | 2/15
|
||||
def build_model_conv1D(config, input_shape, output_dim):
|
||||
model = keras.Sequential([
|
||||
layers.Conv1D(filters=config['filters'], kernel_size=config['kernel_l1'], strides=config['kernel_l1']//2, activation='relu', padding='causal', input_shape=input_shape),
|
||||
layers.MaxPooling1D(pool_size=config['pool_size']),
|
||||
layers.Conv1D(filters=config['filters'] * 2, kernel_size=config['kernel_l2'], strides=config['kernel_l2']//2, activation='relu', padding='causal'),
|
||||
layers.MaxPooling1D(pool_size=config['pool_size']),
|
||||
layers.Flatten(),
|
||||
layers.Dense(config['dense_units'], activation='relu'),
|
||||
layers.Dropout(config['dropout']),
|
||||
layers.Dense(output_dim)
|
||||
])
|
||||
model.compile(optimizer=keras.optimizers.Adam(learning_rate=config['lr']), loss='mse')
|
||||
return model
|
||||
input_shape = X_train.shape[1:]
|
||||
output_dim = Y_train.shape[1]
|
||||
|
||||
def train_model_conv1D(config):
|
||||
X_trainc1D = ray.get(X_train_ref)
|
||||
Y_trainc1D = ray.get(Y_train_ref)
|
||||
X_testc1D = ray.get(X_test_ref)
|
||||
Y_testc1D = ray.get(Y_test_ref)
|
||||
def build_model(trial):
|
||||
"""Builds a Keras model using hyperparameters suggested by Optuna"""
|
||||
filters = trial.suggest_categorical('filters', [32, 64, 128])
|
||||
kernel_l1 = trial.suggest_categorical('kernel_size', [3, 5, 7])
|
||||
# kernel_l2 = trial.suggest_categorical('kernel_size', [3, 5, 7])
|
||||
pool_size = trial.suggest_int('pool_size', 2, min(3, input_shape[0] - 1))
|
||||
dense_units = trial.suggest_categorical('dense_units', [32, 64, 128])
|
||||
dropout = trial.suggest_float('dropout', 0.1, 0.3)
|
||||
lr = trial.suggest_loguniform('lr', 1e-4, 5e-3)
|
||||
batch_size = trial.suggest_categorical('batch_size', [16, 32, 64, 128])
|
||||
|
||||
input_shape = X_trainc1D.shape[1:]
|
||||
output_dim = Y_trainc1D.shape[1]
|
||||
inputs = keras.Input(shape=input_shape)
|
||||
x = layers.Conv1D(filters=filters, kernel_size=kernel_l1, activation='relu', strides=kernel_l1//2, padding='causal')(inputs)
|
||||
x = layers.MaxPooling1D(pool_size=pool_size)(x)
|
||||
# x = layers.Conv1D(filters=filters * 2, kernel_size=kernel_l2, activation='relu', strides=kernel_l2//2, padding='causal')(x)
|
||||
# x = layers.MaxPooling1D(pool_size=pool_size)(x)
|
||||
x = layers.Flatten()(x)
|
||||
x = layers.Dense(dense_units, activation='relu')(x)
|
||||
x = layers.Dropout(dropout)(x)
|
||||
outputs = layers.Dense(output_dim)(x)
|
||||
|
||||
model = build_model_conv1D(config, input_shape, output_dim)
|
||||
model = keras.Model(inputs, outputs)
|
||||
model.compile(optimizer=keras.optimizers.Adam(learning_rate=lr), loss='mse')
|
||||
return model, batch_size
|
||||
|
||||
def objective(trial):
|
||||
"""Objective function for Optuna hyperparameter optimization"""
|
||||
model, batch_size = build_model(trial)
|
||||
# early_stopping = keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)
|
||||
|
||||
model.fit(
|
||||
X_trainc1D, Y_trainc1D,
|
||||
validation_data=(X_testc1D, Y_testc1D),
|
||||
epochs=config['epochs'],
|
||||
batch_size=config['batch_size'],
|
||||
verbose=0
|
||||
X_train, Y_train,
|
||||
validation_data=(X_test, Y_test),
|
||||
epochs=epochs,
|
||||
batch_size=batch_size,
|
||||
verbose=0#,
|
||||
# callbacks=[early_stopping]
|
||||
)
|
||||
|
||||
Y_pred = model.predict(X_testc1D)
|
||||
mse = mean_squared_error(Y_testc1D, Y_pred)
|
||||
tune.report({'mse': mse})
|
||||
Y_pred = model.predict(X_test)
|
||||
mse = mean_squared_error(Y_test, Y_pred)
|
||||
num_params = model.count_params() # Get number of weights in the model
|
||||
trial.set_user_attr("num_params", num_params) # Store it in the trial object
|
||||
|
||||
config_space = {
|
||||
'filters': tune.choice([16, 32, 64]),
|
||||
'kernel_l1': tune.choice([3, 5, 7]),
|
||||
'kernel_l2': tune.choice([3, 5, 7]),
|
||||
'pool_size': tune.choice([2, 3]),
|
||||
'dense_units': tune.choice([32, 64, 128, 256]),
|
||||
'dropout': tune.choice([0.05, 0.15, 0.3]),
|
||||
'lr': tune.choice([0.01, 0.005, 0.001]),
|
||||
'batch_size': tune.choice([16, 32, 64, 128]),
|
||||
'epochs': epochs
|
||||
}
|
||||
total_space = (3*3*3*2*4*3*3*4)
|
||||
return mse
|
||||
|
||||
self.logger.info(f"total_space: {total_space}, num_samples: {int(nsamples*total_space)}")
|
||||
# Run hyperparameter tuning
|
||||
study = optuna.create_study(direction='minimize')
|
||||
study.optimize(objective, n_trials=num_trials)
|
||||
|
||||
scheduler = ASHAScheduler(metric='mse', mode='min', max_t=epochs, grace_period=5, reduction_factor=2)
|
||||
# analysis = tune.run(train_model, config=config_space, num_samples=num_samples, scheduler=scheduler)
|
||||
analysis = tune.run( tune.with_parameters(train_model_conv1D), config=config_space, num_samples=int(nsamples*total_space), scheduler=scheduler, max_concurrent_trials=8 )
|
||||
best_config = analysis.get_best_config(metric='mse', mode='min')
|
||||
best_model = build_model_conv1D(best_config, X_train_orig.shape[1:], Y_train_orig.shape[1])
|
||||
# Get best hyperparameters
|
||||
best_params = study.best_params
|
||||
|
||||
# Train final model with best parameters
|
||||
best_model, best_batch_size = build_model(optuna.trial.FixedTrial(best_params))
|
||||
|
||||
|
||||
ray.internal.free([X_train_ref, Y_train_ref, X_test_ref, Y_test_ref])
|
||||
ray.shutdown()
|
||||
return best_model, analysis
|
||||
return best_model, study, best_batch_size
|
||||
|
||||
def train_and_score_model(self, model, X_train, X_test, Y_train, Y_test):
|
||||
param_dist = self.get_tunable_params(model)
|
||||
|
|
@ -532,25 +518,34 @@ class eNoseTrainer:
|
|||
self.logger.debug(f"Y_train_sample: {Y_train_sample.shape}")
|
||||
self.logger.debug(f"Y_test_sample: {Y_test_sample.shape}")
|
||||
|
||||
optimized_model, analysis = self.search_best_conv1D_v1(X_train_sample, X_test_sample, Y_train_sample, Y_test_sample, epochs=10)
|
||||
model_params = analysis.get_best_config(metric='mse', mode='min')
|
||||
analysis.results_df.to_excel(f"{model_file}.search.xlsx", index=False)
|
||||
best_model, study, best_batch_size = self.search_best_conv1D_v1(X_train_sample, X_test_sample, Y_train_sample, Y_test_sample, epochs=10, num_trials=10)
|
||||
# Save study results to an Excel file
|
||||
trials_data = []
|
||||
for trial in study.trials:
|
||||
trial_info = trial.params.copy()
|
||||
trial_info['mse'] = trial.value
|
||||
trial_info['num_params'] = trial.user_attrs.get("num_params", 0)
|
||||
trials_data.append(trial_info)
|
||||
|
||||
self.logger.info(f"Training Model {model_id} with {model_params}")
|
||||
optimized_model.fit(X_train, Y_train, epochs=epochs, batch_size=model_params['batch_size'], verbose=1)
|
||||
df = pd.DataFrame(trials_data)
|
||||
df.to_excel(f"{model_file}.search.xlsx", index=False)
|
||||
|
||||
Y_train_pred = optimized_model.predict(X_train)
|
||||
Y_test_pred = optimized_model.predict(X_test)
|
||||
self.logger.info(f"Training Model {model_id} with {study.best_params}")
|
||||
early_stopping = keras.callbacks.EarlyStopping(monitor='loss', patience=5, restore_best_weights=True, min_delta=0.0003)
|
||||
best_model.fit(X_train, Y_train, epochs=epochs, batch_size=best_batch_size, verbose=1, callbacks=[early_stopping])
|
||||
|
||||
best_model.save(f"{model_file}.keras")
|
||||
best_model.save_weights(f"{model_file}.weights.h5")
|
||||
|
||||
Y_train_pred = best_model.predict(X_train)
|
||||
Y_test_pred = best_model.predict(X_test)
|
||||
|
||||
mse_train = mean_squared_error(Y_train, Y_train_pred)
|
||||
mae_test = mean_absolute_error(Y_test, Y_test_pred)
|
||||
mse_test = mean_squared_error(Y_test, Y_test_pred)
|
||||
mae_test = mean_absolute_error(Y_test, Y_test_pred)
|
||||
rmse_test = np.sqrt(mse_test)
|
||||
|
||||
|
||||
ts = datetime.now().strftime("%d/%m/%Y %H:%M:%S")
|
||||
optimized_model.save(model_file)
|
||||
optimized_model.save_weights(f"{model_file}.weights.h5")
|
||||
|
||||
newrow = pd.DataFrame( [{"node": node,
|
||||
"ts": ts,
|
||||
|
|
@ -561,7 +556,7 @@ class eNoseTrainer:
|
|||
"Train Ratio": Y_train.shape[0]/Y_xboost.shape[0],
|
||||
"Ratio": self.ratio,
|
||||
"Model": model_id,
|
||||
"Params": json.dumps(model_params),
|
||||
"Params": json.dumps(study.best_params),
|
||||
"Train mse": mse_train,
|
||||
"mse": mse_test,
|
||||
"mae": mae_test,
|
||||
|
|
@ -610,22 +605,34 @@ class eNoseTrainer:
|
|||
self.logger.debug(f"Y_train_sample: {Y_train_sample.shape}")
|
||||
self.logger.debug(f"Y_test_sample: {Y_test_sample.shape}")
|
||||
|
||||
optimized_model, model_params = self.search_best_conv1D_v1(X_train_sample, X_test_sample, Y_train_sample, Y_test_sample, epochs=epochs//3)
|
||||
best_model, study, best_batch_size = self.search_best_conv1D_v1(X_train_sample, X_test_sample, Y_train_sample, Y_test_sample, epochs=10, num_trials=10)
|
||||
# Save study results to an Excel file
|
||||
trials_data = []
|
||||
for trial in study.trials:
|
||||
trial_info = trial.params.copy()
|
||||
trial_info['mse'] = trial.value
|
||||
trial_info['num_params'] = trial.user_attrs.get("num_params", 0)
|
||||
trials_data.append(trial_info)
|
||||
|
||||
self.logger.info(f"Training Model {model_id} with {model_params}")
|
||||
optimized_model.fit(X_train, Y_train, epochs=epochs, batch_size=model_params['batch_size'], verbose=1)
|
||||
df = pd.DataFrame(trials_data)
|
||||
df.to_excel(f"{model_file}.search.xlsx", index=False)
|
||||
|
||||
Y_train_pred = optimized_model.predict(X_train)
|
||||
Y_test_pred = optimized_model.predict(X_test)
|
||||
self.logger.info(f"Training Model {model_id} with {study.best_params}")
|
||||
early_stopping = keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)
|
||||
best_model.fit(X_train, Y_train, epochs=epochs, batch_size=best_batch_size, verbose=1, callbacks=[early_stopping])
|
||||
|
||||
best_model.save(f"{model_file}.keras")
|
||||
best_model.save_weights(f"{model_file}.weights.h5")
|
||||
|
||||
Y_train_pred = best_model.predict(X_train)
|
||||
Y_test_pred = best_model.predict(X_test)
|
||||
|
||||
mse_train = mean_squared_error(Y_train, Y_train_pred)
|
||||
mae_test = mean_absolute_error(Y_test, Y_test_pred)
|
||||
mse_test = mean_squared_error(Y_test, Y_test_pred)
|
||||
mae_test = mean_absolute_error(Y_test, Y_test_pred)
|
||||
rmse_test = np.sqrt(mse_test)
|
||||
|
||||
ts = datetime.now().strftime("%d/%m/%Y %H:%M:%S")
|
||||
optimized_model.save(model_file)
|
||||
optimized_model.save_weights(f"{model_file}.weights.h5")
|
||||
|
||||
newrow = pd.DataFrame( [{"node": node,
|
||||
"ts": ts,
|
||||
|
|
@ -636,7 +643,7 @@ class eNoseTrainer:
|
|||
"Train Ratio": Y_train.shape[0]/Y_xboost.shape[0],
|
||||
"Ratio": self.ratio,
|
||||
"Model": model_id,
|
||||
"Params": json.dumps(model_params),
|
||||
"Params": json.dumps(study.best_params),
|
||||
"Train mse": tmse,
|
||||
"mse": mse,
|
||||
"mae": mae,
|
||||
|
|
|
|||
Loading…
Reference in New Issue