[docs]@classmethoddeffrom_name(cls,name:str)->"Optimiser":lname=name.lower()foroincls:ifo.value[0]==lname:returnoraiseValueError(f"Unknown optimiser name '{name}'; known names: {[o.value[0]foroincls]}")
class_Optimiser(object):""" Wrapper for classes inherited from torch.optim.Optimizer """def__init__(self,params,method:Union[str,Optimiser],lr,max_grad_norm,use_shrinkage=True,**optimiser_args):""" :param params: an iterable of torch.Tensor s or dict s. Specifies what Tensors should be optimized. :param method: the optimiser to use :param lr: learnig rate :param max_grad_norm: gradient norm value beyond which to apply gradient shrinkage :param optimiser_args: keyword arguments to be used in actual torch optimiser """self.method=Optimiser.from_name_or_instance(method)self.params=list(params)# careful: params may be a generatorself.last_ppl=Noneself.lr=lrself.max_grad_norm=max_grad_normself.start_decay=Falseself.optimiserArgs=optimiser_argsself.use_shrinkage=use_shrinkage# instantiate optimiseroptimiser_args=dict(self.optimiserArgs)optimiser_args.update({'lr':self.lr})ifself.method==Optimiser.LBFGS:self.use_shrinkage=Falseself.optimizer=optim.LBFGS(self.params,**optimiser_args)else:cons=self.method.value[1]self.optimizer=cons(self.params,**optimiser_args)defstep(self,loss_backward:Callable):""" :param loss_backward: callable, performs backward step and returns loss :return: loss value """ifself.use_shrinkage:defclosure_with_shrinkage():loss_value=loss_backward()torch.nn.utils.clip_grad_norm_(self.params,self.max_grad_norm)returnloss_valueclosure=closure_with_shrinkageelse:closure=loss_backwardloss=self.optimizer.step(closure)returnloss
[docs]classNNLossEvaluator(ABC):""" Base class defining the interface for training and validation loss evaluation. """
[docs]@abstractmethoddefstart_epoch(self)->None:""" Starts a new epoch, resetting any aggregated values required to ultimately return the epoch's overall training loss (via getEpochTrainLoss) and validation metrics (via getValidationMetrics) """pass
[docs]@abstractmethoddefcompute_train_batch_loss(self,model_output,ground_truth,x,y)->torch.Tensor:""" Computes the loss for the given model outputs and ground truth values for a batch and aggregates the computed loss values such that :meth:``getEpochTrainLoss`` can return an appropriate result for the entire epoch. The original batch tensors X and Y are provided as meta-information only. :param model_output: the model output :param ground_truth: the ground truth values :param x: the original batch input tensor :param y: the original batch output (ground truth) tensor :return: the loss (scalar tensor) """pass
[docs]@abstractmethoddefget_epoch_train_loss(self)->float:""" :return: the epoch's overall training loss (as obtained by collecting data from individual training batch data passed to computeTrainBatchLoss) """pass
[docs]@abstractmethoddefprocess_validation_batch(self,model_output,ground_truth,x,y)->None:""" Processes the given model outputs and ground truth values in order to compute sufficient statistics for velidation metrics, which at the end of the epoch, shall be retrievable via method getValidationMetrics :param model_output: the model output :param ground_truth: the ground truth values :param x: the original batch input tensor :param y: the original batch output (ground truth) tensor :return: the loss (scalar tensor) """pass
[docs]@abstractmethoddefstart_evaluation(self,cuda:bool)->Evaluation:""" Begins the evaluation of a model, returning a (stateful) object which is to perform the necessary computations. :param cuda: whether CUDA is being applied (all tensors/models on the GPU) :return: the evaluation object """pass
[docs]@abstractmethoddefget_validation_metric_name(self)->str:""" :return: the name of the validation metric which is to be used to determine the best model (key for the ordered dictionary returned by method Evaluation.getValidationMetrics) """pass
[docs]classNNLossEvaluatorFixedDim(NNLossEvaluator,ABC):""" Base class defining the interface for training and validation loss evaluation, which uses fixed-dimension outputs and aggregates individual training batch losses that are summed losses per batch (averaging appropriately internally). """
[docs]defcompute_train_batch_loss(self,model_output,ground_truth,x,y)->torch.Tensor:# size of modelOutput and groundTruth: (batchSize, outputDim=numOutputsPerDataPoint)ifself.num_outputs_per_data_pointisNone:output_shape=y.shape[1:]self.num_outputs_per_data_point=functools.reduce(lambdax,y:x*y,output_shape,1)assertself.output_dim_weightsisNoneorlen(self.output_dim_weights)==self.num_outputs_per_data_pointnum_data_points_in_batch=y.shape[0]ifself.output_dim_weightsisNone:# treat all dimensions as equal, applying criterion to entire tensorsloss=self.criterion(model_output,ground_truth)self.num_samples+=num_data_points_in_batch*self.num_outputs_per_data_pointself.total_loss+=loss.item()returnlosselse:# compute loss per dimension and return weighted lossloss_per_dim=torch.zeros(self.num_outputs_per_data_point,device=model_output.device,dtype=torch.float)foroinrange(self.num_outputs_per_data_point):loss_per_dim[o]=self.criterion(model_output[:,o],ground_truth[:,o])weighted_loss=(loss_per_dim*self.output_dim_weights).sum()/self.output_dim_weight_sumself.num_samples+=num_data_points_in_batchself.total_loss+=weighted_loss.item()returnweighted_loss
[docs]defprocess_validation_batch(self,model_output,ground_truth,x,y):ifself.validation_ground_truth_shapeisNone:self.validation_ground_truth_shape=y.shape[1:]# the shape of the output of a single model applicationself.validation_loss_evaluator.start_validation_collection(self.validation_ground_truth_shape)self.validation_loss_evaluator.process_validation_result_batch(model_output,ground_truth)
[docs]@abstractmethoddefget_training_criterion(self)->nn.Module:""" Gets the optimisation criterion (loss function) for training. Standard implementations are available in torch.nn (torch.nn.MSELoss, torch.nn.CrossEntropyLoss, etc.). """pass
[docs]@abstractmethoddefcreate_validation_loss_evaluator(self,cuda:bool)->"ValidationLossEvaluator":""" :param cuda: whether to use CUDA-based tensors :return: the evaluator instance which is to be used to evaluate the model on validation data """pass
[docs]defget_validation_metric_name(self)->str:""" Gets the name of the metric (key of dictionary as returned by the validation loss evaluator's endValidationCollection method), which is defining for the quality of the model and thus determines which epoch's model is considered the best. :return: the name of the metric """pass
[docs]@abstractmethoddefstart_validation_collection(self,ground_truth_shape):""" Initiates validation data collection for a new epoch, appropriately resetting this object's internal state. :param ground_truth_shape: the tensor shape of a single ground truth data point (not including the batch entry dimension) """pass
[docs]@abstractmethoddefprocess_validation_result_batch(self,output,ground_truth):""" Collects, for validation, the given output and ground truth data (tensors holding data on one batch, where the first dimension is the batch entry) :param output: the model's output :param ground_truth: the corresponding ground truth """pass
[docs]@abstractmethoddefend_validation_collection(self)->OrderedDict:""" Computes validation metrics based on the data previously processed. :return: an ordered dictionary with validation metrics """pass
[docs]classNNLossEvaluatorRegression(NNLossEvaluatorFixedDim,ToStringMixin):"""A loss evaluator for (multi-variate) regression."""
def__init__(self,loss_fn:LossFunction=LossFunction.L2LOSS,validation_tensor_transformer:Optional[TensorTransformer]=None,output_dim_weights:Sequence[float]=None,apply_output_dim_weights_in_validation=True,validation_metric_name:Optional[str]=None):""" :param loss_fn: the loss function to use :param validation_tensor_transformer: a transformer which is to be applied to validation tensors (both model outputs and ground truth) prior to computing the validation metrics :param output_dim_weights: vector of weights to apply to then mean loss per output dimension, i.e. for the case where for each data point, the model produces n output dimensions, the mean loss for the i-th dimension is to be computed separately and be scaled with the weight, and the overall loss returned is the weighted average. The weights need not sum to 1 (normalisation is applied). :param apply_output_dim_weights_in_validation: whether output dimension weights are also to be applied to to the metrics computed for validation. Note that this may not be possible if a validationTensorTransformer which changes the output dimensions is used. :param validation_metric_name: the metric to use for model selection during validation; if None, use default depending on lossFn """self.validation_tensor_transformer=validation_tensor_transformerself.output_dim_weights=np.array(output_dim_weights)ifoutput_dim_weightsisnotNoneelseNoneself.apply_output_dim_weights_in_validation=apply_output_dim_weights_in_validationself.validation_metric_name=validation_metric_nameifloss_fnisNone:loss_fn=self.LossFunction.L2LOSStry:self.loss_fn=self.LossFunction(loss_fn)exceptValueError:raiseException(f"The loss function '{loss_fn}' is not supported. "f"Available options are: {[e.valueforeinself.LossFunction]}")
[docs]defget_training_criterion(self):ifself.loss_fnisself.LossFunction.L1LOSS:criterion=nn.L1Loss(reduction='sum')elifself.loss_fnisself.LossFunction.L2LOSSorself.loss_fn==self.LossFunction.MSELOSS:criterion=nn.MSELoss(reduction='sum')elifself.loss_fnisself.LossFunction.SMOOTHL1LOSS:criterion=nn.SmoothL1Loss(reduction='sum')else:raiseAssertionError(f"Loss function {self.loss_fn} defined but instantiation not implemented.")returncriterion
[docs]defstart_validation_collection(self,ground_truth_shape):iflen(ground_truth_shape)!=1:raiseValueError("Outputs that are not vectors are currently unsupported")self.begin_new_validation_collection=True
[docs]defprocess_validation_result_batch(self,output,ground_truth):# apply tensor transformer (if any)ifself.validationTensorTransformerisnotNone:output=self.validationTensorTransformer.transform(output)ground_truth=self.validationTensorTransformer.transform(ground_truth)# check if new collectionifself.begin_new_validation_collection:self.output_dims=ground_truth.shape[-1]self.total_loss_l1=np.zeros(self.output_dims)self.total_loss_l2=np.zeros(self.output_dims)self.allTrueOutputs=Noneself.begin_new_validation_collection=Falseassertlen(output.shape)==2andlen(ground_truth.shape)==2# obtain series of outputs per output dimension: (batch_size, output_size) -> (output_size, batch_size)predicted_output=output.permute(1,0)true_output=ground_truth.permute(1,0)ifself.allTrueOutputsisNone:self.allTrueOutputs=true_outputelse:self.allTrueOutputs=torch.cat((self.allTrueOutputs,true_output),dim=1)foriinrange(self.output_dims):self.total_loss_l1[i]+=self.evaluate_l1(predicted_output[i],true_output[i]).item()self.total_loss_l2[i]+=self.evaluate_l2(predicted_output[i],true_output[i]).item()
[docs]defget_validation_metric_name(self):ifself.validation_metric_nameisnotNone:returnself.validation_metric_nameelse:ifself.loss_fnisself.LossFunction.L1LOSSorself.loss_fnisself.LossFunction.SMOOTHL1LOSS:return"MAE"elifself.loss_fnisself.LossFunction.L2LOSSorself.loss_fnisself.LossFunction.MSELOSS:return"MSE"else:raiseAssertionError(f"No validation metric defined as selection criterion for loss function {self.loss_fn}")
[docs]classNNLossEvaluatorClassification(NNLossEvaluatorFixedDim):"""A loss evaluator for classification"""
[docs]@classmethoddefdefault_for_output_mode(cls,output_mode:ClassificationOutputMode):ifoutput_mode==ClassificationOutputMode.PROBABILITIES:raiseValueError(f"No loss function available for {output_mode}; Either apply log at the end and use "f"{ClassificationOutputMode.LOG_PROBABILITIES} or use a different final activation (e.g. log_softmax) "f"to avoid this type of output.")elifoutput_mode==ClassificationOutputMode.LOG_PROBABILITIES:returncls.NLLelifoutput_mode==ClassificationOutputMode.UNNORMALISED_LOG_PROBABILITIES:returncls.CROSSENTROPYelse:raiseValueError(f"No default specified for {output_mode}")
[docs]classNNOptimiserParams(ToStringMixin):REMOVED_PARAMS={"cuda"}RENAMED_PARAMS={"optimiserClip":"optimiser_clip","lossEvaluator":"loss_evaluator","optimiserLR":"optimiser_lr","earlyStoppingEpochs":"early_stopping_epochs","batchSize":"batch_size","trainFraction":"train_fraction","scaledOutputs":"scaled_outputs","useShrinkage":"use_shrinkage","shrinkageClip":"shrinkage_clip",}def__init__(self,loss_evaluator:NNLossEvaluator=None,gpu:Optional[int]=None,optimiser:Union[str,Optimiser]="adam",optimiser_lr=0.001,early_stopping_epochs=None,batch_size=None,epochs=1000,train_fraction=0.75,scaled_outputs=False,use_shrinkage=True,shrinkage_clip=10.,shuffle=True,optimiser_args:Optional[Dict[str,Any]]=None):""" :param loss_evaluator: the loss evaluator to use :param gpu: the index of the GPU to be used (if CUDA is enabled for the model to be trained); if None, default to first GPU :param optimiser: the optimiser to use :param optimiser_lr: the optimiser's learning rate :param early_stopping_epochs: the number of epochs without validation score improvement after which to abort training and use the best epoch's model (early stopping); if None, never abort training before all epochs are completed :param batch_size: the batch size to use; for algorithms L-BFGS (optimiser='lbfgs'), which do not use batches, leave this at None. If the algorithm uses batches and None is specified, batch size 64 will be used by default. :param train_fraction: the fraction of the data used for training (with the remainder being used for validation). If no validation is to be performed, pass 1.0. :param scaled_outputs: whether to scale all outputs, resulting in computations of the loss function based on scaled values rather than normalised values. Enabling scaling may not be appropriate in cases where there are multiple outputs on different scales/with completely different units. :param use_shrinkage: whether to apply shrinkage to gradients whose norm exceeds ``shrinkageClip``, scaling the gradient down to ``shrinkageClip`` :param shrinkage_clip: the maximum gradient norm beyond which to apply shrinkage (if ``useShrinkage`` is True) :param shuffle: whether to shuffle the training data :param optimiser_args: keyword arguments to be passed on to the actual torch optimiser """ifOptimiser.from_name_or_instance(optimiser)==Optimiser.LBFGS:large_batch_size=1e12ifbatch_sizeisnotNone:log.warning(f"LBFGS does not make use of batches, therefore using large batch size {large_batch_size} "f"to achieve use of a single batch")batch_size=large_batch_sizeelse:ifbatch_sizeisNone:log.debug("No batch size was specified, using batch size 64 by default")batch_size=64self.epochs=epochsself.batch_size=batch_sizeself.optimiser_lr=optimiser_lrself.shrinkage_clip=shrinkage_clipself.optimiser=optimiserself.gpu=gpuself.train_fraction=train_fractionself.scaled_outputs=scaled_outputsself.loss_evaluator=loss_evaluatorself.optimiser_args=optimiser_argsifoptimiser_argsisnotNoneelse{}self.use_shrinkage=use_shrinkageself.early_stopping_epochs=early_stopping_epochsself.shuffle=shuffle@classmethoddef_updated_params(cls,params:dict)->dict:return{cls.RENAMED_PARAMS.get(k,k):vfork,vinparams.items()ifknotincls.REMOVED_PARAMS}def__setstate__(self,state):if"shuffle"notinstate:state["shuffle"]=Trueself.__dict__=self._updated_params(state)
[docs]@classmethoddeffrom_either_dict_or_instance(cls,nn_optimiser_dict_params:dict,nn_optimiser_params:Optional["NNOptimiserParams"]):have_instance=nn_optimiser_paramsisnotNonehave_dict=len(nn_optimiser_dict_params)ifhave_instanceandhave_dict:raiseValueError("Received both a non-empty dictionary and an instance")ifhave_instance:returnnn_optimiser_paramselse:returnNNOptimiserParams.from_dict(nn_optimiser_dict_params)
[docs]classNNOptimiser:log=log.getChild(__qualname__)def__init__(self,params:NNOptimiserParams):""" :param params: parameters """ifparams.loss_evaluatorisNone:raiseValueError("Must provide a loss evaluator")self.params=paramsself.cuda=Noneself.best_epoch=Nonedef__str__(self):returnf"{self.__class__.__name__}[params={self.params}]"
[docs]deffit(self,model:"TorchModel",data:Union[DataUtil,List[DataUtil],TorchDataSetProvider,List[TorchDataSetProvider],TorchDataSet,List[TorchDataSet],Tuple[TorchDataSet,TorchDataSet],List[Tuple[TorchDataSet,TorchDataSet]]],create_torch_module=True)->"TrainingInfo":""" Fits the parameters of the given model to the given data, which can be a list of or single instance of one of the following: * a `DataUtil` or `TorchDataSetProvider` (from which a training set and validation set will be obtained according to the `trainFraction` parameter of this object) * a `TorchDataSet` which shall be used as the training set (for the case where no validation set shall be used) * a tuple with two `TorchDataSet` instances, where the first shall be used as the training set and the second as the validation set :param model: the model to be fitted :param data: the data to use (see variants above) :param create_torch_module: whether to newly create the torch module that is to be trained from the model's factory. If False, (re-)train the existing module. """self.cuda=model.cudaself.log.info(f"Preparing parameter learning of {model} via {self} with cuda={self.cuda}")use_validation=self.params.train_fraction!=1.0defto_data_set_provider(d)->TorchDataSetProvider:ifisinstance(d,TorchDataSetProvider):returndelifisinstance(d,DataUtil):returnTorchDataSetProviderFromDataUtil(d,self.cuda)else:raiseValueError(f"Cannot create a TorchDataSetProvider from {d}")training_log_entries=[]deftraining_log(s):self.log.info(s)training_log_entries.append(s)self._init_cuda()# Set the random seed manually for reproducibility.seed=42torch.manual_seed(seed)ifself.cuda:torchcuda.manual_seed_all(seed)torch.backends.cudnn.benchmark=Falsetorch.backends.cudnn.deterministic=True# obtain data, splitting it into training and validation set(s)validation_sets=[]training_sets=[]output_scalers=[]iftype(data)!=list:data=[data]self.log.info("Obtaining input/output training instances")foridx_data_item,data_iteminenumerate(data):ifisinstance(data_item,TorchDataSet):ifuse_validation:raiseValueError("Passing a TorchDataSet instance is not admissible when validation is enabled (trainFraction != 1.0). ""Pass a TorchDataSetProvider or another representation that supports validation instead.")training_set=data_itemvalidation_set=Noneoutput_scaler=TensorScalerIdentity()eliftype(data_item)==tuple:training_set,validation_set=data_itemoutput_scaler=TensorScalerIdentity()else:data_set_provider=to_data_set_provider(data_item)training_set,validation_set=data_set_provider.provide_split(self.params.train_fraction)output_scaler=data_set_provider.get_output_tensor_scaler()training_sets.append(training_set)ifvalidation_setisnotNone:validation_sets.append(validation_set)output_scalers.append(output_scaler)training_log(f"Data set {idx_data_item+1}/{len(data)}: #train={training_set.size()}, "f"#validation={validation_set.size()ifvalidation_setisnotNoneelse'None'}")training_log("Number of validation sets: %d"%len(validation_sets))torch_model=model.create_torch_module()ifcreate_torch_moduleelsemodel.get_torch_module()ifself.cuda:torch_model.cuda()model.set_torch_module(torch_model)n_params=sum([p.nelement()forpintorch_model.parameters()])self.log.info(f"Learning parameters of {model}")training_log('Number of parameters: %d'%n_params)training_log(f"Starting training process via {self}")loss_evaluator=self.params.loss_evaluatortotal_epochs=Nonebest_val=1e9best_epoch=0optim=_Optimiser(torch_model.parameters(),method=self.params.optimiser,lr=self.params.optimiser_lr,max_grad_norm=self.params.shrinkage_clip,use_shrinkage=self.params.use_shrinkage,**self.params.optimiser_args)best_model_bytes=model.get_module_bytes()loss_evaluation=loss_evaluator.start_evaluation(self.cuda)validation_metric_name=loss_evaluator.get_validation_metric_name()training_loss_values=[]validation_metric_values=[]try:self.log.info(f'Begin training with cuda={self.cuda}')self.log.info('Press Ctrl+C to end training early')forepochinrange(1,self.params.epochs+1):loss_evaluation.start_epoch()epoch_start_time=time.time()# perform training step, processing all the training data oncetrain_loss=self._train(training_sets,torch_model,optim,loss_evaluation,self.params.batch_size,output_scalers)training_loss_values.append(train_loss)# perform validation, computing the mean metrics across all validation sets (if more than one),# and check for new best result according to validation resultsis_new_best=Falseifuse_validation:metrics_sum=Nonemetrics_keys=Nonefori,(validation_set,output_scaler)inenumerate(zip(validation_sets,output_scalers)):metrics=self._evaluate(validation_set,torch_model,loss_evaluation,output_scaler)metrics_array=np.array(list(metrics.values()))ifi==0:metrics_sum=metrics_arraymetrics_keys=metrics.keys()else:metrics_sum+=metrics_arraymetrics_sum/=len(validation_sets)# mean resultsmetrics=dict(zip(metrics_keys,metrics_sum))current_val=metrics[loss_evaluator.get_validation_metric_name()]validation_metric_values.append(current_val)is_new_best=current_val<best_valifis_new_best:best_val=current_valbest_epoch=epochbest_str="best {:s}{:5.6f} from this epoch".format(validation_metric_name,best_val)else:best_str="best {:s}{:5.6f} from epoch {:d}".format(validation_metric_name,best_val,best_epoch)val_str=f' | validation {", ".join(["%s%5.4f"%eforeinmetrics.items()])} | {best_str}'else:val_str=""training_log('Epoch {:3d}/{} completed in {:5.2f}s | train loss {:5.4f}{:s}'.format(epoch,self.params.epochs,(time.time()-epoch_start_time),train_loss,val_str))total_epochs=epochifuse_validation:ifis_new_best:best_model_bytes=model.get_module_bytes()# check for early stoppingnum_epochs_without_improvement=epoch-best_epochifself.params.early_stopping_epochsisnotNoneand \
num_epochs_without_improvement>=self.params.early_stopping_epochs:training_log(f"Stopping early: {num_epochs_without_improvement} epochs without validation metric improvement")breaktraining_log("Training complete")exceptKeyboardInterrupt:training_log('Exiting from training early because of keyboard interrupt')# reload best model according to validation resultsifuse_validation:training_log(f'Best model is from epoch {best_epoch} with {validation_metric_name}{best_val} on validation set')self.best_epoch=best_epochmodel.set_module_bytes(best_model_bytes)returnTrainingInfo(best_epoch=best_epochifuse_validationelseNone,log=training_log_entries,total_epochs=total_epochs,training_loss_sequence=training_loss_values,validation_metric_sequence=validation_metric_values)
def_apply_model(self,model,input:Union[torch.Tensor,Sequence[torch.Tensor]],ground_truth,output_scaler:TensorScaler):ifisinstance(input,torch.Tensor):output=model(input)else:output=model(*input)ifself.params.scaled_outputs:output,ground_truth=self._scaled_values(output,ground_truth,output_scaler)returnoutput,ground_truth@classmethoddef_scaled_values(cls,model_output,ground_truth,output_scaler):scaled_output=output_scaler.denormalise(model_output)scaled_truth=output_scaler.denormalise(ground_truth)returnscaled_output,scaled_truthdef_train(self,data_sets:Sequence[TorchDataSet],model:nn.Module,optim:_Optimiser,loss_evaluation:NNLossEvaluator.Evaluation,batch_size:int,output_scalers:Sequence[TensorScaler]):"""Performs one training epoch"""model.train()fordata_set,output_scalerinzip(data_sets,output_scalers):forX,Yindata_set.iter_batches(batch_size,shuffle=self.params.shuffle):defclosure():model.zero_grad()output,ground_truth=self._apply_model(model,X,Y,output_scaler)loss=loss_evaluation.compute_train_batch_loss(output,ground_truth,X,Y)loss.backward()returnlossoptim.step(closure)returnloss_evaluation.get_epoch_train_loss()def_evaluate(self,data_set:TorchDataSet,model:nn.Module,loss_evaluation:NNLossEvaluator.Evaluation,output_scaler:TensorScaler):"""Evaluates the model on the given data set (a validation set)"""model.eval()forX,Yindata_set.iter_batches(self.params.batch_size,shuffle=False):withtorch.no_grad():output,ground_truth=self._apply_model(model,X,Y,output_scaler)loss_evaluation.process_validation_batch(output,ground_truth,X,Y)returnloss_evaluation.get_validation_metrics()def_init_cuda(self):"""Initialises CUDA (for learning) by setting the appropriate device if necessary"""ifself.cuda:device_count=torchcuda.device_count()ifdevice_count==0:raiseException("CUDA is enabled but no device found")ifself.params.gpuisNone:ifdevice_count>1:log.warning("More than one GPU detected but no GPU index was specified, using GPU 0 by default.")gpu_index=0else:gpu_index=self.params.gputorchcuda.set_device(gpu_index)eliftorchcuda.is_available():self.log.info("NOTE: You have a CUDA device; consider running with cuda=True")
[docs]defplot_all(self)->matplotlib.figure.Figure:""" Plots both the sequence of training loss values and the sequence of validation metric values """ts=self.get_training_loss_series()vs=self.get_validation_metric_series()fig,primary_ax=plt.subplots(1,1)secondary_ax=primary_ax.twinx()training_line=primary_ax.plot(ts,color='blue')validation_line=secondary_ax.plot(vs,color='orange')best_epoc_line=primary_ax.axvline(self.best_epoch,color='black',linestyle='dashed')primary_ax.set_xlabel("epoch")primary_ax.set_ylabel(ts.name)secondary_ax.set_ylabel(vs.name)primary_ax.legend(training_line+validation_line+[best_epoc_line],[ts.name,vs.name,"best epoch"])plt.tight_layout()returnfig