cleaned up code:

added import_data method for importing data, now using the given KDDTest+ and KDDTrain+ data for training instead of only KDDTrain+, a lot of refactoring

code/machine_learning_models/utilities.py

@@ -5,7 +5,6 @@ import pandas as pd
 import seaborn as sns
 from matplotlib import pyplot as plt
 from scipy.io import arff
-from sklearn.base import BaseEstimator
 from sklearn.preprocessing import OrdinalEncoder
 
 show_plots = False
@@ -110,17 +109,19 @@ def plot_features(features, info_text: str = None, model_name=None):
 
 def normalize(df_train, df_test, exclude, numerical_scaler, label_scaler):
 
-	scale_targets = df_train.select_dtypes(include=np.number).drop(columns=exclude).columns
-	df_train[scale_targets] = numerical_scaler.fit_transform(df_train[scale_targets])
+	df_temp = pd.concat([df_train, df_test])
+	scale_targets = df_temp.select_dtypes(include=np.number).drop(columns=exclude).columns
+	numerical_scaler.fit_transform(df_temp[scale_targets])
+
+	df_train[scale_targets] = numerical_scaler.transform(df_train[scale_targets])
 	df_test[scale_targets] = numerical_scaler.transform(df_test[scale_targets])
 
 	labels = df_train.select_dtypes(include=object, exclude=np.number).columns
 	for label in labels:
-		df_train[label] = label_scaler.fit_transform(df_train[label])
+		label_scaler.fit_transform(df_temp[label])
+		df_train[label] = label_scaler.transform(df_train[label])
 		df_test[label] = label_scaler.transform(df_test[label])
 
-
-
 def plot_confusion_matrix(confusion_matrix: List[List[int]], accuracy: float, model_name=None) -> None:
 	if len(confusion_matrix) != 2 or any(len(row) != 2 for row in confusion_matrix):
 		raise ValueError("Confusion matrices must be 2x2")