diff --git a/neural_network_model/transfer_learning.py b/neural_network_model/transfer_learning.py index b67aaa2..07bc817 100644 --- a/neural_network_model/transfer_learning.py +++ b/neural_network_model/transfer_learning.py @@ -878,6 +878,7 @@ def _save_and_display_gradcam( def grad_cam_viz(self, *args, **kwargs): """ Visualize the Grad-CAM heatmap + This method needs the train and predict methods to be run first Keyword Arguments: num_rows {int} -- Number of rows of the subplot grid (default: {None}) num_cols {int} -- Number of columns of the subplot grid (default: {None}) @@ -955,6 +956,84 @@ def grad_cam_viz(self, *args, **kwargs): ) plt.show() + def grad_cam_viz_2(self, **kwargs): + """ + Visualize the Grad-CAM heatmap + Keyword Arguments: + last_conv_layer_name {str} -- Name of the last convolutional layer (default: {"Conv_1"}) + img_size {tuple} -- Size of the image (default: {(224, 224)}) + gard_cam_image_name {str} -- Name of the Grad-CAM image (default: {"transf_cam.jpg"}) + model_path {str} -- path to the model (default: {None}) + img_path {str} -- path to the image (default: {None}) + """ + preprocess_input = tf.keras.applications.mobilenet_v2.preprocess_input + # decode_predictions = tf.keras.applications.mobilenet_v2.decode_predictions + + last_conv_layer_name = kwargs.get("last_conv_layer_name", "Conv_1") + # img_size = kwargs.get("img_size", (224, 224)) + gard_cam_image_name = kwargs.get("gard_cam_image_name", "transf_cam.jpg") + model_path = kwargs.get("model_path", None) + img = kwargs.get("img", None) + cmap = kwargs.get("cmap", "jet") + alpha = kwargs.get("alpha", 0.4) + cam_path = kwargs.get( + "figure_folder_path", Path(__file__).parent / ".." / "figures" + ) + cam_name = kwargs.get("cam_name", "transf_cam.jpg") + + # Remove last layer's softmax + if not model_path: + raise ValueError("model_path is None") + + logger.info(f"Loading the model from {model_path}") + self.model = tf.keras.models.load_model(model_path) + self.model.layers[-1].activation = None + + array = img_to_array(img) + # We add a dimension to transform our array into a "batch" + # of size "size" + array = np.expand_dims(array, axis=0) + + img_array = preprocess_input( + array + ) + heatmap = self._make_gradcam_heatmap( + img_array, self.model, last_conv_layer_name + ) + + # Load the original image + # img = load_img(img) + # img = img_to_array(img) + # Rescale heatmap to a range 0-255 + heatmap = np.uint8(255 * heatmap) + # Use jet colormap to colorize heatmap + jet = cm.get_cmap(cmap) + + # Use RGB values of the colormap + jet_colors = jet(np.arange(256))[:, :3] + jet_heatmap = jet_colors[heatmap] + + # Create an image with RGB colorized heatmap + jet_heatmap = array_to_img(jet_heatmap) + jet_heatmap = jet_heatmap.resize((array.shape[1], array.shape[0])) + jet_heatmap = img_to_array(jet_heatmap) + + # Superimpose the heatmap on original image + superimposed_img = jet_heatmap * alpha + array + superimposed_img = array_to_img(superimposed_img[0]) + + # Save the superimposed image + superimposed_img.save(cam_path / cam_name) + + plt.imshow(plt.imread(cam_path / f"{gard_cam_image_name}")) + plt.tight_layout() + # save the figure + plt.savefig( + Path(__file__).parent / ".." / "figures" / "grad_cam.png", + bbox_inches="tight", + ) + plt.show() + def predict_one_image( self, img_path: str, @@ -1124,6 +1203,113 @@ def predict_image_patch_classes(self, **kwargs): return save_results + def predict_image_patch_classes_2(self, **kwargs): + """ + Predict the classes of the image patches + This function gets the path to an image which is a horizontal and make patches of the image + for each path it does the prediction and save the image and also the original image with the box in + separated folders + :param kwargs: + img_path: path to the image + window_percent: the percentage of the window size + stride: the stride of the window + patch_images_dir: the output directory to save the patch images + img_with_box_dir: the output directory to save the original image with the box + figsize: Tuple the size of the figure + :return: save_results: Dict the dictionary of the results + """ + + img_path = kwargs.get("img_path", None) + figsize = kwargs.get("figsize", (15, 15)) + window_percent = kwargs.get("window_percent", 10) + stride = kwargs.get("stride", 150) + patch_images_dir = kwargs.get("patch_images_dir", Path(__file__).parent / ".." / "dataset" / "patch_images") + img_with_box_dir = kwargs.get("img_with_box_dir", Path(__file__).parent / ".." / "dataset" / + "images_with_box") + fig_show = kwargs.get("fig_show", False) + model_path = kwargs.get("model_path", None) + class_labels_path = kwargs.get("class_labels_path", None) + + if img_path is None: + raise ValueError("img_path is None") + + # Load the image + image = cv2.imread(img_path) + + # Calculate the window size based on the percentage + height, width, _ = image.shape + logger.info(f"Image height, width: {height}, {width}") + window_height = int(height) + window_width = int(width * window_percent / 100) + logger.info(f"Window height, width: {window_height}, {window_width}") + + # Output directory to save patch images + os.makedirs(patch_images_dir, exist_ok=True) + + # Output directory to save patch images + os.makedirs(img_with_box_dir, exist_ok=True) + + save_results = {} + + # Iterate through the image using sliding window + count = 0 + for y in range(0, height - window_height + 1, stride): + for x in tqdm( + range(0, width - window_width + 1 + stride, stride), + desc="Predicting the image patches" + ): + # Extract the patch using the sliding window + patch = image[ + y:y + window_height, + x:x + window_width + ] + + # Save the patch as an image + patch_filename = os.path.join(patch_images_dir, f'patch_{count}.jpg') + cv2.imwrite(patch_filename, patch) + + # Pass the patch to your model for estimation + # Replace the following line with your model's prediction code + predicted_label, class_probabilities = self.predict_one_image( + img_path=patch_filename, + model_path=model_path, + class_labels_path=class_labels_path + ) + + save_results[f'patch_{count}.jpg'] = { + 'predicted_label': predicted_label, + 'class_probabilities': class_probabilities + } + + count += 1 + + # Create a copy of the original image to draw on + image_with_box = image.copy() + + # Draw a single red box + cv2.rectangle( + image_with_box, + (x, y), + (x + window_width, y + window_height), + (0, 0, 255), 4 + ) # Red box + + # Convert BGR image to RGB for matplotlib + image_rgb = cv2.cvtColor(image_with_box, cv2.COLOR_BGR2RGB) + + # Display the image using matplotlib + if fig_show: + plt.figure(figsize=figsize) + plt.imshow(image_with_box) + plt.axis('off') # Turn off axis labels + plt.show() + + # Save the image + _filename = os.path.join(img_with_box_dir, f'patch_{count}.jpg') + cv2.imwrite(_filename, image_rgb) + + return save_results + if __name__ == "__main__": from neural_network_model.process_data import Preprocessing @@ -1140,7 +1326,7 @@ def predict_image_patch_classes(self, **kwargs): # transfer_model.analyze_image_names() # transfer_model.plot_data_images(num_rows=3, num_cols=3, cmap="jet") transfer_model.train_model( - epochs=1, + epochs=5, model_save_path=(Path(__file__).parent / ".." / "deep_model").resolve(), model_name="tf_model_core_1.h5", ) @@ -1153,21 +1339,21 @@ def predict_image_patch_classes(self, **kwargs): # "MildDemented": "Mild", # "VeryMildDemented": "Very Mild", # } - transfer_model.predict_test( - model_path=( - Path(__file__).parent / ".." / "deep_model" / "tf_model_core_1.h5" - ).resolve(), - rotation=90, - y_axis_label_size=12, - x_axis_label_size=12, - title_size=14, - fig_title="Original Confusion Matrix", - conf_matx_font_size=12, - # custom_titles=custom_titles, - cmap="winter", - normalize="true", - ) - transfer_model.grad_cam_viz(num_rows=3, num_cols=2) + # transfer_model.predict_test( + # model_path=( + # Path(__file__).parent / ".." / "deep_model" / "tf_model_core_1.h5" + # ).resolve(), + # rotation=90, + # y_axis_label_size=12, + # x_axis_label_size=12, + # title_size=14, + # fig_title="Original Confusion Matrix", + # conf_matx_font_size=12, + # # custom_titles=custom_titles, + # cmap="winter", + # normalize="true", + # ) + # transfer_model.grad_cam_viz(num_rows=3, num_cols=2) # transfer_model.predict_one_image( # img_path=str( @@ -1180,16 +1366,30 @@ def predict_image_patch_classes(self, **kwargs): # ), # ) + # kwargs_dict = { + # "img_path": str(Path(__file__).parent / ".." / "dataset_core" / "long_core" / "Picture1.png"), + # "window_percent": 5, + # "stride": 150, + # "patch_images_dir": Path(__file__).parent / ".." / "dataset_core" / "patch_images", + # "img_with_box_dir": Path(__file__).parent / ".." / "dataset_core" / "core_images_with_box_red", + # "figsize": (15, 3), + # "fig_show": True, + # "model_path": Path(__file__).parent / ".." / "deep_model" / "tf_model_core_1.h5", + # "class_labels_path": str(Path(__file__).parent / "class_labels.json") + # } + # save_results = transfer_model.predict_image_patch_classes(**kwargs_dict) + # print(save_results) + + img_path = Path(__file__).parent / ".." / "dataset_core" / "patch_images" / "patch_0.jpg" + # load the img from img_path + from PIL import Image + + img = Image.open(img_path) + # Resize the image to the expected shape (224x224) + img = img.resize((224, 224)) kwargs_dict = { - "img_path": str(Path(__file__).parent / ".." / "dataset_core" / "long_core" / "Picture1.png"), - "window_percent": 5, - "stride": 150, - "patch_images_dir": Path(__file__).parent / ".." / "dataset_core" / "patch_images", - "img_with_box_dir": Path(__file__).parent / ".." / "dataset_core" / "core_images_with_box_red", - "figsize": (15, 3), - "fig_show": True, "model_path": Path(__file__).parent / ".." / "deep_model" / "tf_model_core_1.h5", - "class_labels_path": str(Path(__file__).parent / "class_labels.json") + "img": img, + "cmap": "gray", } - save_results = transfer_model.predict_image_patch_classes(**kwargs_dict) - print(save_results) + transfer_model.grad_cam_viz_2(**kwargs_dict)