updating source code with the latest

JordiCorbilla · Dec 16, 2019 · 9417856 · 9417856
1 parent bc89900
commit 9417856
Show file tree

Hide file tree

Showing 27 changed files with 858 additions and 82 deletions.
diff --git a/odir_image_crop.py b/odir_image_crop.py
@@ -1,4 +1,4 @@
-# Copyright 2019 Jordi Corbilla. All Rights Reserved.
+# Copyright 2019-2020 Jordi Corbilla. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/odir_image_crop_job.py b/odir_image_crop_job.py
@@ -1,4 +1,4 @@
-# Copyright 2019 Jordi Corbilla. All Rights Reserved.
+# Copyright 2019-2020 Jordi Corbilla. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/odir_image_resizer.py b/odir_image_resizer.py
@@ -1,4 +1,4 @@
-# Copyright 2019 Jordi Corbilla. All Rights Reserved.
+# Copyright 2019-2020 Jordi Corbilla. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/odir_image_testing_crop_job.py b/odir_image_testing_crop_job.py
@@ -1,4 +1,4 @@
-# Copyright 2019 Jordi Corbilla. All Rights Reserved.
+# Copyright 2019-2020 Jordi Corbilla. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/odir_image_treatment.py b/odir_image_treatment.py
@@ -0,0 +1,91 @@
+# Copyright 2019-2020 Jordi Corbilla. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+import numpy as np
+import tensorflow as tf
+from skimage import exposure
+
+
+class ImageTreatment:
+    def __init__(self, image_size):
+        self.image_size = image_size
+
+    def scaling(self, image, scale_vector):
+        # Resize to 4-D vector
+        image = np.reshape(image, (1, self.image_size, self.image_size, 3))
+        boxes = np.zeros((len(scale_vector), 4), dtype=np.float32)
+        for index, scale in enumerate(scale_vector):
+            x1 = y1 = 0.5 - 0.5 * scale
+            x2 = y2 = 0.5 + 0.5 * scale
+            boxes[index] = np.array([y1, x1, y2, x2], dtype=np.float32)
+        box_ind = np.zeros((len(scale_vector)), dtype=np.int32)
+        crop_size = np.array([self.image_size, self.image_size], dtype=np.int32)
+
+        output = tf.image.crop_and_resize(image, boxes, box_ind, crop_size)
+        output = np.array(output, dtype=np.uint8)
+        return output
+
+    def brightness(self, image, delta):
+        output = tf.image.adjust_brightness(image, delta)
+        output = np.array(output, dtype=np.uint8)
+        return output
+
+    def contrast(self, image, contrast_factor):
+        output = tf.image.adjust_contrast(image, contrast_factor)
+        output = np.array(output, dtype=np.uint8)
+        return output
+
+    def saturation(self, image, saturation_factor):
+        output = tf.image.adjust_saturation(image, saturation_factor)
+        output = np.array(output, dtype=np.uint8)
+        return output
+
+    def hue(self, image, delta):
+        output = tf.image.adjust_hue(image, delta)
+        output = np.array(output, dtype=np.uint8)
+        return output
+
+    def central_crop(self, image, central_fraction):
+        output = tf.image.central_crop(image, central_fraction)
+        output = np.array(output, dtype=np.uint8)
+        return output
+
+    def crop_to_bounding_box(self, image, offset_height, offset_width, target_height, target_width):
+        output = tf.image.crop_to_bounding_box(image, offset_height, offset_width, target_height, target_width)
+        output = tf.image.resize(output, (self.image_size, self.image_size))
+        output = np.array(output, dtype=np.uint8)
+        return output
+
+    def gamma(self, image, gamma):
+        output = tf.image.adjust_gamma(image, gamma)
+        output = np.array(output, dtype=np.uint8)
+        return output
+
+    def rot90(self, image, k):
+        output = tf.image.rot90(image, k)
+        output = np.array(output, dtype=np.uint8)
+        return output
+
+    def rescale_intensity(self, image):
+        p2, p98 = np.percentile(image, (2, 98))
+        img_rescale = exposure.rescale_intensity(image, in_range=(p2, p98))
+        return img_rescale
+
+    def equalize_histogram(self, image):
+        img_eq = exposure.equalize_hist(image)
+        return img_eq
+
+    def equalize_adapthist(self, image):
+        img_adapted = exposure.equalize_adapthist(image, clip_limit=0.03)
+        return img_adapted
diff --git a/odir_inception_testing_inference.py b/odir_inception_testing_inference.py
@@ -0,0 +1,77 @@
+# Copyright 2019 Jordi Corbilla. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import logging.config
+import tensorflow as tf
+from absl import app
+from odir_advance_plotting import Plotter
+from odir_kappa_score import FinalScore
+from odir_normalize_input import Normalizer
+from odir_predictions_writer import Prediction
+import odir
+
+def main(argv):
+    print(tf.version.VERSION)
+    image_size = 224
+    test_run = 'zC'
+
+    # load the data
+    (x_train, y_train), (x_test, y_test) = odir.load_data(image_size, 1)
+
+    class_names = ['Normal', 'Diabetes', 'Glaucoma', 'Cataract', 'AMD', 'Hypertension', 'Myopia', 'Others']
+
+    # plot data input
+    plotter = Plotter(class_names)
+    plotter.plot_input_images(x_train, y_train)
+
+    x_test_drawing = x_test
+
+    # normalize input based on model
+    normalizer = Normalizer()
+    x_test = normalizer.normalize_vgg16(x_test)
+
+    # load one of the test runs
+    model = tf.keras.models.load_model(r'C:\Users\thund\Source\Repos\TFM-ODIR\models\image_classification\modelvgg100.h5')
+    model.summary()
+
+    # display the content of the model
+    baseline_results = model.evaluate(x_test, y_test, verbose=2)
+    for name, value in zip(model.metrics_names, baseline_results):
+        print(name, ': ', value)
+    print()
+
+    # test a prediction
+    test_predictions_baseline = model.predict(x_test)
+    plotter.plot_confusion_matrix_generic(y_test, test_predictions_baseline, test_run, 0)
+
+    # save the predictions
+    prediction_writer = Prediction(test_predictions_baseline, 400)
+    prediction_writer.save()
+    prediction_writer.save_all(y_test)
+
+    # show the final score
+    score = FinalScore()
+    score.output()
+
+    # plot output results
+    plotter.plot_output(test_predictions_baseline, y_test, x_test_drawing)
+
+
+if __name__ == '__main__':
+    # create logger
+    logging.config.fileConfig('logging.conf')
+    logger = logging.getLogger('odir')
+    app.run(main)
diff --git a/odir_kappa_score.py b/odir_kappa_score.py
@@ -1,26 +1,46 @@
+# Copyright 2019 Jordi Corbilla. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
 import csv
 import numpy as np
 from sklearn import metrics
 
-def import_data(filepath):
-    with open(filepath, 'r') as f:
-        reader = csv.reader(f)
-        header = next(reader)
-        pr_data = [[int(row[0])] + list(map(float, row[1:])) for row in reader]
-    pr_data = np.array(pr_data)
-    return pr_data
 
-def odir_metrics(gt_data, pr_data):
-    th = 0.5
-    gt = gt_data.flatten()
-    pr = pr_data.flatten()
-    kappa = metrics.cohen_kappa_score(gt, pr > th)
-    f1 = metrics.f1_score(gt, pr > th, average='micro')
-    auc = metrics.roc_auc_score(gt, pr)
-    final_score = (kappa + f1 + auc) / 3.0
-    return kappa, f1, auc, final_score
+class FinalScore:
+    def odir_metrics(self, gt_data, pr_data):
+        th = 0.5
+        gt = gt_data.flatten()
+        pr = pr_data.flatten()
+        kappa = metrics.cohen_kappa_score(gt, pr > th)
+        f1 = metrics.f1_score(gt, pr > th, average='micro')
+        auc = metrics.roc_auc_score(gt, pr)
+        final_score = (kappa + f1 + auc) / 3.0
+        return kappa, f1, auc, final_score
 
-gt_data = import_data('odir_ground_truth.csv')
-pr_data = import_data('odir_predictions.csv')
-kappa, f1, auc, final_score = odir_metrics(gt_data[:, 1:], pr_data[:, 1:])
-print("kappa score:", kappa, " f-1 score:", f1, " AUC vlaue:", auc, " Final Score:", final_score)
+    def import_data(self, filepath):
+        with open(filepath, 'r') as f:
+            reader = csv.reader(f)
+            header = next(reader)
+            pr_data = [[int(row[0])] + list(map(float, row[1:])) for row in reader]
+        pr_data = np.array(pr_data)
+        return pr_data
+
+    def output(self):
+        gt_data = self.import_data('odir_ground_truth.csv')
+        pr_data = self.import_data('odir_predictions.csv')
+        kappa, f1, auc, final_score = self.odir_metrics(gt_data[:, 1:], pr_data[:, 1:])
+        print("Kappa score:", kappa)
+        print("F-1 score:", f1)
+        print("AUC value:", auc)
+        print("Final Score:", final_score)
diff --git a/odir_load_ground_truth_files.py b/odir_load_ground_truth_files.py
@@ -0,0 +1,59 @@
+# Copyright 2019-2020 Jordi Corbilla. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+import csv
+
+
+class GroundTruthFiles:
+    def __init__(self):
+        self.amd = []
+        self.cataract = []
+        self.diabetes = []
+        self.glaucoma = []
+        self.hypertension = []
+        self.myopia = []
+        self.others = []
+
+    def populate_vectors(self, ground_truth_file):
+        with open(ground_truth_file) as csvDataFile:
+            csv_reader = csv.reader(csvDataFile)
+
+            for row in csv_reader:
+                column_id = row[0]
+                normal = row[1]
+                diabetes = row[2]
+                glaucoma = row[3]
+                cataract = row[4]
+                amd = row[5]
+                hypertension = row[6]
+                myopia = row[7]
+                others = row[8]
+                # just discard the first row
+                if column_id != "ID":
+                    print("Processing image: " + column_id + "_left.jpg")
+                    if diabetes == '1':
+                        self.diabetes.append([column_id, normal, diabetes, glaucoma, cataract, amd, hypertension, myopia, others])
+                    if glaucoma == '1':
+                        self.glaucoma.append([column_id, normal, diabetes, glaucoma, cataract, amd, hypertension, myopia, others])
+                    if cataract == '1':
+                        self.cataract.append([column_id, normal, diabetes, glaucoma, cataract, amd, hypertension, myopia, others])
+                    if amd == '1':
+                        self.amd.append([column_id, normal, diabetes, glaucoma, cataract, amd, hypertension, myopia, others])
+                    if hypertension == '1':
+                        self.hypertension.append([column_id, normal, diabetes, glaucoma, cataract, amd, hypertension, myopia, others])
+                    if myopia == '1':
+                        self.myopia.append([column_id, normal, diabetes, glaucoma, cataract, amd, hypertension, myopia, others])
+                    if others == '1':
+                        self.others.append([column_id, normal, diabetes, glaucoma, cataract, amd, hypertension, myopia, others])
+
diff --git a/odir_model_advanced.py b/odir_model_advanced.py
@@ -1,4 +1,4 @@
-# Copyright 2019 Jordi Corbilla. All Rights Reserved.
+# Copyright 2019-2020 Jordi Corbilla. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -30,7 +30,7 @@ def compile(self):
         model.add(layers.Flatten())
         model.add(layers.Dense(64, activation='relu'))
         model.add(layers.Dense(8, activation='sigmoid'))
-        model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
+        model.compile(optimizer='adam', loss='binary_crossentropy', metrics=self.metrics)
         self.show_summary(model)
         self.plot_summary(model, 'model_advanced.png')
         return model
diff --git a/odir_model_base.py b/odir_model_base.py
@@ -1,4 +1,4 @@
-# Copyright 2019 Jordi Corbilla. All Rights Reserved.
+# Copyright 2019-2020 Jordi Corbilla. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,8 +17,9 @@
 
 
 class ModelBase:
-    def __init__(self, input_shape):
+    def __init__(self, input_shape, metrics):
         self.input_shape = input_shape
+        self.metrics = metrics
 
     def show_summary(self, model):
         model.summary()

diff --git a/odir_model_factory.py b/odir_model_factory.py
@@ -1,4 +1,4 @@
-# Copyright 2019 Jordi Corbilla. All Rights Reserved.
+# Copyright 2019-2020 Jordi Corbilla. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,11 +27,11 @@ class ModelTypes(enum.Enum):
 
 class Factory:
 
-    def __init__(self, input_shape):
+    def __init__(self, input_shape, metrics):
         self.Makers = {
-            ModelTypes.vgg16: Vgg16(input_shape),
-            ModelTypes.inception_v1: InceptionV1(input_shape),
-            ModelTypes.advanced_testing: Advanced(input_shape)
+            ModelTypes.vgg16: Vgg16(input_shape, metrics),
+            ModelTypes.inception_v1: InceptionV1(input_shape, metrics),
+            ModelTypes.advanced_testing: Advanced(input_shape, metrics)
         }
 
     def compile(self, model_type):

diff --git a/odir_model_inception_v1.py b/odir_model_inception_v1.py
@@ -1,4 +1,4 @@
-# Copyright 2019 Jordi Corbilla. All Rights Reserved.
+# Copyright 2019-2020 Jordi Corbilla. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -41,7 +41,7 @@ def compile(self):
         output = Dense(8, activation='sigmoid')(dense_3)
         model = Model([input_img], output)
 
-        model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
+        model.compile(optimizer='adam', loss='binary_crossentropy', metrics=self.metrics)
 
         self.show_summary(model)
         self.plot_summary(model, 'model_inception_v1.png')

diff --git a/odir_model_runner.py b/odir_model_runner.py
@@ -1,4 +1,4 @@
-# Copyright 2019 Jordi Corbilla. All Rights Reserved.
+# Copyright 2019-2020 Jordi Corbilla. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.