Merge pull request #80 from tue-robotics/feat/test

Added test for image_recognition_tensorflow

.travis.yml

@@ -7,9 +7,6 @@ matrix:
   - name: "Kinetic"
     env: ROS_DISTRO=kinetic
 
-  - name: "Melodic"
-    env: ROS_DISTRO=melodic
-
 install:
   - git clone https://github.com/ros-industrial/industrial_ci.git .ci_config
 

image_recognition_tensorflow/CMakeLists.txt

@@ -7,11 +7,16 @@ catkin_python_setup()
 
 catkin_package()
 
-# Test catkin lint
-find_program(CATKIN_LINT catkin_lint REQUIRED)
-execute_process(COMMAND "${CATKIN_LINT}" "-q" "-W2" "${CMAKE_SOURCE_DIR}" RESULT_VARIABLE lint_result)
-if(NOT ${lint_result} EQUAL 0)
-  message(FATAL_ERROR "catkin_lint failed")
+
+if (CATKIN_ENABLE_TESTING)
+    # Test catkin lint
+    find_program(CATKIN_LINT catkin_lint REQUIRED)
+    execute_process(COMMAND "${CATKIN_LINT}" "-q" "-W2" "${CMAKE_SOURCE_DIR}" RESULT_VARIABLE lint_result)
+    if(NOT ${lint_result} EQUAL 0)
+      message(FATAL_ERROR "catkin_lint failed")
+    endif()
+
+    catkin_add_nosetests(test)
 endif()
 
 install(PROGRAMS
@@ -20,4 +25,4 @@ install(PROGRAMS
   scripts/object_recognition_node
   scripts/retrain
   DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION}
-)
+)

image_recognition_tensorflow/scripts/evaluate_classifier

@@ -5,9 +5,7 @@ The script will output a .csv that can be evaluated. """
 
 import argparse
 
-from image_recognition_tensorflow.object_recognizer import ObjectRecognizer
-from image_recognition_util.image_reader import read_annotated
-from image_recognition_util.classification_score_matrix import ClassificationScoreMatrix
+from image_recognition_tensorflow.evaluate_classifier import evaluate_classifier
 
 # Assign description to the help doc
 parser = argparse.ArgumentParser(description='Evaluate the classification performance of a TensorFlow network')
@@ -18,33 +16,10 @@ parser.add_argument('labels_path', type=str, help='Path to a file with the outpu
 parser.add_argument('annotated_dir', type=str, help='Use a the given directory (which must have subdirs for each class)'
                                                     ' and use the annotations to get some metrics about the classifier')
 parser.add_argument('-n', '--top_n', type=int, help='Top N results to display', default=1)
-parser.add_argument('--input-tensor', type=str, help='Tensor name of input image',  default="Cast:0")
+parser.add_argument('--input-tensor', type=str, help='Tensor name of input image', default="Cast:0")
 parser.add_argument('--output-tensor', type=str, help='Tensor name of classifications label', default="final_result:0")
 parser.add_argument('-o', '--output', type=str, help='Classification result output csv. First column is ground truth '
                                                      'label, other columns are the class scores', default="result.csv")
-
 args = parser.parse_args()
-
-object_recognizer = ObjectRecognizer(args.graph_path, args.labels_path, args.input_tensor, args.output_tensor)
-
-classification_score_matrix = ClassificationScoreMatrix(object_recognizer.labels)
-
-print ">> Evaluating classified with labels: {}".format(object_recognizer.labels)
-
-matches = []
-for label, image, filename in read_annotated(args.annotated_dir):
-    if label not in object_recognizer.labels:
-        print ">> Skipping image with label '{}', label not present in classifier".format(label)
-        continue
-
-    scores = object_recognizer.classify(image)
-    match, best_label, best_score = classification_score_matrix.add_classification(label, scores)
-
-    matches.append(match)
-    print "{}({}/{})\t{}\t{} classified as {} ({:.3f})\033[0m".format("\033[92m" if match else "\033[91m",
-                                                                      matches.count(True), len(matches),
-                                                                      filename, label, best_label, best_score)
-
-classification_score_matrix.write_to_file(args.output)
-
-print ">> Final accuracy: {}".format(float(matches.count(True)) / len(matches))
+evaluate_classifier(args.graph_path, args.labels_path, args.annotated_dir, args.output, args.input_tensor,
+                    args.output_tensor)

image_recognition_tensorflow/scripts/retrain

@@ -24,7 +24,8 @@ if __name__ == '__main__':
     parser.add_argument('--flip_left_right', type=bool, default=False,
                         help="""Whether to randomly flip half of the training images horizontally.""")
     parser.add_argument('--random_crop', type=int, default=0,
-                        help="""A percentage determining how much of a margin to randomly crop off the training images.""")
+                        help="""A percentage determining how much of a margin to randomly crop off the training 
+                        images.""")
     parser.add_argument('--random_scale', type=int, default=0,
                         help="""A percentage determining how much to randomly scale up the size of the training images by.""")
     parser.add_argument('--random_brightness', type=int, default=0,

image_recognition_tensorflow/src/image_recognition_tensorflow/evaluate_classifier.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+from image_recognition_tensorflow.object_recognizer import ObjectRecognizer
+from image_recognition_util.classification_score_matrix import ClassificationScoreMatrix
+from image_recognition_util.image_reader import read_annotated
+
+
+def evaluate_classifier(graph_path, labels_path, annotated_dir, output,
+                        input_tensor="Cast:0", output_tensor="final_result:0"):
+    """
+    Evaluate a given classifier against a directory of annotated images.
+    The script will output a .csv that can be evaluated.
+    :param graph_path: Path to graph.pb
+    :param labels_path: Path to labels.txt
+    :param annotated_dir: Directory path where the annotated images are stored, to be verified
+    :param output: Output file
+    :param input_tensor: Input tensor name
+    :param output_tensor: Output tensor name
+    :return: Final accuracy
+    """
+    object_recognizer = ObjectRecognizer(graph_path, labels_path, input_tensor, output_tensor)
+
+    classification_score_matrix = ClassificationScoreMatrix(object_recognizer.labels)
+
+    print("Evaluating classified with labels: {}".format(object_recognizer.labels))
+
+    matches = []
+    for label, image, filename in read_annotated(annotated_dir):
+        if label not in object_recognizer.labels:
+            print("Skipping image with label '{}', label not present in classifier".format(label))
+            continue
+
+        scores = object_recognizer.classify(image)
+        match, best_label, best_score = classification_score_matrix.add_classification(label, scores)
+
+        matches.append(match)
+        print("{}({}/{})\t{}\t{} classified as {} ({:.3f})\033[0m".format("\033[92m" if match else "\033[91m",
+                                                                          matches.count(True), len(matches),
+                                                                          filename, label, best_label, best_score))
+
+    classification_score_matrix.write_to_file(output)
+
+    accuracy = float(matches.count(True)) / len(matches)
+    print("Final accuracy: {}".format(accuracy))
+    return accuracy

image_recognition_tensorflow/src/image_recognition_tensorflow/retrain.py

@@ -4,11 +4,11 @@
 from __future__ import division
 from __future__ import print_function
 
+import errno
 import os
 import sys
 from argparse import Namespace
 
-import errno
 import tensorflow as tf
 from image_recognition_tensorflow.tf_retrain import main as tf_main
 
@@ -25,7 +25,8 @@ def mkdir_p(path):
             raise
 
 
-def main(image_dir, model_dir, output_dir, steps, batch, architecture="inception_v3", flip_left_right=False, random_crop=0, random_scale=0, random_brightness=0):
+def main(image_dir, model_dir, output_dir, steps, batch, architecture="inception_v3", flip_left_right=False,
+         random_crop=0, random_scale=0, random_brightness=0):
     tf.app.flags.FLAGS.image_dir = image_dir
     tf.app.flags.FLAGS.model_dir = model_dir
 
@@ -46,8 +47,7 @@ def main(image_dir, model_dir, output_dir, steps, batch, architecture="inception
     tf.app.flags.FLAGS.random_scale = random_scale
     tf.app.flags.FLAGS.random_brightness = random_brightness
 
-    # Do not pass on the '-s' argument of rqt
-    if "rqt" in sys.argv[0]:
-        sys.argv.pop(1)
+    # Do not pass additional flags to tf (from rqt and nose)
+    sys.argv = [arg for arg in sys.argv if not arg.startswith("-")]
 
     tf_main('')

image_recognition_tensorflow/src/image_recognition_tensorflow/tf_retrain.py

@@ -438,8 +438,7 @@ def _progress(count, block_size, total_size):
     filepath, _ = urllib.request.urlretrieve(data_url, filepath, _progress)
     print()
     statinfo = os.stat(filepath)
-    tf.logging.info('Successfully downloaded', filename, statinfo.st_size,
-                    'bytes.')
+    #tf.logging.info('Successfully downloaded', filename, statinfo.st_size, 'bytes.')
     print('Extracting file from ', filepath)
     tarfile.open(filepath, 'r:gz').extractall(dest_directory)
   else: