diff --git a/.travis.yml b/.travis.yml index d8b415dd..06a6ed9e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,13 @@ language: java install: true -# before_install commands required as of https://www.deps.co/guides/travis-ci-latest-java/ matrix: include: - - jdk: openjdk11 + - jdk: + - openjdk8 + - openjdk11 before_install: - - sudo rm "${JAVA_HOME}/lib/security/cacerts" - - sudo ln -s /etc/ssl/certs/java/cacerts "${JAVA_HOME}/lib/security/cacerts" + - sudo ./fix-java-verts.sh before_cache: - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock diff --git a/GUIEditors.props b/GUIEditors.props index 7e425de1..a55cd406 100644 --- a/GUIEditors.props +++ b/GUIEditors.props @@ -50,6 +50,8 @@ weka.dl4j.inference.Dl4jCNNExplorer=weka.gui.GenericObjectEditor weka.dl4j.inference.ModelOutputDecoder=weka.gui.GenericObjectEditor +weka.dl4j.inference.CustomModelSetup=weka.gui.GenericObjectEditor + weka.dl4j.interpretability.AbstractCNNSaliencyMapWrapper=weka.gui.GenericObjectEditor diff --git a/GenericPropertiesCreator.props b/GenericPropertiesCreator.props index e983e879..88cf44c6 100644 --- a/GenericPropertiesCreator.props +++ b/GenericPropertiesCreator.props @@ -67,5 +67,8 @@ weka.dl4j.inference.Dl4jCNNExplorer =\ weka.dl4j.inference.ModelOutputDecoder =\ weka.dl4j.inference +weka.dl4j.inference.CustomModelSetup =\ + weka.dl4j.inference + weka.dl4j.interpretability.AbstractCNNSaliencyMapWrapper =\ - weka.dl4j.interpretability \ No newline at end of file + weka.dl4j.interpretability diff --git a/build.gradle b/build.gradle index ab0bdaa2..6bd274d6 100644 --- a/build.gradle +++ b/build.gradle @@ -14,8 +14,8 @@ def is_cuda_version_valid = cuda_version in valid_cuda_versions // Set JDK compatibility allprojects { - sourceCompatibility = 1.11 - targetCompatibility = 1.11 + sourceCompatibility = 1.8 + targetCompatibility = 1.8 } /* @@ -335,11 +335,10 @@ task travisTest(type: Test) { } filter { - // Excluded from travis due to taking too long / using too much memory + // Excluded from travis due to taking too long / using too much memory and throwing OOM exception. excludeTestsMatching 'ZooModelTest' excludeTestsMatching 'Dl4jCNNExplorerTest' excludeTestsMatching 'Dl4jMlpFilterTest' - excludeTestsMatching 'ScoreCAMTest' } } diff --git a/docs/examples/dl4j-inference.md b/docs/examples/dl4j-inference.md index 6aca5556..a2018bfb 100644 --- a/docs/examples/dl4j-inference.md +++ b/docs/examples/dl4j-inference.md @@ -94,7 +94,7 @@ see the results in the output panel, again correctly predicting the target class $ java weka.Run .Dl4jCNNExplorer \ -decoder ".ModelOutputDecoder -builtIn VGGFACE" \ -zooModel ".Dl4jVGG -variation VGG16 -pretrained VGGFACE" \ - -i src/test/resources/images/ben_stiller.jpg + -i $WEKA_HOME/packages/wekaDeeplearning4j/src/test/resources/images/ben_stiller.jpg ``` ```bash @@ -125,8 +125,11 @@ which you'd like to experiment with; the process is largely the same as above, w On the `Dl4j Inference` panel, open the `Dl4jCNNExplorer` settings: -- Set `Use serialized model file` to `True` -- Select your previously saved `.model` file as the `Serialized model file` +- Set `Use custom-trained model file` to `True` +- Open the `CustomModelSetup` settings + - Select your previously saved `.model` file as the `Serialized model file` + - Set the input `channels`, `width`, and `height` with the values used to train the model. + These values will be identical to those set on the `ImageInstanceIterator`. - Open the `ModelOutputDecoder` settings: - Set `Built in class map` to `CUSTOM` - Select the `Class map file` on your machine. This can be in two forms: @@ -160,12 +163,16 @@ to classify between different dog breeds isn't going to give accurate answers wh ### Command Line +This example uses a custom-trained model which used an `ImageInstanceIterator` using +`channels`, `width`, and `height` of `3`, `56`, `56`, respectively. These values are explicitly +defined in the `CustomModelSetup`. + ```bash $ java weka.Run .Dl4jCNNExplorer \ - -decoder ".ModelOutputDecoder -builtIn CUSTOM -classMapFile /path/to/classmap.txt" \ - -model-file /path/to/saved/model/Dl4jMlpClassifier.model \ - -use-model-file - -i /path/to/input/image.png + -i path/to/image.jpg \ + -custom-model ".CustomModelSetup -channels 3 -height 56 -width 56 -model-file path/to/customModel.model" \ + -decoder ".ModelOutputDecoder -builtIn CUSTOM -classMapFile path/to/training.arff" \ + -useCustomModel ``` ## Example 4: Saliency Map Generation @@ -245,7 +252,7 @@ know what the class ID may be. ```bash $ java weka.Run .Dl4jCNNExplorer \ - -i "src/test/resources/images/catAndDog.jpg" \ + -i "$WEKA_HOME/packages/wekaDeeplearning4j/src/test/resources/images/catAndDog.jpg" \ -generate-map \ -saliency-map ".WekaScoreCAM -bs 8 -normalize -output output_image.png -target-classes -1" \ -zooModel ".KerasResNet -variation RESNET101" @@ -255,7 +262,7 @@ $ java weka.Run .Dl4jCNNExplorer \ ```bash $ java weka.Run .Dl4jCNNExplorer \ - -i "src/test/resources/images/catAndDog.jpg" \ + -i "$WEKA_HOME/packages/wekaDeeplearning4j/src/test/resources/images/catAndDog.jpg" \ -generate-map \ -saliency-map ".WekaScoreCAM -bs 8 -normalize -output output_image.png -target-classes -1,281" \ -zooModel ".KerasResNet -variation RESNET101" diff --git a/docs/examples/featurize-mnist.md b/docs/examples/featurize-mnist.md index 9f3e1793..eded92ed 100644 --- a/docs/examples/featurize-mnist.md +++ b/docs/examples/featurize-mnist.md @@ -43,7 +43,8 @@ All zoo models have a **default** feature extraction layer, which is typically t activations, hence why it's set to the default (although you can use any intermediary layer). `PoolingType` does not need to be specified when using the default activation layer - the outputs are already the - correct dimensionality. + correct dimensionality (`[batch size, num activations]`). If using an intermediary layer the outputs will + typically be of size `[batch size, width, height, num channels]`. ## Example 1: Default MNIST Minimal The following example walks through using a pretrained ResNet50 (from the Deeplearning4j model zoo) diff --git a/docs/img/inference/Dl4jCNNExplorer_customModel.png b/docs/img/inference/Dl4jCNNExplorer_customModel.png index 00a9b57a..ffb707c5 100644 Binary files a/docs/img/inference/Dl4jCNNExplorer_customModel.png and b/docs/img/inference/Dl4jCNNExplorer_customModel.png differ diff --git a/docs/img/inference/Dl4jCNNExplorer_default.png b/docs/img/inference/Dl4jCNNExplorer_default.png index 97899432..e8739385 100644 Binary files a/docs/img/inference/Dl4jCNNExplorer_default.png and b/docs/img/inference/Dl4jCNNExplorer_default.png differ diff --git a/fix-java-verts.sh b/fix-java-verts.sh new file mode 100755 index 00000000..d28365db --- /dev/null +++ b/fix-java-verts.sh @@ -0,0 +1,9 @@ +# Commands required as of https://www.deps.co/guides/travis-ci-latest-java/ +JAVA_FILE="${JAVA_HOME}/lib/security/cacerts" + +if [ -f "$JAVA_FILE" ]; then + sudo rm -f "${JAVA_HOME}/lib/security/cacerts" + sudo ln -s /etc/ssl/certs/java/cacerts "${JAVA_HOME}/lib/security/cacerts" +else + echo "Couldn't find java certificates folder, ignoring..." +fi \ No newline at end of file diff --git a/src/main/java/weka/classifiers/functions/Dl4jMlpClassifier.java b/src/main/java/weka/classifiers/functions/Dl4jMlpClassifier.java index abaf5e19..0dab9268 100644 --- a/src/main/java/weka/classifiers/functions/Dl4jMlpClassifier.java +++ b/src/main/java/weka/classifiers/functions/Dl4jMlpClassifier.java @@ -31,7 +31,6 @@ import org.apache.commons.io.output.CountingOutputStream; import org.apache.commons.io.output.NullOutputStream; import org.apache.commons.lang3.time.StopWatch; -import org.apache.lucene.util.ThreadInterruptedException; import org.deeplearning4j.datasets.iterator.AsyncDataSetIterator; import org.deeplearning4j.exception.DL4JException; import org.deeplearning4j.exception.DL4JInvalidConfigException; @@ -65,7 +64,7 @@ import weka.dl4j.enums.CacheMode; import weka.dl4j.enums.ConvolutionMode; import weka.dl4j.enums.PoolingType; -import weka.dl4j.enums.PretrainedType; +import weka.dl4j.inference.CustomModelSetup; import weka.dl4j.iterators.instance.*; import weka.dl4j.iterators.instance.api.ConvolutionalIterator; import weka.dl4j.iterators.instance.sequence.text.cnn.CnnTextEmbeddingInstanceIterator; @@ -279,7 +278,7 @@ public class Dl4jMlpClassifier extends RandomizableClassifier implements */ protected ProgressManager progressManager; - private SwingWorker layerSwingWorker; + private SwingWorker layerSwingWorker; public Dl4jMlpClassifier() { if (!s_cudaMultiGPUSet) { @@ -810,6 +809,18 @@ public void setParameterAveragingFrequency(int frequency) { averagingFrequency = frequency; } + /** + * Get the name of the loaded model + * @return Model name + */ + public String getModelName() { + if (useZooModel()) { + return getZooModel().getPrettyName(); + } else { + return "Custom trained Dl4jMlpClassifier"; + } + } + /** * The method used to train the classifier. * @@ -1254,7 +1265,18 @@ protected Instances initFilters(Instances data) throws Exception { return data; } - + public InputType.InputTypeConvolutional getInputShape(CustomModelSetup customModelSetup) { + if (useZooModel()) { + int[] inputShape = getZooModel().getInputShape(); + log.debug("Zoo Model shape for image inference is: " + Arrays.toString(inputShape)); + return new InputType.InputTypeConvolutional(inputShape[1], inputShape[2], inputShape[0]); + } + + return new InputType.InputTypeConvolutional( + customModelSetup.getInputHeight(), + customModelSetup.getInputWidth(), + customModelSetup.getInputChannels()); + } /** * Build the Zoomodel instance @@ -1669,7 +1691,7 @@ public void setZooModel(AbstractZooModel zooModel) { progressManager = new ProgressManager("Parsing model layers..."); progressManager.start(); - layerSwingWorker = new SwingWorker<>() { + layerSwingWorker = new SwingWorker() { @Override protected Layer[] doInBackground() throws Exception { return parseLayers(); @@ -1705,7 +1727,7 @@ private Layer[] parseLayers() { Thread.currentThread().setContextClassLoader( this.getClass().getClassLoader()); ComputationGraph tmpCg = - zooModel.init(dummyNumLabels, getSeed(), zooModel.getShape()[0], isFilterMode()); + zooModel.init(dummyNumLabels, getSeed(), zooModel.getInputShape(), isFilterMode()); tmpCg.init(); tmpLayers = Arrays.stream(tmpCg.getLayers()) @@ -1737,6 +1759,7 @@ public static Dl4jMlpClassifier tryLoadFromFile(File serializedModelFile, Abstra // First try load from the WEKA binary model file try (ObjectInputStream ois = new ObjectInputStream(new FileInputStream(serializedModelFile))) { model = (Dl4jMlpClassifier) ois.readObject(); + model.setCustomNet(); } catch (Exception e) { throw new WekaException("Couldn't load Dl4jMlpClassifier from model file"); } @@ -1786,7 +1809,7 @@ public static Dl4jMlpClassifier loadInferenceModel(File serializedModelFile, Abs Dl4jMlpClassifier model = tryLoadFromFile(serializedModelFile, zooModelType); if (!Utils.notDefaultFileLocation(serializedModelFile)) - model.loadZooModelNoData(2, 1, zooModelType.getShape()[0]); + model.loadZooModelNoData(2, 1, zooModelType.getInputShape()); return model; } @@ -1986,7 +2009,7 @@ public String toString() { * * @return True if zoomodel is not CustomNet */ - protected boolean useZooModel() { + public boolean useZooModel() { return !(zooModel instanceof CustomNet); } diff --git a/src/main/java/weka/core/progress/CommandLineProgressBar.java b/src/main/java/weka/core/progress/CommandLineProgressBar.java index df8731a9..2fd63410 100644 --- a/src/main/java/weka/core/progress/CommandLineProgressBar.java +++ b/src/main/java/weka/core/progress/CommandLineProgressBar.java @@ -1,13 +1,15 @@ package weka.core.progress; import lombok.extern.log4j.Log4j2; +import org.apache.commons.lang.CharUtils; +import org.apache.commons.lang.StringUtils; import java.util.Timer; import java.util.TimerTask; @Log4j2 -/** - * Command line implementation of a progress bar +/* + Command line implementation of a progress bar */ public class CommandLineProgressBar extends AbstractProgressBar { @@ -67,9 +69,9 @@ public void refreshDisplay() { System.err.printf("\r%s: [%s%s%s] %s", getProgressMessage(), - progressRemainingChar.repeat(leftSpace), - progressChar.repeat(currNumDots), - progressRemainingChar.repeat(rightSpace), + StringUtils.repeat(progressRemainingChar, leftSpace), + StringUtils.repeat(progressChar, currNumDots), + StringUtils.repeat(progressRemainingChar, rightSpace), getETAString()); } diff --git a/src/main/java/weka/dl4j/Utils.java b/src/main/java/weka/dl4j/Utils.java index ef448c66..fdeeda4e 100644 --- a/src/main/java/weka/dl4j/Utils.java +++ b/src/main/java/weka/dl4j/Utils.java @@ -496,74 +496,6 @@ public static String defaultFileLocation() { return WekaPackageManager.getPackageHome().getPath(); } - public static void saveNDArray(INDArray array, String filenamePrefix) { - BufferedImage img = Utils.imageFromINDArray(array); - try { - ImageIO.write(img, "png", new File(filenamePrefix + ".png")); - } catch (IOException ex) { - ex.printStackTrace(); - } - } - - /** - * Takes an INDArray containing an image loaded using the native image loader - * libraries associated with DL4J, and converts it into a BufferedImage. - * The INDArray contains the color values split up across three channels (RGB) - * and in the integer range 0-255. - * - * @param array INDArray containing an image in order [N, C, H, W] or [C, H, W] - * @return BufferedImage - */ - public static BufferedImage imageFromINDArray(INDArray array) { - long[] shape = array.shape(); - - boolean is4d = false; - String dimString = "3D"; - - if (shape.length == 4) { - is4d = true; - dimString = "4D"; - } - - log.debug(String.format("Converting %s INDArray to image...", dimString)); - - long height = shape[1]; - long width = shape[2]; - - if (is4d) { - height = shape[2]; - width = shape[3]; - } - - BufferedImage image = new BufferedImage((int) width, (int) height, BufferedImage.TYPE_INT_RGB); - for (int x = 0; x < width; x++) { - for (int y = 0; y < height; y++) { - int red, green, blue; - - if (is4d) { - red = array.getInt(0, 2, y, x); - green = array.getInt(0, 1, y, x); - blue = array.getInt(0, 0, y, x); - } else { - red = array.getInt(2, y, x); - green = array.getInt(1, y, x); - blue = array.getInt(0, y, x); - } - - //handle out of bounds pixel values - red = Math.min(red, 255); - green = Math.min(green, 255); - blue = Math.min(blue, 255); - - red = Math.max(red, 0); - green = Math.max(green, 0); - blue = Math.max(blue, 0); - image.setRGB(x, y, new Color(red, green, blue).getRGB()); - } - } - return image; - } - public static InputType.InputTypeConvolutional decodeCNNShape (int[] shape) { return decodeCNNShape(Arrays.stream(shape).asLongStream().toArray()); } diff --git a/src/main/java/weka/dl4j/inference/CustomModelSetup.java b/src/main/java/weka/dl4j/inference/CustomModelSetup.java new file mode 100644 index 00000000..dc0e0add --- /dev/null +++ b/src/main/java/weka/dl4j/inference/CustomModelSetup.java @@ -0,0 +1,125 @@ +package weka.dl4j.inference; + +import weka.core.Option; +import weka.core.OptionHandler; +import weka.core.OptionMetadata; +import weka.dl4j.Utils; +import weka.dl4j.zoo.AbstractZooModel; + +import java.io.File; +import java.io.Serializable; +import java.util.Enumeration; + +public class CustomModelSetup implements Serializable, OptionHandler { + + /** + * The classifier model this filter is based on. + */ + protected File serializedModelFile = new File(Utils.defaultFileLocation()); + + protected int inputChannels = 3; + + protected int inputWidth = 224; + + protected int inputHeight = 224; + + public void setUseCustomSetup(boolean useCustomSetup) { + if (!useCustomSetup) { + resetModelFilepath(); + } + } + + public void resetModelFilepath() { + setSerializedModelFile(new File(Utils.defaultFileLocation())); + } + + @OptionMetadata( + displayName = "Serialized model file", + description = "Pointer to file of saved Dl4jMlpClassifier", + commandLineParamName = "model-file", + commandLineParamSynopsis = "-model-file ", + displayOrder = 0 + ) + public File getSerializedModelFile() { + return serializedModelFile; + } + + public void setSerializedModelFile(File serializedModelFile) { + this.serializedModelFile = serializedModelFile; + } + + @OptionMetadata( + displayName = "Number of input channels", + description = "Number of channels for input images to this model", + commandLineParamName = "channels", + commandLineParamSynopsis = "-channels ", + displayOrder = 1 + ) + public int getInputChannels() { + return inputChannels; + } + + public void setInputChannels(int inputChannels) { + this.inputChannels = inputChannels; + } + + @OptionMetadata( + displayName = "Input image width", + description = "Width of input images to this model", + commandLineParamName = "width", + commandLineParamSynopsis = "-width ", + displayOrder = 2 + ) + public int getInputWidth() { + return inputWidth; + } + + public void setInputWidth(int inputWidth) { + this.inputWidth = inputWidth; + } + + @OptionMetadata( + displayName = "Input image height", + description = "Height of input images to this model", + commandLineParamName = "height", + commandLineParamSynopsis = "-height ", + displayOrder = 3 + ) + public int getInputHeight() { + return inputHeight; + } + + public void setInputHeight(int inputHeight) { + this.inputHeight = inputHeight; + } + + /** + * Returns an enumeration describing the available options. + * + * @return an enumeration of all the available options. + */ + public Enumeration