Pull request #4: vendor: update dependencies

Merge in SNORT/libml from ~BRASTULT/libml:quantized to master

Squashed commit of the following:

commit d3dc69fcc5988637022c5e7de6a584750ada313f
Author: Brandon Stultz <[email protected]>
Date:   Thu Jun 6 15:30:20 2024 -0400

    build: set version to 1.1.0

commit e452be97f9b12929a6244d6eef7968575e3ba238
Author: Brandon Stultz <[email protected]>
Date:   Wed Jun 12 15:47:45 2024 -0400

    src: add binary_classifier_test

commit 19714264355a545c6c84b1796662dcc2f13cecb0
Author: Brandon Stultz <[email protected]>
Date:   Thu Jun 6 15:30:06 2024 -0400

    examples: fix classifier training

commit 1ff7443b176d70d633c6ba39292ff19b84170ded
Author: Brandon Stultz <[email protected]>
Date:   Thu Jun 6 15:29:32 2024 -0400

    src: update tflite types and copyright

commit 3d67ad226123c07882772716fffa95da09b02361
Author: Brandon Stultz <[email protected]>
Date:   Tue Jun 4 17:54:10 2024 -0400

    cmake: update libraries

commit 188401049288f26d3c7d62454a6498a5ebea8ed7
Author: Brandon Stultz <[email protected]>
Date:   Tue Jun 4 16:52:13 2024 -0400

    vendor: update README

commit 70eaae8c0ebf3dd4ae9a5dc148eb6ec7d1aa25d0
Author: Brandon Stultz <[email protected]>
Date:   Tue Jun 4 16:51:39 2024 -0400

    vendor: update patches

commit 54de7dec985cfc73fc90311eb02733b3305fd636
Author: Brandon Stultz <[email protected]>
Date:   Tue Jun 4 16:43:31 2024 -0400

    vendor: update XNNPACK

commit 2047596954e74c83ee1a6b8b992ea2d6072e2ab5
Author: Brandon Stultz <[email protected]>
Date:   Tue Jun 4 16:37:25 2024 -0400

    vendor: update pthreadpool

commit f5a3871505ba71e35a235750c5bddc79b4ef399d
Author: Brandon Stultz <[email protected]>
Date:   Tue Jun 4 16:32:25 2024 -0400

    vendor: update cpuinfo

commit a7511514224f6cff782757c40b22d1a7c91499fa
Author: Brandon Stultz <[email protected]>
Date:   Tue Jun 4 13:47:56 2024 -0400

    vendor: add ml_dtypes

commit 4fee56a3bc408e1e2ac63e655852477e34046691
Author: Brandon Stultz <[email protected]>
Date:   Tue Jun 4 13:39:21 2024 -0400

    vendor: update ruy

commit aaadf92e568081088c503e9175b111bff0b3aec2
Author: Brandon Stultz <[email protected]>
Date:   Tue Jun 4 13:35:53 2024 -0400

    vendor: update gemmlowp

commit 31d34f161a3db1bf7d79e9b96ad5089b7490ad44
Author: Brandon Stultz <[email protected]>
Date:   Tue Jun 4 13:30:22 2024 -0400

    vendor: update flatbuffers

commit b54497ef45ef49e60fb53b1c79dd85a517a6d7ae
Author: Brandon Stultz <[email protected]>
Date:   Tue Jun 4 12:46:18 2024 -0400

    vendor: update eigen

commit e0078db9d3cbfe47de9a2301416cf1d5ed2abed4
Author: Brandon Stultz <[email protected]>
Date:   Tue Jun 4 12:41:44 2024 -0400

    vendor: update abseil

commit e1b35493e45828e55eef26102850f7359ddbf03a
Author: Brandon Stultz <[email protected]>
Date:   Tue Jun 4 12:29:44 2024 -0400

    vendor: update tensorflow

CMakeLists.txt

@@ -3,8 +3,8 @@ cmake_minimum_required(VERSION 3.16)
 project(libml C CXX)
 
 set(VERSION_MAJOR 1)
-set(VERSION_MINOR 0)
-set(VERSION_PATCH 1)
+set(VERSION_MINOR 1)
+set(VERSION_PATCH 0)
 set(VERSION "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}")
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS True)
@@ -15,6 +15,7 @@ set(CMAKE_CXX_STANDARD 17)
 include(BuildType)
 include(CompilerFlags)
 include(Libraries)
+include(Macros)
 include(InstallPath)
 
 add_subdirectory(src)

cmake/FindCppUTest.cmake

@@ -0,0 +1,30 @@
+find_package(PkgConfig)
+pkg_check_modules(PC_CPPUTEST cpputest)
+
+find_library(CPPUTEST_LIBRARY NAMES CppUTest
+    HINTS ${PC_CPPUTEST_LIBDIR} ${PC_CPPUTEST_LIBRARY_DIRS})
+
+find_library(CPPUTEST_EXT_LIBRARY NAMES CppUTestExt
+    HINTS ${PC_CPPUTEST_LIBDIR} ${PC_CPPUTEST_LIBRARY_DIRS})
+
+find_path(CPPUTEST_INCLUDE_DIRS CppUTest/TestHarness.h
+    HINTS ${PC_CPPUTEST_INCLUDEDIR} ${PC_CPPUTEST_INCLUDE_DIRS})
+
+include(FindPackageHandleStandardArgs)
+
+find_package_handle_standard_args(CppUTest
+    DEFAULT_MSG
+    CPPUTEST_LIBRARY
+    CPPUTEST_EXT_LIBRARY
+    CPPUTEST_INCLUDE_DIRS
+)
+
+if(CPPUTEST_LIBRARY AND CPPUTEST_EXT_LIBRARY)
+    set(CPPUTEST_LIBRARIES ${CPPUTEST_LIBRARY} ${CPPUTEST_EXT_LIBRARY})
+endif()
+
+mark_as_advanced(
+    CPPUTEST_LIBRARY
+    CPPUTEST_EXT_LIBRARY
+    CPPUTEST_INCLUDE_DIRS
+)

cmake/Libraries.cmake

@@ -1,5 +1,6 @@
 set(VENDOR_DIR "${PROJECT_SOURCE_DIR}/vendor" CACHE PATH "")
 set(FETCHCONTENT_QUIET OFF CACHE BOOL "")
+set(CMAKE_EXPORT_PACKAGE_REGISTRY OFF)
 
 if(NOT MSVC)
     add_compile_options(
@@ -12,7 +13,7 @@ if(NOT MSVC)
     )
 
     if(CMAKE_CXX_COMPILER_ID MATCHES "GNU")
-        add_compile_options(-Wno-maybe-uninitialized)
+        add_compile_options(-Wno-maybe-uninitialized -Wno-stringop-overflow -Wno-return-type)
     elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
         add_compile_options(-Wno-unknown-attributes -Wno-deprecated-builtins)
     endif()
@@ -32,14 +33,6 @@ endif()
 set(ABSL_PROPAGATE_CXX_STD ON CACHE BOOL "")
 add_subdirectory("${VENDOR_DIR}/abseil" EXCLUDE_FROM_ALL)
 
-########
-# clog #
-########
-
-set(CLOG_BUILD_TESTS OFF CACHE BOOL "")
-set(CLOG_SOURCE_DIR "${VENDOR_DIR}/cpuinfo/deps/clog" CACHE PATH "")
-add_subdirectory("${CLOG_SOURCE_DIR}" EXCLUDE_FROM_ALL)
-
 ###########
 # cpuinfo #
 ###########
@@ -96,8 +89,18 @@ set(FLATBUFFERS_PROJECT_DIR "${FLATBUFFERS_SOURCE_DIR}" CACHE STRING "")
 # gemmlowp #
 ############
 
-set(GEMMLOWP_SOURCE_DIR "${VENDOR_DIR}/gemmlowp" CACHE PATH "")
-add_subdirectory("${CMAKE_CURRENT_LIST_DIR}/gemmlowp" EXCLUDE_FROM_ALL)
+set(BUILD_TESTING_TMP ${BUILD_TESTING})
+set(BUILD_TESTING OFF)
+
+set(gemmlowp_SOURCE_DIR "${VENDOR_DIR}/gemmlowp/contrib" CACHE PATH "")
+add_subdirectory("${gemmlowp_SOURCE_DIR}" EXCLUDE_FROM_ALL)
+set(gemmlowp_POPULATED ON CACHE BOOL "")
+
+get_target_property(GEMMLOWP_INCLUDE_DIRS gemmlowp INTERFACE_DIRECTORIES)
+add_library(gemmlowp::gemmlowp ALIAS gemmlowp)
+set(GEMMLOWP_LIBRARIES gemmlowp)
+
+set(BUILD_TESTING ${BUILD_TESTING_TMP})
 
 #######
 # ruy #
@@ -149,6 +152,13 @@ get_target_property(NEON2SSE_INCLUDE_DIRS NEON_2_SSE INTERFACE_DIRECTORIES)
 add_library(NEON_2_SSE::NEON_2_SSE ALIAS NEON_2_SSE)
 set(NEON2SSE_LIBRARIES NEON_2_SSE)
 
+#############
+# ml_dtypes #
+#############
+
+set(ML_DTYPES_SOURCE_DIR "${VENDOR_DIR}/ml_dtypes" CACHE PATH "")
+add_subdirectory("${CMAKE_CURRENT_LIST_DIR}/ml_dtypes" EXCLUDE_FROM_ALL)
+
 ###########
 # XNNPACK #
 ###########
@@ -174,3 +184,8 @@ add_subdirectory(
     "${VENDOR_DIR}/tensorflow/tensorflow/lite"
     EXCLUDE_FROM_ALL
 )
+
+if(ENABLE_TESTS)
+    enable_testing()
+    find_package(CppUTest REQUIRED)
+endif()

cmake/Macros.cmake

@@ -0,0 +1,29 @@
+function(add_cpputest testname)
+    if(ENABLE_TESTS)
+        set(multiValueArgs SOURCES INCLUDES LIBS)
+        cmake_parse_arguments(utest "" "" "${multiValueArgs}" "${ARGN}")
+
+        add_executable("${testname}" EXCLUDE_FROM_ALL
+            "${testname}.cc" "${utest_SOURCES}")
+
+        target_include_directories("${testname}" PRIVATE
+            "${CPPUTEST_INCLUDE_DIRS}" "${utest_INCLUDES}")
+
+        target_link_libraries("${testname}" PRIVATE
+            "${CPPUTEST_LIBRARIES}" "${utest_LIBS}")
+
+        add_test("${testname}_build"
+            "${CMAKE_COMMAND}"
+                --build "${CMAKE_BINARY_DIR}"
+                --target "${testname}"
+        )
+
+        set_tests_properties("${testname}_build" PROPERTIES
+            FIXTURES_SETUP "${testname}_fixture")
+
+        add_test(NAME "${testname}" COMMAND ${testname})
+
+        set_tests_properties("${testname}" PROPERTIES
+            FIXTURES_REQUIRED "${testname}_fixture")
+    endif()
+endfunction()

cmake/ml_dtypes/CMakeLists.txt

@@ -0,0 +1,8 @@
+project(ml_dtypes CXX)
+
+add_library(ml_dtypes INTERFACE)
+
+target_include_directories(ml_dtypes INTERFACE
+  "${ML_DTYPES_SOURCE_DIR}"
+  "${ML_DTYPES_SOURCE_DIR}/ml_dtypes"
+)

configure.sh

@@ -45,9 +45,9 @@ while [ $# -ne 0 ]; do
         --prefix=*)
             append_cache_entry CMAKE_INSTALL_PREFIX PATH $optarg
             ;;
-        #--test)
-        #    append_cache_entry ENABLE_TESTS BOOL true
-        #    ;;
+        --test)
+            append_cache_entry ENABLE_TESTS BOOL true
+            ;;
         *)
             echo "Invalid option '$1'.  Try $0 --help to see available options."
             exit 1

examples/classifier/train.py

@@ -58,7 +58,7 @@ def decode_query(str):
         seq[maxlen - arrlen + i] = arr[i]
     X.append(seq)
     Y.append(item['attack'])
-    
+
 #
 # Build Model (Simple LSTM)
 #
@@ -85,9 +85,16 @@ def decode_query(str):
 # Save Model
 #
 
-model.export("model")
+export_archive = tf.keras.export.ExportArchive()
+export_archive.track(model)
+export_archive.add_endpoint(
+    name='serve',
+    fn=model.call,
+    input_signature=[tf.TensorSpec(shape=(1, maxlen), dtype=tf.float32)],
+)
+export_archive.write_out('model')
 
-converter = tf.lite.TFLiteConverter.from_saved_model("model")
+converter = tf.lite.TFLiteConverter.from_saved_model('model')
 
 classifier_model = converter.convert()
 

src/CMakeLists.txt

@@ -61,3 +61,5 @@ install(
     FILES "${CMAKE_BINARY_DIR}/${BUNDLE_NAME}"
     DESTINATION lib
 )
+
+add_subdirectory(test)

src/libml.cc

@@ -1,5 +1,5 @@
 //--------------------------------------------------------------------------
-// Copyright (C) 2023-2023 Cisco and/or its affiliates. All rights reserved.
+// Copyright (C) 2023-2024 Cisco and/or its affiliates. All rights reserved.
 //
 // This program is free software; you can redistribute it and/or modify it
 // under the terms of the GNU General Public License Version 2 as published

src/libml.h

@@ -1,5 +1,5 @@
 //--------------------------------------------------------------------------
-// Copyright (C) 2023-2023 Cisco and/or its affiliates. All rights reserved.
+// Copyright (C) 2023-2024 Cisco and/or its affiliates. All rights reserved.
 //
 // This program is free software; you can redistribute it and/or modify it
 // under the terms of the GNU General Public License Version 2 as published
@@ -25,7 +25,7 @@
 
 const char* libml_version();
 
-namespace tflite
+namespace tflite::impl
 {
     class FlatBufferModel;
     class Interpreter;
@@ -45,6 +45,6 @@ class BinaryClassifier
 private:
     std::string src;
     size_t input_size = 0;
-    std::unique_ptr<tflite::FlatBufferModel> model;
-    std::unique_ptr<tflite::Interpreter> interpreter;
+    std::unique_ptr<tflite::impl::FlatBufferModel> model;
+    std::unique_ptr<tflite::impl::Interpreter> interpreter;
 };

src/test/CMakeLists.txt

@@ -0,0 +1,11 @@
+add_cpputest(binary_classifier_test
+    INCLUDES
+        "${PROJECT_SOURCE_DIR}/src"
+    LIBS
+        libml_api_static
+)
+
+file(
+    COPY test.model
+    DESTINATION "${CMAKE_CURRENT_BINARY_DIR}"
+)

src/test/binary_classifier_test.cc

@@ -0,0 +1,47 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2024-2024 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+// binary_classifier_test.cc author Brandon Stultz <[email protected]>
+
+#include <string>
+
+#include "libml.h"
+
+#include "CppUTest/CommandLineTestRunner.h"
+#include "CppUTest/TestHarness.h"
+
+TEST_GROUP(binary_classifier_test_group) {};
+
+TEST(binary_classifier_test_group, model_check)
+{
+    BinaryClassifier classifier;
+
+    CHECK(classifier.buildFromFile("test.model"));
+
+    std::string input = "foo=bar' OR 1=1;--";
+    float output = 0.0;
+
+    CHECK(classifier.run(input.c_str(), input.length(), output));
+
+    CHECK(output > 0.95);
+}
+
+int main(int argc, char* argv[])
+{
+    MemoryLeakWarningPlugin::turnOffNewDeleteOverloads();
+    return CommandLineTestRunner::RunAllTests(argc, argv);
+}

src/util.cc

@@ -1,5 +1,5 @@
 //--------------------------------------------------------------------------
-// Copyright (C) 2023-2023 Cisco and/or its affiliates. All rights reserved.
+// Copyright (C) 2023-2024 Cisco and/or its affiliates. All rights reserved.
 //
 // This program is free software; you can redistribute it and/or modify it
 // under the terms of the GNU General Public License Version 2 as published

src/util.h

@@ -1,5 +1,5 @@
 //--------------------------------------------------------------------------
-// Copyright (C) 2023-2023 Cisco and/or its affiliates. All rights reserved.
+// Copyright (C) 2023-2024 Cisco and/or its affiliates. All rights reserved.
 //
 // This program is free software; you can redistribute it and/or modify it
 // under the terms of the GNU General Public License Version 2 as published