From f459fcb4ec5a18c0af5835ed82fe0b5919fac845 Mon Sep 17 00:00:00 2001 From: Stefano Bonicatti Date: Wed, 25 Sep 2019 14:10:23 +0200 Subject: [PATCH 01/28] Move the test config files in their own directory This is have better separation between them and the python tests. BUCK files have been updated accordingly. Changes to the internal generateCopyFileTarget function were needed. The function now supports a base folder to be set so that the regex or file path is appended to that base path, without having it included in the destination. It will also not use a library target anymore, but a custom one so that's possible to set properties with custom names. --- cmake/utilities.cmake | 35 ++++++++++--- osquery/config/tests/BUCK | 4 +- osquery/config/tests/CMakeLists.txt | 6 ++- osquery/distributed/tests/BUCK | 2 +- osquery/filesystem/BUCK | 2 +- osquery/filesystem/CMakeLists.txt | 3 +- osquery/remote/enroll/BUCK | 2 +- osquery/remote/enroll/CMakeLists.txt | 3 +- osquery/remote/transports/BUCK | 2 +- osquery/remote/transports/CMakeLists.txt | 3 +- osquery/tables/events/tests/BUCK | 2 +- osquery/tables/networking/tests/BUCK | 4 +- .../tables/networking/tests/CMakeLists.txt | 6 ++- osquery/tables/system/tests/BUCK | 12 ++--- osquery/tables/system/tests/CMakeLists.txt | 19 ++++--- osquery/utils/aws/tests/BUCK | 2 +- osquery/utils/aws/tests/CMakeLists.txt | 3 +- plugins/config/parsers/tests/BUCK | 10 ++-- plugins/config/parsers/tests/CMakeLists.txt | 11 ++-- plugins/logger/tests/BUCK | 4 +- plugins/logger/tests/CMakeLists.txt | 6 ++- tools/tests/BUCK | 41 --------------- tools/tests/CMakeLists.txt | 14 +++--- tools/tests/configs/BUCK | 47 ++++++++++++++++++ tools/tests/{ => configs}/aws/config | 0 tools/tests/{ => configs}/aws/credentials | 0 tools/tests/{ => configs}/test.badconfig | 0 tools/tests/{ => configs}/test.config | 0 .../test.config.d/extra_options.conf | 0 .../{ => configs}/test.config.d/osquery.conf | 0 tools/tests/{ => configs}/test.plist | 0 tools/tests/{ => configs}/test_airport.plist | 0 .../{ => configs}/test_airport_legacy.plist | 0 tools/tests/{ => configs}/test_alf.plist | 0 .../{ => configs}/test_alf_nested_path.plist | Bin tools/tests/{ => configs}/test_array.plist | 0 tools/tests/{ => configs}/test_atc_db.db | Bin tools/tests/{ => configs}/test_binary.plist | Bin tools/tests/{ => configs}/test_cert.pem | 0 tools/tests/{ => configs}/test_client.key | 0 tools/tests/{ => configs}/test_client.pem | 0 .../{ => configs}/test_enroll_secret.txt | 0 tools/tests/{ => configs}/test_hashing.bin | Bin tools/tests/{ => configs}/test_hosts.txt | 0 tools/tests/{ => configs}/test_hosts_ics.txt | 0 tools/tests/{ => configs}/test_info.plist | 0 .../tests/{ => configs}/test_inline_pack.conf | 0 .../tests/{ => configs}/test_killswitch.conf | 0 .../test_killswitch_incorrect_key.conf | 0 .../test_killswitch_incorrect_value.conf | 0 .../test_killswitch_no_table.conf | 0 tools/tests/{ => configs}/test_launchd.plist | 0 .../{ => configs}/test_noninline_packs.conf | 0 tools/tests/{ => configs}/test_pack.conf | 0 .../tests/{ => configs}/test_parse_items.conf | 0 tools/tests/{ => configs}/test_protocols.txt | 0 tools/tests/{ => configs}/test_server.key | 0 tools/tests/{ => configs}/test_server.pem | 0 tools/tests/{ => configs}/test_server_ca.pem | 0 .../{ => configs}/test_startup_items.plist | 0 tools/tests/{ => configs}/test_xattrs.txt | 0 tools/tests/{ => configs}/view_test.conf | 0 tools/tests/{ => configs}/view_test2.conf | 0 63 files changed, 144 insertions(+), 99 deletions(-) create mode 100644 tools/tests/configs/BUCK rename tools/tests/{ => configs}/aws/config (100%) rename tools/tests/{ => configs}/aws/credentials (100%) rename tools/tests/{ => configs}/test.badconfig (100%) rename tools/tests/{ => configs}/test.config (100%) rename tools/tests/{ => configs}/test.config.d/extra_options.conf (100%) rename tools/tests/{ => configs}/test.config.d/osquery.conf (100%) rename tools/tests/{ => configs}/test.plist (100%) rename tools/tests/{ => configs}/test_airport.plist (100%) rename tools/tests/{ => configs}/test_airport_legacy.plist (100%) rename tools/tests/{ => configs}/test_alf.plist (100%) rename tools/tests/{ => configs}/test_alf_nested_path.plist (100%) rename tools/tests/{ => configs}/test_array.plist (100%) rename tools/tests/{ => configs}/test_atc_db.db (100%) rename tools/tests/{ => configs}/test_binary.plist (100%) rename tools/tests/{ => configs}/test_cert.pem (100%) rename tools/tests/{ => configs}/test_client.key (100%) rename tools/tests/{ => configs}/test_client.pem (100%) rename tools/tests/{ => configs}/test_enroll_secret.txt (100%) rename tools/tests/{ => configs}/test_hashing.bin (100%) rename tools/tests/{ => configs}/test_hosts.txt (100%) rename tools/tests/{ => configs}/test_hosts_ics.txt (100%) rename tools/tests/{ => configs}/test_info.plist (100%) rename tools/tests/{ => configs}/test_inline_pack.conf (100%) rename tools/tests/{ => configs}/test_killswitch.conf (100%) rename tools/tests/{ => configs}/test_killswitch_incorrect_key.conf (100%) rename tools/tests/{ => configs}/test_killswitch_incorrect_value.conf (100%) rename tools/tests/{ => configs}/test_killswitch_no_table.conf (100%) rename tools/tests/{ => configs}/test_launchd.plist (100%) rename tools/tests/{ => configs}/test_noninline_packs.conf (100%) rename tools/tests/{ => configs}/test_pack.conf (100%) rename tools/tests/{ => configs}/test_parse_items.conf (100%) rename tools/tests/{ => configs}/test_protocols.txt (100%) rename tools/tests/{ => configs}/test_server.key (100%) rename tools/tests/{ => configs}/test_server.pem (100%) rename tools/tests/{ => configs}/test_server_ca.pem (100%) rename tools/tests/{ => configs}/test_startup_items.plist (100%) rename tools/tests/{ => configs}/test_xattrs.txt (100%) rename tools/tests/{ => configs}/view_test.conf (100%) rename tools/tests/{ => configs}/view_test2.conf (100%) diff --git a/cmake/utilities.cmake b/cmake/utilities.cmake index d4db3c75a48..80ec33545e0 100644 --- a/cmake/utilities.cmake +++ b/cmake/utilities.cmake @@ -124,14 +124,23 @@ function(generateUnsupportedPlatformSourceFile) set(unsupported_platform_source_file "${source_file}" PARENT_SCOPE) endfunction() -function(generateCopyFileTarget name type relative_file_paths destination) - set(source_base_path "${CMAKE_CURRENT_SOURCE_DIR}") +function(generateCopyFileTarget name base_path type relative_file_paths destination) + + if(base_path) + set(base_path "${base_path}/") + else() + set(base_path "${CMAKE_CURRENT_SOURCE_DIR}/") + endif() if(type STREQUAL "REGEX") - file(GLOB_RECURSE relative_file_paths RELATIVE "${source_base_path}" "${source_base_path}/${relative_file_paths}") + if(base_path) + file(GLOB_RECURSE relative_file_paths RELATIVE "${base_path}" "${base_path}${relative_file_paths}") + else() + file(GLOB_RECURSE relative_file_paths "${base_path}${relative_file_paths}") + endif() endif() - add_library("${name}" INTERFACE) + add_custom_target("${name}") foreach(file ${relative_file_paths}) get_filename_component(intermediate_directory "${file}" DIRECTORY) @@ -148,7 +157,16 @@ function(generateCopyFileTarget name type relative_file_paths destination) list(APPEND created_directories "${destination}/${directory}") endforeach() - add_custom_target("${name}_create_dirs" DEPENDS "${created_directories}") + list(APPEND "create_dirs_deps" + "${created_directories}" + "${destination}" + ) + + add_custom_target("${name}_create_dirs" DEPENDS "${create_dirs_deps}") + add_custom_command( + OUTPUT "${destination}" + COMMAND "${CMAKE_COMMAND}" -E make_directory "${destination}" + ) foreach(file ${relative_file_paths}) @@ -160,16 +178,17 @@ function(generateCopyFileTarget name type relative_file_paths destination) add_custom_command( OUTPUT "${destination}/${file}" - COMMAND "${CMAKE_COMMAND}" -E copy "${source_base_path}/${file}" "${destination}/${file}" + COMMAND "${CMAKE_COMMAND}" -E copy "${base_path}${file}" "${destination}/${file}" ) list(APPEND copied_files "${destination}/${file}") endforeach() - add_custom_target("${name}_copy_files" DEPENDS "${name}_create_dirs" "${copied_files}") + add_custom_target("${name}_copy_files" DEPENDS "${copied_files}") + add_dependencies("${name}_copy_files" "${name}_create_dirs") add_dependencies("${name}" "${name}_copy_files") - set_target_properties("${name}" PROPERTIES INTERFACE_BINARY_DIR "${destination}") + set_target_properties("${name}" PROPERTIES FILES_DESTINATION_DIR "${destination}") endfunction() function(add_osquery_executable) diff --git a/osquery/config/tests/BUCK b/osquery/config/tests/BUCK index 02327497d4e..c6299ee8956 100644 --- a/osquery/config/tests/BUCK +++ b/osquery/config/tests/BUCK @@ -14,7 +14,7 @@ osquery_cxx_test( ], env = { "TEST_CONF_FILES_DIR": "$(location {})".format( - osquery_target("tools/tests:conf_files"), + osquery_target("tools/tests/configs:conf_files"), ), }, visibility = ["PUBLIC"], @@ -42,7 +42,7 @@ osquery_cxx_test( ], env = { "TEST_CONF_FILES_DIR": "$(location {})".format( - osquery_target("tools/tests:conf_files"), + osquery_target("tools/tests/configs:conf_files"), ), }, visibility = ["PUBLIC"], diff --git a/osquery/config/tests/CMakeLists.txt b/osquery/config/tests/CMakeLists.txt index fe823ece780..8da026b6894 100644 --- a/osquery/config/tests/CMakeLists.txt +++ b/osquery/config/tests/CMakeLists.txt @@ -45,10 +45,11 @@ function(generateOsqueryConfigTestsTest) plugins_database_ephemeral specs_tables osquery_config_tests_testutils - osquery_tools_tests_conffiles thirdparty_googletest ) + add_dependencies(osquery_config_tests-test osquery_tools_tests_conffiles) + endfunction() function(generateOsqueryConfigTestsPacksTest) @@ -69,10 +70,11 @@ function(generateOsqueryConfigTestsPacksTest) plugins_database_ephemeral specs_tables osquery_config_tests_testutils - osquery_tools_tests_conffiles thirdparty_googletest ) + add_dependencies(osquery_config_tests_packs-test osquery_tools_tests_conffiles) + endfunction() osqueryConfigTestsMain() diff --git a/osquery/distributed/tests/BUCK b/osquery/distributed/tests/BUCK index a8c4acb4240..4d37fdbe2a3 100644 --- a/osquery/distributed/tests/BUCK +++ b/osquery/distributed/tests/BUCK @@ -14,7 +14,7 @@ osquery_cxx_test( ], env = { "TEST_CONF_FILES_DIR": "$(location {})".format( - osquery_target("tools/tests:test_files"), + osquery_target("tools/tests/configs:test_files"), ), }, visibility = ["PUBLIC"], diff --git a/osquery/filesystem/BUCK b/osquery/filesystem/BUCK index 1ce471b9699..8b3a4cd4990 100644 --- a/osquery/filesystem/BUCK +++ b/osquery/filesystem/BUCK @@ -98,7 +98,7 @@ osquery_cxx_test( ], env = { "TEST_CONF_FILES_DIR": "$(location {})".format( - osquery_target("tools/tests:plist_files"), + osquery_target("tools/tests/configs:plist_files"), ), }, platform_srcs = [ diff --git a/osquery/filesystem/CMakeLists.txt b/osquery/filesystem/CMakeLists.txt index 8e9bbdc23f2..05bd136ccb8 100644 --- a/osquery/filesystem/CMakeLists.txt +++ b/osquery/filesystem/CMakeLists.txt @@ -123,9 +123,10 @@ function(generateOsqueryFilesystemTest) specs_tables osquery_filesystem_mockfilestructure osquery_filesystem - osquery_tools_tests_plistfiles thirdparty_googletest ) + + add_dependencies(osquery_filesystem_filesystemtests-test osquery_tools_tests_plistfiles) endfunction() osqueryFilesystemMain() diff --git a/osquery/remote/enroll/BUCK b/osquery/remote/enroll/BUCK index 435a774744b..cf1831afc0f 100644 --- a/osquery/remote/enroll/BUCK +++ b/osquery/remote/enroll/BUCK @@ -41,7 +41,7 @@ osquery_cxx_test( ], env = { "TEST_CONF_FILES_DIR": "$(location {})".format( - osquery_target("tools/tests:test_files"), + osquery_target("tools/tests/configs:test_files"), ), }, platform_srcs = [ diff --git a/osquery/remote/enroll/CMakeLists.txt b/osquery/remote/enroll/CMakeLists.txt index fedb1120485..da16505e693 100644 --- a/osquery/remote/enroll/CMakeLists.txt +++ b/osquery/remote/enroll/CMakeLists.txt @@ -75,8 +75,9 @@ function(generateOsqueryRemoteenrolltestsTest) plugins_database_ephemeral tests_helper thirdparty_googletest - osquery_tools_tests_testfiles ) + + add_dependencies(osquery_remote_enroll_remoteenrolltests-test osquery_tools_tests_testfiles) endfunction() osqueryRemoteEnrollTlsenrollMain() diff --git a/osquery/remote/transports/BUCK b/osquery/remote/transports/BUCK index 78eac2a9f12..df93849a996 100644 --- a/osquery/remote/transports/BUCK +++ b/osquery/remote/transports/BUCK @@ -30,7 +30,7 @@ osquery_cxx_test( name = "remote_transports_tls_tests", env = { "TEST_CONF_FILES_DIR": "$(location {})".format( - osquery_target("tools/tests:test_files"), + osquery_target("tools/tests/configs:test_files"), ), }, platform_srcs = [ diff --git a/osquery/remote/transports/CMakeLists.txt b/osquery/remote/transports/CMakeLists.txt index 670c8f76895..b1e5b181326 100644 --- a/osquery/remote/transports/CMakeLists.txt +++ b/osquery/remote/transports/CMakeLists.txt @@ -64,8 +64,9 @@ function(generateOsqueryRemoteTransportsRemotetransportstlstestsTest) plugins_database_ephemeral tests_helper thirdparty_googletest - osquery_tools_tests_testfiles ) + + add_dependencies(osquery_remote_transports_remotetransportstlstests-test osquery_tools_tests_testfiles) endfunction() osqueryRemoteTransportsMain() diff --git a/osquery/tables/events/tests/BUCK b/osquery/tables/events/tests/BUCK index 7975e8f3e3a..993aced3e33 100644 --- a/osquery/tables/events/tests/BUCK +++ b/osquery/tables/events/tests/BUCK @@ -15,7 +15,7 @@ osquery_cxx_test( ], env = { "TEST_CONF_FILES_DIR": "$(location {})".format( - osquery_target("tools/tests:test_files"), + osquery_target("tools/tests/configs:test_files"), ), }, visibility = ["PUBLIC"], diff --git a/osquery/tables/networking/tests/BUCK b/osquery/tables/networking/tests/BUCK index 7ab032a4644..ebdd756ddbd 100644 --- a/osquery/tables/networking/tests/BUCK +++ b/osquery/tables/networking/tests/BUCK @@ -16,7 +16,7 @@ osquery_cxx_test( ], env = { "TEST_CONF_FILES_DIR": "$(location {})".format( - osquery_target("tools/tests:test_files"), + osquery_target("tools/tests/configs:test_files"), ), }, visibility = ["PUBLIC"], @@ -40,7 +40,7 @@ osquery_cxx_test( name = "wifi_tests", env = { "TEST_CONF_FILES_DIR": "$(location {})".format( - osquery_target("tools/tests:plist_files"), + osquery_target("tools/tests/configs:plist_files"), ), }, platform_srcs = [ diff --git a/osquery/tables/networking/tests/CMakeLists.txt b/osquery/tables/networking/tests/CMakeLists.txt index bc1d1f8d4bb..5e7355d5918 100644 --- a/osquery/tables/networking/tests/CMakeLists.txt +++ b/osquery/tables/networking/tests/CMakeLists.txt @@ -32,8 +32,9 @@ function(generateOsqueryTablesNetworkingTestsNetworkingtablestestsTest) plugins_database_ephemeral thirdparty_boost thirdparty_googletest - osquery_tools_tests_testfiles ) + + add_dependencies(osquery_tables_networking_tests_networkingtablestests-test osquery_tools_tests_testfiles) endfunction() function(generateOsqueryTablesNetworkingTestsWifitestsTest) @@ -55,8 +56,9 @@ function(generateOsqueryTablesNetworkingTestsWifitestsTest) plugins_database_ephemeral thirdparty_boost thirdparty_googletest - osquery_tools_tests_plistfiles ) + + add_dependencies(osquery_tables_networking_tests_wifitests-test osquery_tools_tests_plistfiles) endfunction() function(generateOsqueryTablesNetworkingTestsIptablestestsTest) diff --git a/osquery/tables/system/tests/BUCK b/osquery/tables/system/tests/BUCK index ad1f956bd4b..92bf81a0799 100644 --- a/osquery/tables/system/tests/BUCK +++ b/osquery/tables/system/tests/BUCK @@ -206,7 +206,7 @@ osquery_cxx_test( name = "apps_tests", env = { "TEST_CONF_FILES_DIR": "$(location {})".format( - osquery_target("tools/tests:plist_files"), + osquery_target("tools/tests/configs:plist_files"), ), }, platform_srcs = [ @@ -261,7 +261,7 @@ osquery_cxx_test( name = "certificates_tests", env = { "TEST_CONF_FILES_DIR": "$(location {})".format( - osquery_target("tools/tests:test_files"), + osquery_target("tools/tests/configs:test_files"), ), }, platform_srcs = [ @@ -298,7 +298,7 @@ osquery_cxx_test( name = "extended_attributes_tests", env = { "TEST_CONF_FILES_DIR": "$(location {})".format( - osquery_target("tools/tests:test_files"), + osquery_target("tools/tests/configs:test_files"), ), }, platform_srcs = [ @@ -328,7 +328,7 @@ osquery_cxx_test( name = "firewall_tests", env = { "TEST_CONF_FILES_DIR": "$(location {})".format( - osquery_target("tools/tests:plist_files"), + osquery_target("tools/tests/configs:plist_files"), ), }, platform_srcs = [ @@ -358,7 +358,7 @@ osquery_cxx_test( name = "launchd_tests", env = { "TEST_CONF_FILES_DIR": "$(location {})".format( - osquery_target("tools/tests:plist_files"), + osquery_target("tools/tests/configs:plist_files"), ), }, platform_srcs = [ @@ -463,7 +463,7 @@ osquery_cxx_test( name = "startup_items_tests", env = { "TEST_CONF_FILES_DIR": "$(location {})".format( - osquery_target("tools/tests:plist_files"), + osquery_target("tools/tests/configs:plist_files"), ), }, platform_srcs = [ diff --git a/osquery/tables/system/tests/CMakeLists.txt b/osquery/tables/system/tests/CMakeLists.txt index 83e254ecb98..944f8ba97cb 100644 --- a/osquery/tables/system/tests/CMakeLists.txt +++ b/osquery/tables/system/tests/CMakeLists.txt @@ -230,8 +230,9 @@ function(generateOsqueryTablesSystemTestsAppstestsTest) osquery_utils_conversions plugins_database_ephemeral thirdparty_googletest - osquery_tools_tests_plistfiles ) + + add_dependencies(osquery_tables_system_tests_appstests-test osquery_tools_tests_plistfiles) endfunction() function(generateOsqueryTablesSystemTestsAsltestsTest) @@ -269,8 +270,9 @@ function(generateOsqueryTablesSystemTestsMacOsCertificatestestsTest) osquery_utils_conversions plugins_database_ephemeral thirdparty_googletest - osquery_tools_tests_testfiles ) + + add_dependencies(osquery_tables_system_tests_certificatestests-test osquery_tools_tests_testfiles) endfunction() function(generateOsqueryTablesSystemTestsExtendedattributestestsTest) @@ -289,8 +291,9 @@ function(generateOsqueryTablesSystemTestsExtendedattributestestsTest) osquery_utils_conversions plugins_database_ephemeral thirdparty_googletest - osquery_tools_tests_testfiles ) + + add_dependencies(osquery_tables_system_tests_extendedattributestests-test osquery_tools_tests_testfiles) endfunction() function(generateOsqueryTablesSystemTestsFirewalltestsTest) @@ -309,8 +312,9 @@ function(generateOsqueryTablesSystemTestsFirewalltestsTest) osquery_utils_conversions plugins_database_ephemeral thirdparty_googletest - osquery_tools_tests_plistfiles ) + + add_dependencies(osquery_tables_system_tests_firewalltests-test osquery_tools_tests_plistfiles) endfunction() function(generateOsqueryTablesSystemTestsLaunchdtestsTest) @@ -329,8 +333,9 @@ function(generateOsqueryTablesSystemTestsLaunchdtestsTest) osquery_utils_conversions plugins_database_ephemeral thirdparty_googletest - osquery_tools_tests_plistfiles ) + + add_dependencies(osquery_tables_system_tests_launchdtests-test osquery_tools_tests_plistfiles) endfunction() function(generateOsqueryTablesSystemTestsMdfindtestsTest) @@ -406,8 +411,9 @@ function(generateOsqueryTablesSystemTestsStartupitemstestsTest) osquery_utils_conversions plugins_database_ephemeral thirdparty_googletest - osquery_tools_tests_plistfiles ) + + add_dependencies(osquery_tables_system_tests_startupitemstests-test osquery_tools_tests_plistfiles) endfunction() function(generateOsqueryTablesSystemTestsSignaturetestsTest) @@ -493,7 +499,6 @@ function(generateOsqueryTablesSystemTestsAugeasTestsTest) add_dependencies(osquery_tables_system_tests_augeastests-test osquery_tables_system_tests_augeaslenses) add_dependencies(osquery_tables_system_tests_augeaslenses thirdparty_augeas) - add_dependencies(osquery_tables_system_tests_augeaslenses osquery_tools_tests_testconfigsfolder) endfunction() diff --git a/osquery/utils/aws/tests/BUCK b/osquery/utils/aws/tests/BUCK index 0fe97ad749f..ce1addaaff9 100644 --- a/osquery/utils/aws/tests/BUCK +++ b/osquery/utils/aws/tests/BUCK @@ -14,7 +14,7 @@ osquery_cxx_test( ], env = { "TEST_CONF_FILES_DIR": "$(location {})".format( - osquery_target("tools/tests:aws_files"), + osquery_target("tools/tests/configs:aws_files"), ), }, visibility = ["PUBLIC"], diff --git a/osquery/utils/aws/tests/CMakeLists.txt b/osquery/utils/aws/tests/CMakeLists.txt index 4a5c803e125..e7f3f50086c 100644 --- a/osquery/utils/aws/tests/CMakeLists.txt +++ b/osquery/utils/aws/tests/CMakeLists.txt @@ -26,8 +26,9 @@ function(generateOsqueryUtilsAwsTestsTest) plugins_database_ephemeral specs_tables thirdparty_googletest - osquery_tools_tests_awsfiles ) + + add_dependencies(osquery_utils_aws_tests-test osquery_tools_tests_awsfiles) endfunction() osqueryUtilsAwsTestsMain() diff --git a/plugins/config/parsers/tests/BUCK b/plugins/config/parsers/tests/BUCK index d540f964740..6f47bb0be14 100644 --- a/plugins/config/parsers/tests/BUCK +++ b/plugins/config/parsers/tests/BUCK @@ -14,7 +14,7 @@ osquery_cxx_test( ], env = { "TEST_CONF_FILES_DIR": "$(location {})".format( - osquery_target("tools/tests:conf_files"), + osquery_target("tools/tests/configs:conf_files"), ), }, visibility = ["PUBLIC"], @@ -44,7 +44,7 @@ osquery_cxx_test( ], env = { "TEST_CONF_FILES_DIR": "$(location {})".format( - osquery_target("tools/tests:conf_files"), + osquery_target("tools/tests/configs:conf_files"), ), }, visibility = ["PUBLIC"], @@ -74,7 +74,7 @@ osquery_cxx_test( ], env = { "TEST_CONF_FILES_DIR": "$(location {})".format( - osquery_target("tools/tests:conf_files"), + osquery_target("tools/tests/configs:conf_files"), ), }, visibility = ["PUBLIC"], @@ -105,7 +105,7 @@ osquery_cxx_test( ], env = { "TEST_CONF_FILES_DIR": "$(location {})".format( - osquery_target("tools/tests:conf_files"), + osquery_target("tools/tests/configs:conf_files"), ), }, visibility = ["PUBLIC"], @@ -136,7 +136,7 @@ osquery_cxx_test( ], env = { "TEST_CONF_FILES_DIR": "$(location {})".format( - osquery_target("tools/tests:conf_files"), + osquery_target("tools/tests/configs:conf_files"), ), }, visibility = ["PUBLIC"], diff --git a/plugins/config/parsers/tests/CMakeLists.txt b/plugins/config/parsers/tests/CMakeLists.txt index 5dfe9f3347c..4a4f63f78d3 100644 --- a/plugins/config/parsers/tests/CMakeLists.txt +++ b/plugins/config/parsers/tests/CMakeLists.txt @@ -32,9 +32,10 @@ function(generatePluginsConfigParsersTestsDecoratorstestsTest) plugins_config_parsers plugins_database_ephemeral specs_tables - osquery_tools_tests_conffiles thirdparty_googletest ) + + add_dependencies(plugins_config_parsers_tests_decoratorstests-test osquery_tools_tests_conffiles) endfunction() function(generatePluginsConfigParsersTestsEventsparsertestsTest) @@ -58,10 +59,10 @@ function(generatePluginsConfigParsersTestsEventsparsertestsTest) plugins_config_parsers plugins_database_ephemeral specs_tables - osquery_tools_tests_conffiles thirdparty_googletest ) + add_dependencies(plugins_config_parsers_tests_eventsparsertests-test osquery_tools_tests_conffiles) endfunction() function(generatePluginsConfigParsersTestsFilepathstestsTest) @@ -85,9 +86,10 @@ function(generatePluginsConfigParsersTestsFilepathstestsTest) plugins_config_parsers plugins_database_ephemeral specs_tables - osquery_tools_tests_conffiles thirdparty_googletest ) + + add_dependencies(plugins_config_parsers_tests_filepathstests-test osquery_tools_tests_conffiles) endfunction() function(generatePluginsConfigParsersTestsOptionstestsTest) @@ -112,9 +114,10 @@ function(generatePluginsConfigParsersTestsOptionstestsTest) plugins_config_parsers plugins_database_ephemeral specs_tables - osquery_tools_tests_conffiles thirdparty_googletest ) + + add_dependencies(plugins_config_parsers_tests_optionstests-test osquery_tools_tests_conffiles) endfunction() function(generatePluginsConfigParsersTestsViewstestsTest) diff --git a/plugins/logger/tests/BUCK b/plugins/logger/tests/BUCK index 2edc4ae3282..fb6528e4b2c 100644 --- a/plugins/logger/tests/BUCK +++ b/plugins/logger/tests/BUCK @@ -130,7 +130,7 @@ osquery_cxx_test( ], env = { "TEST_CONF_FILES_DIR": "$(location {})".format( - osquery_target("tools/tests:test_files"), + osquery_target("tools/tests/configs:test_files"), ), }, visibility = ["PUBLIC"], @@ -159,7 +159,7 @@ osquery_cxx_test( name = "syslog_logger_tests", env = { "TEST_CONF_FILES_DIR": "$(location {})".format( - osquery_target("tools/tests:test_files"), + osquery_target("tools/tests/configs:test_files"), ), }, platform_srcs = [ diff --git a/plugins/logger/tests/CMakeLists.txt b/plugins/logger/tests/CMakeLists.txt index c3cb750c840..42f67de8adc 100644 --- a/plugins/logger/tests/CMakeLists.txt +++ b/plugins/logger/tests/CMakeLists.txt @@ -142,8 +142,9 @@ function(generatePluginsLoggerTestsTlsloggertestsTest) specs_tables thirdparty_googletest thirdparty_gflags - osquery_tools_tests_testfiles ) + + add_dependencies(plugins_logger_tests_tlsloggertests-test osquery_tools_tests_testfiles) endfunction() function(generatePluginsLoggerTestsSyslogloggertestsTest) @@ -169,8 +170,9 @@ function(generatePluginsLoggerTestsSyslogloggertestsTest) thirdparty_googletest thirdparty_gflags osquery_remote_tests_remotetestutils - osquery_tools_tests_testfiles ) + + add_dependencies(plugins_logger_tests_syslogloggertests-test osquery_tools_tests_testfiles) endfunction() pluginsLoggerTestsMain() diff --git a/tools/tests/BUCK b/tools/tests/BUCK index 32b4cb6ff2d..d58633811f4 100644 --- a/tools/tests/BUCK +++ b/tools/tests/BUCK @@ -4,7 +4,6 @@ # This source code is licensed as defined on the LICENSE file found in the # root directory of this source tree. -load("//tools/build_defs/oss/osquery:native.bzl", "osquery_filegroup") load("//tools/build_defs/oss/osquery:python.bzl", "osquery_python_library") osquery_python_library( @@ -15,43 +14,3 @@ osquery_python_library( base_module = "osquery.tools.tests", visibility = ["PUBLIC"], ) - -osquery_filegroup( - name = "conf_files", - srcs = glob([ - "*.conf", - ]), - visibility = ["PUBLIC"], -) - -osquery_filegroup( - name = "config_files", - srcs = glob([ - "*.config", - ]), - visibility = ["PUBLIC"], -) - -osquery_filegroup( - name = "plist_files", - srcs = glob([ - "*.plist", - ]), - visibility = ["PUBLIC"], -) - -osquery_filegroup( - name = "test_files", - srcs = glob([ - "*", - ]), - visibility = ["PUBLIC"], -) - -osquery_filegroup( - name = "aws_files", - srcs = glob([ - "aws/*", - ]), - visibility = ["PUBLIC"], -) diff --git a/tools/tests/CMakeLists.txt b/tools/tests/CMakeLists.txt index 863e95aecee..39807849d71 100644 --- a/tools/tests/CMakeLists.txt +++ b/tools/tests/CMakeLists.txt @@ -5,12 +5,14 @@ # the LICENSE file found in the root directory of this source tree. function(osqueryToolsTestsMain) - add_custom_target(osquery_tools_tests_testconfigsfolder COMMAND "${CMAKE_COMMAND}" -E make_directory "${CMAKE_BINARY_DIR}/test_configs") - generateCopyFileTarget("osquery_tools_tests_conffiles" "REGEX" "*.conf" "${TEST_CONFIGS_DIR}") - generateCopyFileTarget("osquery_tools_tests_configfiles" "REGEX" "*.config" "${TEST_CONFIGS_DIR}") - generateCopyFileTarget("osquery_tools_tests_plistfiles" "REGEX" "*.plist" "${TEST_CONFIGS_DIR}") - generateCopyFileTarget("osquery_tools_tests_testfiles" "REGEX" "*" "${TEST_CONFIGS_DIR}") - generateCopyFileTarget("osquery_tools_tests_awsfiles" "REGEX" "aws/*" "${TEST_CONFIGS_DIR}") + if(OSQUERY_BUILD_TESTS) + set(configs_base_path "${CMAKE_CURRENT_SOURCE_DIR}/configs") + generateCopyFileTarget("osquery_tools_tests_conffiles" "${configs_base_path}" "REGEX" "*.conf" "${TEST_CONFIGS_DIR}") + generateCopyFileTarget("osquery_tools_tests_configfiles" "${configs_base_path}" "REGEX" "*.config" "${TEST_CONFIGS_DIR}") + generateCopyFileTarget("osquery_tools_tests_plistfiles" "${configs_base_path}" "REGEX" "*.plist" "${TEST_CONFIGS_DIR}") + generateCopyFileTarget("osquery_tools_tests_testfiles" "${configs_base_path}" "REGEX" "*" "${TEST_CONFIGS_DIR}") + generateCopyFileTarget("osquery_tools_tests_awsfiles" "${configs_base_path}" "REGEX" "aws/*" "${TEST_CONFIGS_DIR}") + endif() endfunction() osqueryToolsTestsMain() diff --git a/tools/tests/configs/BUCK b/tools/tests/configs/BUCK new file mode 100644 index 00000000000..b17f515d25f --- /dev/null +++ b/tools/tests/configs/BUCK @@ -0,0 +1,47 @@ +# Copyright (c) 2014-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed as defined on the LICENSE file found in the +# root directory of this source tree. + +load("//tools/build_defs/oss/osquery:native.bzl", "osquery_filegroup") + +osquery_filegroup( + name = "conf_files", + srcs = glob([ + "*.conf", + ]), + visibility = ["PUBLIC"], +) + +osquery_filegroup( + name = "config_files", + srcs = glob([ + "*.config", + ]), + visibility = ["PUBLIC"], +) + +osquery_filegroup( + name = "plist_files", + srcs = glob([ + "*.plist", + ]), + visibility = ["PUBLIC"], +) + +osquery_filegroup( + name = "test_files", + srcs = glob([ + "*", + ]), + visibility = ["PUBLIC"], +) + +osquery_filegroup( + name = "aws_files", + srcs = glob([ + "aws/*", + ]), + visibility = ["PUBLIC"], +) \ No newline at end of file diff --git a/tools/tests/aws/config b/tools/tests/configs/aws/config similarity index 100% rename from tools/tests/aws/config rename to tools/tests/configs/aws/config diff --git a/tools/tests/aws/credentials b/tools/tests/configs/aws/credentials similarity index 100% rename from tools/tests/aws/credentials rename to tools/tests/configs/aws/credentials diff --git a/tools/tests/test.badconfig b/tools/tests/configs/test.badconfig similarity index 100% rename from tools/tests/test.badconfig rename to tools/tests/configs/test.badconfig diff --git a/tools/tests/test.config b/tools/tests/configs/test.config similarity index 100% rename from tools/tests/test.config rename to tools/tests/configs/test.config diff --git a/tools/tests/test.config.d/extra_options.conf b/tools/tests/configs/test.config.d/extra_options.conf similarity index 100% rename from tools/tests/test.config.d/extra_options.conf rename to tools/tests/configs/test.config.d/extra_options.conf diff --git a/tools/tests/test.config.d/osquery.conf b/tools/tests/configs/test.config.d/osquery.conf similarity index 100% rename from tools/tests/test.config.d/osquery.conf rename to tools/tests/configs/test.config.d/osquery.conf diff --git a/tools/tests/test.plist b/tools/tests/configs/test.plist similarity index 100% rename from tools/tests/test.plist rename to tools/tests/configs/test.plist diff --git a/tools/tests/test_airport.plist b/tools/tests/configs/test_airport.plist similarity index 100% rename from tools/tests/test_airport.plist rename to tools/tests/configs/test_airport.plist diff --git a/tools/tests/test_airport_legacy.plist b/tools/tests/configs/test_airport_legacy.plist similarity index 100% rename from tools/tests/test_airport_legacy.plist rename to tools/tests/configs/test_airport_legacy.plist diff --git a/tools/tests/test_alf.plist b/tools/tests/configs/test_alf.plist similarity index 100% rename from tools/tests/test_alf.plist rename to tools/tests/configs/test_alf.plist diff --git a/tools/tests/test_alf_nested_path.plist b/tools/tests/configs/test_alf_nested_path.plist similarity index 100% rename from tools/tests/test_alf_nested_path.plist rename to tools/tests/configs/test_alf_nested_path.plist diff --git a/tools/tests/test_array.plist b/tools/tests/configs/test_array.plist similarity index 100% rename from tools/tests/test_array.plist rename to tools/tests/configs/test_array.plist diff --git a/tools/tests/test_atc_db.db b/tools/tests/configs/test_atc_db.db similarity index 100% rename from tools/tests/test_atc_db.db rename to tools/tests/configs/test_atc_db.db diff --git a/tools/tests/test_binary.plist b/tools/tests/configs/test_binary.plist similarity index 100% rename from tools/tests/test_binary.plist rename to tools/tests/configs/test_binary.plist diff --git a/tools/tests/test_cert.pem b/tools/tests/configs/test_cert.pem similarity index 100% rename from tools/tests/test_cert.pem rename to tools/tests/configs/test_cert.pem diff --git a/tools/tests/test_client.key b/tools/tests/configs/test_client.key similarity index 100% rename from tools/tests/test_client.key rename to tools/tests/configs/test_client.key diff --git a/tools/tests/test_client.pem b/tools/tests/configs/test_client.pem similarity index 100% rename from tools/tests/test_client.pem rename to tools/tests/configs/test_client.pem diff --git a/tools/tests/test_enroll_secret.txt b/tools/tests/configs/test_enroll_secret.txt similarity index 100% rename from tools/tests/test_enroll_secret.txt rename to tools/tests/configs/test_enroll_secret.txt diff --git a/tools/tests/test_hashing.bin b/tools/tests/configs/test_hashing.bin similarity index 100% rename from tools/tests/test_hashing.bin rename to tools/tests/configs/test_hashing.bin diff --git a/tools/tests/test_hosts.txt b/tools/tests/configs/test_hosts.txt similarity index 100% rename from tools/tests/test_hosts.txt rename to tools/tests/configs/test_hosts.txt diff --git a/tools/tests/test_hosts_ics.txt b/tools/tests/configs/test_hosts_ics.txt similarity index 100% rename from tools/tests/test_hosts_ics.txt rename to tools/tests/configs/test_hosts_ics.txt diff --git a/tools/tests/test_info.plist b/tools/tests/configs/test_info.plist similarity index 100% rename from tools/tests/test_info.plist rename to tools/tests/configs/test_info.plist diff --git a/tools/tests/test_inline_pack.conf b/tools/tests/configs/test_inline_pack.conf similarity index 100% rename from tools/tests/test_inline_pack.conf rename to tools/tests/configs/test_inline_pack.conf diff --git a/tools/tests/test_killswitch.conf b/tools/tests/configs/test_killswitch.conf similarity index 100% rename from tools/tests/test_killswitch.conf rename to tools/tests/configs/test_killswitch.conf diff --git a/tools/tests/test_killswitch_incorrect_key.conf b/tools/tests/configs/test_killswitch_incorrect_key.conf similarity index 100% rename from tools/tests/test_killswitch_incorrect_key.conf rename to tools/tests/configs/test_killswitch_incorrect_key.conf diff --git a/tools/tests/test_killswitch_incorrect_value.conf b/tools/tests/configs/test_killswitch_incorrect_value.conf similarity index 100% rename from tools/tests/test_killswitch_incorrect_value.conf rename to tools/tests/configs/test_killswitch_incorrect_value.conf diff --git a/tools/tests/test_killswitch_no_table.conf b/tools/tests/configs/test_killswitch_no_table.conf similarity index 100% rename from tools/tests/test_killswitch_no_table.conf rename to tools/tests/configs/test_killswitch_no_table.conf diff --git a/tools/tests/test_launchd.plist b/tools/tests/configs/test_launchd.plist similarity index 100% rename from tools/tests/test_launchd.plist rename to tools/tests/configs/test_launchd.plist diff --git a/tools/tests/test_noninline_packs.conf b/tools/tests/configs/test_noninline_packs.conf similarity index 100% rename from tools/tests/test_noninline_packs.conf rename to tools/tests/configs/test_noninline_packs.conf diff --git a/tools/tests/test_pack.conf b/tools/tests/configs/test_pack.conf similarity index 100% rename from tools/tests/test_pack.conf rename to tools/tests/configs/test_pack.conf diff --git a/tools/tests/test_parse_items.conf b/tools/tests/configs/test_parse_items.conf similarity index 100% rename from tools/tests/test_parse_items.conf rename to tools/tests/configs/test_parse_items.conf diff --git a/tools/tests/test_protocols.txt b/tools/tests/configs/test_protocols.txt similarity index 100% rename from tools/tests/test_protocols.txt rename to tools/tests/configs/test_protocols.txt diff --git a/tools/tests/test_server.key b/tools/tests/configs/test_server.key similarity index 100% rename from tools/tests/test_server.key rename to tools/tests/configs/test_server.key diff --git a/tools/tests/test_server.pem b/tools/tests/configs/test_server.pem similarity index 100% rename from tools/tests/test_server.pem rename to tools/tests/configs/test_server.pem diff --git a/tools/tests/test_server_ca.pem b/tools/tests/configs/test_server_ca.pem similarity index 100% rename from tools/tests/test_server_ca.pem rename to tools/tests/configs/test_server_ca.pem diff --git a/tools/tests/test_startup_items.plist b/tools/tests/configs/test_startup_items.plist similarity index 100% rename from tools/tests/test_startup_items.plist rename to tools/tests/configs/test_startup_items.plist diff --git a/tools/tests/test_xattrs.txt b/tools/tests/configs/test_xattrs.txt similarity index 100% rename from tools/tests/test_xattrs.txt rename to tools/tests/configs/test_xattrs.txt diff --git a/tools/tests/view_test.conf b/tools/tests/configs/view_test.conf similarity index 100% rename from tools/tests/view_test.conf rename to tools/tests/configs/view_test.conf diff --git a/tools/tests/view_test2.conf b/tools/tests/configs/view_test2.conf similarity index 100% rename from tools/tests/view_test2.conf rename to tools/tests/configs/view_test2.conf From a5f095893da91c9dcdb7b306cbe0e3dfc65f54d1 Mon Sep 17 00:00:00 2001 From: Stefano Bonicatti Date: Wed, 16 Oct 2019 15:07:19 +0200 Subject: [PATCH 02/28] Change and add some python packages in the CI Dockerfile - Add the bdist_wheel package to have pip optimize packages. - Update Python 2 packages to Python 3 - Use pexpect==3.3 --- tools/ci/osquery-ubuntu18.04-toolchain.dockerfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/ci/osquery-ubuntu18.04-toolchain.dockerfile b/tools/ci/osquery-ubuntu18.04-toolchain.dockerfile index ee5f0989947..3d99562a5f3 100644 --- a/tools/ci/osquery-ubuntu18.04-toolchain.dockerfile +++ b/tools/ci/osquery-ubuntu18.04-toolchain.dockerfile @@ -15,18 +15,18 @@ RUN apt update -q -y && apt upgrade -q -y && apt install -q -y --no-install-reco flex \ bison \ xz-utils \ - python-setuptools \ - python-pexpect \ - python-psutil \ - python-pip \ - python-six \ + python3-setuptools \ + python3-psutil \ + python3-pip \ + python3-six \ rpm \ dpkg-dev \ file \ elfutils \ + python3-wheel \ && dpkg -i linux-base_1.0_all.deb linux-firmware_1.0_all.deb linux-generic_1.0_all.deb \ && apt clean && rm -rf /var/lib/apt/lists/* \ -&& sudo pip install timeout_decorator +&& sudo pip3 install timeout_decorator thrift==0.11.0 osquery pexpect==3.3 RUN cd ~ && wget https://github.com/Kitware/CMake/releases/download/v3.14.6/cmake-3.14.6-Linux-x86_64.tar.gz \ && sudo tar xvf cmake-3.14.6-Linux-x86_64.tar.gz -C /usr/local --strip 1 && rm cmake-3.14.6-Linux-x86_64.tar.gz \ && wget https://github.com/osquery/osquery-toolchain/releases/download/1.0.0/osquery-toolchain-1.0.0.tar.xz \ From d27c77738b654d2388478ab2e91e78b586c657a7 Mon Sep 17 00:00:00 2001 From: Stefano Bonicatti Date: Wed, 25 Sep 2019 14:19:49 +0200 Subject: [PATCH 03/28] Restore python tests in CMake Restored all the python tests under tools/tests. Some tests have been modified so that they can be relocated and don't expect too specific hardcoded paths. Fixed the gen_api function in genapi.py missing the path to the spec file when evaluating blacklisted tables. Updated the wiki docs with the optionally needed packages to run the python tests. Updated azure-pipelines.yml to install the needed packages for the python tests. Add an init process so that reaping works in the CI Docker image --- azure-pipelines.yml | 10 ++- cmake/globals.cmake | 2 +- cmake/utilities.cmake | 8 +-- docs/wiki/development/building.md | 15 ++++ libraries/cmake/facebook/modules/api.cmake | 33 +++++---- .../cmake/facebook/thrift/CMakeLists.txt | 1 + libraries/cmake/source/thrift/CMakeLists.txt | 1 + osquery/CMakeLists.txt | 11 +++ osquery/extensions/thrift/CMakeLists.txt | 1 + specs/CMakeLists.txt | 2 + tools/codegen/CMakeLists.txt | 25 +++---- tools/codegen/genapi.py | 2 +- tools/codegen/gentable.py | 2 +- tools/tests/BUCK | 2 +- tools/tests/CMakeLists.txt | 69 +++++++++++++++++++ tools/tests/test_additional.py | 2 +- tools/tests/test_base.py | 24 ++++--- tools/tests/test_example_queries.py | 10 +-- tools/tests/test_osqueryi.py | 11 ++- tools/tests/test_release.py | 3 - 20 files changed, 162 insertions(+), 72 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3977dbab840..df65e73811e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -20,8 +20,8 @@ jobs: vmImage: 'Ubuntu-16.04' container: - image: trailofbits/osquery:ubuntu-18.04-toolchain-v4 - options: --privileged + image: trailofbits/osquery:ubuntu-18.04-toolchain-391938a2 + options: --privileged --init timeoutInMinutes: 120 @@ -199,6 +199,7 @@ jobs: - script: | brew upgrade brew install ccache flex bison + pip3 install setuptools pexpect psutil timeout_decorator six thrift==0.11.0 osquery sudo xcode-select -s /Applications/Xcode_10.3.app/Contents/Developer displayName: "Install Homebrew and prerequisites" timeoutInMinutes: 20 @@ -328,6 +329,11 @@ jobs: - checkout: self + - powershell: | + $python3_path = ((Get-Item C:\hostedtoolcache\windows\Python\3*\x64) | Sort-Object -Descending)[0].FullName + & $python3_path\python -m pip install setuptools psutil timeout_decorator thrift==0.11.0 osquery pywin32 + displayName: Install tests prerequisites + - powershell: | mkdir $(Build.BinariesDirectory)\build displayName: "Create build folder" diff --git a/cmake/globals.cmake b/cmake/globals.cmake index 57889216235..3e21184c1d1 100644 --- a/cmake/globals.cmake +++ b/cmake/globals.cmake @@ -19,7 +19,7 @@ endif() # This is the destination for the remotely imported Python modules, used when # setting up the PYTHONPATH folder -set(PYTHON_PATH "${CMAKE_BINARY_DIR}/python_path") +set(OSQUERY_PYTHON_PATH "${CMAKE_BINARY_DIR}/python_path") # TODO(alessandro): Add missing defines: PLATFORM_FREEBSD if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") diff --git a/cmake/utilities.cmake b/cmake/utilities.cmake index 80ec33545e0..c9ceb54713e 100644 --- a/cmake/utilities.cmake +++ b/cmake/utilities.cmake @@ -102,9 +102,6 @@ endfunction() function(findPythonExecutablePath) find_package(Python2 COMPONENTS Interpreter REQUIRED) find_package(Python3 COMPONENTS Interpreter REQUIRED) - - set(EX_TOOL_PYTHON2_EXECUTABLE_PATH "${Python2_EXECUTABLE}" PARENT_SCOPE) - set(EX_TOOL_PYTHON3_EXECUTABLE_PATH "${Python3_EXECUTABLE}" PARENT_SCOPE) endfunction() function(generateBuildTimeSourceFile file_path content) @@ -249,7 +246,6 @@ function(generateSpecialTargets) # Used to generate all the files necessary to have a complete view of the project in the IDE add_custom_target(prepare_for_ide) - set(excluded_folders "libraries/cmake/source" ) @@ -260,12 +256,12 @@ function(generateSpecialTargets) endif() add_custom_target(format_check - COMMAND ${command_prefix} ${EX_TOOL_PYTHON2_EXECUTABLE_PATH} ${CMAKE_SOURCE_DIR}/tools/formatting/format-check.py --exclude-folders ${excluded_folders} origin/master + COMMAND ${command_prefix} "${Python2_EXECUTABLE}" ${CMAKE_SOURCE_DIR}/tools/formatting/format-check.py --exclude-folders ${excluded_folders} origin/master WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" VERBATIM ) add_custom_target(format - COMMAND ${command_prefix} ${EX_TOOL_PYTHON2_EXECUTABLE_PATH} ${CMAKE_SOURCE_DIR}/tools/formatting/git-clang-format.py --exclude-folders ${excluded_folders} -f --style=file + COMMAND ${command_prefix} "${Python2_EXECUTABLE}" ${CMAKE_SOURCE_DIR}/tools/formatting/git-clang-format.py --exclude-folders ${excluded_folders} -f --style=file WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" VERBATIM ) diff --git a/docs/wiki/development/building.md b/docs/wiki/development/building.md index 47cee2b1d73..be50216b89b 100644 --- a/docs/wiki/development/building.md +++ b/docs/wiki/development/building.md @@ -24,6 +24,10 @@ The root folder is assumed to be `/home/` # Install the prerequisites sudo apt install --no-install-recommends git python python3 bison flex make +# Optional: install python tests prerequisites +sudo apt install --no-install-recommends python-pip python-setuptools python-psutil python-pexpect python-six +pip install timeout-decorator + # Download and install the osquery toolchain wget https://github.com/osquery/osquery-toolchain/releases/download/1.0.0/osquery-toolchain-1.0.0.tar.xz sudo tar xvf osquery-toolchain-1.0.0.tar.xz -C /usr/local @@ -55,6 +59,9 @@ Please ensure [Homebrew](https://brew.sh/) has been installed, first. Then do th # Install prerequisites xcode-select --install brew install git cmake python@2 python + +# Optional: install python tests prerequisites +pip install setuptools pexpect psutil timeout_decorator six ``` **Step 2: Download and build** @@ -93,6 +100,14 @@ Note: It may be easier to install these prerequisites using [Chocolatey](https:/ - [Strawberry Perl](http://strawberryperl.com/) for the OpenSSL formula. It is recommended to install it to the default destination path. - [7-Zip](https://www.7-zip.org/) if building the Chocolatey package. +**Optional: Install python tests prerequisites** +Python 2 is assumed to be installed in `C:\Python27` + +```PowerShell +# Using a PowerShell console +C:\Python27\python.exe -m pip install setuptools psutil timeout_decorator six +``` + **Step 2: Download and build** ```PowerShell diff --git a/libraries/cmake/facebook/modules/api.cmake b/libraries/cmake/facebook/modules/api.cmake index 81778242379..52685ee39b0 100644 --- a/libraries/cmake/facebook/modules/api.cmake +++ b/libraries/cmake/facebook/modules/api.cmake @@ -10,7 +10,7 @@ function(downloadRemoteFile identifier base_url file_name hash) set(destination "${CMAKE_CURRENT_BINARY_DIR}/${file_name}") set(url "${base_url}/${file_name}") - set(command_prefix "${EX_TOOL_PYTHON3_EXECUTABLE_PATH}") + set(command_prefix "${Python3_EXECUTABLE}") add_custom_command( OUTPUT "${destination}" @@ -161,15 +161,17 @@ function(importThirdPartyHeaderOnlyLibrary library_type name version hash anchor endfunction() # Initializes the PYTHONPATH folder in the binary directory, used to run the codegen scripts -function(initializePythonPathFolder) - add_custom_command( - OUTPUT "${PYTHON_PATH}" - COMMAND "${CMAKE_COMMAND}" -E make_directory "${PYTHON_PATH}" - COMMENT "Initializing custom PYTHONPATH: ${PYTHON_PATH}" - ) - - add_custom_target(thirdparty_pythonpath DEPENDS "${PYTHON_PATH}") - add_custom_target(thirdparty_python_modules) +function(initializePythonPathFolder) + if(NOT TARGET thirdparty_python_modules) + add_custom_command( + OUTPUT "${OSQUERY_PYTHON_PATH}" + COMMAND "${CMAKE_COMMAND}" -E make_directory "${OSQUERY_PYTHON_PATH}" + COMMENT "Initializing custom PYTHONPATH: ${OSQUERY_PYTHON_PATH}" + ) + + add_custom_target(thirdparty_pythonpath DEPENDS "${OSQUERY_PYTHON_PATH}") + add_custom_target(thirdparty_python_modules) + endif() endfunction() # Imports a remote Python module inside the PYTHONPATH folder (previously initialized @@ -178,7 +180,7 @@ function(importRemotePythonModule identifier base_url file_name hash) set(target_name "thirdparty_pythonmodule_${identifier}") downloadRemoteFile("${target_name}" "${base_url}" "${file_name}" "${hash}") - extractLocalArchive("${target_name}" "${PYTHON_PATH}/${identifier}" "${downloadRemoteFile_destination}" "${PYTHON_PATH}") + extractLocalArchive("${target_name}" "${OSQUERY_PYTHON_PATH}/${identifier}" "${downloadRemoteFile_destination}" "${OSQUERY_PYTHON_PATH}") add_dependencies("${target_name}_extractor" thirdparty_pythonpath) add_osquery_library("${target_name}" INTERFACE) @@ -199,15 +201,12 @@ function(importFacebookLibrary library_name) endfunction() # Make sure that globals.cmake and options.cmake have been included -if("${PYTHON_PATH}" STREQUAL "") - message(FATAL_ERROR "The PYTHON_PATH variable was not found. Has globals.cmake been included?") +if("${OSQUERY_PYTHON_PATH}" STREQUAL "") + message(FATAL_ERROR "The OSQUERY_PYTHON_PATH variable was not found. Has globals.cmake been included?") endif() if("${THIRD_PARTY_REPOSITORY_URL}" STREQUAL "") message(FATAL_ERROR "The THIRD_PARTY_REPOSITORY_URL variable was not found. Has options.cmake been included?") endif() -if(NOT PYTHON_PATH_FOLDER_INITIALIZED) - initializePythonPathFolder() - set(PYTHON_PATH_FOLDER_INITIALIZED true) -endif() +initializePythonPathFolder() \ No newline at end of file diff --git a/libraries/cmake/facebook/thrift/CMakeLists.txt b/libraries/cmake/facebook/thrift/CMakeLists.txt index 295aa7b5f30..c711db8dc0e 100644 --- a/libraries/cmake/facebook/thrift/CMakeLists.txt +++ b/libraries/cmake/facebook/thrift/CMakeLists.txt @@ -34,6 +34,7 @@ function(thrifMain) importThirdPartyBinaryLibrary("${name}" "${version}" "${hash}" "${anchor_file}" ${additional_libraries}) target_link_libraries("thirdparty_${name}" INTERFACE thirdparty_boost) + endfunction() thrifMain() diff --git a/libraries/cmake/source/thrift/CMakeLists.txt b/libraries/cmake/source/thrift/CMakeLists.txt index 73281e0f0c0..4f3379df18b 100644 --- a/libraries/cmake/source/thrift/CMakeLists.txt +++ b/libraries/cmake/source/thrift/CMakeLists.txt @@ -109,6 +109,7 @@ function(thriftMain) "${library_root}/src" "${CMAKE_CURRENT_SOURCE_DIR}/config" ) + endfunction() thriftMain() diff --git a/osquery/CMakeLists.txt b/osquery/CMakeLists.txt index 272ad4cb17a..4ccb1995fed 100644 --- a/osquery/CMakeLists.txt +++ b/osquery/CMakeLists.txt @@ -64,6 +64,17 @@ function(generateOsqueryd) osquery_cxx_settings osquery_main ) + + set(osqueryi_ext "") + if(PLATFORM_WINDOWS) + set(osqueryi_ext ".exe") + endif() + + add_custom_target(create_osqueryi ALL DEPENDS osqueryi${osqueryi_ext}) + add_custom_command(OUTPUT osqueryi${osqueryi_ext} + COMMAND ${CMAKE_COMMAND} -E create_symlink osqueryd osqueryi${osqueryi_ext} + ) + add_dependencies(create_osqueryi osqueryd) endfunction() osqueryMain() diff --git a/osquery/extensions/thrift/CMakeLists.txt b/osquery/extensions/thrift/CMakeLists.txt index 245f25c3eed..daac00f2add 100644 --- a/osquery/extensions/thrift/CMakeLists.txt +++ b/osquery/extensions/thrift/CMakeLists.txt @@ -29,6 +29,7 @@ function(generateOsqueryExtensionsThriftOsquerycpp2) ) generateIncludeNamespace(osquery_extensions_thrift_osquerycpp2 "" "FILE_ONLY" ${public_header_files}) + endfunction() osqueryExtensionsThriftMain() diff --git a/specs/CMakeLists.txt b/specs/CMakeLists.txt index 6ee51896e6b..792e2638191 100644 --- a/specs/CMakeLists.txt +++ b/specs/CMakeLists.txt @@ -19,6 +19,8 @@ function(specsMain) endforeach() target_link_libraries(specs_tables INTERFACE ${table_target_list}) + + generateCopyFileTarget("specs_table_files" "" "REGEX" "*.table" "${TEST_CONFIGS_DIR}/specs") endfunction() function(generateForeignTables spec_file_list) diff --git a/tools/codegen/CMakeLists.txt b/tools/codegen/CMakeLists.txt index 39fb94db1a2..ac9e717f22a 100644 --- a/tools/codegen/CMakeLists.txt +++ b/tools/codegen/CMakeLists.txt @@ -12,26 +12,26 @@ endfunction() # Generates a runnable gentable.py script; unfortunately, upstream has scattered # the required files around, so we need to piece them together function(generateGentable) - set(gentable_working_directory "${CMAKE_CURRENT_BINARY_DIR}/gentable") - - set(module_path "osquery/tools/tests") - generatePythonModulePath("${gentable_working_directory}" "${module_path}" "gentable_dependency_") + set(module_path "osquery_tests/tools/tests") + generatePythonModulePath("${OSQUERY_PYTHON_PATH}" "${module_path}" "osquery_pythonpath_") - set(utils_file_path "${gentable_working_directory}/${module_path}/utils.py") + set(utils_file_path "${OSQUERY_PYTHON_PATH}/${module_path}/utils.py") add_custom_command( OUTPUT "${utils_file_path}" - COMMAND "${CMAKE_COMMAND}" -E create_symlink "${CMAKE_SOURCE_DIR}/tools/tests/utils.py" "${utils_file_path}" + COMMAND "${CMAKE_COMMAND}" -E copy "${CMAKE_SOURCE_DIR}/tools/tests/utils.py" "${utils_file_path}" DEPENDS "${generatePythonModulePath_rootTarget}" ) add_custom_target("codegen_gentable_utils_dependency" DEPENDS "${utils_file_path}") + set(gentable_working_directory "${CMAKE_CURRENT_BINARY_DIR}/gentable") + generatePythonModulePath("${CMAKE_CURRENT_BINARY_DIR}" "gentable" "gentable_dependency_") set(gentable_file_path "${gentable_working_directory}/gentable.py") set(source_base_path "${CMAKE_CURRENT_SOURCE_DIR}") add_custom_command( OUTPUT "${gentable_file_path}" COMMAND "${CMAKE_COMMAND}" -E create_symlink "${source_base_path}/gentable.py" "${gentable_file_path}" - DEPENDS "codegen_gentable_utils_dependency" "thirdparty_python_modules" + DEPENDS "codegen_gentable_utils_dependency" "thirdparty_python_modules" "${generatePythonModulePath_rootTarget}" ) add_custom_target("codegen_gentable" DEPENDS "${gentable_file_path}") @@ -128,15 +128,10 @@ function(generateTables category) set(optional_foreign_parameter "--foreign") endif() - set(env_path_separator ":") - if(DEFINED PLATFORM_WINDOWS) - set(env_path_separator ";") - endif() - add_custom_command( OUTPUT "${generated_source_code}" COMMAND "${CMAKE_COMMAND}" -E make_directory "${source_code_intermediate_directories}" - COMMAND "${CMAKE_COMMAND}" -E env "PYTHONPATH=${gentable_working_directory}${env_path_separator}${PYTHON_PATH}" "${EX_TOOL_PYTHON3_EXECUTABLE_PATH}" "${gentable_file_path}" --templates "${CMAKE_SOURCE_DIR}/tools/codegen/templates" ${optional_foreign_parameter} "${table_spec}" "${generated_source_code}" + COMMAND "${CMAKE_COMMAND}" -E env "PYTHONPATH=${OSQUERY_PYTHON_PATH}" "${Python3_EXECUTABLE}" "${gentable_file_path}" --templates "${CMAKE_SOURCE_DIR}/tools/codegen/templates" ${optional_foreign_parameter} "${table_spec}" "${generated_source_code}" WORKING_DIRECTORY "${gentable_working_directory}" DEPENDS codegen_gentable "${table_spec}" COMMENT "Generating code for table ${category}/${table_name}..." @@ -146,7 +141,7 @@ function(generateTables category) add_custom_command( OUTPUT "${generated_header_code}" COMMAND "${CMAKE_COMMAND}" -E make_directory "${generated_headers_folder}" - COMMAND "${CMAKE_COMMAND}" -E env "PYTHONPATH=${gentable_working_directory}${env_path_separator}${PYTHON_PATH}" "${EX_TOOL_PYTHON3_EXECUTABLE_PATH}" "${gentable_file_path}" --header --templates "${CMAKE_SOURCE_DIR}/tools/codegen/templates" ${optional_foreign_parameter} "${table_spec}" "${generated_header_code}" + COMMAND "${CMAKE_COMMAND}" -E env "PYTHONPATH=${OSQUERY_PYTHON_PATH}" "${Python3_EXECUTABLE}" "${gentable_file_path}" --header --templates "${CMAKE_SOURCE_DIR}/tools/codegen/templates" ${optional_foreign_parameter} "${table_spec}" "${generated_header_code}" WORKING_DIRECTORY "${gentable_working_directory}" DEPENDS codegen_gentable "${table_spec}" COMMENT "Generating header for table ${category}/${table_name}..." @@ -197,7 +192,7 @@ function(generateTableCategoryAmalgamation category_name) add_custom_command( OUTPUT "${amalgamation_file}" - COMMAND "${CMAKE_COMMAND}" -E env "PYTHONPATH=${PYTHON_PATH}" "${EX_TOOL_PYTHON3_EXECUTABLE_PATH}" "${CMAKE_SOURCE_DIR}/tools/codegen/amalgamate.py" --templates "${CMAKE_SOURCE_DIR}/tools/codegen/templates" ${amalgamation_type} --sources "${generateTables_output}" --output "${amalgamation_file}" + COMMAND "${CMAKE_COMMAND}" -E env "PYTHONPATH=${OSQUERY_PYTHON_PATH}" "${Python3_EXECUTABLE}" "${CMAKE_SOURCE_DIR}/tools/codegen/amalgamate.py" --templates "${CMAKE_SOURCE_DIR}/tools/codegen/templates" ${amalgamation_type} --sources "${generateTables_output}" --output "${amalgamation_file}" COMMENT "Generating amalgamation file for the ${category_name} tables..." DEPENDS "${category_spec_files}" ${generateTables_targetList} "thirdparty_python_modules" ) diff --git a/tools/codegen/genapi.py b/tools/codegen/genapi.py index c314e8be3a0..edddcb38e4b 100755 --- a/tools/codegen/genapi.py +++ b/tools/codegen/genapi.py @@ -207,7 +207,7 @@ def gen_api(tables_path, profile={}): table_spec = gen_spec(tree) table_profile = profile.get("%s.%s" % (platform, name), {}) table_spec["profile"] = NoIndent(table_profile) - table_spec["blacklisted"] = is_blacklisted(table_spec["name"], + table_spec["blacklisted"] = is_blacklisted(table_spec["name"], path=spec_file, blacklist=blacklist) categories[platform]["tables"].append(table_spec) categories = [{"key": k, "name": v["name"], "tables": v["tables"]} diff --git a/tools/codegen/gentable.py b/tools/codegen/gentable.py index 2e4dbf9d30e..fa15b040aec 100644 --- a/tools/codegen/gentable.py +++ b/tools/codegen/gentable.py @@ -16,7 +16,7 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -from osquery.tools.tests import utils +from osquery_tests.tools.tests import utils # the log format for the logging module LOG_FORMAT = "%(levelname)s [Line %(lineno)d]: %(message)s" diff --git a/tools/tests/BUCK b/tools/tests/BUCK index d58633811f4..67aaf6a8ad4 100644 --- a/tools/tests/BUCK +++ b/tools/tests/BUCK @@ -11,6 +11,6 @@ osquery_python_library( srcs = [ "utils.py", ], - base_module = "osquery.tools.tests", + base_module = "osquery_tests.tools.tests", visibility = ["PUBLIC"], ) diff --git a/tools/tests/CMakeLists.txt b/tools/tests/CMakeLists.txt index 39807849d71..d3ae450ed4b 100644 --- a/tools/tests/CMakeLists.txt +++ b/tools/tests/CMakeLists.txt @@ -12,6 +12,75 @@ function(osqueryToolsTestsMain) generateCopyFileTarget("osquery_tools_tests_plistfiles" "${configs_base_path}" "REGEX" "*.plist" "${TEST_CONFIGS_DIR}") generateCopyFileTarget("osquery_tools_tests_testfiles" "${configs_base_path}" "REGEX" "*" "${TEST_CONFIGS_DIR}") generateCopyFileTarget("osquery_tools_tests_awsfiles" "${configs_base_path}" "REGEX" "aws/*" "${TEST_CONFIGS_DIR}") + generatePythonTests() + endif() +endfunction() + +function(addPythonTest) + set(oneValueArgs NAME SCRIPT) + set(multiValueArgs EXTRA_ARGS) + cmake_parse_arguments(PARSE_ARGV 0 osquery_test "" "${oneValueArgs}" "${multiValueArgs}") + + get_filename_component(script_name "${osquery_test_SCRIPT}" NAME) + + configure_file("${osquery_test_SCRIPT}" ${script_name} COPYONLY) + + set(python_test_command + "${CMAKE_COMMAND}" -E env "PYTHONPATH=${OSQUERY_PYTHON_PATH}" + "${Python3_EXECUTABLE}" -u ${script_name} --verbose --build "${CMAKE_BINARY_DIR}" --test-configs-dir "${TEST_CONFIGS_DIR}" "${osquery_test_EXTRA_ARGS}") + + add_test(NAME ${osquery_test_NAME} + COMMAND ${python_test_command} + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") + + set_tests_properties(${osquery_test_NAME} PROPERTIES TIMEOUT 30) +endfunction() + +function(preparePythonTestsEnvironment) + # Common files and scripts needed by the tests + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/test_base.py" test_base.py COPYONLY) + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/utils.py" utils.py COPYONLY) + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/test_http_server.py" test_http_server.py COPYONLY) + configure_file("${CMAKE_SOURCE_DIR}/tools/deployment/osquery.example.conf" "${TEST_CONFIGS_DIR}/osquery.example.conf" COPYONLY) + + add_custom_target(osquery_tools_tests_pythontests ALL) + + # The test test_example_queries requires to use codegen scripts, let's copy them in the binary dir + generateCopyFileTarget("osquery_tools_codegen_scripts" "${CMAKE_SOURCE_DIR}/tools/codegen" "REGEX" "gen*.py" "${CMAKE_CURRENT_BINARY_DIR}") + + # Also prepare a proper module path for utils.py + if(PLATFORM_WINDOWS) + configure_file("${CMAKE_SOURCE_DIR}/tools/tests/winexpect.py" winexpect.py COPYONLY) + endif() + + add_dependencies(osquery_tools_tests_pythontests osqueryd) + add_dependencies(osquery_tools_tests_pythontests osquery_tools_tests_testfiles) + add_dependencies(osquery_tools_tests_pythontests osquery_tools_codegen_scripts) + add_dependencies(osquery_tools_tests_pythontests specs_table_files) + add_dependencies(osquery_tools_tests_pythontests codegen_gentable_utils_dependency) +endfunction() + +function(generatePythonTests) + + preparePythonTestsEnvironment() + + # Tests + addPythonTest(NAME tools_tests_testosqueryd SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_osqueryd.py") + + addPythonTest(NAME tools_tests_testosqueryi SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_osqueryi.py") + + addPythonTest(NAME tools_tests_testrelease SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_release.py") + + if(DEFINED PLATFORM_WINDOWS) + addPythonTest(NAME tools_tests_testwindowsservice SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_windows_service.py") + else() + addPythonTest(NAME tools_tests_testadditional SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_additional.py") + + addPythonTest(NAME tools_tests_testexamplequeries SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_example_queries.py") + + # The following tests need to be restored when extensions work again + addPythonTest(NAME tools_tests_testextensions SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_extensions.py") + set_tests_properties(tools_tests_testextensions PROPERTIES DISABLED TRUE) endif() endfunction() diff --git a/tools/tests/test_additional.py b/tools/tests/test_additional.py index d4d105b8414..8c1273e35e6 100755 --- a/tools/tests/test_additional.py +++ b/tools/tests/test_additional.py @@ -19,7 +19,7 @@ class AdditionalFeatureTests(test_base.ProcessGenerator, unittest.TestCase): @test_base.flaky def test_query_packs(self): - query_pack_path = test_base.CONFIG_DIR + "/test_pack.conf" + query_pack_path = test_base.TEST_CONFIGS_DIR + "/test_pack.conf" utils.write_config({ "queries": { "simple_test": { diff --git a/tools/tests/test_base.py b/tools/tests/test_base.py index 5d84ac9f935..ea6363daf5d 100644 --- a/tools/tests/test_base.py +++ b/tools/tests/test_base.py @@ -39,10 +39,6 @@ def timeout(*args, **kwargs): else: import pexpect -# While this path can be variable, in practice is lives statically. -OSQUERY_DEPENDENCIES = os.getenv("OSQUERY_DEPS", "/usr/local/osquery") -sys.path = [OSQUERY_DEPENDENCIES + "/lib/python2.7/site-packages"] + sys.path - if os.name != "nt": try: from pexpect.replwrap import REPLWrapper @@ -546,14 +542,22 @@ def wrapper(this): class Tester(object): def __init__(self): - global ARGS, CONFIG, CONFIG_DIR + global ARGS, CONFIG, CONFIG_DIR, TEST_CONFIGS_DIR parser = argparse.ArgumentParser( description=("osquery python integration testing.")) + + parser.add_argument( + "--test-configs-dir", + required=True, + help="Directory where the config files the test may use are" + ) + parser.add_argument( "--config", metavar="FILE", default=None, help="Use special options from a config.") + parser.add_argument( "--verbose", default=False, @@ -578,6 +582,7 @@ def __init__(self): utils.reset_dir(CONFIG_DIR) CONFIG = read_config(ARGS.config) if ARGS.config else DEFAULT_CONFIG + TEST_CONFIGS_DIR = ARGS.test_configs_dir @timeout_decorator.timeout(20 * 60) def run(self): @@ -711,14 +716,11 @@ def getLatestInfoLog(base): def loadThriftFromBuild(build_dir): - '''Find and import the thrift-generated python interface.''' - thrift_path = build_dir + "/generated/gen-py" + '''Import the thrift-generated python interface.''' try: - sys.path = [thrift_path, thrift_path + "/osquery"] + sys.path - from osquery import ExtensionManager, Extension + from osquery.extensions import ExtensionManager, Extension EXClient.setUp(ExtensionManager, Extension) except ImportError as e: - print("Cannot import osquery thrift API from %s" % (thrift_path)) + print("Cannot import osquery thrift API") print("Exception: %s" % (str(e))) - print("You must first run: make") exit(1) diff --git a/tools/tests/test_example_queries.py b/tools/tests/test_example_queries.py index f64ffdcddc4..9cba3e7097f 100755 --- a/tools/tests/test_example_queries.py +++ b/tools/tests/test_example_queries.py @@ -17,7 +17,6 @@ import test_base import utils - class ExampleQueryTests(test_base.QueryTester): @test_base.flaky def test_cross_platform_queries(self): @@ -35,12 +34,11 @@ def test_utility_queries(self): self._execute_set(PLATFORM_EXAMPLES["utility"]) if __name__ == '__main__': + module = test_base.Tester() + # Import the API generation code for example query introspection. - SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) - SOURCE_DIR = os.path.abspath(SCRIPT_DIR + "/../../") - sys.path.append(SOURCE_DIR + "/tools/codegen") from genapi import gen_api - API = gen_api(SOURCE_DIR + "/specs") + API = gen_api("%s/specs" % (test_base.TEST_CONFIGS_DIR)) # Organize example queries by platform PLATFORM_EXAMPLES = {} @@ -54,8 +52,6 @@ def test_utility_queries(self): "select * from %s limit 1" % table["name"] ] - module = test_base.Tester() - # Find and import the thrift-generated python interface test_base.loadThriftFromBuild(test_base.ARGS.build) diff --git a/tools/tests/test_osqueryi.py b/tools/tests/test_osqueryi.py index b1030c17da5..395cac52b51 100755 --- a/tools/tests/test_osqueryi.py +++ b/tools/tests/test_osqueryi.py @@ -39,7 +39,7 @@ def test_config_check_success(self): self.binary, "--config_check", "--database_path=%s" % (self.dbpath), - "--config_path=%s/test.config" % test_base.SCRIPT_DIR, + "--config_path=%s/test.config" % test_base.TEST_CONFIGS_DIR, "--extensions_autoload=", "--verbose", ], @@ -51,7 +51,7 @@ def test_config_check_success(self): def test_config_dump(self): '''Test that config raw output is dumped when requested''' - config = os.path.join(test_base.SCRIPT_DIR, "test_noninline_packs.conf") + config = os.path.join(test_base.TEST_CONFIGS_DIR, "test_noninline_packs.conf") proc = test_base.TimeoutRunner([ self.binary, "--config_dump", @@ -97,7 +97,7 @@ def test_config_check_failure_valid_path(self): "--extensions_autoload=", "--verbose", "--database_path=%s" % (self.dbpath), - "--config_path=%s" % os.path.join(test_base.SCRIPT_DIR, "test.badconfig") + "--config_path=%s" % os.path.join(test_base.TEST_CONFIGS_DIR, "test.badconfig") ], SHELL_TIMEOUT) self.assertEqual(proc.proc.poll(), 1) @@ -121,11 +121,10 @@ def test_config_check_failure_missing_plugin(self): def test_config_check_example(self): '''Test that the example config passes''' - example_path = os.path.join("deployment", "osquery.example.conf") proc = test_base.TimeoutRunner([ self.binary, "--config_check", - "--config_path=%s" % os.path.join(test_base.SCRIPT_DIR, "..", example_path), + "--config_path=%s" % os.path.join(test_base.TEST_CONFIGS_DIR, "osquery.example.conf"), "--extensions_autoload=", "--verbose", ], @@ -245,7 +244,7 @@ def test_config_bad_json(self): @test_base.flaky def test_atc(self): local_osquery_instance = test_base.OsqueryWrapper(self.binary, - args={"config_path": "test.config"}) + args={"config_path": "%s" % os.path.join(test_base.TEST_CONFIGS_DIR, "test.config") }) result = local_osquery_instance.run_query('SELECT a_number FROM test_atc') self.assertEqual(result, [{'a_number':'314159'}]) diff --git a/tools/tests/test_release.py b/tools/tests/test_release.py index 873dddb987f..b956cfc5e5a 100755 --- a/tools/tests/test_release.py +++ b/tools/tests/test_release.py @@ -69,9 +69,6 @@ def test_no_local_link(self): self.assertEqual(proc, 1) if __name__ == '__main__': - SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) - SOURCE_DIR = os.path.abspath(SCRIPT_DIR + "/../../") - module = test_base.Tester() # Find and import the thrift-generated python interface test_base.loadThriftFromBuild(test_base.ARGS.build) From e64e0ed05b6ae56a8767dbb3f9e0ec148b5caac0 Mon Sep 17 00:00:00 2001 From: Stefano Bonicatti Date: Mon, 21 Oct 2019 22:02:20 +0200 Subject: [PATCH 04/28] test_osqueryi.py depends on the presence of the osqueryi link --- tools/tests/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/tests/CMakeLists.txt b/tools/tests/CMakeLists.txt index d3ae450ed4b..f0a1f74dfb9 100644 --- a/tools/tests/CMakeLists.txt +++ b/tools/tests/CMakeLists.txt @@ -58,6 +58,7 @@ function(preparePythonTestsEnvironment) add_dependencies(osquery_tools_tests_pythontests osquery_tools_codegen_scripts) add_dependencies(osquery_tools_tests_pythontests specs_table_files) add_dependencies(osquery_tools_tests_pythontests codegen_gentable_utils_dependency) + add_dependencies(osquery_tools_tests_pythontests create_osqueryi) endfunction() function(generatePythonTests) From 41b41269aa2b597209281d9f6134475e07fea415 Mon Sep 17 00:00:00 2001 From: Stefano Bonicatti Date: Wed, 16 Oct 2019 22:01:52 +0200 Subject: [PATCH 05/28] Fix test_http_server.py and test_windows_service.py - Update to Python3. - Scripts now accepts the test configs directory as an argument. - Update findOsquerydBinary to support Ninja builds - Fix test_windows_service Administrator detection - Avoid a ResourceWarning on BufferedReader when killing the osquery process --- tools/tests/test_base.py | 5 ++- tools/tests/test_http_server.py | 50 ++++++++++++++------- tools/tests/test_windows_service.py | 68 ++++++++++++++++++----------- 3 files changed, 79 insertions(+), 44 deletions(-) diff --git a/tools/tests/test_base.py b/tools/tests/test_base.py index ea6363daf5d..1dc2d986dad 100644 --- a/tools/tests/test_base.py +++ b/tools/tests/test_base.py @@ -531,7 +531,7 @@ def wrapper(this): i = 1 for exc in exceptions: print("Test (attempt %d) %s::%s failed: %s" % - (i, this.__class__.__name__, gen.__name__, str(exc[0]))) + (i, this.__class__.__name__, gen.__name__, str(exc))) i += 1 if len(exceptions) > 0: raise exceptions[0] @@ -542,7 +542,7 @@ def wrapper(this): class Tester(object): def __init__(self): - global ARGS, CONFIG, CONFIG_DIR, TEST_CONFIGS_DIR + global ARGS, CONFIG, CONFIG_DIR, TEST_CONFIGS_DIR, BUILD_DIR parser = argparse.ArgumentParser( description=("osquery python integration testing.")) @@ -583,6 +583,7 @@ def __init__(self): utils.reset_dir(CONFIG_DIR) CONFIG = read_config(ARGS.config) if ARGS.config else DEFAULT_CONFIG TEST_CONFIGS_DIR = ARGS.test_configs_dir + BUILD_DIR = ARGS.build @timeout_decorator.timeout(20 * 60) def run(self): diff --git a/tools/tests/test_http_server.py b/tools/tests/test_http_server.py index 28a70cdc2bb..134d9c530c8 100755 --- a/tools/tests/test_http_server.py +++ b/tools/tests/test_http_server.py @@ -14,12 +14,12 @@ import ssl import string import sys -import thread +import _thread import threading # Create a simple TLS/HTTP server. -from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer -from urlparse import parse_qs +from http.server import BaseHTTPRequestHandler, HTTPServer +from urllib.parse import parse_qs # Script run directory, used for default values SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -29,11 +29,11 @@ HTTP_SERVER_PERSIST = False HTTP_SERVER_TIMEOUT = 10 HTTP_SERVER_VERBOSE = False -HTTP_SERVER_CERT = SCRIPT_DIR + "/test_server.pem" -HTTP_SERVER_KEY = SCRIPT_DIR + "/test_server.key" -HTTP_SERVER_CA = SCRIPT_DIR + "/test_server_ca.pem" +HTTP_SERVER_CERT = "test_server.pem" +HTTP_SERVER_KEY = "test_server.key" +HTTP_SERVER_CA = "test_server_ca.pem" HTTP_SERVER_USE_ENROLL_SECRET = True -HTTP_SERVER_ENROLL_SECRET = SCRIPT_DIR + "/test_enroll_secret.txt" +HTTP_SERVER_ENROLL_SECRET = "test_enroll_secret.txt" # Global accessor value for arguments passed to the server ARGS = None @@ -167,7 +167,7 @@ def do_HEAD(self): def do_POST(self): debug("RealSimpleHandler::post %s" % self.path) self._set_headers() - content_len = int(self.headers.getheader('content-length', 0)) + content_len = int(self.headers.get('content-length', 0)) body = self.rfile.read(content_len) request = json.loads(body) @@ -335,7 +335,7 @@ def _push_request(self, command, request): def _reply(self, response): debug("Replying: %s" % (str(response))) - self.wfile.write(json.dumps(response)) + self.wfile.write(json.dumps(response).encode()) def handler(): @@ -385,7 +385,6 @@ def run_http_server(bind_port=80, **kwargs): except KeyboardInterrupt: sys.exit(0) - if __name__ == '__main__': parser = argparse.ArgumentParser( description=("osquery python https server for client TLS testing.")) @@ -414,17 +413,17 @@ def run_http_server(bind_port=80, **kwargs): parser.add_argument( "--cert", metavar="CERT_FILE", - default=HTTP_SERVER_CERT, + default=None, help="TLS server cert.") parser.add_argument( "--key", metavar="PRIVATE_KEY_FILE", - default=HTTP_SERVER_KEY, + default=None, help="TLS server cert private key.") parser.add_argument( "--ca", metavar="CA_FILE", - default=HTTP_SERVER_CA, + default=None, help="TLS server CA list for client-auth.") parser.add_argument( @@ -437,12 +436,31 @@ def run_http_server(bind_port=80, **kwargs): metavar="SECRET_FILE", default=HTTP_SERVER_ENROLL_SECRET, help="File containing enrollment secret") + parser.add_argument( + "--test-configs-dir", + required=True, + help="Directory where the script will search for configuration files it needs") parser.add_argument( "port", metavar="PORT", type=int, help="Bind to which local TCP port.") - args = { + args = parser.parse_args() + + if args.cert is None: + args.cert = "%s/%s" % (args.test_configs_dir, HTTP_SERVER_CERT) + + if args.key is None: + args.key = "%s/%s" % (args.test_configs_dir, HTTP_SERVER_KEY) + + if args.ca is None: + args.ca = "%s/%s" % (args.test_configs_dir, HTTP_SERVER_CA) + + if args.enroll_secret is None: + args.enroll_secret = "%s/%s" % (args.test_configs_dir, HTTP_SERVER_ENROLL_SECRET) + + nonempty_args = { k: v - for k, v in vars(parser.parse_args()).items() if v is not None + for k, v in vars(args).items() if v is not None } - run_http_server(args['port'], **args) + + run_http_server(nonempty_args['port'], **nonempty_args) diff --git a/tools/tests/test_windows_service.py b/tools/tests/test_windows_service.py index eada6066b6a..3b0f262560d 100644 --- a/tools/tests/test_windows_service.py +++ b/tools/tests/test_windows_service.py @@ -17,6 +17,8 @@ import time import threading import unittest +import ctypes +import copy from signal import SIGTERM @@ -87,9 +89,9 @@ def assertUserIsAdmin(): if os.name != 'nt': sys.exit(-1) - try: - os.listdir('\\Windows\\Temp') - except WindowsError: + + if not ctypes.windll.shell32.IsUserAnAdmin(): + print("User is not an Admin. Please run this script as an Administrator.") sys.exit(-1) @@ -102,10 +104,11 @@ def sc(*args): ['sc.exe'] + list(args), stderr=subprocess.PIPE, stdout=subprocess.PIPE) - except subprocess.CalledProcessError, err: + except subprocess.CalledProcessError as err: return (err.returncode, err.output) - out, _ = p.communicate() + stdout, _ = p.communicate() + out = stdout.decode() out = [x.strip() for x in out.split('\r\n') if x.strip() is not ''] if len(out) >= 1: @@ -125,15 +128,18 @@ def sc(*args): def findOsquerydBinary(): - script_root = os.path.split(os.path.abspath(__file__))[0] - build_root = os.path.abspath( - os.path.join(script_root, '..', '..', 'build', 'windows10', 'osquery')) - path = os.path.join(build_root, 'Release', 'osqueryd.exe') + path = os.path.abspath("%s/osquery/Release/osqueryd.exe" % (test_base.BUILD_DIR)) if os.path.exists(path): return path - path = os.path.join(build_root, 'RelWithDebInfo', 'osqueryd.exe') + + path = os.path.abspath("%s/osquery/RelWithDebInfo/osqueryd.exe" % (test_base.BUILD_DIR)) if os.path.exists(path): return path + + path = os.path.abspath("%s/osquery/osqueryd.exe" % (test_base.BUILD_DIR)) + if os.path.exists(path): + return path + sys.exit(-1) @@ -203,7 +209,7 @@ def killOsqueryProcesses(): # that matches our regex, stopping it, and then deleting the service def cleanOsqueryServices(): service_args = POWERSHELL_ARGS + ['$(Get-Service osqueryd_test_*).Name'] - services = subprocess.check_output(service_args).split() + services = subprocess.check_output(service_args).decode().split() # No services found on the system if len(services) == 0: @@ -217,6 +223,14 @@ def isServiceStopped(): test_base.expectTrue(isServiceStopped) uninstallService(service) +def prepareTlsServerArgs(): + tls_server_args = copy.deepcopy(TLS_SERVER_ARGS) + tls_server_args["ca"] = "%s/%s" % (test_base.TEST_CONFIGS_DIR, TLS_SERVER_ARGS["ca"]) + tls_server_args["key"] = "%s/%s" % (test_base.TEST_CONFIGS_DIR, TLS_SERVER_ARGS["key"]) + tls_server_args["cert"] = "%s/%s" % (test_base.TEST_CONFIGS_DIR, TLS_SERVER_ARGS["cert"]) + tls_server_args["enroll_secret"] = "%s/%s" % (test_base.TEST_CONFIGS_DIR, TLS_SERVER_ARGS["enroll_secret"]) + return tls_server_args + class OsquerydTest(unittest.TestCase): @@ -225,6 +239,7 @@ class OsquerydTest(unittest.TestCase): def setUp(self): # Ensure that no residual processes are alive before starting cleanOsqueryServices() + tls_server_args = prepareTlsServerArgs() self.test_instance = random.randint(0, 65535) self.tmp_dir = os.path.join(tempfile.gettempdir(), @@ -245,38 +260,39 @@ def setUp(self): self.flagfile = os.path.join(self.tmp_dir, 'osquery.flags') # Write out our mock configuration files - with open(self.config_path, 'wb') as fd: + with open(self.config_path, 'w') as fd: fd.write(CONFIG_FILE) - with open(self.flagfile, 'wb') as fd: + with open(self.flagfile, 'w') as fd: fd.write( FLAGS_FILE.format(self.log_path, self.pidfile, - test_http_server.HTTP_SERVER_CA, - test_http_server.HTTP_SERVER_ENROLL_SECRET)) + tls_server_args["ca"], + tls_server_args["enroll_secret"])) # Start the test TLS server to add more internal services self.http_server_ = threading.Thread( target=test_http_server.run_http_server, args=(443, ), - kwargs=TLS_SERVER_ARGS) + kwargs=tls_server_args) self.http_server_.daemon = True self.http_server_.start() def runDaemon(self, *args): try: - p = subprocess.Popen( + with subprocess.Popen( [self.bin_path] + list(args), stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE) as p: + + start = time.time() + while p.poll() is None: + if time.time() - start > 5: + p.kill() + break + time.sleep(1) - start = time.time() - while p.poll() is None: - if time.time() - start > 5: - p.kill() - break - time.sleep(1) + return (p.stdout.read(), p.stderr.read()) - return (p.stdout.read(), p.stderr.read()) except subprocess.CalledProcessError: return ('', '') @@ -302,7 +318,7 @@ def test_1_install_run_stop_uninstall_windows_service(self): '--database_path', self.database_path, '--logger_path', self.log_path, '--pidfile', self.pidfile) - self.assertNotEqual(stderr.find('is already running'), -1) + self.assertNotEqual(stderr.decode().find('is already running'), -1) if code == 0: code = stopService(name) From 354b4d666ace5ab5aee4dd81f4282dd0e80f9f6f Mon Sep 17 00:00:00 2001 From: SS Date: Wed, 23 Oct 2019 17:01:28 -0400 Subject: [PATCH 06/28] Some Python 2 to Python 3 updates - Fixed a ResourceWarning due to a child killed but not yet reaped. --- azure-pipelines.yml | 2 +- tools/tests/test_base.py | 2 ++ tools/tests/test_osqueryd.py | 4 ++-- tools/tests/test_osqueryi.py | 10 +++++----- tools/tests/test_release.py | 4 ++-- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index df65e73811e..89822add34c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -199,7 +199,7 @@ jobs: - script: | brew upgrade brew install ccache flex bison - pip3 install setuptools pexpect psutil timeout_decorator six thrift==0.11.0 osquery + pip3 install setuptools pexpect==3.3 psutil timeout_decorator six thrift==0.11.0 osquery sudo xcode-select -s /Applications/Xcode_10.3.app/Contents/Developer displayName: "Install Homebrew and prerequisites" timeoutInMinutes: 20 diff --git a/tools/tests/test_base.py b/tools/tests/test_base.py index 1dc2d986dad..7fb28741fab 100644 --- a/tools/tests/test_base.py +++ b/tools/tests/test_base.py @@ -157,6 +157,7 @@ def run_query(self, query): ''' query = query + ';' # Extra semicolon causes no harm result = self.run_command(query) + result = result.decode("utf-8") # On Mac, the query appears first in the string. Remove it if so. result = re.sub(re.escape(query), '', result).strip() result_lines = result.splitlines() @@ -284,6 +285,7 @@ def kill(self, children=False): if self.proc: try: os.kill(self.pid, sig) + self.proc.wait() # == -sig.value on posix except: pass self.proc = None diff --git a/tools/tests/test_osqueryd.py b/tools/tests/test_osqueryd.py index 76a7d209228..ea87421289a 100755 --- a/tools/tests/test_osqueryd.py +++ b/tools/tests/test_osqueryd.py @@ -150,7 +150,7 @@ def test_5_daemon_sigint(self): @test_base.flaky def test_6_logger_mode(self): logger_path = test_base.getTestDirectory(test_base.CONFIG_DIR) - test_mode = 0754 # Strange mode that should never exist + test_mode = 0o754 # Strange mode that should never exist daemon = self._run_daemon( { "disable_watchdog": True, @@ -187,7 +187,7 @@ def results_exists(): # TODO: Add ACL checks for Windows logs if pth.find('.log') > 0 and os.name != "nt": rpath = os.path.realpath(pth) - mode = os.stat(rpath).st_mode & 0777 + mode = os.stat(rpath).st_mode & 0o777 self.assertEqual(mode, test_mode) daemon.kill() diff --git a/tools/tests/test_osqueryi.py b/tools/tests/test_osqueryi.py index 395cac52b51..2e0e7efe27b 100755 --- a/tools/tests/test_osqueryi.py +++ b/tools/tests/test_osqueryi.py @@ -44,7 +44,7 @@ def test_config_check_success(self): "--verbose", ], SHELL_TIMEOUT) - self.assertEqual(proc.stdout, "") + self.assertEqual(proc.stdout, b"") print(proc.stdout) print(proc.stderr) self.assertEqual(proc.proc.poll(), 0) @@ -68,7 +68,7 @@ def test_config_dump(self): if os.name == "nt": actual = actual.replace('\r', '') - self.assertEqual(actual, '{"%s": %s}\n' % (config, content)) + self.assertEqual(actual.decode("utf-8"), '{"%s": %s}\n' % (config, content)) print (proc.stderr) self.assertEqual(proc.proc.poll(), 0) @@ -129,7 +129,7 @@ def test_config_check_example(self): "--verbose", ], SHELL_TIMEOUT) - self.assertEqual(proc.stdout, "") + self.assertEqual(proc.stdout, b"") print (proc.stdout) print (proc.stderr) self.assertEqual(proc.proc.poll(), 0) @@ -190,9 +190,9 @@ def test_json_output(self): SHELL_TIMEOUT ) if os.name == "nt": - self.assertEqual(proc.stdout, "[\r\n {\"0\":\"0\"}\r\n]\r\n") + self.assertEqual(proc.stdout, b"[\r\n {\"0\":\"0\"}\r\n]\r\n") else: - self.assertEqual(proc.stdout, "[\n {\"0\":\"0\"}\n]\n") + self.assertEqual(proc.stdout, b"[\n {\"0\":\"0\"}\n]\n") print(proc.stdout) print(proc.stderr) self.assertEqual(proc.proc.poll(), 0) diff --git a/tools/tests/test_release.py b/tools/tests/test_release.py index b956cfc5e5a..0f888ecb40c 100755 --- a/tools/tests/test_release.py +++ b/tools/tests/test_release.py @@ -29,12 +29,12 @@ class ReleaseTests(test_base.QueryTester): @test_base.flaky def test_pack_queries(self): packs = {} - PACKS_DIR = SOURCE_DIR + "/packs" + PACKS_DIR = test_base.BUILD_DIR + "/package/linux/packs" for root, dirs, files in os.walk(PACKS_DIR): for name in files: with open(os.path.join(PACKS_DIR, name), 'r') as fh: content = fh.read() - content = string.replace(content, "\\\n", "") + content = content.replace("\\\n", "") packs[name] = json.loads(content) for name, pack in packs.items(): if "queries" not in pack: From 140f60fc09ea395d9e3a42322d7f0ce00eb7ca2c Mon Sep 17 00:00:00 2001 From: scoders Date: Thu, 24 Oct 2019 13:00:16 -0400 Subject: [PATCH 07/28] Fix for magic table in test_example_queries.py Magic tables requires a 'where' clause: kolide/fleet#1040 --- specs/posix/magic.table | 3 +++ 1 file changed, 3 insertions(+) diff --git a/specs/posix/magic.table b/specs/posix/magic.table index 1cb30d27a9f..d02d0b01529 100644 --- a/specs/posix/magic.table +++ b/specs/posix/magic.table @@ -8,3 +8,6 @@ schema([ Column("mime_encoding", TEXT, "MIME encoding data from libmagic"), ]) implementation("system/magic@genMagicData") +examples([ + "select * from magic where path = '.'", +]) From f92fa0b92c36ac30728ac47790af3bc967e0e5ec Mon Sep 17 00:00:00 2001 From: Stefano Bonicatti Date: Thu, 24 Oct 2019 17:42:37 +0200 Subject: [PATCH 08/28] Fix osqueryi link on Windows --- osquery/CMakeLists.txt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/osquery/CMakeLists.txt b/osquery/CMakeLists.txt index 4ccb1995fed..b342a3a252e 100644 --- a/osquery/CMakeLists.txt +++ b/osquery/CMakeLists.txt @@ -65,14 +65,15 @@ function(generateOsqueryd) osquery_main ) - set(osqueryi_ext "") + set(osquery_ext "") if(PLATFORM_WINDOWS) - set(osqueryi_ext ".exe") + set(osquery_ext ".exe") endif() - add_custom_target(create_osqueryi ALL DEPENDS osqueryi${osqueryi_ext}) - add_custom_command(OUTPUT osqueryi${osqueryi_ext} - COMMAND ${CMAKE_COMMAND} -E create_symlink osqueryd osqueryi${osqueryi_ext} + add_custom_target(create_osqueryi ALL DEPENDS osqueryi${osquery_ext}) + add_custom_command(OUTPUT osqueryi${osquery_ext} + COMMAND "${CMAKE_COMMAND}" -E create_symlink osqueryd${osquery_ext} osqueryi${osquery_ext} + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" ) add_dependencies(create_osqueryi osqueryd) endfunction() From 0ee62be1d71969a572f747e625ea394b6f7ac0be Mon Sep 17 00:00:00 2001 From: Stefano Bonicatti Date: Thu, 24 Oct 2019 17:42:51 +0200 Subject: [PATCH 09/28] Uniform how osqueryd and osqueryi binaries are retrieved --- tools/tests/test_base.py | 46 +++++++++++++++++------------ tools/tests/test_windows_service.py | 18 +---------- 2 files changed, 28 insertions(+), 36 deletions(-) diff --git a/tools/tests/test_base.py b/tools/tests/test_base.py index 7fb28741fab..c1e9342c688 100644 --- a/tools/tests/test_base.py +++ b/tools/tests/test_base.py @@ -325,26 +325,34 @@ def isDead(self, pid, timeout=5): def getLatestOsqueryBinary(binary): - if os.name == "posix": - return os.path.join(ARGS.build, "osquery", binary) - - release_path = os.path.abspath( - os.path.join(ARGS.build, "osquery", "Release", "{}.exe".format(binary))) - relwithdebinfo_path = os.path.abspath( - os.path.join(ARGS.build, "osquery", "RelWithDebInfo", "{}.exe".format(binary))) - - if os.path.exists(release_path) and os.path.exists(relwithdebinfo_path): - if os.stat(release_path).st_mtime > os.stat( - relwithdebinfo_path).st_mtime: - return release_path - else: - return relwithdebinfo_path - elif os.path.exists(release_path): - return release_path - elif os.path.exists(relwithdebinfo_path): - return relwithdebinfo_path + + if os.name == "nt": + normal_release_path = os.path.abspath(os.path.join(BUILD_DIR, "osquery", "{}.exe".format(binary))) + + if os.path.exists(normal_release_path): + return normal_release_path + + msbuild_release_path = os.path.abspath( + os.path.join(BUILD_DIR, "osquery", "Release", "{}.exe".format(binary))) + msbuild_relwithdebinfo_path = os.path.abspath( + os.path.join(BUILD_DIR, "osquery", "RelWithDebInfo", "{}.exe".format(binary))) + + if os.path.exists(msbuild_release_path) and os.path.exists(msbuild_relwithdebinfo_path): + if os.stat(msbuild_release_path).st_mtime > os.stat( + msbuild_relwithdebinfo_path).st_mtime: + return msbuild_release_path + else: + return msbuild_relwithdebinfo_path + elif os.path.exists(msbuild_release_path): + return msbuild_release_path + elif os.path.exists(msbuild_relwithdebinfo_path): + return msbuild_relwithdebinfo_path else: - return None + normal_release_path = os.path.abspath(os.path.join(BUILD_DIR, "osquery", binary)) + if os.path.exists(normal_release_path): + return normal_release_path + + return None class ProcessGenerator(object): diff --git a/tools/tests/test_windows_service.py b/tools/tests/test_windows_service.py index 3b0f262560d..f382c5aaa4a 100644 --- a/tools/tests/test_windows_service.py +++ b/tools/tests/test_windows_service.py @@ -127,22 +127,6 @@ def sc(*args): return (-1, 'UNKNOWN') -def findOsquerydBinary(): - path = os.path.abspath("%s/osquery/Release/osqueryd.exe" % (test_base.BUILD_DIR)) - if os.path.exists(path): - return path - - path = os.path.abspath("%s/osquery/RelWithDebInfo/osqueryd.exe" % (test_base.BUILD_DIR)) - if os.path.exists(path): - return path - - path = os.path.abspath("%s/osquery/osqueryd.exe" % (test_base.BUILD_DIR)) - if os.path.exists(path): - return path - - sys.exit(-1) - - def installService(name, path): return sc('create', name, 'binPath=', path) @@ -245,7 +229,7 @@ def setUp(self): self.tmp_dir = os.path.join(tempfile.gettempdir(), 'osquery-test-python-{}'.format( self.test_instance)) - self.bin_path = findOsquerydBinary() + self.bin_path = test_base.getLatestOsqueryBinary("osqueryd") if os.path.exists(self.tmp_dir): shutil.rmtree(self.tmp_dir) From 2b0b4a7e8835e6fdde527d952700746438669434 Mon Sep 17 00:00:00 2001 From: Stefano Bonicatti Date: Thu, 24 Oct 2019 17:43:08 +0200 Subject: [PATCH 10/28] Set the osquery current working directory to the test configs dir --- tools/tests/test_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/tests/test_base.py b/tools/tests/test_base.py index c1e9342c688..1523b435081 100644 --- a/tools/tests/test_base.py +++ b/tools/tests/test_base.py @@ -139,9 +139,9 @@ def __init__(self, command='../osqueryi', args={}, env={}): command = command + " " + " ".join( ["--%s=%s" % (k, v) for k, v in options.items()]) if os.name == "nt": - proc = WinExpectSpawn(command, env=env) + proc = WinExpectSpawn(command, env=env, cwd=TEST_CONFIGS_DIR) else: - proc = pexpect.spawn(command, env=env) + proc = pexpect.spawn(command, env=env, cwd=TEST_CONFIGS_DIR) super().__init__( proc, From 78df8a1d660af8b270b5ac1163625a0480762212 Mon Sep 17 00:00:00 2001 From: Stefano Bonicatti Date: Thu, 24 Oct 2019 21:47:16 +0200 Subject: [PATCH 11/28] Other Python 2 to Python 3 fixes for test_osqueryi.py --- tools/tests/CMakeLists.txt | 1 + tools/tests/test_osqueryi.py | 28 ++++++++++++++-------------- tools/tests/winexpect.py | 8 +++++--- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/tools/tests/CMakeLists.txt b/tools/tests/CMakeLists.txt index f0a1f74dfb9..6deeaee605f 100644 --- a/tools/tests/CMakeLists.txt +++ b/tools/tests/CMakeLists.txt @@ -69,6 +69,7 @@ function(generatePythonTests) addPythonTest(NAME tools_tests_testosqueryd SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_osqueryd.py") addPythonTest(NAME tools_tests_testosqueryi SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_osqueryi.py") + set_tests_properties(tools_tests_testosqueryi PROPERTIES TIMEOUT 120) addPythonTest(NAME tools_tests_testrelease SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_release.py") diff --git a/tools/tests/test_osqueryi.py b/tools/tests/test_osqueryi.py index 2e0e7efe27b..5383fa85eae 100755 --- a/tools/tests/test_osqueryi.py +++ b/tools/tests/test_osqueryi.py @@ -45,8 +45,8 @@ def test_config_check_success(self): ], SHELL_TIMEOUT) self.assertEqual(proc.stdout, b"") - print(proc.stdout) - print(proc.stderr) + print(proc.stdout.decode()) + print(proc.stderr.decode()) self.assertEqual(proc.proc.poll(), 0) def test_config_dump(self): @@ -63,13 +63,13 @@ def test_config_dump(self): content = "" with open(config, 'r') as fh: content = fh.read() - actual = proc.stdout + actual = proc.stdout.decode() if os.name == "nt": actual = actual.replace('\r', '') - self.assertEqual(actual.decode("utf-8"), '{"%s": %s}\n' % (config, content)) - print (proc.stderr) + self.assertEqual(actual, '{"%s": %s}\n' % (config, content)) + print (proc.stderr.decode()) self.assertEqual(proc.proc.poll(), 0) @test_base.flaky @@ -85,8 +85,8 @@ def test_config_check_failure_invalid_path(self): ], SHELL_TIMEOUT) self.assertNotEqual(proc.stderr, "") - print(proc.stdout) - print(proc.stderr) + print(proc.stdout.decode()) + print(proc.stderr.decode()) self.assertEqual(proc.proc.poll(), 1) def test_config_check_failure_valid_path(self): @@ -101,7 +101,7 @@ def test_config_check_failure_valid_path(self): ], SHELL_TIMEOUT) self.assertEqual(proc.proc.poll(), 1) - self.assertNotEqual(proc.stderr, "") + self.assertNotEqual(proc.stderr, b"") def test_config_check_failure_missing_plugin(self): # Finally with a missing config plugin @@ -114,7 +114,7 @@ def test_config_check_failure_missing_plugin(self): "--config_plugin=does_not_exist" ], SHELL_TIMEOUT) - self.assertNotEqual(proc.stderr, "") + self.assertNotEqual(proc.stderr, b"") self.assertNotEqual(proc.proc.poll(), 0) # Also do not accept a SIGSEG self.assertEqual(proc.proc.poll(), EXIT_CATASTROPHIC) @@ -130,8 +130,8 @@ def test_config_check_example(self): ], SHELL_TIMEOUT) self.assertEqual(proc.stdout, b"") - print (proc.stdout) - print (proc.stderr) + print (proc.stdout.decode()) + print (proc.stderr.decode()) self.assertEqual(proc.proc.poll(), 0) def test_meta_commands(self): @@ -193,8 +193,8 @@ def test_json_output(self): self.assertEqual(proc.stdout, b"[\r\n {\"0\":\"0\"}\r\n]\r\n") else: self.assertEqual(proc.stdout, b"[\n {\"0\":\"0\"}\n]\n") - print(proc.stdout) - print(proc.stderr) + print(proc.stdout.decode()) + print(proc.stderr.decode()) self.assertEqual(proc.proc.poll(), 0) @test_base.flaky @@ -231,7 +231,7 @@ def test_foreign_tables(self): @test_base.flaky def test_time_using_all(self): self.osqueryi.run_command(' ') - result = self.osqueryi.run_command('.all time') + result = self.osqueryi.run_command('.all time').decode() self.assertNotEqual(result.rstrip(), "Error querying table: time") @test_base.flaky diff --git a/tools/tests/winexpect.py b/tools/tests/winexpect.py index 518c103fc90..50e9e9df8f0 100644 --- a/tools/tests/winexpect.py +++ b/tools/tests/winexpect.py @@ -46,7 +46,9 @@ def run_command(self, command): if not command: return res try: - self.child.proc.stdin.write(command + '\r\n') + command = command + '\r\n' + print(command) + self.child.proc.stdin.write(command.encode()) self.child.proc.stdin.flush() # Wait for stderr/stdout to populate for at most timeout seconds @@ -55,12 +57,12 @@ def run_command(self, command): break time.sleep(1) while not self.child.out_queue.empty(): - l = self.child.out_queue.get_nowait() + l = self.child.out_queue.get_nowait().decode() res += l except Exception as e: print('[-] Failed to communicate with client: {}'.format(e)) - return res + return res.encode() class WinExpectSpawn(object): From 4abd08370e98c62cce5637d3bde38a8ecd848537 Mon Sep 17 00:00:00 2001 From: Stefano Bonicatti Date: Thu, 24 Oct 2019 22:52:13 +0200 Subject: [PATCH 12/28] Increase test timeout to test_windows_service.py --- tools/tests/CMakeLists.txt | 1 + tools/tests/test_windows_service.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/tests/CMakeLists.txt b/tools/tests/CMakeLists.txt index 6deeaee605f..5e2a8fb0926 100644 --- a/tools/tests/CMakeLists.txt +++ b/tools/tests/CMakeLists.txt @@ -75,6 +75,7 @@ function(generatePythonTests) if(DEFINED PLATFORM_WINDOWS) addPythonTest(NAME tools_tests_testwindowsservice SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_windows_service.py") + set_tests_properties(tools_tests_testwindowsservice PROPERTIES TIMEOUT 120) else() addPythonTest(NAME tools_tests_testadditional SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_additional.py") diff --git a/tools/tests/test_windows_service.py b/tools/tests/test_windows_service.py index f382c5aaa4a..43bec4c2924 100644 --- a/tools/tests/test_windows_service.py +++ b/tools/tests/test_windows_service.py @@ -308,7 +308,6 @@ def test_1_install_run_stop_uninstall_windows_service(self): code = stopService(name) self.assertEqual(code, 0) - test_base.expectTrue(serviceDead) self.assertTrue(serviceDead()) _, output = queryService(name) From 99b8c3375f45ce35f208eafaa0718c7edbeef9b5 Mon Sep 17 00:00:00 2001 From: Stefano Bonicatti Date: Fri, 25 Oct 2019 00:20:33 +0200 Subject: [PATCH 13/28] Fix again osqueryi link --- osquery/CMakeLists.txt | 2 +- tools/tests/winexpect.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/osquery/CMakeLists.txt b/osquery/CMakeLists.txt index b342a3a252e..4d6ec34aea9 100644 --- a/osquery/CMakeLists.txt +++ b/osquery/CMakeLists.txt @@ -73,7 +73,7 @@ function(generateOsqueryd) add_custom_target(create_osqueryi ALL DEPENDS osqueryi${osquery_ext}) add_custom_command(OUTPUT osqueryi${osquery_ext} COMMAND "${CMAKE_COMMAND}" -E create_symlink osqueryd${osquery_ext} osqueryi${osquery_ext} - WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + WORKING_DIRECTORY "$" ) add_dependencies(create_osqueryi osqueryd) endfunction() diff --git a/tools/tests/winexpect.py b/tools/tests/winexpect.py index 50e9e9df8f0..d07aba8467b 100644 --- a/tools/tests/winexpect.py +++ b/tools/tests/winexpect.py @@ -47,7 +47,6 @@ def run_command(self, command): return res try: command = command + '\r\n' - print(command) self.child.proc.stdin.write(command.encode()) self.child.proc.stdin.flush() From 2f74414e96f2106a238a1daee7cd8be62ed5791b Mon Sep 17 00:00:00 2001 From: Stefano Bonicatti Date: Fri, 25 Oct 2019 14:08:48 +0200 Subject: [PATCH 14/28] Fix QueryTester not always getting the correct osqueryi binary --- tools/tests/test_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/tests/test_base.py b/tools/tests/test_base.py index 1523b435081..94245f52806 100644 --- a/tools/tests/test_base.py +++ b/tools/tests/test_base.py @@ -627,7 +627,7 @@ def expect(functional, expected, interval=0.01, timeout=4): class QueryTester(ProcessGenerator, unittest.TestCase): def setUp(self): - self.binary = os.path.join(ARGS.build, "osquery", "osqueryi") + self.binary = getLatestOsqueryBinary("osqueryi") self.daemon = self._run_daemon({ # The set of queries will hammer the daemon process. "disable_watchdog": True, From fc07d452aad35d74926ac698c333cb9c4b59968c Mon Sep 17 00:00:00 2001 From: Stefano Bonicatti Date: Fri, 25 Oct 2019 15:48:11 +0200 Subject: [PATCH 15/28] Change the retry system for expectTrue, try_open and open The timeout concept was confusing, because it wasn't really waiting for that amount of time and then interrupting the tentatives. It was just trying timeout/interval times which by default, in the expectTrue case, was 800 times, which is a lot and compounds to a lot if the function tested did some tentatives too. timeout is now called attemps, and the default values have been changed to something a bit saner. --- tools/tests/test_additional.py | 2 +- tools/tests/test_base.py | 17 ++++++++--------- tools/tests/test_release.py | 2 +- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/tools/tests/test_additional.py b/tools/tests/test_additional.py index 8c1273e35e6..a44079b40a0 100755 --- a/tools/tests/test_additional.py +++ b/tools/tests/test_additional.py @@ -49,7 +49,7 @@ def test_query_packs(self): # Introspect into the daemon's query packs. client = test_base.EXClient(daemon.options["extensions_socket"]) - test_base.expectTrue(client.try_open) + test_base.expectTrue(client.try_open, attempts=2, interval=5) self.assertTrue(client.open()) em = client.getEM() diff --git a/tools/tests/test_base.py b/tools/tests/test_base.py index 94245f52806..2e430f5fdc3 100644 --- a/tools/tests/test_base.py +++ b/tools/tests/test_base.py @@ -458,23 +458,23 @@ def close(self): if self.transport: self.transport.close() - def try_open(self, timeout=0.1, interval=0.01): + def try_open(self, attempts=10, interval=0.5): '''Try to open, on success, close the UNIX domain socket.''' - did_open = self.open(timeout, interval) + did_open = self.open(attempts, interval) if did_open: self.close() return did_open - def open(self, timeout=0.1, interval=0.01): + def open(self, attempts=10, interval=0.5): '''Attempt to open the UNIX domain socket.''' delay = 0 - while delay < timeout: + for i in range(0, attempts): try: self.transport.open() return True except Exception as e: pass - delay += interval + time.sleep(interval) return False @@ -640,7 +640,7 @@ def setUp(self): # The sets of example tests will use the extensions APIs. self.client = EXClient(self.daemon.options["extensions_socket"]) - expectTrue(self.client.try_open) + expectTrue(self.client.try_open, attempts=2, interval=5) self.assertTrue(self.client.open()) self.em = self.client.getEM() @@ -687,14 +687,13 @@ def __exit__(self, type, value, traceback): pass -def expectTrue(functional, interval=0.01, timeout=8): +def expectTrue(functional, interval=1, attempts=10): """Helper function to run a function with expected latency""" delay = 0 - while delay < timeout: + for i in range(0, attempts): if functional(): return True time.sleep(interval) - delay += interval return False diff --git a/tools/tests/test_release.py b/tools/tests/test_release.py index 0f888ecb40c..f41b815411e 100755 --- a/tools/tests/test_release.py +++ b/tools/tests/test_release.py @@ -71,5 +71,5 @@ def test_no_local_link(self): if __name__ == '__main__': module = test_base.Tester() # Find and import the thrift-generated python interface - test_base.loadThriftFromBuild(test_base.ARGS.build) + test_base.loadThriftFromBuild(test_base.BUILD_DIR) module.run() From a8040cd3eb3daa1d424d5999785617ed8a0c639a Mon Sep 17 00:00:00 2001 From: Stefano Bonicatti Date: Fri, 25 Oct 2019 15:53:37 +0200 Subject: [PATCH 16/28] test_release.py should not run on Windows --- tools/tests/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/tests/CMakeLists.txt b/tools/tests/CMakeLists.txt index 5e2a8fb0926..c452a7d0ab2 100644 --- a/tools/tests/CMakeLists.txt +++ b/tools/tests/CMakeLists.txt @@ -71,12 +71,12 @@ function(generatePythonTests) addPythonTest(NAME tools_tests_testosqueryi SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_osqueryi.py") set_tests_properties(tools_tests_testosqueryi PROPERTIES TIMEOUT 120) - addPythonTest(NAME tools_tests_testrelease SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_release.py") - if(DEFINED PLATFORM_WINDOWS) addPythonTest(NAME tools_tests_testwindowsservice SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_windows_service.py") set_tests_properties(tools_tests_testwindowsservice PROPERTIES TIMEOUT 120) else() + addPythonTest(NAME tools_tests_testrelease SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_release.py") + addPythonTest(NAME tools_tests_testadditional SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_additional.py") addPythonTest(NAME tools_tests_testexamplequeries SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_example_queries.py") From cc05e92742f642ae2f6f02ad41b10a44eefd15cc Mon Sep 17 00:00:00 2001 From: Stefano Bonicatti Date: Fri, 25 Oct 2019 17:06:16 +0200 Subject: [PATCH 17/28] Increase default timeout to 60 seconds for all python tests --- tools/tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/tests/CMakeLists.txt b/tools/tests/CMakeLists.txt index c452a7d0ab2..3bc839e0e63 100644 --- a/tools/tests/CMakeLists.txt +++ b/tools/tests/CMakeLists.txt @@ -33,7 +33,7 @@ function(addPythonTest) COMMAND ${python_test_command} WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") - set_tests_properties(${osquery_test_NAME} PROPERTIES TIMEOUT 30) + set_tests_properties(${osquery_test_NAME} PROPERTIES TIMEOUT 60) endfunction() function(preparePythonTestsEnvironment) From 9238a73bece6d517ed8e9d4c5a344a8d620c3586 Mon Sep 17 00:00:00 2001 From: Stefano Bonicatti Date: Fri, 25 Oct 2019 18:23:42 +0200 Subject: [PATCH 18/28] Increase interval between service alive/dead checks --- tools/tests/test_windows_service.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tools/tests/test_windows_service.py b/tools/tests/test_windows_service.py index 43bec4c2924..f9f16bbc3ff 100644 --- a/tools/tests/test_windows_service.py +++ b/tools/tests/test_windows_service.py @@ -166,13 +166,13 @@ def serviceStarted(service_name): def startService(name, *argv): start_ = sc('start', name, *argv) - test_base.expectTrue(serviceAlive) + test_base.expectTrue(serviceAlive, interval=2) return start_[0] def stopService(name): stop_ = sc('stop', name) - test_base.expectTrue(serviceDead) + test_base.expectTrue(serviceDead, interval=2) return stop_[0] @@ -291,7 +291,7 @@ def test_1_install_run_stop_uninstall_windows_service(self): self.assertEqual(code, 0) # Ensure the service is online before proceeding - test_base.expectTrue(serviceAlive) + self.assertTrue(serviceAlive()) _, output = queryService(name) self.assertEqual(output, '4RUNNING') @@ -331,19 +331,15 @@ def test_2_thrash_windows_service(self): code = startService(name, '--flagfile', self.flagfile) self.assertEqual(code, 0) - - test_base.expectTrue(serviceAlive) self.assertTrue(serviceAlive()) for _ in range(5): status = restartService(name, '--flagfile', self.flagfile) self.assertTrue(status) - test_base.expectTrue(serviceAlive) self.assertTrue(serviceAlive()) code = stopService(name) self.assertEqual(code, 0) - test_base.expectTrue(serviceDead) self.assertTrue(serviceDead()) _, output = queryService(name) From fddc23b2c7c505d84476e8f5c708ded3b1d8d118 Mon Sep 17 00:00:00 2001 From: Stefano Bonicatti Date: Sat, 26 Oct 2019 17:27:02 +0200 Subject: [PATCH 19/28] Fix psutil.NoSuchProcess error psutil caches the results, so when accessing a previously found process, it might hand a dead process handle, so accessing it throws an exception. Handle the exception and ignore the process. --- tools/tests/test_windows_service.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tools/tests/test_windows_service.py b/tools/tests/test_windows_service.py index f9f16bbc3ff..e4086bd8e96 100644 --- a/tools/tests/test_windows_service.py +++ b/tools/tests/test_windows_service.py @@ -141,10 +141,15 @@ def queryService(name): def getOsqueryProcs(): - return [ - p.pid for p in psutil.process_iter() if p.name() == 'osqueryd.exe' - ] + processes_pid = [] + for p in psutil.process_iter(): + try: + if p.name() == 'osqueryd.exe': + processes_pid.append(p.pid) + except (psutil.NoSuchProcess, psutil.ZombieProcess): + pass + return processes_pid def serviceAlive(): return len(getOsqueryProcs()) == 2 From 2e3f2e72a6fc5934845aaf7e963e141e705ccc74 Mon Sep 17 00:00:00 2001 From: Stefano Bonicatti Date: Sat, 26 Oct 2019 21:11:01 +0200 Subject: [PATCH 20/28] Rework ProcRunner functions retries --- tools/tests/test_base.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/tools/tests/test_base.py b/tools/tests/test_base.py index 2e430f5fdc3..b0b7840aabf 100644 --- a/tools/tests/test_base.py +++ b/tools/tests/test_base.py @@ -198,7 +198,7 @@ class ProcRunner(object): this class wrapper. ''' - def __init__(self, name, path, _args=[], interval=0.02, silent=False): + def __init__(self, name, path, _args=[], interval=1, silent=False): self.started = False self.proc = None self.name = name @@ -230,34 +230,32 @@ def run(self): try: while self.proc.poll() is None: self.started = True - time.sleep(self.interval) + time.sleep(0.1) self.started = True self.retcode = -1 if self.proc is None else self.proc.poll() self.proc = None except Exception as e: return - def requireStarted(self, timeout=2): + def requireStarted(self, attempts=5): delay = 0 - while delay < timeout: + for i in range(attempts): if self.started is True: break - time.sleep(self.interval * 10) - delay += self.interval * 10 + time.sleep(self.interval) - def getChildren(self, timeout=1): + def getChildren(self, attempts=5): '''Get the child pids.''' self.requireStarted() if not self.proc: return [] try: proc = psutil.Process(pid=self.proc.pid) - delay = 0 + attempt = 0 while len(proc.children()) == 0: - if delay > timeout: + if attempt > attempts: return [] time.sleep(self.interval) - delay += self.interval return [p.pid for p in proc.children()] except: pass @@ -290,20 +288,20 @@ def kill(self, children=False): pass self.proc = None - def isAlive(self, timeout=3): + def isAlive(self, attempts=10): self.requireStarted() '''Check if the process is alive.''' - delay = 0 + attempt = 0 while self.proc is None: - if delay > timeout: + if attempt > attempts: break time.sleep(self.interval) - delay += self.interval + attempt += 1 if self.proc is None: return False return self.proc.poll() is None - def isDead(self, pid, timeout=5): + def isDead(self, pid, attempts=10): self.requireStarted() '''Check if the process was killed. @@ -316,11 +314,11 @@ def isDead(self, pid, timeout=5): except psutil.NoSuchProcess as e: return True delay = 0 - while delay < timeout: + for i in range(attempts): if not proc.is_running(): return True time.sleep(self.interval) - delay += self.interval + return False From 14a94bffbd54301ec3b8303c2c3473da2049c7cb Mon Sep 17 00:00:00 2001 From: Stefano Bonicatti Date: Sun, 27 Oct 2019 16:31:06 +0100 Subject: [PATCH 21/28] Run test_release.py only on Release builds --- tools/tests/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/tests/CMakeLists.txt b/tools/tests/CMakeLists.txt index 3bc839e0e63..e89b3f20fc6 100644 --- a/tools/tests/CMakeLists.txt +++ b/tools/tests/CMakeLists.txt @@ -75,7 +75,9 @@ function(generatePythonTests) addPythonTest(NAME tools_tests_testwindowsservice SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_windows_service.py") set_tests_properties(tools_tests_testwindowsservice PROPERTIES TIMEOUT 120) else() - addPythonTest(NAME tools_tests_testrelease SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_release.py") + if("${CMAKE_BUILD_TYPE}" STREQUAL "Release") + addPythonTest(NAME tools_tests_testrelease SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_release.py") + endif() addPythonTest(NAME tools_tests_testadditional SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_additional.py") From b6c3d79c5e00a6d39567deabffb3fbe0296d7377 Mon Sep 17 00:00:00 2001 From: Stefano Bonicatti Date: Sun, 27 Oct 2019 23:21:37 +0100 Subject: [PATCH 22/28] Increase timeout of test_examplequeries.py --- tools/tests/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/tests/CMakeLists.txt b/tools/tests/CMakeLists.txt index e89b3f20fc6..e8fe0a47fa0 100644 --- a/tools/tests/CMakeLists.txt +++ b/tools/tests/CMakeLists.txt @@ -82,6 +82,7 @@ function(generatePythonTests) addPythonTest(NAME tools_tests_testadditional SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_additional.py") addPythonTest(NAME tools_tests_testexamplequeries SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_example_queries.py") + set_tests_properties(tools_tests_testexamplequeries PROPERTIES TIMEOUT 120) # The following tests need to be restored when extensions work again addPythonTest(NAME tools_tests_testextensions SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_extensions.py") From 3d5194131104fdd9199a75e834d994a92e310223 Mon Sep 17 00:00:00 2001 From: Stefano Bonicatti Date: Mon, 28 Oct 2019 23:40:27 +0100 Subject: [PATCH 23/28] CMake: Fix generateCopyFileTarget not copying files when they change --- cmake/utilities.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/utilities.cmake b/cmake/utilities.cmake index c9ceb54713e..37f30825e05 100644 --- a/cmake/utilities.cmake +++ b/cmake/utilities.cmake @@ -176,6 +176,7 @@ function(generateCopyFileTarget name base_path type relative_file_paths destinat add_custom_command( OUTPUT "${destination}/${file}" COMMAND "${CMAKE_COMMAND}" -E copy "${base_path}${file}" "${destination}/${file}" + DEPENDS "${base_path}${file}" ) list(APPEND copied_files "${destination}/${file}") endforeach() From a9b0d21264494c56b4900068a070a4ef95e94666 Mon Sep 17 00:00:00 2001 From: Stefano Bonicatti Date: Mon, 28 Oct 2019 23:43:58 +0100 Subject: [PATCH 24/28] Add example queries to extended_attributes table spec --- specs/darwin/extended_attributes.table | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/specs/darwin/extended_attributes.table b/specs/darwin/extended_attributes.table index 00b2a6712b2..470de068d95 100644 --- a/specs/darwin/extended_attributes.table +++ b/specs/darwin/extended_attributes.table @@ -8,3 +8,7 @@ schema([ Column("base64", INTEGER, "1 if the value is base64 encoded else 0"), ]) implementation("extended_attributes@genXattr") +examples([ + "select * from extended_attributes where path = '/usr'", + "select * from extended_attributes where directory = '/'" +]) From e70d1e9234d23efb966ab8adb3db41722b029758 Mon Sep 17 00:00:00 2001 From: Stefano Bonicatti Date: Wed, 30 Oct 2019 18:28:49 +0100 Subject: [PATCH 25/28] Update doc --- docs/wiki/development/building.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/wiki/development/building.md b/docs/wiki/development/building.md index be50216b89b..87bcdd8939c 100644 --- a/docs/wiki/development/building.md +++ b/docs/wiki/development/building.md @@ -25,8 +25,8 @@ The root folder is assumed to be `/home/` sudo apt install --no-install-recommends git python python3 bison flex make # Optional: install python tests prerequisites -sudo apt install --no-install-recommends python-pip python-setuptools python-psutil python-pexpect python-six -pip install timeout-decorator +sudo apt install --no-install-recommends python3-pip python3-setuptools python3-psutil python3-six python3-wheel +pip3 install timeout_decorator thrift==0.11.0 osquery pexpect==3.3 # Download and install the osquery toolchain wget https://github.com/osquery/osquery-toolchain/releases/download/1.0.0/osquery-toolchain-1.0.0.tar.xz @@ -61,7 +61,7 @@ xcode-select --install brew install git cmake python@2 python # Optional: install python tests prerequisites -pip install setuptools pexpect psutil timeout_decorator six +pip3 install setuptools pexpect==3.3 psutil timeout_decorator six thrift==0.11.0 osquery ``` **Step 2: Download and build** @@ -101,11 +101,11 @@ Note: It may be easier to install these prerequisites using [Chocolatey](https:/ - [7-Zip](https://www.7-zip.org/) if building the Chocolatey package. **Optional: Install python tests prerequisites** -Python 2 is assumed to be installed in `C:\Python27` +Python 3 is assumed to be installed in `C:\Program Files\Python37` ```PowerShell # Using a PowerShell console -C:\Python27\python.exe -m pip install setuptools psutil timeout_decorator six +& 'C:\Program Files\Python37\python.exe' -m pip install setuptools psutil timeout_decorator thrift==0.11.0 osquery pywin32 ``` **Step 2: Download and build** From c646ffa38b67f5f7870e5590b56938909b49324f Mon Sep 17 00:00:00 2001 From: SS Date: Thu, 31 Oct 2019 17:50:56 +0100 Subject: [PATCH 26/28] Couple of fixes for test_base.py - attempt variable has to be increased each loop, otherwise we end in an infinite loop. - Corrected example_extension.ext path. --- tools/tests/test_base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/tests/test_base.py b/tools/tests/test_base.py index b0b7840aabf..0ffcb5fe00c 100644 --- a/tools/tests/test_base.py +++ b/tools/tests/test_base.py @@ -255,6 +255,7 @@ def getChildren(self, attempts=5): while len(proc.children()) == 0: if attempt > attempts: return [] + attempt += 1 time.sleep(self.interval) return [p.pid for p in proc.children()] except: @@ -388,11 +389,11 @@ def _run_daemon(self, def _run_extension(self, timeout=0, path=None, silent=False): '''Spawn an osquery extension (example_extension)''' - global ARGS, CONFIG + global CONFIG, BUILD_DIR config = copy.deepcopy(CONFIG) config["options"]["extensions_socket"] += str( random.randint(1000, 9999)) - binary = os.path.join(ARGS.build, "osquery", "example_extension.ext") + binary = os.path.join(BUILD_DIR, "osquery", "examples", "example_extension.ext") if path is not None: config["options"]["extensions_socket"] = path extension = ProcRunner( From 10992bf08f7b584c62b0fed963acf3c62091b5e2 Mon Sep 17 00:00:00 2001 From: scoders-tob <55897633+scoders-tob@users.noreply.github.com> Date: Mon, 4 Nov 2019 13:25:08 -0500 Subject: [PATCH 27/28] enable example extension + tests --- osquery/CMakeLists.txt | 1 + osquery/examples/CMakeLists.txt | 11 +++++++ osquery/examples/example_extension.cpp | 11 ++++--- tools/tests/CMakeLists.txt | 2 +- tools/tests/test_extensions.py | 42 +++++++++++++------------- 5 files changed, 40 insertions(+), 27 deletions(-) create mode 100644 osquery/examples/CMakeLists.txt diff --git a/osquery/CMakeLists.txt b/osquery/CMakeLists.txt index 4d6ec34aea9..0c19a8e0c62 100644 --- a/osquery/CMakeLists.txt +++ b/osquery/CMakeLists.txt @@ -30,6 +30,7 @@ function(osqueryMain) add_subdirectory("ev2") add_subdirectory("experimental") add_subdirectory("system") + add_subdirectory("examples") generateOsqueryHeaders() generateOsqueryd() diff --git a/osquery/examples/CMakeLists.txt b/osquery/examples/CMakeLists.txt new file mode 100644 index 00000000000..a181b319be5 --- /dev/null +++ b/osquery/examples/CMakeLists.txt @@ -0,0 +1,11 @@ +include(${CMAKE_SOURCE_DIR}/external/cmake/cmakelibs.cmake) + +function(osqueryExtensionsExampleMain) + addOsqueryExtension(example_extension example_extension.cpp) + target_link_libraries(example_extension PRIVATE + osquery_sdk_pluginsdk + osquery_extensions_implthrift + ) +endfunction() + +osqueryExtensionsExampleMain() diff --git a/osquery/examples/example_extension.cpp b/osquery/examples/example_extension.cpp index 0737e9b6aba..a2613fc6875 100644 --- a/osquery/examples/example_extension.cpp +++ b/osquery/examples/example_extension.cpp @@ -6,7 +6,8 @@ * the LICENSE file found in the root directory of this source tree. */ -#include +#include +#include #include using namespace osquery; @@ -72,10 +73,10 @@ class ComplexExampleTable : public TablePlugin { auto r = make_table_row(); // Use the basic 'force' flag to check implicit SQL usage. - auto flags = - SQL("select default_value from osquery_flags where name = 'force'"); - if (flags.rows().size() > 0) { - r["flag_test"] = flags.rows().back().at("default_value"); + auto flags = SQL::selectFrom( + {"default_value"}, "osquery_flags", "name", EQUALS, "force"); + if (flags.size() > 0) { + r["flag_test"] = flags[0]["default_value"]; } std::string content; diff --git a/tools/tests/CMakeLists.txt b/tools/tests/CMakeLists.txt index e8fe0a47fa0..56c2d327490 100644 --- a/tools/tests/CMakeLists.txt +++ b/tools/tests/CMakeLists.txt @@ -86,7 +86,7 @@ function(generatePythonTests) # The following tests need to be restored when extensions work again addPythonTest(NAME tools_tests_testextensions SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/test_extensions.py") - set_tests_properties(tools_tests_testextensions PROPERTIES DISABLED TRUE) + set_tests_properties(tools_tests_testextensions PROPERTIES TIMEOUT 120) endif() endfunction() diff --git a/tools/tests/test_extensions.py b/tools/tests/test_extensions.py index 1531da3e9f8..e8f93d92b07 100755 --- a/tools/tests/test_extensions.py +++ b/tools/tests/test_extensions.py @@ -46,7 +46,7 @@ def test_2_daemon_api(self): # Get a python-based thrift client client = test_base.EXClient(daemon.options["extensions_socket"]) - self.assertTrue(client.open(timeout=EXTENSION_TIMEOUT)) + self.assertTrue(client.open()) em = client.getEM() # List the number of extensions @@ -78,7 +78,7 @@ def test_3_example_extension(self): # Get a python-based thrift client client = test_base.EXClient(daemon.options["extensions_socket"]) - self.assertTrue(client.open(timeout=EXTENSION_TIMEOUT)) + self.assertTrue(client.open()) em = client.getEM() # Make sure there are no extensions registered @@ -95,7 +95,7 @@ def test_3_example_extension(self): # Now that an extension has started, check extension list result = test_base.expect(em.extensions, 1) self.assertEqual(len(result), 1) - ex_uuid = result.keys()[0] + ex_uuid = list(result.keys())[0] ex_data = result[ex_uuid] self.assertEqual(ex_data.name, "example") self.assertEqual(ex_data.version, "0.0.1") @@ -104,7 +104,7 @@ def test_3_example_extension(self): # Get a python-based thrift client to the extension's service client2 = test_base.EXClient(daemon.options["extensions_socket"], uuid=ex_uuid) - self.assertTrue(client2.open(timeout=EXTENSION_TIMEOUT)) + self.assertTrue(client2.open()) ex = client2.getEX() self.assertEqual(ex.ping().code, 0) @@ -147,7 +147,7 @@ def test_4_extension_dies(self): # Get a python-based thrift client client = test_base.EXClient(daemon.options["extensions_socket"]) - self.assertTrue(client.open(timeout=EXTENSION_TIMEOUT)) + self.assertTrue(client.open()) em = client.getEM() # Make sure there are no extensions registered @@ -207,7 +207,7 @@ def test_5_extension_timeout(self): # Get a python-based thrift client client = test_base.EXClient(extension.options["extensions_socket"]) test_base.expectTrue(client.try_open) - self.assertTrue(client.open(timeout=EXTENSION_TIMEOUT)) + self.assertTrue(client.open()) em = client.getEM() # The waiting extension should have connected to the daemon. @@ -215,13 +215,13 @@ def test_5_extension_timeout(self): self.assertEqual(len(result), 1) client.close() - daemon.kill(True) + daemon.kill() extension.kill() @test_base.flaky def test_6_extensions_autoload(self): loader = test_base.Autoloader( - [test_base.ARGS.build + "/osquery/example_extension.ext"]) + [test_base.BUILD_DIR + "/osquery/examples/example_extension.ext"]) daemon = self._run_daemon({ "disable_watchdog": True, "extensions_timeout": EXTENSION_TIMEOUT, @@ -231,7 +231,7 @@ def test_6_extensions_autoload(self): # Get a python-based thrift client client = test_base.EXClient(daemon.options["extensions_socket"]) - self.assertTrue(client.open(timeout=EXTENSION_TIMEOUT)) + self.assertTrue(client.open()) em = client.getEM() # The waiting extension should have connected to the daemon. @@ -243,7 +243,7 @@ def test_6_extensions_autoload(self): @test_base.flaky def test_6_extensions_directory_autoload(self): - utils.copy_file(test_base.ARGS.build + "/osquery/example_extension.ext", + utils.copy_file(test_base.BUILD_DIR + "/osquery/examples/example_extension.ext", test_base.CONFIG_DIR) loader = test_base.Autoloader([test_base.CONFIG_DIR]) daemon = self._run_daemon({ @@ -255,7 +255,7 @@ def test_6_extensions_directory_autoload(self): # Get a python-based thrift client client = test_base.EXClient(daemon.options["extensions_socket"]) - self.assertTrue(client.open(timeout=EXTENSION_TIMEOUT)) + self.assertTrue(client.open()) em = client.getEM() # The waiting extension should have connected to the daemon. @@ -268,7 +268,7 @@ def test_6_extensions_directory_autoload(self): @test_base.flaky def test_7_extensions_autoload_watchdog(self): loader = test_base.Autoloader( - [test_base.ARGS.build + "/osquery/example_extension.ext"]) + [test_base.BUILD_DIR + "/osquery/examples/example_extension.ext"]) daemon = self._run_daemon({ "extensions_timeout": EXTENSION_TIMEOUT, "extensions_autoload": loader.path, @@ -277,7 +277,7 @@ def test_7_extensions_autoload_watchdog(self): # Get a python-based thrift client client = test_base.EXClient(daemon.options["extensions_socket"]) - self.assertTrue(client.open(timeout=EXTENSION_TIMEOUT)) + self.assertTrue(client.open()) em = client.getEM() # The waiting extension should have connected to the daemon. @@ -290,7 +290,7 @@ def test_7_extensions_autoload_watchdog(self): @test_base.flaky def test_8_external_config(self): loader = test_base.Autoloader( - [test_base.ARGS.build + "/osquery/example_extension.ext"]) + [test_base.BUILD_DIR + "/osquery/examples/example_extension.ext"]) daemon = self._run_daemon({ "extensions_autoload": loader.path, "extensions_timeout": EXTENSION_TIMEOUT, @@ -300,7 +300,7 @@ def test_8_external_config(self): # Get a python-based thrift client client = test_base.EXClient(daemon.options["extensions_socket"]) - self.assertTrue(client.open(timeout=EXTENSION_TIMEOUT)) + self.assertTrue(client.open()) em = client.getEM() # The waiting extension should have connected to the daemon. @@ -328,17 +328,17 @@ def test_9_external_config_update(self): # Get a python-based thrift client to the manager and extension. client = test_base.EXClient(extension.options["extensions_socket"]) test_base.expectTrue(client.try_open) - self.assertTrue(client.open(timeout=EXTENSION_TIMEOUT)) + self.assertTrue(client.open()) em = client.getEM() # Need the manager to request the extension's UUID. result = test_base.expect(em.extensions, 1) - self.assertTrue(result is not None) - ex_uuid = result.keys()[0] + self.assertTrue(len(result), 1) + ex_uuid = list(result.keys())[0] client2 = test_base.EXClient(extension.options["extensions_socket"], uuid=ex_uuid) test_base.expectTrue(client2.try_open) - self.assertTrue(client2.open(timeout=EXTENSION_TIMEOUT)) + self.assertTrue(client2.open()) ex = client2.getEX() # Trigger an async update from the extension. @@ -362,7 +362,7 @@ def test_9_external_config_update(self): @test_base.flaky def test_91_extensions_settings(self): loader = test_base.Autoloader( - [test_base.ARGS.build + "/osquery/example_extension.ext"]) + [test_base.BUILD_DIR + "/osquery/examples/example_extension.ext"]) daemon = self._run_daemon({ "disable_watchdog": True, "extensions_timeout": EXTENSION_TIMEOUT, @@ -372,7 +372,7 @@ def test_91_extensions_settings(self): # Get a python-based thrift client for the manager (core). client = test_base.EXClient(daemon.options["extensions_socket"]) - self.assertTrue(client.open(timeout=EXTENSION_TIMEOUT)) + self.assertTrue(client.open()) em = client.getEM() # The waiting extension should have connected to the daemon. From f4de2336b7af44693db7f5ab9c84e7257f39b523 Mon Sep 17 00:00:00 2001 From: SS Date: Wed, 13 Nov 2019 17:13:33 -0500 Subject: [PATCH 28/28] adding tests for extension group example --- osquery/CMakeLists.txt | 1 + .../extension_group_example/CMakeLists.txt | 35 ++++++++++++ .../src/complex_example.cpp | 54 +++++++++++++++++++ .../src/complex_example.h | 21 ++++++++ .../extension_group_example/src/example.cpp | 13 ++--- .../extension_group_example/src/example.h | 5 +- .../osquery_extension_group_main.cpp.in | 2 +- tools/tests/test_extensions.py | 49 +++++++++++++++++ 8 files changed, 171 insertions(+), 9 deletions(-) create mode 100644 osquery/examples/extension_group_example/CMakeLists.txt create mode 100644 osquery/examples/extension_group_example/src/complex_example.cpp create mode 100644 osquery/examples/extension_group_example/src/complex_example.h diff --git a/osquery/CMakeLists.txt b/osquery/CMakeLists.txt index 0c19a8e0c62..c5660d8625f 100644 --- a/osquery/CMakeLists.txt +++ b/osquery/CMakeLists.txt @@ -31,6 +31,7 @@ function(osqueryMain) add_subdirectory("experimental") add_subdirectory("system") add_subdirectory("examples") + add_subdirectory("examples/extension_group_example") generateOsqueryHeaders() generateOsqueryd() diff --git a/osquery/examples/extension_group_example/CMakeLists.txt b/osquery/examples/extension_group_example/CMakeLists.txt new file mode 100644 index 00000000000..613096d6589 --- /dev/null +++ b/osquery/examples/extension_group_example/CMakeLists.txt @@ -0,0 +1,35 @@ +include(${CMAKE_SOURCE_DIR}/external/cmake/cmakelibs.cmake) + +project(ext_group_example) + +function(osqueryExtensionGroupExampleMain) + set(project_common_include_dirs + "${CMAKE_CURRENT_SOURCE_DIR}/src" + ) + + set(example_table_sources + "${CMAKE_CURRENT_SOURCE_DIR}/src/example.h" + "${CMAKE_CURRENT_SOURCE_DIR}/src/example.cpp" + ) + + set(complex_example_table_sources + "${CMAKE_CURRENT_SOURCE_DIR}/src/complex_example.h" + "${CMAKE_CURRENT_SOURCE_DIR}/src/complex_example.cpp" + ) + + addOsqueryExtensionEx("ExampleTable" "table" "example" + SOURCES ${example_table_sources} + INCLUDEDIRS ${project_common_include_dirs} + MAININCLUDES example.h + ) + + addOsqueryExtensionEx("ComplexExampleTable" "table" "complex_example" + SOURCES ${complex_example_table_sources} + INCLUDEDIRS ${project_common_include_dirs} + MAININCLUDES complex_example.h + ) + + generateOsqueryExtensionGroup() +endfunction() + +osqueryExtensionGroupExampleMain() diff --git a/osquery/examples/extension_group_example/src/complex_example.cpp b/osquery/examples/extension_group_example/src/complex_example.cpp new file mode 100644 index 00000000000..845df7e1a49 --- /dev/null +++ b/osquery/examples/extension_group_example/src/complex_example.cpp @@ -0,0 +1,54 @@ +/** + * Copyright (c) 2014-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed in accordance with the terms specified in + * the LICENSE file found in the root directory of this source tree. + */ + +#include "complex_example.h" + +using namespace osquery; + +/** + * @brief A more 'complex' example table is provided to assist with tests. + * + * This table will access options and flags known to the extension. + * An extension should not assume access to any CLI flags- rather, access is + * provided via the osquery-meta table: osquery_flags. + * + * There is no API/C++ wrapper to provide seamless use of flags yet. + * We can force an implicit query to the manager though. + * + * Database access should be mediated by the *Database functions. + * Direct use of the "database" registry will lead to undefined behavior. + */ + +TableColumns ComplexExampleTable::columns() const { + return { + std::make_tuple("flag_test", TEXT_TYPE, ColumnOptions::DEFAULT), + std::make_tuple("database_test", TEXT_TYPE, ColumnOptions::DEFAULT), + }; +} + +TableRows ComplexExampleTable::generate(QueryContext& request) { + auto r = make_table_row(); + + // Use the basic 'force' flag to check implicit SQL usage. + auto flags = SQL::selectFrom( + {"default_value"}, "osquery_flags", "name", EQUALS, "force"); + if (flags.size() > 0) { + r["flag_test"] = flags[0]["default_value"]; + } + + std::string content; + setDatabaseValue(kPersistentSettings, "complex_example", "1"); + if (getDatabaseValue(kPersistentSettings, "complex_example", content)) { + r["database_test"] = content; + } + + TableRows result; + result.push_back(std::move(r)); + return result; +} + diff --git a/osquery/examples/extension_group_example/src/complex_example.h b/osquery/examples/extension_group_example/src/complex_example.h new file mode 100644 index 00000000000..f9fa0cfaf63 --- /dev/null +++ b/osquery/examples/extension_group_example/src/complex_example.h @@ -0,0 +1,21 @@ +/** + * Copyright (c) 2014-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed in accordance with the terms specified in + * the LICENSE file found in the root directory of this source tree. + */ + +#pragma once + +#include +#include +#include + +namespace osquery { +class ComplexExampleTable : public TablePlugin { + private: + TableColumns columns() const; + TableRows generate(QueryContext& request); +}; +} // namespace osquery diff --git a/osquery/examples/extension_group_example/src/example.cpp b/osquery/examples/extension_group_example/src/example.cpp index 907a715f8fd..be82e6abc54 100644 --- a/osquery/examples/extension_group_example/src/example.cpp +++ b/osquery/examples/extension_group_example/src/example.cpp @@ -11,18 +11,19 @@ namespace osquery { TableColumns ExampleTable::columns() const { return { - std::make_tuple("example_text", TEXT_TYPE, ColumnOptions::DEFAULT), - std::make_tuple("example_integer", INTEGER_TYPE, ColumnOptions::DEFAULT), + std::make_tuple("example_text", TEXT_TYPE, ColumnOptions::DEFAULT), + std::make_tuple("example_integer", INTEGER_TYPE, ColumnOptions::DEFAULT), }; } -QueryData ExampleTable::generate(QueryContext& request) { - static_cast(request); +TableRows ExampleTable::generate(QueryContext& request) { + TableRows results; - Row r; + auto r = make_table_row(); r["example_text"] = "example"; r["example_integer"] = INTEGER(1); - return {r}; + results.push_back(std::move(r)); + return results; } } // namespace osquery diff --git a/osquery/examples/extension_group_example/src/example.h b/osquery/examples/extension_group_example/src/example.h index fd225aeda04..4bab6ce6aaa 100644 --- a/osquery/examples/extension_group_example/src/example.h +++ b/osquery/examples/extension_group_example/src/example.h @@ -8,13 +8,14 @@ #pragma once -#include #include +#include +#include namespace osquery { class ExampleTable : public TablePlugin { private: TableColumns columns() const; - QueryData generate(QueryContext& request); + TableRows generate(QueryContext& request); }; } // namespace osquery diff --git a/tools/codegen/templates/osquery_extension_group_main.cpp.in b/tools/codegen/templates/osquery_extension_group_main.cpp.in index a61b3afaa0c..aa8c306c30c 100644 --- a/tools/codegen/templates/osquery_extension_group_main.cpp.in +++ b/tools/codegen/templates/osquery_extension_group_main.cpp.in @@ -25,6 +25,6 @@ int main(int argc, char* argv[]) { } // Finally wait for a signal / interrupt to shutdown. - runner.waitForShutdown(); + runner.waitThenShutdown(); return 0; } diff --git a/tools/tests/test_extensions.py b/tools/tests/test_extensions.py index e8f93d92b07..2982a011ae3 100755 --- a/tools/tests/test_extensions.py +++ b/tools/tests/test_extensions.py @@ -396,6 +396,55 @@ def test_91_extensions_settings(self): client.close() daemon.kill(True) + @test_base.flaky + def test_10_extension_group(self): + loader = test_base.Autoloader( + [test_base.BUILD_DIR + "/osquery/examples/extension_group_example/osquery_extension_group.ext"]) + daemon = self._run_daemon({ + "disable_watchdog": True, + "extensions_timeout": EXTENSION_TIMEOUT, + "extensions_autoload": loader.path, + }) + self.assertTrue(daemon.isAlive()) + + # Get a python-based thrift client for the manager (core). + client = test_base.EXClient(daemon.options["extensions_socket"]) + self.assertTrue(client.open()) + em = client.getEM() + + # The waiting extension should have connected to the daemon. + # This expect statement will block with a short timeout. + result = test_base.expect(em.extensions, 1) + self.assertEqual(len(result), 1) + + # Make sure the extension includes a custom registry plugin + result = em.query("select * from example") + if len(result.response) == 0: + time.sleep(0.5) + result = em.query("select * from example") + self.assertEqual(result.status.code, 0) + self.assertEqual(len(result.response), 1) + self.assertTrue("example_text" in result.response[0]) + self.assertTrue("example_integer" in result.response[0]) + self.assertEqual(result.response[0]["example_text"], "example") + self.assertEqual(result.response[0]["example_integer"], "1") + + # The 'complex_example' table reports several columns. + # Each is a 'test_type', check each expected value. + result = em.query("select * from complex_example") + if len(result.response) == 0: + # There is a brief race between register and registry broadcast + # That fast external client fight when querying tables. + # Other config/logger plugins have wrappers to retry/wait. + time.sleep(0.5) + result = em.query("select * from complex_example") + + self.assertEqual(result.response[0]['flag_test'], 'false') + self.assertEqual(result.response[0]['database_test'], '1') + + client.close() + daemon.kill(True) + if __name__ == "__main__": test_base.assertPermissions() module = test_base.Tester()