Merge pull request #5 from ykwd/dev

Add more unit tests
KnightKingWalk · Nov 19, 2019 · 7e21523 · 7e21523
2 parents 1bff8ee + c860786
commit 7e21523
Show file tree

Hide file tree

Showing 20 changed files with 1,373 additions and 193 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -4,3 +4,6 @@
 [submodule "ext/gemini"]
 	path = ext/gemini
 	url = https://github.com/thu-pacman/GeminiGraph.git
+[submodule "ext/gtest"]
+	path = ext/gtest
+	url = https://github.com/google/googletest.git
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -2,8 +2,8 @@ cmake_minimum_required(VERSION 2.8)
 
 project(KnightKing)
 
-set(WITH_UNIT_TEST TRUE)
 set(KTK_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
+option(WITH_TESTS "Build unit test programs" OFF)
 
 include(knightking.cmake)
 
@@ -21,12 +21,12 @@ if(MPI_FOUND)
     include_directories(${MPI_INCLUDE_PATH})
 endif()
 
-if(WITH_UNIT_TEST)
+if(WITH_TESTS)
     enable_testing()
-    find_package(GTest REQUIRED)
-    if (GTEST_FOUND)
-        include_directories(${GTEST_INCLUDE_DIRS})
-    endif()
+    include_directories("${CMAKE_CURRENT_SOURCE_DIR}/ext/gtest/googletest/include")
+    add_subdirectory(ext/gtest)
+    link_directories(${CMAKE_BINARY_DIR}/lib)
+    set(GTEST_LIBRARIES "gtest" "gtest_main")
 endif()
 
 #use c++11

diff --git a/README.md b/README.md
@@ -57,6 +57,14 @@ The compiled executable files will be installed at the "bin" directory:
 ls ./bin
 ```
 
+[Optional] Run unit tests:
+
+```
+cmake -DWITH_TESTS=on ..
+ctest
+```
+note that the unit test may take about one hour.
+
 ### Run Built-in Applications
 
 Here we take node2vec as an example to show how to run the built-in applications. The usage of other three applications is quite similar.

diff --git a/ext/gtest b/ext/gtest
diff --git a/include/graph.hpp b/include/graph.hpp
@@ -33,6 +33,7 @@
 #include <type_traits>
 #include <thread>
 #include <random>
+#include <functional>
 #include <unistd.h>
 #include <sys/mman.h>
 #include <limits.h>

diff --git a/include/walk.hpp b/include/walk.hpp
@@ -118,7 +118,9 @@ class WalkEngine : public GraphEngine<edge_data_t>
 #ifdef COLLECT_WALK_SEQUENCE
     std::vector<std::vector<Footprint> > footprints;
 #endif
-
+#ifdef COLLECT_WALKER_INIT_STATE
+    std::vector<Walker<walker_data_t> > walker_init_state;
+#endif
     template<typename T>
     T* alloc_walker_array()
     {
@@ -185,6 +187,9 @@ class WalkEngine : public GraphEngine<edge_data_t>
         local_walker_num = 0;
 #ifdef COLLECT_WALK_SEQUENCE
         footprints.resize(this->worker_num);
+#endif
+#ifdef COLLECT_WALKER_INIT_STATE
+        walker_init_state.clear();
 #endif
         this->set_msg_buffer(walker_num, sizeof(walker_msg_t));
         this->template distributed_execute<walker_t>(
@@ -217,6 +222,13 @@ class WalkEngine : public GraphEngine<edge_data_t>
                 walker_init_state_func(local_walkers[w_i].data, local_walkers[w_i].dst_vertex_id);
             }
         }
+        #ifdef COLLECT_WALKER_INIT_STATE
+        walker_init_state.clear();
+        for (walker_id_t w_i = 0; w_i < local_walker_num; w_i ++)
+        {
+            walker_init_state.push_back(local_walkers[w_i].data);
+        }
+        #endif
     }
 
     void free_walkers(
@@ -686,11 +698,11 @@ class WalkEngine : public GraphEngine<edge_data_t>
                                         }
                                     }
 
+                                    walker.step ++;
                                     if (walker_update_state_func != nullptr)
                                     {
                                         walker_update_state_func(walker, current_v, ac_edge);
                                     }
-                                    walker.step ++;
 
                                     if (output_flag)
                                     {
@@ -976,8 +988,8 @@ class WalkEngine : public GraphEngine<edge_data_t>
                                             {
                                                 if (this->is_local_vertex(cd->candidate->neighbour))
                                                 {
-                                                    walker_update_state_func(p->data, current_v, cd->candidate);
                                                     p->data.step ++;
+                                                    walker_update_state_func(p->data, current_v, cd->candidate);
                                                     p->dst_vertex_id = cd->candidate->neighbour;
 
                                                     if (output_flag)
@@ -1086,8 +1098,8 @@ class WalkEngine : public GraphEngine<edge_data_t>
                                 {
                                     if (cd->accepted || cd->randval <= dynamic_comp_func(walker, remote_response_cache[walker_idx], current_v, cd->candidate))
                                     {
-                                        walker_update_state_func(walker, current_v, cd->candidate);
                                         walker.step ++;
+                                        walker_update_state_func(walker, current_v, cd->candidate);
                                         this->emit(cd->candidate->neighbour, walker, worker_id);
 
                                         if (output_flag)
@@ -1204,4 +1216,31 @@ class WalkEngine : public GraphEngine<edge_data_t>
         send_thread.join();
     }
 #endif
+#ifdef COLLECT_WALKER_INIT_STATE
+    void collect_walker_init_state(std::vector<Walker<walker_data_t> > &output)
+    {
+        if (get_mpi_rank() != 0)
+        {
+            MPI_Send(walker_init_state.data(), walker_init_state.size() * sizeof(Walker<walker_data_t>), get_mpi_data_type<char>(), 0, Tag_Msg, MPI_COMM_WORLD);
+        } else
+        {
+            output = walker_init_state;
+            for (partition_id_t p_i = 1; p_i < get_mpi_size(); p_i++)
+            {
+                int recv_size = 0;
+                MPI_Status recv_status;
+                MPI_Probe(p_i, Tag_Msg, MPI_COMM_WORLD, &recv_status);
+                MPI_Get_count(&recv_status, get_mpi_data_type<char>(), &recv_size);
+                int recv_n = recv_size / sizeof(Walker<walker_data_t>);
+                std::vector<Walker<walker_data_t> > recv_data(recv_n);
+                MPI_Recv(recv_data.data(), recv_size, get_mpi_data_type<char>(), p_i, Tag_Msg, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+                for (auto x : recv_data)
+                {
+                    output.push_back(x);
+                }
+            }
+            std::sort(output.begin(), output.end(), [](const Walker<walker_data_t> a, const Walker<walker_data_t> b) { return a.id < b.id; });
+        }
+    }
+#endif
 };
diff --git a/knightking.cmake b/knightking.cmake
@@ -17,7 +17,7 @@ foreach(prog "test_storage")
     add_test("${prog}" "${KTK_RUNTIME_OUTPUT_DIRECTORY}/${prog}")
 endforeach(prog)
 
-foreach(prog "test_graph" "test_path" "test_outlier" "test_deepwalk" "test_ppr" "test_node2vec")
+foreach(prog "test_graph" "test_path" "test_walker" "test_bound" "test_outlier" "test_deepwalk" "test_ppr" "test_metapath" "test_node2vec")
     add_test("${prog}" "./bin/${prog}")
     add_test("distributed_${prog}" "mpirun" "-n" "2" "${KTK_RUNTIME_OUTPUT_DIRECTORY}/${prog}")
 endforeach(prog)
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -1,4 +1,4 @@
-IF(WITH_UNIT_TEST)
+IF(WITH_TESTS)
     add_subdirectory(tests)
 ENDIF()
 add_subdirectory(apps)

diff --git a/src/apps/metapath.cpp b/src/apps/metapath.cpp
@@ -74,7 +74,7 @@ int main(int argc, char** argv)
         run(&graph, &opt);
     } else if(opt.static_comp.compare("unweighted") == 0)
     {
-        WalkEngine<int, MetapathState> graph;
+        WalkEngine<UnweightedMetaData, MetapathState> graph;
         run(&graph, &opt);
     } else
     {

diff --git a/src/apps/metapath.hpp b/src/apps/metapath.hpp
@@ -53,7 +53,7 @@ std::vector<std::vector<scheme_mask_t> > get_scheme_mask(std::vector<std::vector
 }
 
 template<typename walker_state_t>
-std::function<real_t(vertex_id_t, AdjUnit<int>*)> get_metapath_static_comp(WalkEngine<int, walker_state_t> *graph)
+std::function<real_t(vertex_id_t, AdjUnit<UnweightedMetaData>*)> get_metapath_static_comp(WalkEngine<UnweightedMetaData, walker_state_t> *graph)
 {
     return nullptr;
 }
@@ -67,16 +67,6 @@ std::function<real_t(vertex_id_t, AdjUnit<WeightedMetaData>*)> get_metapath_stat
     return static_comp;
 }
 
-int get_edge_meta(AdjUnit<int> *edge)
-{
-    return edge->data;
-}
-
-int get_edge_meta(AdjUnit<WeightedMetaData> *edge)
-{
-    return edge->data.meta_info;
-}
-
 template<typename edge_data_t>
 void metapath(WalkEngine<edge_data_t, MetapathState> *graph, std::vector<std::vector<std::vector<bool> > > schemes, walker_id_t walker_num, step_t walk_length)
 {
@@ -91,7 +81,7 @@ void metapath(WalkEngine<edge_data_t, MetapathState> *graph, std::vector<std::ve
             vertex_masks[v_id] = 0;
             for (auto p = graph->csr->adj_lists[v_id].begin; p < graph->csr->adj_lists[v_id].end; p++)
             {
-                vertex_masks[v_id] |= (1 << get_edge_meta(p));
+                vertex_masks[v_id] |= (1 << p->data.get_meta());
             }
             return 0;
         }
@@ -116,7 +106,7 @@ void metapath(WalkEngine<edge_data_t, MetapathState> *graph, std::vector<std::ve
         get_metapath_static_comp(graph),
         [&] (Walker<MetapathState> &walker, vertex_id_t current_v, AdjUnit<edge_data_t> *edge)
         {
-            if (schemes[walker.data.scheme_id][walker.data.state][get_edge_meta(edge)])
+            if (schemes[walker.data.scheme_id][walker.data.state][edge->data.get_meta()])
             {
                 return 1.0;
             } else

diff --git a/src/apps/metascheme.hpp b/src/apps/metascheme.hpp
@@ -30,16 +30,40 @@
 
 #include "type.hpp"
 
+typedef int scheme_id_t;
+typedef int meta_state_t;
+
 struct MetapathState
 {
-    int scheme_id;
-    int state;
+    scheme_id_t scheme_id;
+    meta_state_t state;
 };
 
 struct WeightedMetaData
 {
     real_t weight;
-    int meta_info;
+    meta_state_t meta_info;
+    meta_state_t get_meta()
+    {
+        return meta_info;
+    }
+    real_t get_weight()
+    {
+        return weight;
+    }
+};
+
+struct UnweightedMetaData
+{
+    meta_state_t meta_info;
+    meta_state_t get_meta()
+    {
+        return meta_info;
+    }
+    real_t get_weight()
+    {
+        return 1.0;
+    }
 };
 
 std::vector<std::vector<std::vector<bool> > > read_metapath_schemes(const char* path)

diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt
@@ -1,13 +1,17 @@
 SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DCMAKE_BUILD_TYPE=Debug")
+SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ggdb")
 SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUNIT_TEST")
 SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DCOLLECT_WALK_SEQUENCE")
-SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ggdb")
+SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DCOLLECT_WALKER_INIT_STATE")
 
 add_test_exec(test_storage)
 add_test_exec(test_graph)
 add_test_exec(test_path)
+add_test_exec(test_walker)
 add_test_exec(test_outlier)
+add_test_exec(test_bound)
 
 add_test_exec(test_deepwalk)
 add_test_exec(test_ppr)
 add_test_exec(test_node2vec)
+add_test_exec(test_metapath)