From 2af2028e03a6c9d48f90738b998fbb71c1545f49 Mon Sep 17 00:00:00 2001 From: Patrik Huber Date: Thu, 29 Mar 2018 21:13:24 +0100 Subject: [PATCH] Update eos to latest master (v0.15.0) & update 4dface accordingly We make a few unnecessary core::Image conversions, but overall, it seems to run at least as fast or even faster (which is expected) Added a few 'const's too. --- CMakeLists.txt | 14 ++++++++------ apps/4dface.cpp | 49 ++++++++++++++++++++++++++---------------------- apps/helpers.hpp | 34 ++++----------------------------- external/eos | 2 +- 4 files changed, 40 insertions(+), 59 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e4e516..82219d4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ project(4dface) -cmake_minimum_required(VERSION 2.8.10) +cmake_minimum_required(VERSION 2.8.12) set(4dface_VERSION_MAJOR 0) -set(4dface_VERSION_MINOR 2) +set(4dface_VERSION_MINOR 3) set(4dface_VERSION_PATCH 0) set(4dface_VERSION ${4dface_VERSION_MAJOR}.${4dface_VERSION_MINOR}.${4dface_VERSION_PATCH}) @@ -43,7 +43,7 @@ set(CPACK_PACKAGE_VERSION_PATCH "${4dface_VERSION_PATCH}") include(CPack) # Find dependencies: -find_package(OpenCV 2.4.3 REQUIRED core imgproc highgui objdetect) # in 3.0: require videoio? Plus the thing Michael fixed for eos? +find_package(OpenCV 3 REQUIRED core imgcodecs imgproc highgui videoio objdetect) message(STATUS "OpenCV include dir found at ${OpenCV_INCLUDE_DIRS}") message(STATUS "OpenCV library dir found at ${OpenCV_LIB_DIR}") set_target_properties(${OpenCV_LIBS} PROPERTIES MAP_IMPORTED_CONFIG_RELWITHDEBINFO RELEASE) @@ -82,10 +82,11 @@ set(superviseddescent_DIR "${CMAKE_SOURCE_DIR}/external/superviseddescent") set(eos_INCLUDE_DIR "${eos_DIR}/include") set(superviseddescent_INCLUDE_DIR "${superviseddescent_DIR}/include") -set(cereal_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/external/eos/3rdparty/cereal-1.1.1/include") +set(cereal_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/external/eos/3rdparty/cereal/include") set(glm_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/external/eos/3rdparty/glm") set(nanoflann_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/external/eos/3rdparty/nanoflann/include") set(eigen3_nnls_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/external/eos/3rdparty/eigen3-nnls/src") +set(toml11_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/external/eos/3rdparty/toml11") # The new model is not in the repository, download it manually for now: if(NOT EXISTS "face_landmarks_model_rcr_68.bin") @@ -110,6 +111,7 @@ include_directories(${cereal_INCLUDE_DIR}) include_directories(${glm_INCLUDE_DIR}) include_directories(${nanoflann_INCLUDE_DIR}) include_directories(${eigen3_nnls_INCLUDE_DIR}) +include_directories(${toml11_INCLUDE_DIR}) add_executable(4dface apps/4dface.cpp apps/helpers.hpp) target_link_libraries(4dface ${OpenCV_LIBS} ${Boost_LIBRARIES}) @@ -117,8 +119,8 @@ target_link_libraries(4dface ${OpenCV_LIBS} ${Boost_LIBRARIES}) # install targets: install(TARGETS 4dface DESTINATION bin) install(FILES ${eos_DIR}/share/sfm_shape_3448.bin DESTINATION share) -install(FILES ${eos_DIR}/share/ibug2did.txt DESTINATION share) -install(FILES ${eos_DIR}/share/model_contours.json DESTINATION share) +install(FILES ${eos_DIR}/share/ibug_to_sfm.txt DESTINATION share) +install(FILES ${eos_DIR}/share/sfm_model_contours.json DESTINATION share) install(FILES ${eos_DIR}/share/sfm_3448_edge_topology.json DESTINATION share) install(FILES ${eos_DIR}/share/expression_blendshapes_3448.bin DESTINATION share) install(FILES ${CMAKE_BINARY_DIR}/face_landmarks_model_rcr_68.bin DESTINATION share) diff --git a/apps/4dface.cpp b/apps/4dface.cpp index e3b2442..ce4237f 100644 --- a/apps/4dface.cpp +++ b/apps/4dface.cpp @@ -21,14 +21,16 @@ #include "eos/core/Landmark.hpp" #include "eos/core/LandmarkMapper.hpp" +#include "eos/core/Mesh.hpp" #include "eos/fitting/fitting.hpp" -#include "eos/fitting/orthographic_camera_estimation_linear.hpp" #include "eos/fitting/contour_correspondence.hpp" #include "eos/fitting/closest_edge_fitting.hpp" #include "eos/fitting/RenderingParameters.hpp" #include "eos/render/utils.hpp" #include "eos/render/render.hpp" #include "eos/render/texture_extraction.hpp" +#include "eos/render/draw_utils.hpp" +#include "eos/core/Image_opencv_interop.hpp" #include "rcr/model.hpp" #include "cereal/cereal.hpp" @@ -82,9 +84,9 @@ int main(int argc, char *argv[]) "full path to OpenCV's face detector (haarcascade_frontalface_alt2.xml)") ("landmarkdetector,l", po::value(&landmarkdetector)->required()->default_value("../share/face_landmarks_model_rcr_68.bin"), "learned landmark detection model") - ("mapping,p", po::value(&mappingsfile)->required()->default_value("../share/ibug2did.txt"), + ("mapping,p", po::value(&mappingsfile)->required()->default_value("../share/ibug_to_sfm.txt"), "landmark identifier to model vertex number mapping") - ("model-contour,c", po::value(&contourfile)->required()->default_value("../share/model_contours.json"), + ("model-contour,c", po::value(&contourfile)->required()->default_value("../share/sfm_model_contours.json"), "file with model contour indices") ("edge-topology,e", po::value(&edgetopologyfile)->required()->default_value("../share/sfm_3448_edge_topology.json"), "file with model's precomputed edge topology") @@ -109,11 +111,11 @@ int main(int argc, char *argv[]) } // Load the Morphable Model and the LandmarkMapper: - morphablemodel::MorphableModel morphable_model = morphablemodel::load_model(modelfile.string()); - core::LandmarkMapper landmark_mapper = mappingsfile.empty() ? core::LandmarkMapper() : core::LandmarkMapper(mappingsfile); + const morphablemodel::MorphableModel morphable_model = morphablemodel::load_model(modelfile.string()); + const core::LandmarkMapper landmark_mapper = mappingsfile.empty() ? core::LandmarkMapper() : core::LandmarkMapper(mappingsfile.string()); - fitting::ModelContour model_contour = contourfile.empty() ? fitting::ModelContour() : fitting::ModelContour::load(contourfile.string()); - fitting::ContourLandmarks ibug_contour = fitting::ContourLandmarks::load(mappingsfile.string()); + const fitting::ModelContour model_contour = contourfile.empty() ? fitting::ModelContour() : fitting::ModelContour::load(contourfile.string()); + const fitting::ContourLandmarks ibug_contour = fitting::ContourLandmarks::load(mappingsfile.string()); rcr::detection_model rcr_model; // Load the landmark detection model: @@ -145,9 +147,9 @@ int main(int argc, char *argv[]) return EXIT_FAILURE; } - vector blendshapes = morphablemodel::load_blendshapes(blendshapesfile.string()); + const morphablemodel::Blendshapes blendshapes = morphablemodel::load_blendshapes(blendshapesfile.string()); - morphablemodel::EdgeTopology edge_topology = morphablemodel::load_edge_topology(edgetopologyfile.string()); + const morphablemodel::EdgeTopology edge_topology = morphablemodel::load_edge_topology(edgetopologyfile.string()); cv::namedWindow("video", 1); cv::namedWindow("render", 1); @@ -206,34 +208,37 @@ int main(int argc, char *argv[]) // Fit the 3DMM: fitting::RenderingParameters rendering_params; vector shape_coefficients, blendshape_coefficients; - vector image_points; - render::Mesh mesh; - std::tie(mesh, rendering_params) = fitting::fit_shape_and_pose(morphable_model, blendshapes, rcr_to_eos_landmark_collection(current_landmarks), landmark_mapper, unmodified_frame.cols, unmodified_frame.rows, edge_topology, ibug_contour, model_contour, 3, 5, 15.0f, boost::none, shape_coefficients, blendshape_coefficients, image_points); + vector image_points; + core::Mesh mesh; + std::tie(mesh, rendering_params) = fitting::fit_shape_and_pose(morphable_model, blendshapes, rcr_to_eos_landmark_collection(current_landmarks), landmark_mapper, unmodified_frame.cols, unmodified_frame.rows, edge_topology, ibug_contour, model_contour, 3, 5, 15.0f, cpp17::nullopt, shape_coefficients, blendshape_coefficients, image_points); // Draw the 3D pose of the face: draw_axes_topright(glm::eulerAngles(rendering_params.get_rotation())[0], glm::eulerAngles(rendering_params.get_rotation())[1], glm::eulerAngles(rendering_params.get_rotation())[2], frame); // Wireframe rendering of mesh of this frame (non-averaged): - draw_wireframe(frame, mesh, rendering_params.get_modelview(), rendering_params.get_projection(), fitting::get_opencv_viewport(frame.cols, frame.rows)); + render::draw_wireframe(frame, mesh, rendering_params.get_modelview(), rendering_params.get_projection(), fitting::get_opencv_viewport(frame.cols, frame.rows)); // Extract the texture using the fitted mesh from this frame: - Mat affine_cam = fitting::get_3x4_affine_camera_matrix(rendering_params, frame.cols, frame.rows); - Mat isomap = render::extract_texture(mesh, affine_cam, unmodified_frame, true, render::TextureInterpolation::NearestNeighbour, 512); + const Eigen::Matrix affine_cam = fitting::get_3x4_affine_camera_matrix(rendering_params, frame.cols, frame.rows); + const Mat isomap = core::to_mat(render::extract_texture(mesh, affine_cam, core::from_mat(unmodified_frame), true, render::TextureInterpolation::NearestNeighbour, 512)); // Merge the isomaps - add the current one to the already merged ones: - Mat merged_isomap = isomap_averaging.add_and_merge(isomap); + const Mat merged_isomap = isomap_averaging.add_and_merge(isomap); // Same for the shape: shape_coefficients = pca_shape_merging.add_and_merge(shape_coefficients); - auto merged_shape = morphable_model.get_shape_model().draw_sample(shape_coefficients) + morphablemodel::to_matrix(blendshapes) * Mat(blendshape_coefficients); - render::Mesh merged_mesh = morphablemodel::sample_to_mesh(merged_shape, morphable_model.get_color_model().get_mean(), morphable_model.get_shape_model().get_triangle_list(), morphable_model.get_color_model().get_triangle_list(), morphable_model.get_texture_coordinates()); + const Eigen::VectorXf merged_shape = + morphable_model.get_shape_model().draw_sample(shape_coefficients) + + to_matrix(blendshapes) * + Eigen::Map(blendshape_coefficients.data(), blendshape_coefficients.size()); + const core::Mesh merged_mesh = morphablemodel::sample_to_mesh(merged_shape, morphable_model.get_color_model().get_mean(), morphable_model.get_shape_model().get_triangle_list(), morphable_model.get_color_model().get_triangle_list(), morphable_model.get_texture_coordinates()); // Render the model in a separate window using the estimated pose, shape and merged texture: - Mat rendering; + core::Image4u rendering; auto modelview_no_translation = rendering_params.get_modelview(); modelview_no_translation[3][0] = 0; modelview_no_translation[3][1] = 0; std::tie(rendering, std::ignore) = render::render(merged_mesh, modelview_no_translation, glm::ortho(-130.0f, 130.0f, -130.0f, 130.0f), 256, 256, render::create_mipmapped_texture(merged_isomap), true, false, false); - cv::imshow("render", rendering); + cv::imshow("render", core::to_mat(rendering)); cv::imshow("video", frame); auto key = cv::waitKey(30); @@ -244,8 +249,8 @@ int main(int argc, char *argv[]) } if (key == 's') { // save an obj + current merged isomap to the disk: - render::Mesh neutral_expression = morphablemodel::sample_to_mesh(morphable_model.get_shape_model().draw_sample(shape_coefficients), morphable_model.get_color_model().get_mean(), morphable_model.get_shape_model().get_triangle_list(), morphable_model.get_color_model().get_triangle_list(), morphable_model.get_texture_coordinates()); - render::write_textured_obj(neutral_expression, "current_merged.obj"); + const core::Mesh neutral_expression = morphablemodel::sample_to_mesh(morphable_model.get_shape_model().draw_sample(shape_coefficients), morphable_model.get_color_model().get_mean(), morphable_model.get_shape_model().get_triangle_list(), morphable_model.get_color_model().get_triangle_list(), morphable_model.get_texture_coordinates()); + core::write_textured_obj(neutral_expression, "current_merged.obj"); cv::imwrite("current_merged.isomap.png", merged_isomap); } } diff --git a/apps/helpers.hpp b/apps/helpers.hpp index b869625..2bda4a9 100644 --- a/apps/helpers.hpp +++ b/apps/helpers.hpp @@ -32,6 +32,8 @@ #include "glm/mat4x4.hpp" #include "glm/gtc/matrix_transform.hpp" +#include "Eigen/Core" + #include "opencv2/core/core.hpp" #include @@ -75,8 +77,8 @@ cv::Rect rescale_facebox(cv::Rect facebox, float scaling, float translation_y) */ auto rcr_to_eos_landmark_collection(const rcr::LandmarkCollection& landmark_collection) { - eos::core::LandmarkCollection eos_landmark_collection; - std::transform(begin(landmark_collection), end(landmark_collection), std::back_inserter(eos_landmark_collection), [](auto&& lm) { return eos::core::Landmark{ lm.name, lm.coordinates }; }); + eos::core::LandmarkCollection eos_landmark_collection; + std::transform(begin(landmark_collection), end(landmark_collection), std::back_inserter(eos_landmark_collection), [](auto&& lm) { return eos::core::Landmark{ lm.name, { lm.coordinates[0], lm.coordinates[1] } }; }); return eos_landmark_collection; }; @@ -113,34 +115,6 @@ cv::Rect make_bbox_square(cv::Rect bounding_box) return cv::Rect(center_x - box_size / 2.0, center_y - box_size / 2.0, box_size, box_size); }; -/** - * Draws the given mesh as wireframe into the image. - * - * It does backface culling, i.e. draws only vertices in CCW order. - * - * @param[in] image An image to draw into. - * @param[in] mesh The mesh to draw. - * @param[in] modelview Model-view matrix to draw the mesh. - * @param[in] projection Projection matrix to draw the mesh. - * @param[in] viewport Viewport to draw the mesh. - * @param[in] colour Colour of the mesh to be drawn. - */ -void draw_wireframe(cv::Mat image, const eos::render::Mesh& mesh, glm::mat4x4 modelview, glm::mat4x4 projection, glm::vec4 viewport, cv::Scalar colour = cv::Scalar(0, 255, 0, 255)) -{ - for (const auto& triangle : mesh.tvi) - { - const auto p1 = glm::project({ mesh.vertices[triangle[0]][0], mesh.vertices[triangle[0]][1], mesh.vertices[triangle[0]][2] }, modelview, projection, viewport); - const auto p2 = glm::project({ mesh.vertices[triangle[1]][0], mesh.vertices[triangle[1]][1], mesh.vertices[triangle[1]][2] }, modelview, projection, viewport); - const auto p3 = glm::project({ mesh.vertices[triangle[2]][0], mesh.vertices[triangle[2]][1], mesh.vertices[triangle[2]][2] }, modelview, projection, viewport); - if (eos::render::detail::are_vertices_ccw_in_screen_space(glm::vec2(p1), glm::vec2(p2), glm::vec2(p3))) - { - cv::line(image, cv::Point(p1.x, p1.y), cv::Point(p2.x, p2.y), colour); - cv::line(image, cv::Point(p2.x, p2.y), cv::Point(p3.x, p3.y), colour); - cv::line(image, cv::Point(p3.x, p3.y), cv::Point(p1.x, p1.y), colour); - } - } -}; - /** * @brief Merges isomaps from a live video with a weighted averaging, based * on the view angle of each vertex to the camera. diff --git a/external/eos b/external/eos index dc84b9a..cb8731c 160000 --- a/external/eos +++ b/external/eos @@ -1 +1 @@ -Subproject commit dc84b9ae2b93dcfee097890772a7c07241834b17 +Subproject commit cb8731c95a2f2a4101628c066ebb5d646e00a518