feat(encryption): add kms key management

src/replica/CMakeLists.txt

@@ -59,7 +59,6 @@ set(MY_SRC_SEARCH_MODE "GLOB")
 
 set(MY_PROJ_LIBS 
     absl::strings
-    curl
     dsn_replication_common
     dsn.failure_detector
     dsn.block_service
@@ -69,6 +68,7 @@ set(MY_PROJ_LIBS
     dsn_nfs
     dsn_dist_cmd
     dsn_http
+    curl
     dsn_runtime
     dsn_aio
     dsn_meta_server

src/replica/kms_client.cpp

@@ -83,6 +83,7 @@ dsn::error_s PegasusKMSClient::DecryptEncryptionKey(const std::string &encryptio
         }
         long http_status;
         client.get_http_status(http_status);
+        LOG_INFO("http status = ({})", http_status);
         if (http_status == 200) {
             j = nlohmann::json::parse(resp);
         }
@@ -143,6 +144,7 @@ dsn::error_s PegasusKMSClient::GenerateEncryptionKeyFromKMS(const std::string &k
 
         long http_status;
         client.get_http_status(http_status);
+        LOG_INFO("http status = ({})", http_status);
         if (http_status == 200) {
             j = nlohmann::json::parse(resp);
             nlohmann::json jsonObject = j.at(0);

src/replica/replica_stub.cpp

@@ -592,28 +592,25 @@ void replica_stub::initialize(bool clear /* = false*/)
 }
 
 dsn::error_s store_kms_key(std::string data_dir, std::string encryption_key, std::string iv, std::string key_version){
-    std::string path = ::absl::StrCat(data_dir, "/replica_encrypted_key");
-    std::ofstream in(file_name, std::ios::binary);
-    if(in.is_open()){
-        in << server_key << std::endl;
-        in << iv << std::endl;
-        in << key_version << std::endl;
+    replica_kms_info kms_info(encryption_key, iv, key_version);
+    auto err = kms_info.store(data_dir);
+    if(dsn::ERR_OK == err){
+        return dsn::error_s::ok();
     } else{
-        return dsn::error_s::make(ERR_FILE_OPERATION_FAILED, "Can't open replica_encrypted_key file to write");
+        return dsn::error_s::make(err , "Can't open replica_encrypted_key file to write");
     }
-
-    return dsn::error_s::ok();
 }
 
-void get_kms_key(std::string data_dir, std::string encryption_key, std::string iv, std::string key_version){
-    std::string path = ::absl::StrCat(data_dir, "/replica_encrypted_key");
-    std::ofstream in(path, std::ios::binary);
-    if(!in.bad()){
-        getline(in, encryption_key);
-        getline(in, iv);
-        getline(in, key_version);
+void get_kms_key(std::string data_dir, std::string *encryption_key, std::string *iv, std::string *key_version){
+    replica_kms_info kms_info;
+    auto err = kms_info.load(data_dir);
+    LOG_INFO("encryption_key = {}, iv = {}, key_version = {}", kms_info.encryption_key, kms_info.iv, kms_info.key_version);
+    *encryption_key = kms_info.encryption_key;
+    *iv = kms_info.iv;
+    *key_version = kms_info.key_version;
+    if(dsn::ERR_OK != err){
+        CHECK(err, "Can't open replica_encrypted_key file to read");
     }
-
 }
 
 void replica_stub::initialize(const replication_options &opts, bool clear /* = false*/)
@@ -649,8 +646,9 @@ void replica_stub::initialize(const replication_options &opts, bool clear /* = f
     std::string server_key;
     //get and store eek from kms
     if (key_provider) {
-        get_kms_key(_options.data_dirs[0], encryption_key, iv, key_version);
-        if(encryption_key == nullptr){
+        get_kms_key(_options.data_dirs[0], &encryption_key, &iv, &key_version);
+        LOG_INFO("encryption_key = {}, iv = {}, key_version = {}", encryption_key, iv, key_version);
+        if(encryption_key.empty()){
             CHECK(key_provider, "invalid kms url ({})", FLAGS_hadoop_kms_url);
             CHECK(key_provider->GenerateEncryptionKey(&encryption_key, &iv, &key_version), "get encryption key failed");
         }
@@ -662,7 +660,7 @@ void replica_stub::initialize(const replication_options &opts, bool clear /* = f
     _fs_manager.initialize(_options.data_dirs, _options.data_dir_tags);
 
     if (key_provider) {
-        CHECK(store_kms_key(encryption_key), "Cant store kms key");
+        CHECK(store_kms_key(_options.data_dirs[0], encryption_key, iv, key_version), "Cant store kms key");
     }
 
     // TODO(yingchun): remove the slog related code.

src/replica/replication_app_base.cpp

@@ -65,6 +65,7 @@ namespace dsn {
 namespace replication {
 
 const std::string replica_init_info::kInitInfo = ".init-info";
+const std::string replica_kms_info::kFileName = "/replica_encrypted_key";
 
 namespace {
 error_code write_blob_to_file(const std::string &fname, const blob &data)
@@ -149,6 +150,71 @@ std::string replica_init_info::to_string()
     return oss.str();
 }
 
+error_code replica_kms_info::load(const std::string &dir)
+{
+    std::string info_path = utils::filesystem::path_combine(dir, kFileName);
+    LOG_AND_RETURN_NOT_TRUE(ERROR,
+                            utils::filesystem::path_exists(info_path),
+                            ERR_PATH_NOT_FOUND,
+                            "file({}) not exist",
+                            info_path);
+    LOG_AND_RETURN_NOT_OK(
+        ERROR, load_json(info_path), "load replica_kms_info from {} failed", info_path);
+    LOG_INFO("load replica_kms_info from {} ", info_path);
+    return ERR_OK;
+}
+
+error_code replica_kms_info::store(const std::string &dir)
+{
+    uint64_t start = dsn_now_ns();
+    std::string info_path = utils::filesystem::path_combine(dir, kFileName);
+    LOG_AND_RETURN_NOT_OK(ERROR,
+                          store_json(info_path),
+                          "store replica_kms_info to {} failed, time_used_ns = {}",
+                          info_path,
+                          dsn_now_ns() - start);
+    LOG_INFO("store replica_kms_info to {} succeed, time_used_ns = {}",
+             info_path,
+             dsn_now_ns() - start);
+    return ERR_OK;
+}
+
+error_code replica_kms_info::load_json(const std::string &fname)
+{
+    std::string data;
+    auto s = rocksdb::ReadFileToString(
+        dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive), fname, &data);
+    LOG_AND_RETURN_NOT_TRUE(ERROR, s.ok(), ERR_FILE_OPERATION_FAILED, "read file {} failed", fname);
+    LOG_AND_RETURN_NOT_TRUE(ERROR,
+                            json::json_forwarder<replica_kms_info>::decode(
+                                blob::create_from_bytes(std::move(data)), *this),
+                            ERR_FILE_OPERATION_FAILED,
+                            "decode json from file {} failed",
+                            fname);
+    return ERR_OK;
+}
+
+error_code replica_kms_info::store_json(const std::string &fname)
+{
+    const blob &data = json::json_forwarder<replica_kms_info>::encode(*this);
+    std::string tmp_fname = fname + ".tmp";
+    auto cleanup = defer([tmp_fname]() { utils::filesystem::remove_path(tmp_fname); });
+    auto s =
+        rocksdb::WriteStringToFile(dsn::utils::PegasusEnv(dsn::utils::FileDataType::kNonSensitive),
+                                   rocksdb::Slice(data.data(), data.length()),
+                                   tmp_fname,
+                                   /* should_sync */ true);
+    LOG_AND_RETURN_NOT_TRUE(
+        ERROR, s.ok(), ERR_FILE_OPERATION_FAILED, "write file {} failed", tmp_fname);
+    LOG_AND_RETURN_NOT_TRUE(ERROR,
+                            utils::filesystem::rename_path(tmp_fname, fname),
+                            ERR_FILE_OPERATION_FAILED,
+                            "move file from {} to {} failed",
+                            tmp_fname,
+                            fname);
+    return ERR_OK;
+}
+
 error_code replica_app_info::load(const std::string &fname)
 {
     std::string data;

src/replica/replication_app_base.h

@@ -80,6 +80,28 @@ class replica_init_info
     error_code store_json(const std::string &fname);
 };
 
+class replica_kms_info
+{
+public:
+    std::string encryption_key;
+    std::string iv;
+    std::string key_version;
+    DEFINE_JSON_SERIALIZATION(encryption_key,
+                              iv,
+                              key_version)
+    static const std::string kFileName;
+
+public:
+    replica_kms_info(const std::string& e_key = "", const std::string& i = "", const std::string& k_version = "")
+            : encryption_key(e_key), iv(i), key_version(k_version) {}
+    error_code load(const std::string &dir) WARN_UNUSED_RESULT;
+    error_code store(const std::string &dir);
+
+private:
+    error_code load_json(const std::string &fname);
+    error_code store_json(const std::string &fname);
+};
+
 class replica_app_info
 {
 private:

src/utils/CMakeLists.txt

@@ -31,7 +31,7 @@ set(MY_SRC_SEARCH_MODE "GLOB")
 
 set(MY_BOOST_LIBS Boost::system Boost::filesystem)
 
-set(MY_PROJ_LIBS dsn_http rocksdb)
+set(MY_PROJ_LIBS absl::strings dsn_http curl gssapi_krb5 rocksdb)
 
 # Extra files that will be installed
 set(MY_BINPLACES "")