Fix cpp test scripts

.gitignore

@@ -50,4 +50,6 @@ test/project_path.h
 hash_hex/
 
 *.csv
-toolkit/plugins
+toolkit/plugins
+
+*.sealed

README.md

@@ -1,4 +1,4 @@
-# Fidelius - 基于可信执行环境的熠智隐私计算中间件
+# Fidelius - YeeZ Privacy Computing
 English / [中文](doc/README_ZH.md)
 
 # Fidelius - YeeZ Privacy Computing

test/integrate/test.sh

@@ -0,0 +1,11 @@
+for i in `seq 1 20`; do
+    rm -rf *.json
+    rm -rf *.sealed
+    rm -rf *.output
+    ../../bin/test_scheduler
+    echo "Test $i"
+    echo "Test $i"
+    echo "Test $i"
+    echo "Test $i"
+    echo "Test $i"
+done

test/toolkit/scheduler/common.hpp

@@ -55,7 +55,8 @@ namespace cluster {
 
         static nlohmann::json fid_terminus(nlohmann::json kwargs)
         {
-            spdlog::trace("fid_terminus");
+            spdlog::trace("fid_terminus starts");
+            spdlog::trace(kwargs.dump()); 
 
             nlohmann::json ret;
 
@@ -73,10 +74,13 @@ namespace cluster {
             }
 
             std::string output = execute_cmd(cmd);
+            spdlog::trace(output); 
 
             ret["cmd"] = cmd;
             ret["output"] = output;
 
+            spdlog::trace("fid_terminus ends");
+
             return ret;
         }
 
@@ -257,6 +261,7 @@ namespace cluster {
                 }
             }
             std::string output = execute_cmd(cmd);
+            spdlog::trace(output);
 
             ret["cmd"] = cmd;
             ret["output"] = output;

test/toolkit/scheduler/job_step.hpp

@@ -10,11 +10,17 @@
 
 #include <list>
 #include <fstream>
+#include <mutex>
 
 #include <boost/algorithm/string.hpp>
 
-namespace cluster {
-    class JobStep {
+namespace cluster
+{
+    class JobStep
+    {
+    public: 
+        static std::mutex mutex; 
+
     public:
         static void remove_files(std::vector<std::string> file_list)
         {
@@ -25,39 +31,48 @@ namespace cluster {
                 std::string cmd = std::string{"rm -rf "} + iter;
                 Common::execute_cmd(cmd);
             }
-
         }
 
         static nlohmann::json gen_key(std::string crypto, std::string shukey_file)
         {
-            spdlog::trace("gen_key");
+            spdlog::trace("gen_key starts");
 
             nlohmann::json param;
             param["crypto"] = crypto;
             param["gen-key"] = "";
             param["no-password"] = "";
             param["output"] = shukey_file;
-//            nlohmann::json param = nlohmann::json::parse(R"(
-//                {
-//                    "crypto": crypto,
-//                    "gen-key": "",
-//                    "no-password": "",
-//                    "output": shukey_file
-//                }
-//            )");
+            //            nlohmann::json param = nlohmann::json::parse(R"(
+            //                {
+            //                    "crypto": crypto,
+            //                    "gen-key": "",
+            //                    "no-password": "",
+            //                    "output": shukey_file
+            //                }
+            //            )");
+            // JobStep::mutex.lock();
             Common::fid_terminus(param);
-            std::ifstream f(shukey_file);
-            nlohmann::json data = nlohmann::json::parse(f);
+            // JobStep::mutex.unlock(); 
+
+            // JobStep::mutex.lock();
+            spdlog::trace("shukey_file={}", shukey_file); 
+            std::ifstream ifs(shukey_file);
+            nlohmann::json data = nlohmann::json::parse(ifs);
+            ifs.close(); 
+            // JobStep::mutex.unlock();
+
+            spdlog::trace("gen_key ends");
+
             return data;
         }
 
         static nlohmann::json seal_data(
-                std::string crypto,
-                std::string data_url,
-                std::string plugin_url,
-                std::string sealed_data_url,
-                std::string sealed_output,
-                std::string data_key_file)
+            std::string crypto,
+            std::string data_url,
+            std::string plugin_url,
+            std::string sealed_data_url,
+            std::string sealed_output,
+            std::string data_key_file)
         {
             spdlog::trace("seal_data");
 
@@ -74,37 +89,44 @@ namespace cluster {
 
         static std::string read_sealed_output(std::string filepath, std::string field)
         {
-            spdlog::trace("read_sealed_output");
-
             std::ifstream ifs(filepath);
+            if (!ifs)
+            {
+                throw std::runtime_error("fail to open file stream");
+            }
 
             std::string line;
             while (std::getline(ifs, line))
             {
-                std::istringstream iss(line);
-                std::string iss_data;
-                std::getline(iss, iss_data, '=');
+                std::vector<std::string> line_split;
+                boost::split(line_split, line, boost::is_any_of("="));
                 std::string key, value;
-                iss >> key >> value;
+                key = line_split[0];
+                value = line_split[1];
                 boost::trim(key);
                 boost::trim(value);
+                if (value.front() == '"')
+                {
+                    value.erase(0, 1);             // erase the first character
+                    value.erase(value.size() - 1); // erase the last character
+                }
                 if (key == field)
                 {
                     return value;
                 }
             }
+            ifs.close(); 
 
             // FIXME: should report error here
             return std::string{""};
         }
 
         static nlohmann::json forward_message(
-                std::string crypto,
-                std::string shukey_file,
-                std::string dian_pkey,
-                std::string enclave_hash,
-                std::string forward_result
-                )
+            std::string crypto,
+            std::string shukey_file,
+            std::string dian_pkey,
+            std::string enclave_hash,
+            std::string forward_result)
         {
             spdlog::trace("forward_message starts");
 
@@ -119,11 +141,15 @@ namespace cluster {
             {
                 param["use-enclave-hash"] = enclave_hash;
             }
+
+            // JobStep::mutex.lock();
             Common::fid_terminus(param);
+            // JobStep::mutex.unlock();
 
             spdlog::trace("forward_message: get forward result");
             std::ifstream ifs(forward_result);
             nlohmann::json output = nlohmann::json::parse(ifs);
+            ifs.close(); 
 
             spdlog::trace("forward_message ends");
 
@@ -159,28 +185,32 @@ namespace cluster {
             return ret;
         }
 
-        static std::string read_parser_hash(std::string parser_url)
+        static std::string read_parser_hash(std::string name, std::string parser_url)
         {
             spdlog::trace("read_parser_hash");
 
             nlohmann::json param;
             param["enclave"] = parser_url;
-            param["output"] = "info.json";
+            std::string name_url = name + "-info.json";
+            param["output"] = name_url;
 
+            // JobStep::mutex.lock();
             nlohmann::json r = Common::fid_dump(param);
+            // JobStep::mutex.unlock(); 
 
-            std::ifstream ifs("info.json");
+            std::ifstream ifs(name_url);
             nlohmann::json data = nlohmann::json::parse(ifs);
+            ifs.close(); 
 
             return data["enclave-hash"];
         }
 
         static nlohmann::json generate_request(
-                std::string crypto,
-                std::string input_param,
-                std::string shukey_file,
-                std::string param_output_url,
-                nlohmann::json config)
+            std::string crypto,
+            std::string input_param,
+            std::string shukey_file,
+            std::string param_output_url,
+            nlohmann::json config)
         {
             spdlog::trace("generate_request starts");
 
@@ -195,11 +225,15 @@ namespace cluster {
             std::string r;
             if (config.contains("request-use-js") && config["request-use-js"] != "")
             {
+                // JobStep::mutex.lock();
                 nlohmann::json r = CommonJs::fid_terminus(param);
+                // JobStep::mutex.unlock();
             }
             else
             {
+                // JobStep::mutex.lock();
                 nlohmann::json r = Common::fid_terminus(param);
+                // JobStep::mutex.unlock();
             }
 
             std::string abs_param_output_url = Common::current_dir / std::filesystem::path(param_output_url);
@@ -211,28 +245,29 @@ namespace cluster {
             }
 
             nlohmann::json ret = nlohmann::json::parse(ifs);
+            ifs.close(); 
 
             spdlog::trace("generate_request ends");
 
             return ret;
         }
 
         static nlohmann::json fid_analyzer_tg(
-                nlohmann::json shukey_json,
-                nlohmann::json rq_forward_json,
-                nlohmann::json algo_shu_info,
-                nlohmann::json algo_forward_json,
-                std::string enclave_hash,
-                std::vector<nlohmann::json> input_data,
-                std::string parser_url,
-                std::string dian_pkey,
-                nlohmann::json model,
-                std::string crypto,
-                nlohmann::json param_json,
-                std::vector<std::string> flat_kgt_pkey_list,
-                std::vector<uint64_t> allowances,
-                std::string parser_input_file,
-                std::string parser_output_file)
+            nlohmann::json shukey_json,
+            nlohmann::json rq_forward_json,
+            nlohmann::json algo_shu_info,
+            nlohmann::json algo_forward_json,
+            std::string enclave_hash,
+            std::vector<nlohmann::json> input_data,
+            std::string parser_url,
+            std::string dian_pkey,
+            nlohmann::json model,
+            std::string crypto,
+            nlohmann::json param_json,
+            std::vector<std::string> flat_kgt_pkey_list,
+            std::vector<uint64_t> allowances,
+            std::string parser_input_file,
+            std::string parser_output_file)
         {
             spdlog::trace("fid_analyzer_tg starts");
 
@@ -268,12 +303,15 @@ namespace cluster {
             param["output"] = parser_output_file;
             nlohmann::json r = Common::fid_analyzer(param);
 
-            try {
+            try
+            {
                 std::ifstream ifs(parser_output_file);
                 spdlog::trace("fid_analyzer_tg ends");
-                return nlohmann::json::parse(ifs);
+                nlohmann::json ret = nlohmann::json::parse(ifs); 
+                ifs.close(); 
+                return ret; 
             }
-            catch (const std::exception& e)
+            catch (const std::exception &e)
             {
                 // do nothing
                 spdlog::error(e.what());
@@ -283,4 +321,4 @@ namespace cluster {
     };
 }
 
-#endif //YPC_JOB_STEP_HPP
+#endif // YPC_JOB_STEP_HPP