From eebb545b40b60e0f27a4f1e0f3c9ff3ec71fb303 Mon Sep 17 00:00:00 2001 From: adityaruplaha <30696515+adityaruplaha@users.noreply.github.com> Date: Tue, 30 Jun 2020 23:25:08 +0530 Subject: [PATCH] common/param_package: Implement lists and nested data structures. Implements the handling of arbitrarily complex data structures. The following are supported: - Lists (like `key:[val1|val2|...]`) - Dictionaries/Maps (like `key:[k1:v1,k2:[v2a,v2b]]`) Lists of Dictionaries/Maps look a bit different: ``` key:[k1:v1,k2:[v2a,v2b]|a:x,b:y] \_______________/ \_____/ dict1 dict2 ``` --- src/common/param_package.cpp | 212 ++++++++++++++++++++++++++++++++++- src/common/param_package.h | 37 ++++++ 2 files changed, 248 insertions(+), 1 deletion(-) diff --git a/src/common/param_package.cpp b/src/common/param_package.cpp index 3a218efbc..15645999a 100644 --- a/src/common/param_package.cpp +++ b/src/common/param_package.cpp @@ -2,9 +2,12 @@ // Licensed under GPLv2 or any later version // Refer to the license.txt file included. +#include #include +#include #include #include +#include #include "common/logging/log.h" #include "common/param_package.h" #include "common/string_util.h" @@ -13,6 +16,10 @@ namespace Common { constexpr char KEY_VALUE_SEPARATOR = ':'; constexpr char PARAM_SEPARATOR = ','; +constexpr char LIST_SEPARATOR = '|'; + +// Matches text enclosed in square brackets. +const std::regex LIST_MATCH(R"(\[(?:\[??[^\[]*?\]))"); constexpr char ESCAPE_CHARACTER = '$'; constexpr char KEY_VALUE_SEPARATOR_ESCAPE[] = "$0"; @@ -28,8 +35,11 @@ ParamPackage::ParamPackage(const std::string& serialized) { return; } + std::vector lookup{}; + std::string s = PlaceholderifyData(serialized, lookup); + std::vector pairs; - Common::SplitString(serialized, PARAM_SEPARATOR, pairs); + Common::SplitString(s, PARAM_SEPARATOR, pairs); for (const std::string& pair : pairs) { std::vector key_value; @@ -47,6 +57,10 @@ ParamPackage::ParamPackage(const std::string& serialized) { Set(key_value[0], std::move(key_value[1])); } + + for (auto& [key, value] : data) { + ReplacePlaceholders(value, lookup); + } } ParamPackage::ParamPackage(std::initializer_list list) : data(list) {} @@ -71,6 +85,8 @@ std::string ParamPackage::Serialize() const { return result; } +// Getters + std::string ParamPackage::Get(const std::string& key, const std::string& default_value) const { auto pair = data.find(key); if (pair == data.end()) { @@ -81,6 +97,28 @@ std::string ParamPackage::Get(const std::string& key, const std::string& default return pair->second; } +std::vector ParamPackage::Get(const std::string& key, + const std::vector& default_value) const { + auto pair = data.find(key); + if (pair == data.end()) { + LOG_DEBUG(Common, "key {} not found", key); + return default_value; + } + + std::string val = ""; + + if (pair->second.front() == '[' && pair->second.back() == ']') { + val = pair->second.substr(1, pair->second.size() - 2); + } else { + LOG_ERROR(Common, "failed to convert {} to vector", pair->second); + return default_value; + } + + std::vector vec; + Common::SplitString(val, LIST_SEPARATOR, vec); + return vec; +} + int ParamPackage::Get(const std::string& key, int default_value) const { auto pair = data.find(key); if (pair == data.end()) { @@ -96,6 +134,24 @@ int ParamPackage::Get(const std::string& key, int default_value) const { } } +std::vector ParamPackage::Get(const std::string& key, + const std::vector& default_value) const { + auto vec0 = Get(key, std::vector{}); + if (vec0.empty()) { + return default_value; + } + std::vector vec{}; + for (auto& str : vec0) { + try { + vec.emplace_back(std::stoi(str)); + } catch (const std::logic_error&) { + LOG_ERROR(Common, "failed to convert {} to int", str); + return default_value; + } + } + return vec; +} + float ParamPackage::Get(const std::string& key, float default_value) const { auto pair = data.find(key); if (pair == data.end()) { @@ -111,18 +167,141 @@ float ParamPackage::Get(const std::string& key, float default_value) const { } } +std::vector ParamPackage::Get(const std::string& key, + const std::vector& default_value) const { + auto vec0 = Get(key, std::vector{}); + if (vec0.empty()) { + return default_value; + } + std::vector vec{}; + for (auto& str : vec0) { + try { + vec.emplace_back(std::stof(str)); + } catch (const std::logic_error&) { + LOG_ERROR(Common, "failed to convert {} to float", str); + return default_value; + } + } + return vec; +} + +ParamPackage ParamPackage::Get(const std::string& key, const ParamPackage& default_value) const { + auto pair = data.find(key); + if (pair == data.end()) { + LOG_DEBUG(Common, "key {} not found", key); + return default_value; + } + + std::string val = ""; + + if (pair->second.front() == '[' && pair->second.back() == ']') { + val = pair->second.substr(1, pair->second.size() - 2); + // Check that the data is actually a ParamPackage to prevent sending garbage data. + // Placeholderify is necessary becuase we only want to check the outer layer. + std::vector lookup{}; + auto chk = PlaceholderifyData(val, lookup); + if (chk.find({LIST_SEPARATOR}) != std::string::npos) { + LOG_ERROR(Common, "{} is a vector, not a ParamPackage", pair->second); + return default_value; + } + if (chk.find({PARAM_SEPARATOR}) == std::string::npos) { + LOG_ERROR(Common, "{} is a unit vector of a primitive type, not a ParamPackage", + pair->second); + return default_value; + } + } else { + LOG_ERROR(Common, "{} is not a ParamPackage", pair->second); + return default_value; + } + + return ParamPackage{val}; +} + +std::vector ParamPackage::Get(const std::string& key, + const std::vector& default_value) const { + auto pair = data.find(key); + if (pair == data.end()) { + LOG_DEBUG(Common, "key {} not found", key); + return default_value; + } + + std::string val = ""; + + std::vector lookup{}; + if (pair->second.front() == '[' && pair->second.back() == ']') { + val = pair->second.substr(1, pair->second.size() - 2); + // Check that the data is actually a ParamPackage to prevent sending garbage data. + // Placeholderify is necessary becuase we only want to check the outer layer. + val = PlaceholderifyData(val, lookup); + if (val.find({PARAM_SEPARATOR}) == std::string::npos) { + LOG_ERROR(Common, "{} is a vector of a primitive type, not a ParamPackage", + pair->second); + return default_value; + } + } else { + LOG_ERROR(Common, "{} is not a vector", pair->second); + return default_value; + } + + std::vector vec{}; + Common::SplitString(val, LIST_SEPARATOR, vec); + std::for_each(vec.begin(), vec.end(), + [&lookup, this](std::string& in) { ReplacePlaceholders(in, lookup); }); + std::vector packs{}; + std::transform(vec.begin(), vec.end(), std::back_inserter(packs), + [](std::string& in) { return ParamPackage(in); }); + return packs; +} + +// Setters + void ParamPackage::Set(const std::string& key, std::string value) { data.insert_or_assign(key, std::move(value)); } +void ParamPackage::Set(const std::string& key, std::vector value) { + data.insert_or_assign(key, + "[" + boost::algorithm::join(value, std::string{LIST_SEPARATOR}) + "]"); +} + void ParamPackage::Set(const std::string& key, int value) { data.insert_or_assign(key, std::to_string(value)); } +void ParamPackage::Set(const std::string& key, std::vector value) { + std::vector vec{}; + std::transform(value.begin(), value.end(), std::back_inserter(vec), + [](float a) -> std::string const { return std::to_string(a); }); + data.insert_or_assign(key, + "[" + boost::algorithm::join(vec, std::string{LIST_SEPARATOR}) + "]"); +} + void ParamPackage::Set(const std::string& key, float value) { data.insert_or_assign(key, std::to_string(value)); } +void ParamPackage::Set(const std::string& key, std::vector value) { + std::vector vec{}; + std::transform(value.begin(), value.end(), std::back_inserter(vec), + [](float a) -> std::string { return std::to_string(a); }); + data.insert_or_assign(key, + "[" + boost::algorithm::join(vec, std::string{LIST_SEPARATOR}) + "]"); +} + +void ParamPackage::Set(const std::string& key, ParamPackage value) { + data.insert_or_assign(key, std::move(value.Serialize())); +} + +void ParamPackage::Set(const std::string& key, std::vector value) { + std::vector vec{}; + std::transform(value.begin(), value.end(), std::back_inserter(vec), + [](ParamPackage& a) -> std::string { return a.Serialize(); }); + data.insert_or_assign(key, + "[" + boost::algorithm::join(vec, std::string{LIST_SEPARATOR}) + "]"); +} + +// Other methods + bool ParamPackage::Has(const std::string& key) const { return data.find(key) != data.end(); } @@ -151,4 +330,35 @@ ParamPackage::DataType::const_iterator ParamPackage::end() const { return data.end(); } +// Static helper methods + +std::string ParamPackage::PlaceholderifyData(const std::string& str, + std::vector& lookup) { + std::smatch m; + std::string s = str; + while (std::regex_search(s, m, LIST_MATCH)) { + for (auto x : m) { + s.replace(s.find(x.str()), x.length(), "##" + std::to_string(lookup.size())); + lookup.emplace_back(x.str()); + } + } + return s; +} + +void ParamPackage::ReplacePlaceholders(std::string& str, const std::vector& lookup) { + std::smatch m; + while (std::regex_search(str, m, std::regex(R"(##([\d]+))"))) { + for (auto x : m) { + str.replace(str.find("##" + x.str()), 2 + x.length(), lookup[std::stoi(x.str())]); + } + } +} + +std::string ParamPackage::ReplacePlaceholders(const std::string& str, + const std::vector& lookup) { + std::string s = str; + ReplacePlaceholders(s, lookup); + return s; +} + } // namespace Common diff --git a/src/common/param_package.h b/src/common/param_package.h index 1fffb5035..e6cec107e 100644 --- a/src/common/param_package.h +++ b/src/common/param_package.h @@ -7,6 +7,7 @@ #include #include #include +#include namespace Common { @@ -25,12 +26,30 @@ public: ParamPackage& operator=(ParamPackage&& other) = default; std::string Serialize() const; + + // Getters std::string Get(const std::string& key, const std::string& default_value) const; + std::vector Get(const std::string& key, + const std::vector& default_value) const; int Get(const std::string& key, int default_value) const; + std::vector Get(const std::string& key, const std::vector& default_value) const; float Get(const std::string& key, float default_value) const; + std::vector Get(const std::string& key, const std::vector& default_value) const; + ParamPackage Get(const std::string& key, const ParamPackage& default_value) const; + std::vector Get(const std::string& key, + const std::vector& default_value) const; + + // Setters void Set(const std::string& key, std::string value); + void Set(const std::string& key, std::vector value); void Set(const std::string& key, int value); + void Set(const std::string& key, std::vector value); void Set(const std::string& key, float value); + void Set(const std::string& key, std::vector value); + void Set(const std::string& key, ParamPackage value); + void Set(const std::string& key, std::vector value); + + // Other methods bool Has(const std::string& key) const; void Erase(const std::string& key); void Clear(); @@ -43,6 +62,24 @@ public: private: DataType data; + + /** + * Replace complex data structures with placeholders, and generate the lookup table. + * The lookup table is emptied before being used. + */ + static std::string PlaceholderifyData(const std::string& str, std::vector& lookup); + + /** + * Replace placeholders (in-place) with original data obtained from a lookup table. + */ + static void ReplacePlaceholders(std::string& str, const std::vector& lookup); + + /** + * Replace placeholders with original data obtained from a lookup table and returns the + * resultant string. + */ + static std::string ReplacePlaceholders(const std::string& str, + const std::vector& lookup); }; } // namespace Common