common/param_package: Implement lists and nested data structures.

Implements the handling of arbitrarily complex data structures.
The following are supported:
- Lists (like `key:[val1|val2|...]`)
- Dictionaries/Maps (like `key:[k1:v1,k2:[v2a,v2b]]`)

Lists of Dictionaries/Maps look a bit different:
```
key:[k1:v1,k2:[v2a,v2b]|a:x,b:y]
     \_______________/  \_____/
           dict1         dict2
```

src/common/param_package.cpp

@@ -2,9 +2,12 @@
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
+#include <algorithm>
 #include <array>
+#include <regex>
 #include <utility>
 #include <vector>
+#include <boost/algorithm/string/join.hpp>
 #include "common/logging/log.h"
 #include "common/param_package.h"
 #include "common/string_util.h"
@@ -13,6 +16,10 @@ namespace Common {
 
 constexpr char KEY_VALUE_SEPARATOR = ':';
 constexpr char PARAM_SEPARATOR = ',';
+constexpr char LIST_SEPARATOR = '|';
+
+// Matches text enclosed in square brackets.
+const std::regex LIST_MATCH(R"(\[(?:\[??[^\[]*?\]))");
 
 constexpr char ESCAPE_CHARACTER = '$';
 constexpr char KEY_VALUE_SEPARATOR_ESCAPE[] = "$0";
@@ -28,8 +35,11 @@ ParamPackage::ParamPackage(const std::string& serialized) {
         return;
     }
 
+    std::vector<std::string> lookup{};
+    std::string s = PlaceholderifyData(serialized, lookup);
+
     std::vector<std::string> pairs;
-    Common::SplitString(serialized, PARAM_SEPARATOR, pairs);
+    Common::SplitString(s, PARAM_SEPARATOR, pairs);
 
     for (const std::string& pair : pairs) {
         std::vector<std::string> key_value;
@@ -47,6 +57,10 @@ ParamPackage::ParamPackage(const std::string& serialized) {
 
         Set(key_value[0], std::move(key_value[1]));
     }
+
+    for (auto& [key, value] : data) {
+        ReplacePlaceholders(value, lookup);
+    }
 }
 
 ParamPackage::ParamPackage(std::initializer_list<DataType::value_type> list) : data(list) {}
@@ -71,6 +85,8 @@ std::string ParamPackage::Serialize() const {
     return result;
 }
 
+// Getters
+
 std::string ParamPackage::Get(const std::string& key, const std::string& default_value) const {
     auto pair = data.find(key);
     if (pair == data.end()) {
@@ -81,6 +97,28 @@ std::string ParamPackage::Get(const std::string& key, const std::string& default
     return pair->second;
 }
 
+std::vector<std::string> ParamPackage::Get(const std::string& key,
+                                           const std::vector<std::string>& default_value) const {
+    auto pair = data.find(key);
+    if (pair == data.end()) {
+        LOG_DEBUG(Common, "key {} not found", key);
+        return default_value;
+    }
+
+    std::string val = "";
+
+    if (pair->second.front() == '[' && pair->second.back() == ']') {
+        val = pair->second.substr(1, pair->second.size() - 2);
+    } else {
+        LOG_ERROR(Common, "failed to convert {} to vector", pair->second);
+        return default_value;
+    }
+
+    std::vector<std::string> vec;
+    Common::SplitString(val, LIST_SEPARATOR, vec);
+    return vec;
+}
+
 int ParamPackage::Get(const std::string& key, int default_value) const {
     auto pair = data.find(key);
     if (pair == data.end()) {
@@ -96,6 +134,24 @@ int ParamPackage::Get(const std::string& key, int default_value) const {
     }
 }
 
+std::vector<int> ParamPackage::Get(const std::string& key,
+                                   const std::vector<int>& default_value) const {
+    auto vec0 = Get(key, std::vector<std::string>{});
+    if (vec0.empty()) {
+        return default_value;
+    }
+    std::vector<int> vec{};
+    for (auto& str : vec0) {
+        try {
+            vec.emplace_back(std::stoi(str));
+        } catch (const std::logic_error&) {
+            LOG_ERROR(Common, "failed to convert {} to int", str);
+            return default_value;
+        }
+    }
+    return vec;
+}
+
 float ParamPackage::Get(const std::string& key, float default_value) const {
     auto pair = data.find(key);
     if (pair == data.end()) {
@@ -111,18 +167,141 @@ float ParamPackage::Get(const std::string& key, float default_value) const {
     }
 }
 
+std::vector<float> ParamPackage::Get(const std::string& key,
+                                     const std::vector<float>& default_value) const {
+    auto vec0 = Get(key, std::vector<std::string>{});
+    if (vec0.empty()) {
+        return default_value;
+    }
+    std::vector<float> vec{};
+    for (auto& str : vec0) {
+        try {
+            vec.emplace_back(std::stof(str));
+        } catch (const std::logic_error&) {
+            LOG_ERROR(Common, "failed to convert {} to float", str);
+            return default_value;
+        }
+    }
+    return vec;
+}
+
+ParamPackage ParamPackage::Get(const std::string& key, const ParamPackage& default_value) const {
+    auto pair = data.find(key);
+    if (pair == data.end()) {
+        LOG_DEBUG(Common, "key {} not found", key);
+        return default_value;
+    }
+
+    std::string val = "";
+
+    if (pair->second.front() == '[' && pair->second.back() == ']') {
+        val = pair->second.substr(1, pair->second.size() - 2);
+        // Check that the data is actually a ParamPackage to prevent sending garbage data.
+        // Placeholderify is necessary becuase we only want to check the outer layer.
+        std::vector<std::string> lookup{};
+        auto chk = PlaceholderifyData(val, lookup);
+        if (chk.find({LIST_SEPARATOR}) != std::string::npos) {
+            LOG_ERROR(Common, "{} is a vector, not a ParamPackage", pair->second);
+            return default_value;
+        }
+        if (chk.find({PARAM_SEPARATOR}) == std::string::npos) {
+            LOG_ERROR(Common, "{} is a unit vector of a primitive type, not a ParamPackage",
+                      pair->second);
+            return default_value;
+        }
+    } else {
+        LOG_ERROR(Common, "{} is not a ParamPackage", pair->second);
+        return default_value;
+    }
+
+    return ParamPackage{val};
+}
+
+std::vector<ParamPackage> ParamPackage::Get(const std::string& key,
+                                            const std::vector<ParamPackage>& default_value) const {
+    auto pair = data.find(key);
+    if (pair == data.end()) {
+        LOG_DEBUG(Common, "key {} not found", key);
+        return default_value;
+    }
+
+    std::string val = "";
+
+    std::vector<std::string> lookup{};
+    if (pair->second.front() == '[' && pair->second.back() == ']') {
+        val = pair->second.substr(1, pair->second.size() - 2);
+        // Check that the data is actually a ParamPackage to prevent sending garbage data.
+        // Placeholderify is necessary becuase we only want to check the outer layer.
+        val = PlaceholderifyData(val, lookup);
+        if (val.find({PARAM_SEPARATOR}) == std::string::npos) {
+            LOG_ERROR(Common, "{} is a vector of a primitive type, not a ParamPackage",
+                      pair->second);
+            return default_value;
+        }
+    } else {
+        LOG_ERROR(Common, "{} is not a vector", pair->second);
+        return default_value;
+    }
+
+    std::vector<std::string> vec{};
+    Common::SplitString(val, LIST_SEPARATOR, vec);
+    std::for_each(vec.begin(), vec.end(),
+                  [&lookup, this](std::string& in) { ReplacePlaceholders(in, lookup); });
+    std::vector<ParamPackage> packs{};
+    std::transform(vec.begin(), vec.end(), std::back_inserter(packs),
+                   [](std::string& in) { return ParamPackage(in); });
+    return packs;
+}
+
+// Setters
+
 void ParamPackage::Set(const std::string& key, std::string value) {
     data.insert_or_assign(key, std::move(value));
 }
 
+void ParamPackage::Set(const std::string& key, std::vector<std::string> value) {
+    data.insert_or_assign(key,
+                          "[" + boost::algorithm::join(value, std::string{LIST_SEPARATOR}) + "]");
+}
+
 void ParamPackage::Set(const std::string& key, int value) {
     data.insert_or_assign(key, std::to_string(value));
 }
 
+void ParamPackage::Set(const std::string& key, std::vector<int> value) {
+    std::vector<std::string> vec{};
+    std::transform(value.begin(), value.end(), std::back_inserter(vec),
+                   [](float a) -> std::string const { return std::to_string(a); });
+    data.insert_or_assign(key,
+                          "[" + boost::algorithm::join(vec, std::string{LIST_SEPARATOR}) + "]");
+}
+
 void ParamPackage::Set(const std::string& key, float value) {
     data.insert_or_assign(key, std::to_string(value));
 }
 
+void ParamPackage::Set(const std::string& key, std::vector<float> value) {
+    std::vector<std::string> vec{};
+    std::transform(value.begin(), value.end(), std::back_inserter(vec),
+                   [](float a) -> std::string { return std::to_string(a); });
+    data.insert_or_assign(key,
+                          "[" + boost::algorithm::join(vec, std::string{LIST_SEPARATOR}) + "]");
+}
+
+void ParamPackage::Set(const std::string& key, ParamPackage value) {
+    data.insert_or_assign(key, std::move(value.Serialize()));
+}
+
+void ParamPackage::Set(const std::string& key, std::vector<ParamPackage> value) {
+    std::vector<std::string> vec{};
+    std::transform(value.begin(), value.end(), std::back_inserter(vec),
+                   [](ParamPackage& a) -> std::string { return a.Serialize(); });
+    data.insert_or_assign(key,
+                          "[" + boost::algorithm::join(vec, std::string{LIST_SEPARATOR}) + "]");
+}
+
+// Other methods
+
 bool ParamPackage::Has(const std::string& key) const {
     return data.find(key) != data.end();
 }
@@ -151,4 +330,35 @@ ParamPackage::DataType::const_iterator ParamPackage::end() const {
     return data.end();
 }
 
+// Static helper methods
+
+std::string ParamPackage::PlaceholderifyData(const std::string& str,
+                                             std::vector<std::string>& lookup) {
+    std::smatch m;
+    std::string s = str;
+    while (std::regex_search(s, m, LIST_MATCH)) {
+        for (auto x : m) {
+            s.replace(s.find(x.str()), x.length(), "##" + std::to_string(lookup.size()));
+            lookup.emplace_back(x.str());
+        }
+    }
+    return s;
+}
+
+void ParamPackage::ReplacePlaceholders(std::string& str, const std::vector<std::string>& lookup) {
+    std::smatch m;
+    while (std::regex_search(str, m, std::regex(R"(##([\d]+))"))) {
+        for (auto x : m) {
+            str.replace(str.find("##" + x.str()), 2 + x.length(), lookup[std::stoi(x.str())]);
+        }
+    }
+}
+
+std::string ParamPackage::ReplacePlaceholders(const std::string& str,
+                                              const std::vector<std::string>& lookup) {
+    std::string s = str;
+    ReplacePlaceholders(s, lookup);
+    return s;
+}
+
 } // namespace Common

src/common/param_package.h

@@ -7,6 +7,7 @@
 #include <initializer_list>
 #include <map>
 #include <string>
+#include <vector>
 
 namespace Common {
 
@@ -25,12 +26,30 @@ public:
     ParamPackage& operator=(ParamPackage&& other) = default;
 
     std::string Serialize() const;
+
+    // Getters
     std::string Get(const std::string& key, const std::string& default_value) const;
+    std::vector<std::string> Get(const std::string& key,
+                                 const std::vector<std::string>& default_value) const;
     int Get(const std::string& key, int default_value) const;
+    std::vector<int> Get(const std::string& key, const std::vector<int>& default_value) const;
     float Get(const std::string& key, float default_value) const;
+    std::vector<float> Get(const std::string& key, const std::vector<float>& default_value) const;
+    ParamPackage Get(const std::string& key, const ParamPackage& default_value) const;
+    std::vector<ParamPackage> Get(const std::string& key,
+                                  const std::vector<ParamPackage>& default_value) const;
+
+    // Setters
     void Set(const std::string& key, std::string value);
+    void Set(const std::string& key, std::vector<std::string> value);
     void Set(const std::string& key, int value);
+    void Set(const std::string& key, std::vector<int> value);
     void Set(const std::string& key, float value);
+    void Set(const std::string& key, std::vector<float> value);
+    void Set(const std::string& key, ParamPackage value);
+    void Set(const std::string& key, std::vector<ParamPackage> value);
+
+    // Other methods
     bool Has(const std::string& key) const;
     void Erase(const std::string& key);
     void Clear();
@@ -43,6 +62,24 @@ public:
 
 private:
     DataType data;
+
+    /**
+     * Replace complex data structures with placeholders, and generate the lookup table.
+     * The lookup table is emptied before being used.
+     */
+    static std::string PlaceholderifyData(const std::string& str, std::vector<std::string>& lookup);
+
+    /**
+     * Replace placeholders (in-place) with original data obtained from a lookup table.
+     */
+    static void ReplacePlaceholders(std::string& str, const std::vector<std::string>& lookup);
+
+    /**
+     * Replace placeholders with original data obtained from a lookup table and returns the
+     * resultant string.
+     */
+    static std::string ReplacePlaceholders(const std::string& str,
+                                           const std::vector<std::string>& lookup);
 };
 
 } // namespace Common