add support for quotes in the config naming to match TOML standard (#967)

This PR is to further support for TOML. To allow and generate quoted
names in config files including those separated by the parent separator.

like 
```toml
"sub"."sub2".value=1
'sub'.'sub.sub'.value=2
```

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

include/CLI/Config.hpp

@@ -37,6 +37,8 @@ std::string ini_join(const std::vector<std::string> &args,
                      char stringQuote = '"',
                      char literalQuote = '\'');
 
+void clean_name_string(std::string &name, const std::string &keyChars);
+
 std::vector<std::string> generate_parents(const std::string &section, std::string &name, char parentSeparator);
 
 /// assuming non default segments do a check on the close and open of the segments in a configItem structure

include/CLI/impl/Config_inl.hpp

@@ -122,23 +122,19 @@ generate_parents(const std::string &section, std::string &name, char parentSepar
     std::vector<std::string> parents;
     if(detail::to_lower(section) != "default") {
         if(section.find(parentSeparator) != std::string::npos) {
-            parents = detail::split(section, parentSeparator);
+            parents = detail::split_up(section, parentSeparator);
         } else {
             parents = {section};
         }
     }
     if(name.find(parentSeparator) != std::string::npos) {
-        std::vector<std::string> plist = detail::split(name, parentSeparator);
+        std::vector<std::string> plist = detail::split_up(name, parentSeparator);
         name = plist.back();
-        detail::remove_quotes(name);
         plist.pop_back();
         parents.insert(parents.end(), plist.begin(), plist.end());
     }
-
     // clean up quotes on the parents
-    for(auto &parent : parents) {
+    detail::remove_quotes(parents);
-        detail::remove_quotes(parent);
-    }
     return parents;
 }
 
@@ -218,10 +214,10 @@ inline std::vector<ConfigItem> ConfigBase::from_config(std::istream &input) cons
     char aSep = (isINIArray && arraySeparator == ' ') ? ',' : arraySeparator;
     int currentSectionIndex{0};
 
+    std::string line_sep_chars{parentSeparatorChar, commentChar, valueDelimiter};
     while(getline(input, buffer)) {
         std::vector<std::string> items_buffer;
         std::string name;
-        bool literalName{false};
         line = detail::trim_copy(buffer);
         std::size_t len = line.length();
         // lines have to be at least 3 characters to have any meaning to CLI just skip the rest
@@ -275,8 +271,21 @@ inline std::vector<ConfigItem> ConfigBase::from_config(std::istream &input) cons
             continue;
         }
         std::size_t search_start = 0;
-        if(line.front() == stringQuote || line.front() == literalQuote || line.front() == '`') {
+        if(line.find_first_of("\"'`") != std::string::npos) {
-            search_start = detail::close_sequence(line, 0, line.front());
+            while(search_start < line.size()) {
+                auto test_char = line[search_start];
+                if(test_char == '\"' || test_char == '\'' || test_char == '`') {
+                    search_start = detail::close_sequence(line, search_start, line[search_start]);
+                    ++search_start;
+                } else if(test_char == valueDelimiter || test_char == commentChar) {
+                    --search_start;
+                    break;
+                } else if(test_char == ' ' || test_char == '\t' || test_char == parentSeparatorChar) {
+                    ++search_start;
+                } else {
+                    search_start = line.find_first_of(line_sep_chars, search_start);
+                }
+            }
         }
         // Find = in string, split and recombine
         auto delimiter_pos = line.find_first_of(valueDelimiter, search_start + 1);
@@ -290,7 +299,7 @@ inline std::vector<ConfigItem> ConfigBase::from_config(std::istream &input) cons
             std::string item = detail::trim_copy(line.substr(delimiter_pos + 1, std::string::npos));
             bool mlquote =
                 (item.compare(0, 3, multiline_literal_quote) == 0 || item.compare(0, 3, multiline_string_quote) == 0);
-            if(!mlquote && comment_pos != std::string::npos && !literalName) {
+            if(!mlquote && comment_pos != std::string::npos) {
                 auto citems = detail::split_up(item, commentChar);
                 item = detail::trim_copy(citems.front());
             }
@@ -365,9 +374,10 @@ inline std::vector<ConfigItem> ConfigBase::from_config(std::istream &input) cons
             name = detail::trim_copy(line.substr(0, comment_pos));
             items_buffer = {"true"};
         }
+        std::vector<std::string> parents;
         try {
-            literalName = detail::process_quoted_string(name, stringQuote, literalQuote);
+            parents = detail::generate_parents(currentSection, name, parentSeparatorChar);
-
+            detail::process_quoted_string(name);
             // clean up quotes on the items and check for escaped strings
             for(auto &it : items_buffer) {
                 detail::process_quoted_string(it, stringQuote, literalQuote);
@@ -375,13 +385,7 @@ inline std::vector<ConfigItem> ConfigBase::from_config(std::istream &input) cons
         } catch(const std::invalid_argument &ia) {
             throw CLI::ParseError(ia.what(), CLI::ExitCodes::InvalidError);
         }
-        std::vector<std::string> parents;
+
-        if(literalName) {
-            std::string noname{};
-            parents = detail::generate_parents(currentSection, noname, parentSeparatorChar);
-        } else {
-            parents = detail::generate_parents(currentSection, name, parentSeparatorChar);
-        }
         if(parents.size() > maximumLayers) {
             continue;
         }
@@ -418,6 +422,23 @@ inline std::vector<ConfigItem> ConfigBase::from_config(std::istream &input) cons
     return output;
 }
 
+CLI11_INLINE std::string &clean_name_string(std::string &name, const std::string &keyChars) {
+    if(name.find_first_of(keyChars) != std::string::npos || (name.front() == '[' && name.back() == ']') ||
+       (name.find_first_of("'`\"\\") != std::string::npos)) {
+        if(name.find_first_of('\'') == std::string::npos) {
+            name.insert(0, 1, '\'');
+            name.push_back('\'');
+        } else {
+            if(detail::has_escapable_character(name)) {
+                name = detail::add_escaped_characters(name);
+            }
+            name.insert(0, 1, '\"');
+            name.push_back('\"');
+        }
+    }
+    return name;
+}
+
 CLI11_INLINE std::string
 ConfigBase::to_config(const App *app, bool default_also, bool write_description, std::string prefix) const {
     std::stringstream out;
@@ -429,6 +450,14 @@ ConfigBase::to_config(const App *app, bool default_also, bool write_description,
     commentTest.push_back(commentChar);
     commentTest.push_back(parentSeparatorChar);
 
+    std::string keyChars = commentTest;
+    keyChars.push_back(literalQuote);
+    keyChars.push_back(stringQuote);
+    keyChars.push_back(arrayStart);
+    keyChars.push_back(arrayEnd);
+    keyChars.push_back(valueDelimiter);
+    keyChars.push_back(arraySeparator);
+
     std::vector<std::string> groups = app->get_groups();
     bool defaultUsed = false;
     groups.insert(groups.begin(), std::string("Options"));
@@ -498,24 +527,7 @@ ConfigBase::to_config(const App *app, bool default_also, bool write_description,
                         out << '\n';
                         out << commentLead << detail::fix_newlines(commentLead, opt->get_description()) << '\n';
                     }
-                    if(single_name.find_first_of(commentTest) != std::string::npos ||
+                    clean_name_string(single_name, keyChars);
-                       single_name.compare(0, 3, multiline_string_quote) == 0 ||
-                       single_name.compare(0, 3, multiline_literal_quote) == 0 ||
-                       (single_name.front() == '[' && single_name.back() == ']') ||
-                       (single_name.find_first_of(stringQuote) != std::string::npos) ||
-                       (single_name.find_first_of(literalQuote) != std::string::npos) ||
-                       (single_name.find_first_of('`') != std::string::npos)) {
-                        if(single_name.find_first_of(literalQuote) == std::string::npos) {
-                            single_name.insert(0, 1, literalQuote);
-                            single_name.push_back(literalQuote);
-                        } else {
-                            if(detail::has_escapable_character(single_name)) {
-                                single_name = detail::add_escaped_characters(single_name);
-                            }
-                            single_name.insert(0, 1, stringQuote);
-                            single_name.push_back(stringQuote);
-                        }
-                    }
 
                     std::string name = prefix + single_name;
 
@@ -554,22 +566,29 @@ ConfigBase::to_config(const App *app, bool default_also, bool write_description,
             if(!default_also && (subcom->count_all() == 0)) {
                 continue;
             }
+            std::string subname = subcom->get_name();
+            clean_name_string(subname, keyChars);
+
             if(subcom->get_configurable() && app->got_subcommand(subcom)) {
                 if(!prefix.empty() || app->get_parent() == nullptr) {
-                    out << '[' << prefix << subcom->get_name() << "]\n";
+
+                    out << '[' << prefix << subname << "]\n";
                 } else {
-                    std::string subname = app->get_name() + parentSeparatorChar + subcom->get_name();
+                    std::string appname = app->get_name();
+                    clean_name_string(appname, keyChars);
+                    subname = appname + parentSeparatorChar + subname;
                     const auto *p = app->get_parent();
                     while(p->get_parent() != nullptr) {
-                        subname = p->get_name() + parentSeparatorChar + subname;
+                        std::string pname = p->get_name();
+                        clean_name_string(pname, keyChars);
+                        subname = pname + parentSeparatorChar + subname;
                         p = p->get_parent();
                     }
                     out << '[' << subname << "]\n";
                 }
                 out << to_config(subcom, default_also, write_description, "");
             } else {
-                out << to_config(
+                out << to_config(subcom, default_also, write_description, prefix + subname + parentSeparatorChar);
-                    subcom, default_also, write_description, prefix + subcom->get_name() + parentSeparatorChar);
             }
         }
     }

tests/ConfigFileTest.cpp

@@ -501,6 +501,38 @@ TEST_CASE("StringBased: Layers2LevelChange", "[config]") {
     CHECK(checkSections(output));
 }
 
+TEST_CASE("StringBased: Layers2LevelChangeInQuotes", "[config]") {
+    std::stringstream ofile;
+
+    ofile << "simple = true\n\n";
+    ofile << "[\"other\".\"sub2\".cmd]\n";
+    ofile << "[other.\"sub3\".\"cmd\"]\n";
+    ofile << "absolute_newest = true\n";
+    ofile.seekg(0, std::ios::beg);
+
+    std::vector<CLI::ConfigItem> output = CLI::ConfigINI().from_config(ofile);
+
+    // 2 flags and 5 openings and 5 closings
+    CHECK(output.size() == 12u);
+    CHECK(checkSections(output));
+}
+
+TEST_CASE("StringBased: Layers2LevelChangeInQuotesWithDot", "[config]") {
+    std::stringstream ofile;
+
+    ofile << "simple = true\n\n";
+    ofile << "[\"other\".\"sub2.cmd\"]\n";
+    ofile << "[other.\"sub3.cmd\"]\n";
+    ofile << "absolute_newest = true\n";
+    ofile.seekg(0, std::ios::beg);
+
+    std::vector<CLI::ConfigItem> output = CLI::ConfigINI().from_config(ofile);
+
+    // 2 flags and 3 openings and 3 closings
+    CHECK(output.size() == 8u);
+    CHECK(checkSections(output));
+}
+
 TEST_CASE("StringBased: Layers3LevelChange", "[config]") {
     std::stringstream ofile;
 
@@ -1583,6 +1615,45 @@ TEST_CASE_METHOD(TApp, "IniLayeredDotSection", "[config]") {
     CHECK(three == 0);
 }
 
+TEST_CASE_METHOD(TApp, "IniLayeredDotSectionInQuotes", "[config]") {
+
+    TempFile tmpini{"TestIniTmp.ini"};
+
+    app.set_config("--config", tmpini);
+
+    {
+        std::ofstream out{tmpini};
+        out << "[default]" << std::endl;
+        out << "val=1" << std::endl;
+        out << "['subcom']" << std::endl;
+        out << "val=2" << std::endl;
+        out << "['subcom'.\"subsubcom\"]" << std::endl;
+        out << "val=3" << std::endl;
+    }
+
+    int one{0}, two{0}, three{0};
+    app.add_option("--val", one);
+    auto *subcom = app.add_subcommand("subcom");
+    subcom->add_option("--val", two);
+    auto *subsubcom = subcom->add_subcommand("subsubcom");
+    subsubcom->add_option("--val", three);
+
+    run();
+
+    CHECK(one == 1);
+    CHECK(two == 2);
+    CHECK(three == 3);
+
+    CHECK(0U == subcom->count());
+    CHECK(!*subcom);
+
+    three = 0;
+    // check maxlayers
+    app.get_config_formatter_base()->maxLayers(1);
+    run();
+    CHECK(three == 0);
+}
+
 TEST_CASE_METHOD(TApp, "IniLayeredCustomSectionSeparator", "[config]") {
 
     TempFile tmpini{"TestIniTmp.ini"};
@@ -1674,6 +1745,138 @@ TEST_CASE_METHOD(TApp, "IniSubcommandConfigurable", "[config]") {
     CHECK(app.got_subcommand(subcom));
 }
 
+TEST_CASE_METHOD(TApp, "IniSubcommandConfigurableInQuotes", "[config]") {
+
+    TempFile tmpini{"TestIniTmp.ini"};
+
+    app.set_config("--config", tmpini);
+
+    {
+        std::ofstream out{tmpini};
+        out << "[default]" << std::endl;
+        out << "val=1" << std::endl;
+        out << "[subcom]" << std::endl;
+        out << "val=2" << std::endl;
+        out << "\"subsubcom\".'val'=3" << std::endl;
+    }
+
+    int one{0}, two{0}, three{0};
+    app.add_option("--val", one);
+    auto *subcom = app.add_subcommand("subcom");
+    subcom->configurable();
+    subcom->add_option("--val", two);
+    auto *subsubcom = subcom->add_subcommand("subsubcom");
+    subsubcom->add_option("--val", three);
+
+    run();
+
+    CHECK(one == 1);
+    CHECK(two == 2);
+    CHECK(three == 3);
+
+    CHECK(1U == subcom->count());
+    CHECK(*subcom);
+    CHECK(app.got_subcommand(subcom));
+}
+
+TEST_CASE_METHOD(TApp, "IniSubcommandConfigurableInQuotesAlias", "[config]") {
+
+    TempFile tmpini{"TestIniTmp.ini"};
+
+    app.set_config("--config", tmpini);
+
+    {
+        std::ofstream out{tmpini};
+        out << "[default]" << std::endl;
+        out << "val=1" << std::endl;
+        out << "[subcom]" << std::endl;
+        out << "val=2" << std::endl;
+        out << R"("sub\tsub\t.com".'val'=3)" << std::endl;
+    }
+
+    int one{0}, two{0}, three{0};
+    app.add_option("--val", one);
+    auto *subcom = app.add_subcommand("subcom");
+    subcom->configurable();
+    subcom->add_option("--val", two);
+    auto *subsubcom = subcom->add_subcommand("subsubcom")->alias("sub\tsub\t.com");
+    subsubcom->add_option("--val", three);
+
+    run();
+
+    CHECK(one == 1);
+    CHECK(two == 2);
+    CHECK(three == 3);
+
+    CHECK(1U == subcom->count());
+    CHECK(*subcom);
+    CHECK(app.got_subcommand(subcom));
+}
+
+TEST_CASE_METHOD(TApp, "IniSubcommandConfigurableInQuotesAliasWithEquals", "[config]") {
+
+    TempFile tmpini{"TestIniTmp.ini"};
+
+    app.set_config("--config", tmpini);
+
+    {
+        std::ofstream out{tmpini};
+        out << "[default]" << std::endl;
+        out << "val=1" << std::endl;
+        out << "[subcom]" << std::endl;
+        out << "val=2" << std::endl;
+        out << R"("sub=sub=.com".'val'=3)" << std::endl;
+    }
+
+    int one{0}, two{0}, three{0};
+    app.add_option("--val", one);
+    auto *subcom = app.add_subcommand("subcom");
+    subcom->configurable();
+    subcom->add_option("--val", two);
+    auto *subsubcom = subcom->add_subcommand("subsubcom")->alias("sub=sub=.com");
+    subsubcom->add_option("--val", three);
+
+    run();
+
+    CHECK(one == 1);
+    CHECK(two == 2);
+    CHECK(three == 3);
+
+    CHECK(1U == subcom->count());
+    CHECK(*subcom);
+    CHECK(app.got_subcommand(subcom));
+}
+
+TEST_CASE_METHOD(TApp, "IniSubcommandConfigurableInQuotesAliasWithComment", "[config]") {
+
+    TempFile tmpini{"TestIniTmp.ini"};
+
+    app.set_config("--config", tmpini);
+
+    {
+        std::ofstream out{tmpini};
+        out << "[default]" << std::endl;
+        out << "val=1" << std::endl;
+        out << "[subcom]" << std::endl;
+        out << "val=2" << std::endl;
+        out << R"("sub#sub;.com".'val'=3)" << std::endl;
+    }
+
+    int one{0}, two{0}, three{0};
+    app.add_option("--val", one);
+    auto *subcom = app.add_subcommand("subcom");
+    subcom->configurable();
+    subcom->add_option("--val", two);
+    auto *subsubcom = subcom->add_subcommand("subsubcom")->alias("sub#sub;.com");
+    subsubcom->add_option("--val", three);
+
+    run();
+
+    CHECK(one == 1);
+    CHECK(two == 2);
+    CHECK(three == 3);
+}
+
 TEST_CASE_METHOD(TApp, "IniSubcommandConfigurablePreParse", "[config]") {
 
     TempFile tmpini{"TestIniTmp.ini"};
@@ -2181,6 +2384,57 @@ TEST_CASE_METHOD(TApp, "IniShort", "[config]") {
     CHECK(3 == key);
 }
 
+TEST_CASE_METHOD(TApp, "IniShortQuote1", "[config]") {
+
+    TempFile tmpini{"TestIniTmp.ini"};
+
+    int key{0};
+    app.add_option("--flag,-f", key);
+    app.set_config("--config", tmpini);
+
+    {
+        std::ofstream out{tmpini};
+        out << "\"f\"=3" << std::endl;
+    }
+
+    REQUIRE_NOTHROW(run());
+    CHECK(3 == key);
+}
+
+TEST_CASE_METHOD(TApp, "IniShortQuote2", "[config]") {
+
+    TempFile tmpini{"TestIniTmp.ini"};
+
+    int key{0};
+    app.add_option("--flag,-f", key);
+    app.set_config("--config", tmpini);
+
+    {
+        std::ofstream out{tmpini};
+        out << "'f'=3" << std::endl;
+    }
+
+    REQUIRE_NOTHROW(run());
+    CHECK(3 == key);
+}
+
+TEST_CASE_METHOD(TApp, "IniShortQuote3", "[config]") {
+
+    TempFile tmpini{"TestIniTmp.ini"};
+
+    int key{0};
+    app.add_option("--flag,-f", key);
+    app.set_config("--config", tmpini);
+
+    {
+        std::ofstream out{tmpini};
+        out << "`f`=3" << std::endl;
+    }
+
+    REQUIRE_NOTHROW(run());
+    CHECK(3 == key);
+}
+
 TEST_CASE_METHOD(TApp, "IniDefaultPath", "[config]") {
 
     TempFile tmpini{"../TestIniTmp.ini"};
@@ -3388,6 +3642,23 @@ TEST_CASE_METHOD(TApp, "IniOutputSubsubcom", "[config]") {
     CHECK_THAT(str, Contains("other.sub2.newest=true"));
 }
 
+TEST_CASE_METHOD(TApp, "IniOutputSubsubcomWithDot", "[config]") {
+
+    app.add_flag("--simple");
+    auto *subcom = app.add_subcommand("other");
+    subcom->add_flag("--newer");
+    auto *subsubcom = subcom->add_subcommand("sub2.bb");
+    subsubcom->add_flag("--newest");
+    app.config_formatter(std::make_shared<CLI::ConfigINI>());
+    args = {"--simple", "other", "--newer", "sub2.bb", "--newest"};
+    run();
+
+    std::string str = app.config_to_str();
+    CHECK_THAT(str, Contains("simple=true"));
+    CHECK_THAT(str, Contains("other.newer=true"));
+    CHECK_THAT(str, Contains("other.'sub2.bb'.newest=true"));
+}
+
 TEST_CASE_METHOD(TApp, "IniOutputSubsubcomCustomSep", "[config]") {
 
     app.add_flag("--simple");
@@ -3406,6 +3677,42 @@ TEST_CASE_METHOD(TApp, "IniOutputSubsubcomCustomSep", "[config]") {
     CHECK_THAT(str, Contains("other|sub2|newest=true"));
 }
 
+TEST_CASE_METHOD(TApp, "IniOutputSubsubcomCustomSepWithInternalSep", "[config]") {
+
+    app.add_flag("--simple");
+    auto *subcom = app.add_subcommand("other");
+    subcom->add_flag("--newer");
+    auto *subsubcom = subcom->add_subcommand("sub2|BB");
+    subsubcom->add_flag("--newest");
+    app.config_formatter(std::make_shared<CLI::ConfigINI>());
+    app.get_config_formatter_base()->parentSeparator('|');
+    args = {"--simple", "other", "--newer", "sub2|BB", "--newest"};
+    run();
+
+    std::string str = app.config_to_str();
+    CHECK_THAT(str, Contains("simple=true"));
+    CHECK_THAT(str, Contains("other|newer=true"));
+    CHECK_THAT(str, Contains("other|'sub2|BB'|newest=true"));
+}
+
+TEST_CASE_METHOD(TApp, "IniOutputSubsubcomCustomSepWithInternalQuote", "[config]") {
+
+    app.add_flag("--simple");
+    auto *subcom = app.add_subcommand("other");
+    subcom->add_flag("--newer");
+    auto *subsubcom = subcom->add_subcommand("sub2'BB");
+    subsubcom->add_flag("--newest");
+    app.config_formatter(std::make_shared<CLI::ConfigINI>());
+    app.get_config_formatter_base()->parentSeparator('|');
+    args = {"--simple", "other", "--newer", "sub2'BB", "--newest"};
+    run();
+
+    std::string str = app.config_to_str();
+    CHECK_THAT(str, Contains("simple=true"));
+    CHECK_THAT(str, Contains("other|newer=true"));
+    CHECK_THAT(str, Contains("other|\"sub2'BB\"|newest=true"));
+}
+
 TEST_CASE_METHOD(TApp, "IniOutputSubsubcomConfigurable", "[config]") {
 
     app.add_flag("--simple");

tests/FuzzFailTest.cpp

@@ -50,7 +50,7 @@ TEST_CASE("file_fail") {
     CLI::FuzzApp fuzzdata;
     auto app = fuzzdata.generateApp();
 
-    int index = GENERATE(range(1, 5));
+    int index = GENERATE(range(1, 6));
     auto parseData = loadFailureFile("fuzz_file_fail", index);
     std::stringstream out(parseData);
     try {

tests/fuzzFail/fuzz_file_fail5

@@ -0,0 +1 @@
+"\uasdwrapｦｦｦ-"ｦｦ-"--confi矣矣矣矣矣矣矣矣矣矣矣矣矣矣矣矣矣矣矣矣矣矣矣矣矣矣矣矣矣矣矣矣矣.矣矣矣矣矣矣矣矣矣矣矣矣矣矣矣矣矣矣矣g