Fix Json quoting and UTF-8 handling

Sming/Core/Data/Format/Formatter.cpp

@@ -59,6 +59,9 @@ unsigned escapeControls(String& value, Options options)
 			extra += 1; // "\"
 		} else if(uint8_t(c) < 0x20) {
 			extra += options[Option::unicode] ? 5 : 3; // "\uNNNN" or "\xnn"
+		} else if(options[Option::utf8] && (c & 0x80)) {
+			// Characters such as £ (0xa3) are escaped to 0xc2 0xa3 in UTF-8
+			extra += 1; // '\xc2' prefix
 		}
 	}
 	if(extra == 0) {
@@ -89,6 +92,9 @@ unsigned escapeControls(String& value, Options options)
 			}
 			*out++ = hexchar(uint8_t(c) >> 4);
 			*out++ = hexchar(uint8_t(c) & 0x0f);
+		} else if(options[Option::utf8] && (c & 0x80)) {
+			*out++ = '\xc2';
+			*out++ = c;
 		} else {
 			*out++ = c;
 		}

Sming/Core/Data/Format/Formatter.h

@@ -20,6 +20,7 @@ namespace Format
 {
 enum class Option {
 	unicode, //< Use unicode escapes \uNNNN, otherwise hex \xNN
+	utf8,	///< Convert extended ASCII to UTF8
 	doublequote,
 	singlequote,
 	backslash,

Sming/Core/Data/Format/Json.cpp

@@ -55,6 +55,19 @@ bool IsValidUtf8(const char* str, unsigned length)
 	return true;
 }
 
+void escapeText(String& value)
+{
+	escapeControls(value, Option::unicode | Option::utf8 | Option::doublequote | Option::backslash);
+	if(!IsValidUtf8(value.c_str(), value.length())) {
+		debug_w("Invalid UTF8: %s", value.c_str());
+		for(unsigned i = 0; i < value.length(); ++i) {
+			char& c = value[i];
+			if(c < 0x20 || uint8_t(c) > 127)
+				c = '_';
+		}
+	}
+}
+
 } // namespace
 
 /*
@@ -66,14 +79,18 @@ bool IsValidUtf8(const char* str, unsigned length)
  */
 void Json::escape(String& value) const
 {
-	escapeControls(value, Option::unicode | Option::doublequote | Option::backslash);
-	if(!IsValidUtf8(value.c_str(), value.length())) {
-		debug_w("Invalid UTF8: %s", value.c_str());
-		for(unsigned i = 0; i < value.length(); ++i) {
-			char& c = value[i];
-			if(c < 0x20 || uint8_t(c) > 127)
-				c = '_';
-		}
+	escapeText(value);
+}
+
+void Json::quote(String& value) const
+{
+	escapeText(value);
+	auto len = value.length();
+	if(value.setLength(len + 2)) {
+		auto s = value.begin();
+		memmove(s + 1, s, len);
+		s[0] = '"';
+		s[len + 1] = '"';
 	}
 }
 

Sming/Core/Data/Format/Json.h

@@ -20,6 +20,7 @@ class Json : public Standard
 {
 public:
 	void escape(String& value) const override;
+	void quote(String& value) const override;
 
 	MimeType mimeType() const override
 	{

tests/HostTests/modules/Formatter.cpp

@@ -10,28 +10,39 @@ class FormatterTest : public TestGroup
 
 	void execute() override
 	{
+		// Note: \xa3 is unicode for £
 		DEFINE_FSTR_LOCAL(text1, "A JSON\ntest string\twith escapes\x12\0\n"
-								 "Worth \"maybe\" \xc2\xa3"
-								 "0.53.")
+								 "Worth \"maybe\" \xa3 0.53. Yen \xa5 5bn.")
 
 		TEST_CASE("JSON")
 		{
 			DEFINE_FSTR_LOCAL(text1b, "A JSON\\ntest string\\twith escapes\\u0012\\u0000\\n"
-									  "Worth \\\"maybe\\\" \xc2\xa3"
-									  "0.53.")
+									  "Worth \\\"maybe\\\" \xc2\xa3 0.53. Yen \xc2\xa5 5bn.")
 
 			Serial << text1 << endl;
 			String s(text1);
 			Format::json.escape(s);
 			REQUIRE_EQ(s, text1b);
+
+			s = text1;
+			Format::json.quote(s);
+			String quoted = String('"') + text1b + '"';
+			REQUIRE_EQ(s, quoted);
 		}
 
 		TEST_CASE("C++")
 		{
-			DEFINE_FSTR_LOCAL(text1b, "A JSON\\ntest string\\twith escapes\\x12\\0\\nWorth \\\"maybe\\\" £0.53.")
-
+			DEFINE_FSTR_LOCAL(
+				text1a, "A JSON\\ntest string\\twith escapes\\x12\\0\\nWorth \\\"maybe\\\" \xa3 0.53. Yen \xa5 5bn.")
 			String s(text1);
 			Format::escapeControls(s, Format::Option::doublequote | Format::Option::backslash);
+			REQUIRE_EQ(s, text1a);
+
+			DEFINE_FSTR_LOCAL(
+				text1b,
+				"A JSON\\ntest string\\twith escapes\\x12\\0\\nWorth \\\"maybe\\\" \xc2\xa3 0.53. Yen \xc2\xa5 5bn.")
+			s = text1;
+			Format::escapeControls(s, Format::Option::utf8 | Format::Option::doublequote | Format::Option::backslash);
 			REQUIRE_EQ(s, text1b);
 		}
 	}