diff --git a/.gitmodules b/.gitmodules index 9d62de59ac..a1404fb3cd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -217,6 +217,10 @@ path = Sming/Libraries/CS5460/CS5460 url = https://github.com/xxzl0130/CS5460.git ignore = dirty +[submodule "Libraries.CsvReader"] + path = Sming/Libraries/CsvReader + url = https://github.com/mikee47/CsvReader + ignore = dirty [submodule "Libraries.DFRobotDFPlayerMini"] path = Sming/Libraries/DFRobotDFPlayerMini url = https://github.com/DFRobot/DFRobotDFPlayerMini.git diff --git a/Sming/Core/Data/CsvReader.cpp b/Sming/Core/Data/CsvReader.cpp deleted file mode 100644 index 569ce02a0b..0000000000 --- a/Sming/Core/Data/CsvReader.cpp +++ /dev/null @@ -1,124 +0,0 @@ -/**** - * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. - * Created 2015 by Skurydin Alexey - * http://github.com/SmingHub/Sming - * All files of the Sming Core are provided under the LGPL v3 license. - * - * CsvReader.cpp - * - * @author: 2021 - Mikee47 - * - ****/ - -#include "CsvReader.h" -#include - -void CsvReader::reset() -{ - source->seekFrom(0, SeekOrigin::Start); - if(!userHeadingsProvided) { - readRow(); - headings = row; - } - row = nullptr; -} - -bool CsvReader::readRow() -{ - constexpr size_t blockSize{512}; - - String buffer(std::move(reinterpret_cast(row))); - constexpr char quoteChar{'"'}; - enum class FieldKind { - unknown, - quoted, - unquoted, - }; - FieldKind fieldKind{}; - bool escape{false}; - bool quote{false}; - char lc{'\0'}; - unsigned writepos{0}; - - while(true) { - if(buffer.length() == maxLineLength) { - debug_w("[CSV] Line buffer limit reached %u", maxLineLength); - return false; - } - size_t buflen = std::min(writepos + blockSize, maxLineLength); - if(!buffer.setLength(buflen)) { - debug_e("[CSV] Out of memory %u", buflen); - return false; - } - auto len = source->readBytes(buffer.begin() + writepos, buflen - writepos); - if(len == 0) { - if(writepos == 0) { - return false; - } - buffer.setLength(writepos); - row = std::move(buffer); - return true; - } - buflen = writepos + len; - unsigned readpos = writepos; - - for(; readpos < buflen; ++readpos) { - char c = buffer[readpos]; - if(escape) { - switch(c) { - case 'n': - c = '\n'; - break; - case 'r': - c = '\r'; - break; - case 't': - c = '\t'; - break; - default:; - // Just accept character - } - escape = false; - } else { - if(fieldKind == FieldKind::unknown) { - if(c == quoteChar) { - fieldKind = FieldKind::quoted; - quote = true; - lc = '\0'; - continue; - } - fieldKind = FieldKind::unquoted; - } - if(c == quoteChar) { - quote = !quote; - if(fieldKind == FieldKind::quoted) { - if(lc == quoteChar) { - buffer[writepos++] = c; - lc = '\0'; - } else { - lc = c; - } - continue; - } - } else if(c == '\\') { - escape = true; - continue; - } else if(!quote) { - if(c == fieldSeparator) { - c = '\0'; - fieldKind = FieldKind::unknown; - } else if(c == '\r') { - continue; - } else if(c == '\n') { - source->seekFrom(readpos + 1 - buflen, SeekOrigin::Current); - buffer.setLength(writepos); - row = std::move(buffer); - return true; - } - } - } - buffer[writepos++] = c; - lc = c; - } - } -} diff --git a/Sming/Core/Data/CsvReader.h b/Sming/Core/Data/CsvReader.h index 596a62a27e..2fcd76036a 100644 --- a/Sming/Core/Data/CsvReader.h +++ b/Sming/Core/Data/CsvReader.h @@ -12,132 +12,4 @@ #pragma once -#include "Stream/DataSourceStream.h" -#include "CStringArray.h" -#include - -/** - * @brief Class to parse a CSV file - * - * Spec: https://www.ietf.org/rfc/rfc4180.txt - * - * 1. Each record is located on a separate line - * 2. Line ending for last record in the file is optional - * 3. Field headings are provided either in the source data or in constructor (but not both) - * 4. Fields separated with ',' and whitespace considered part of field content - * 5. Fields may or may not be quoted - if present, will be removed during parsing - * 6. Fields may contain line breaks, quotes or commas - * 7. Quotes may be escaped thus "" if field itself is quoted - * - * Additional features: - * - * - Line breaks can be \n or \r\n - * - Escapes codes within fields will be converted: \n \r \t \", \\ - * - Field separator can be changed in constructor - */ -class CsvReader -{ -public: - /** - * @brief Construct a CSV reader - * @param source Stream to read CSV text from - * @param fieldSeparator - * @param headings Required if source data does not contain field headings as first row - * @param maxLineLength Limit size of buffer to guard against malformed data - */ - CsvReader(IDataSourceStream* source, char fieldSeparator = ',', const CStringArray& headings = nullptr, - size_t maxLineLength = 2048) - : source(source), fieldSeparator(fieldSeparator), userHeadingsProvided(headings), maxLineLength(maxLineLength), - headings(headings) - { - reset(); - } - - /** - * @brief Reset reader to start of CSV file - * - * Cursor is set to 'before start'. - * Call 'next()' to fetch first record. - */ - void reset(); - - /** - * @brief Seek to next record - */ - bool next() - { - return readRow(); - } - - /** - * @brief Get number of columns - */ - unsigned count() const - { - return headings.count(); - } - - /** - * @brief Get a value from the current row - * @param index Column index, starts at 0 - * @retval const char* nullptr if index is not valid - */ - const char* getValue(unsigned index) - { - return row[index]; - } - - /** - * @brief Get a value from the current row - * @param index Column name - * @retval const char* nullptr if name is not found - */ - const char* getValue(const char* name) - { - return getValue(getColumn(name)); - } - - /** - * @brief Get index of column given its name - * @param name Column name to find - * @retval int -1 if name is not found - */ - int getColumn(const char* name) - { - return headings.indexOf(name); - } - - /** - * @brief Determine if row is valid - */ - explicit operator bool() const - { - return bool(row); - } - - /** - * @brief Get headings - */ - const CStringArray& getHeadings() const - { - return headings; - } - - /** - * @brief Get current row - */ - const CStringArray& getRow() const - { - return row; - } - -private: - bool readRow(); - - std::unique_ptr source; - char fieldSeparator; - bool userHeadingsProvided; - size_t maxLineLength; - CStringArray headings; - CStringArray row; -}; +static_assert(false, "CsvReader class has been moved to the CsvReader library."); diff --git a/Sming/Libraries/CsvReader b/Sming/Libraries/CsvReader new file mode 160000 index 0000000000..8f4d416442 --- /dev/null +++ b/Sming/Libraries/CsvReader @@ -0,0 +1 @@ +Subproject commit 8f4d416442292927d15fe00d80130fb2fc7d8bb6 diff --git a/docs/source/upgrading/5.1-5.2.rst b/docs/source/upgrading/5.1-5.2.rst index 96f7fc41a4..cc8815a84f 100644 --- a/docs/source/upgrading/5.1-5.2.rst +++ b/docs/source/upgrading/5.1-5.2.rst @@ -91,3 +91,13 @@ These will now fail to compile. Copy construction and assignment has been explicitly deleted so avoid unintentional side-effects. Objects should always be passed by reference. + + +**CsvReader library** + +The :cpp:type:`CsvReader` class has been moved out of ``Core/Data`` and into :library:`CsvReader` +which has additional capabilities. Changes to existing code: + +- Add ``CsvReader`` to your project's :cpp:envvar:`COMPONENT_DEPENDS` +- Change ``#include `` to ``#include `` +- Change ``CsvReader`` class to :cpp:class:`CSV::Reader` diff --git a/samples/Basic_Templates/app/CsvTemplate.h b/samples/Basic_Templates/app/CsvTemplate.h index b29b4ae349..cdf56fdd8a 100644 --- a/samples/Basic_Templates/app/CsvTemplate.h +++ b/samples/Basic_Templates/app/CsvTemplate.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include /** @@ -35,5 +35,5 @@ class CsvTemplate : public SectionTemplate } private: - CsvReader csv; + CSV::Reader csv; }; diff --git a/samples/Basic_Templates/app/application.cpp b/samples/Basic_Templates/app/application.cpp index efca95a384..a2bbae2d40 100644 --- a/samples/Basic_Templates/app/application.cpp +++ b/samples/Basic_Templates/app/application.cpp @@ -1,7 +1,6 @@ #include #include #include -#include #include "CsvTemplate.h" namespace @@ -51,7 +50,7 @@ void printCars() void printClassics(const FlashString& templateSource, Format::Formatter& formatter) { // The CSV data source - CsvReader csv(new FileStream(Filename::classics_csv)); + CSV::Reader csv(new FileStream(Filename::classics_csv)); // Use a regular SectionTemplate class to process the template SectionTemplate tmpl(new FSTR::Stream(templateSource)); diff --git a/samples/Basic_Templates/component.mk b/samples/Basic_Templates/component.mk index 092ca88129..bcb5ca8f81 100644 --- a/samples/Basic_Templates/component.mk +++ b/samples/Basic_Templates/component.mk @@ -1,3 +1,4 @@ +COMPONENT_DEPENDS := CsvReader HWCONFIG := basic_templates DISABLE_NETWORK := 1 diff --git a/tests/HostTests/modules/TemplateStream.cpp b/tests/HostTests/modules/TemplateStream.cpp index 66f1a6c9bc..36c725dd5a 100644 --- a/tests/HostTests/modules/TemplateStream.cpp +++ b/tests/HostTests/modules/TemplateStream.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #ifdef ARCH_HOST #include @@ -23,12 +22,6 @@ DEFINE_FSTR_LOCAL(template3_1, "Document Title