diff --git a/.appveyor/build.cmd b/.appveyor/build.cmd deleted file mode 100644 index c3beab0f5a..0000000000 --- a/.appveyor/build.cmd +++ /dev/null @@ -1,46 +0,0 @@ -REM Windows build script - -subst Z: %APPVEYOR_BUILD_FOLDER% -set SMING_HOME=Z:\Sming - -if "%build_compiler%" == "udk" set ESP_HOME=%UDK_ROOT% -if "%build_compiler%" == "eqt" set ESP_HOME=%EQT_ROOT% - -cd /d %SMING_HOME% -gcc -v - -set MAKE_PARALLEL=make -j2 - -REM Move samples and tests into directory outside of the Sming repo. -set SMING_PROJECTS_DIR=%APPVEYOR_BUILD_FOLDER%\.. -move ..\samples %SMING_PROJECTS_DIR% -move ..\tests %SMING_PROJECTS_DIR% - -REM This will build the Basic_Blink application and most of the framework Components -cd %SMING_PROJECTS_DIR%/samples/Basic_Blink -make help -make list-config -%MAKE_PARALLEL% || goto :error - -cd %SMING_HOME% - -if "%SMING_ARCH%" == "Host" ( - - REM Build a couple of basic applications - %MAKE_PARALLEL% Basic_Serial Basic_ProgMem STRICT=1 V=1 || goto :error - - REM Run basic tests - %MAKE_PARALLEL% tests || goto :error - -) else ( - - %MAKE_PARALLEL% Basic_Ssl || goto :error - %MAKE_PARALLEL% Basic_SmartConfig || goto :error - -) - -goto :EOF - -:error -echo Failed with error #%errorlevel%. -exit /b %errorlevel% diff --git a/.appveyor/install.cmd b/.appveyor/install.cmd deleted file mode 100644 index 8f06ce2d35..0000000000 --- a/.appveyor/install.cmd +++ /dev/null @@ -1,28 +0,0 @@ -REM Windows install script - -rmdir /s /q c:\MinGW -curl -Lo MinGW.7z %SMINGTOOLS%/MinGW-2020-10-19.7z -7z -oC:\ x MinGW.7z - -goto :%SMING_ARCH% - -:Esp8266 - - REM Old toolchain - set TOOLCHAIN=esp-udk-win32.7z - curl -LO %SMINGTOOLS%/%TOOLCHAIN% - 7z -o%UDK_ROOT% x %TOOLCHAIN% - - REM New toolchain - mkdir %EQT_ROOT% - set TOOLCHAIN=x86_64-w64-mingw32.xtensa-lx106-elf-e6a192b.201211.zip - curl -LO %SMINGTOOLS%/%TOOLCHAIN% - 7z -o%EQT_ROOT% x %TOOLCHAIN% - - goto :EOF - - -:Host - - goto :EOF - diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 216059625b..da50404adc 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -17,7 +17,7 @@ on: jobs: analyze: name: Analyze - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: fail-fast: false @@ -36,11 +36,6 @@ jobs: # a pull request then we can checkout the head. fetch-depth: 2 - # If this run was triggered by a pull request event, then checkout - # the head of the pull request instead of the merge commit. - - run: git checkout HEAD^2 - if: ${{ github.event_name == 'pull_request' }} - # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v1 @@ -66,15 +61,10 @@ jobs: - if: matrix.language == 'cpp' || matrix.language == 'c' name: Build Sming C/C++ run: | - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - sudo apt-get update -y - sudo apt-get install gcc-9-multilib g++-9-multilib - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 60 --slave /usr/bin/g++ g++ /usr/bin/g++-9 - env - cd Sming - export SMING_HOME=$(pwd) - cd ../samples/Basic_Blink - make SMING_ARCH=Host - + sudo Tools/install.sh host + . Tools/export.sh + cd samples/Basic_Blink + make -j3 SMING_ARCH=Host + - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v1 diff --git a/.gitignore b/.gitignore index f112376880..a30086cd31 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ language.settings.xml nbproject .*.swp .vscode +*.code-workspace GTAGS GRTAGS GPATH diff --git a/.gitmodules b/.gitmodules index ad249f7315..85003018ee 100644 --- a/.gitmodules +++ b/.gitmodules @@ -49,6 +49,10 @@ path = Sming/Components/http-parser url = https://github.com/nodejs/http-parser.git ignore = dirty +[submodule "IFS"] + path = Sming/Components/IFS + url = https://github.com/mikee47/IFS.git + ignore = dirty [submodule "libyuarel"] path = Sming/Components/libyuarel url = https://github.com/jacketizer/libyuarel.git @@ -59,7 +63,7 @@ ignore = dirty [submodule "rboot"] path = Sming/Components/rboot/rboot - url = https://github.com/raburton/rboot.git + url = https://github.com/mikee47/rboot ignore = dirty [submodule "spiffs"] path = Sming/Components/spiffs/spiffs @@ -89,7 +93,7 @@ ignore = dirty [submodule "Esp8266.lwip2"] path = Sming/Arch/Esp8266/Components/lwip2/lwip2 - url = https://github.com/d-a-v/esp82xx-nonos-linklayer.git + url = https://github.com/mikee47/esp82xx-nonos-linklayer.git ignore = dirty [submodule "Esp8266.new-pwm"] path = Sming/Arch/Esp8266/Components/driver/new-pwm @@ -187,6 +191,14 @@ path = Sming/Libraries/DHTesp url = https://github.com/beegee-tokyo/DHTesp.git ignore = dirty +[submodule "Libraries.flatbuffers"] + path = Sming/Libraries/flatbuffers/src + url = https://github.com/google/flatbuffers.git + ignore = dirty +[submodule "Libraries.GoogleCast"] + path = Sming/Libraries/GoogleCast + url = https://github.com/slaff/Sming-GoogleCast.git + ignore = dirty [submodule "Libraries.HardwareSPI"] path = Sming/Libraries/HardwareSPI url = https://github.com/mikee47/HardwareSPI @@ -207,6 +219,10 @@ path = Sming/Libraries/libsodium/libsodium url = https://github.com/jedisct1/libsodium.git ignore = dirty +[submodule "Libraries.MDNS"] + path = Sming/Libraries/MDNS + url = https://github.com/mikee47/Sming-MDNS + ignore = dirty [submodule "Libraries.modbusino"] path = Sming/Libraries/modbusino/modbusino url = https://github.com/kmihaylov/modbusino.git @@ -223,6 +239,10 @@ path = Sming/Libraries/MultipartParser/multipart-parser url = https://github.com/iafonov/multipart-parser-c.git ignore = dirty +[submodule "Libraries.nanopb"] + path = Sming/Libraries/nanopb/nanopb + url = https://github.com/nanopb/nanopb.git + ignore = dirty [submodule "Libraries.RapidXML"] path = Sming/Libraries/RapidXML url = https://github.com/mikee47/Sming-RapidXML diff --git a/.travis.yml b/.travis.yml index 1228bc56d0..97b2443e0f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -56,11 +56,11 @@ addons: update: true packages: - xmlstarlet -install: ".travis/install.sh" -script: ".travis/build.sh" +install: "Tools/travis/install.sh" +script: "Tools/travis/build.sh" deploy: provider: script - script: ".travis/deploy.sh $TRAVIS_TAG" + script: "Tools/travis/deploy.sh $TRAVIS_TAG" skip_cleanup: true on: tags: true diff --git a/.travis/tools/esptool2 b/.travis/tools/esptool2 deleted file mode 100755 index 7de8b1f191..0000000000 Binary files a/.travis/tools/esptool2 and /dev/null differ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 19039ad569..a7f6067b20 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,7 +32,7 @@ __Sming Contributing flow__: 4. Document your code - As a bare minimum, please include a `README.rst` or `README.md` file. See :doc:`/contribute/documentation` for further information. + As a bare minimum, please include a `README.rst` or `README.md` file. See :doc:`/information/develop/documentation` for further information. 5. Commit changes diff --git a/README.md b/README.md index 098cf7b11a..102408e219 100644 --- a/README.md +++ b/README.md @@ -67,17 +67,17 @@ Please note Version 4 documentation is at [sming.readthedocs.io](https://sming.r * Async TCP and UDP stack based on [LWIP](http://savannah.nongnu.org/projects/lwip/). * With clients supporting: HTTP, MQTT, WebSockets and SMTP. * And servers for: DNS, FTP, HTTP(+ WebSockets), Telnet. - * With [SSL support](https://sming.readthedocs.io/en/latest/framework/core/network/ssl.html) for all network clients and servers. Based on [axTLS](http://axtls.sourceforge.net/) and [BearSSL](https://www.bearssl.org/). - * Out of the box support for OTA over HTTPS. + * With [SSL support](https://sming.readthedocs.io/en/latest/_inc/Sming/Components/ssl/index.html) for all network clients and servers. Based on [axTLS](http://axtls.sourceforge.net/) and [BearSSL](https://www.bearssl.org/). + * Over-The-Air(OTA) firmware upgrades via HTTP(S) and MQTT(S). * ESP8266 specific features. * Integrated boot loader [rboot](https://sming.readthedocs.io/en/latest/_inc/Sming/Components/rboot/index.html) with support for 1MB ROMs, OTA firmware updating and ROM switching. - * [Crash handlers](https://sming.readthedocs.io/en/latest/information/debugging/index.html) for analyzing/handling system restarts due to fatal errors or WDT resets. + * [Crash handlers](https://sming.readthedocs.io/en/latest/information/debugging.html) for analyzing/handling system restarts due to fatal errors or WDT resets. * [PWM support](https://sming.readthedocs.io/en/latest/_inc/Sming/Arch/Esp8266/Components/driver/index.html) based on [Stefan Bruens PWM](https://github.com/StefanBruens/ESP8266_new_pwm.git). * Optional [custom heap allocation](https://sming.readthedocs.io/en/latest/_inc/Sming/Arch/Esp8266/Components/heap/index.html) based on [Umm Malloc](https://github.com/rhempel/umm_malloc.git). * Based on Espressif NONOS SDK. Officially suppored NON SDK version is >= 3.0.1. * Support for a [thin No-Wifi-SDK](https://sming.readthedocs.io/en/latest/_inc/Sming/Arch/Esp8266/Components/esp_no_wifi/index.html). Helpful when a project does not require WiFi (or networking) and reduces code size and memory usage signficantly. * Linux/Windows features - * Sming has a [host emulator](https://sming.readthedocs.io/en/latest/arch/host/host-emulator.html) that allows libraries and sample applications to be compiled on a Linux/Windows host system and be tested before uploading them to an actual microcontroller. + * Sming has a [host emulator](https://sming.readthedocs.io/en/latest/arch/host/index.html) that allows libraries and sample applications to be compiled on a Linux/Windows host system and be tested before uploading them to an actual microcontroller. ## Compatibility @@ -117,7 +117,7 @@ Supported SDK: ESP-IDF v4.1 ### Stable -- [Sming V4.2.0](https://github.com/SmingHub/Sming/releases/tag/4.2.0) - great new features, performance and stability improvements. +- [Sming V4.3.0](https://github.com/SmingHub/Sming/releases/tag/4.3.0) - great new features, performance and stability improvements. ### Long Term Support (LTS) @@ -136,7 +136,7 @@ git clone https://github.com/SmingHub/Sming.git ## Getting Started -Sming supports multiple architectures. Choose the architecture of your choice to [install the needed development software](https://sming.readthedocs.io/en/latest/getting-started.html) and toolchain(s). +Sming supports multiple architectures. Choose the architecture of your choice to [install the needed development software](https://sming.readthedocs.io/en/latest/getting-started) and toolchain(s). You can also try Sming without installing anything locally. We have an [interactive tutorial](https://www.katacoda.com/slaff/scenarios/sming-host-emulator) that can be run directly from your browser. @@ -196,7 +196,7 @@ Serial.println("Hello Sming! Let's do smart things."); ### Connect to WiFi ```c++ WifiStation.enable(true); -WifiStation.config("LOCAL-NETWORK", "123456789087"); // Put you SSID and Password here +WifiStation.config("LOCAL-NETWORK", "123456789087"); // Put your SSID and password here ``` ### Read DHT22 sensor diff --git a/Sming/Arch/Esp32/Components/esp32/component.mk b/Sming/Arch/Esp32/Components/esp32/component.mk index 2bf48d5e6c..bb207b3ad9 100644 --- a/Sming/Arch/Esp32/Components/esp32/component.mk +++ b/Sming/Arch/Esp32/Components/esp32/component.mk @@ -252,7 +252,9 @@ SDK_PROJECT_PATH := $(COMPONENT_PATH)/project SDK_CONFIG_DEFAULTS := $(SDK_PROJECT_PATH)/sdkconfig.defaults SDKCONFIG_MAKEFILE ?= $(SDK_PROJECT_PATH)/sdkconfig +ifeq ($(MAKE_DOCS),) -include $(SDKCONFIG_MAKEFILE) +endif export SDKCONFIG_MAKEFILE # sub-makes (like bootloader) will reuse this path $(SDK_BUILD_BASE) $(SDK_COMPONENT_LIBDIR): diff --git a/Sming/Arch/Esp32/Components/esp32/project/sdkconfig.defaults.debug b/Sming/Arch/Esp32/Components/esp32/project/sdkconfig.defaults.debug index af39f856b3..758d125f83 100644 --- a/Sming/Arch/Esp32/Components/esp32/project/sdkconfig.defaults.debug +++ b/Sming/Arch/Esp32/Components/esp32/project/sdkconfig.defaults.debug @@ -17,6 +17,7 @@ CONFIG_LWIP_TCPIP_TASK_STACK_SIZE=8192 # Mandatory Sming framework changes CONFIG_ESP_TIMER_TASK_STACK_SIZE=8192 +CONFIG_ESP_SYSTEM_EVENT_TASK_STACK_SIZE=8192 # The bootloader logs all type of messages CONFIG_BOOTLOADER_LOG_LEVEL_NONE= diff --git a/Sming/Arch/Esp32/Components/esp32/project/sdkconfig.defaults.release b/Sming/Arch/Esp32/Components/esp32/project/sdkconfig.defaults.release index 7f2468230e..d1c7ab696f 100644 --- a/Sming/Arch/Esp32/Components/esp32/project/sdkconfig.defaults.release +++ b/Sming/Arch/Esp32/Components/esp32/project/sdkconfig.defaults.release @@ -17,6 +17,7 @@ CONFIG_LWIP_TCPIP_TASK_STACK_SIZE=8192 # Mandatory Sming framework changes CONFIG_ESP_TIMER_TASK_STACK_SIZE=8192 +CONFIG_ESP_SYSTEM_EVENT_TASK_STACK_SIZE=8192 # The bootloader logs only errors CONFIG_BOOTLOADER_LOG_LEVEL_NONE= diff --git a/Sming/Arch/Esp32/Components/esp32/sdk/partitions/base.csv b/Sming/Arch/Esp32/Components/esp32/sdk/partitions/base.csv deleted file mode 100644 index 97c8e0148e..0000000000 --- a/Sming/Arch/Esp32/Components/esp32/sdk/partitions/base.csv +++ /dev/null @@ -1,5 +0,0 @@ -# Name, Type, SubType, Offset, Size, Flags -# Note: if you have increased the bootloader size, make sure to update the offsets to avoid overlap -nvs, data, nvs, 0x9000, 0x6000, -phy_init, data, phy, 0xf000, 0x1000, -factory, app, factory, 0x10000, 0x1F0000, diff --git a/Sming/Arch/Esp32/Components/esp32/src/startup.cpp b/Sming/Arch/Esp32/Components/esp32/src/startup.cpp index 602dd46c13..cdb1ff521b 100644 --- a/Sming/Arch/Esp32/Components/esp32/src/startup.cpp +++ b/Sming/Arch/Esp32/Components/esp32/src/startup.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #ifndef ESP32_STACK_SIZE #define ESP32_STACK_SIZE 16384U @@ -63,6 +64,7 @@ void main(void*) esp_init_flash(); esp_init_wifi(); ets_init_tasks(); + Storage::initialize(); System.initialize(); init(); diff --git a/Sming/Arch/Esp32/Components/esp_spiffs/README.rst b/Sming/Arch/Esp32/Components/esp_spiffs/README.rst deleted file mode 100644 index 9869b71f45..0000000000 --- a/Sming/Arch/Esp32/Components/esp_spiffs/README.rst +++ /dev/null @@ -1,4 +0,0 @@ -Esp32 SPIFFS -============ - -SPIFFS implementation for ESP32 devices. diff --git a/Sming/Arch/Esp32/Components/esp_spiffs/component.mk b/Sming/Arch/Esp32/Components/esp_spiffs/component.mk deleted file mode 100644 index 12ce0a9792..0000000000 --- a/Sming/Arch/Esp32/Components/esp_spiffs/component.mk +++ /dev/null @@ -1 +0,0 @@ -COMPONENT_DEPENDS := spiffs diff --git a/Sming/Arch/Esp32/Components/esp_spiffs/spiffs_config.c b/Sming/Arch/Esp32/Components/esp_spiffs/spiffs_config.c deleted file mode 100644 index c8e5f095e9..0000000000 --- a/Sming/Arch/Esp32/Components/esp_spiffs/spiffs_config.c +++ /dev/null @@ -1,37 +0,0 @@ -/**** - * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. - * Created 2015 by Skurydin Alexey - * http://github.com/SmingHub/Sming - * All files of the Sming Core are provided under the LGPL v3 license. - * - * rboot-overrides.c - * - ****/ - -#include "spiffs_sming.h" -#include -#include - -/* - * rBoot uses different spiffs organization and we need to override this method - * during application compile time in order to make automatic - * mounting with `spiffs_mount()` work as expected. - */ -spiffs_config spiffs_get_storage_config() -{ - spiffs_config cfg = {0}; - - const esp_partition_t* partition = - esp_partition_find_first(ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_SPIFFS, NULL); - - if(partition == NULL) { - debug_w("No SPIFFS partition registered"); - } else { - cfg.phys_addr = partition->address; - cfg.phys_size = partition->size; - debug_w("SPIFFS partition found at 0x%08x, size 0x%08x", cfg.phys_addr, cfg.phys_size); - // TODO: Check partition->flash_chip is valid? - } - - return cfg; -} diff --git a/Sming/Arch/Esp32/Components/sming-arch/component.mk b/Sming/Arch/Esp32/Components/sming-arch/component.mk index 70ecd54254..d97a67fcac 100644 --- a/Sming/Arch/Esp32/Components/sming-arch/component.mk +++ b/Sming/Arch/Esp32/Components/sming-arch/component.mk @@ -18,7 +18,6 @@ COMPONENT_DEPENDS := \ driver \ heap \ fatfs \ - esp_spiffs \ esp32 \ gdbstub \ esptool diff --git a/Sming/Arch/Esp32/Components/spi_flash/flashmem.cpp b/Sming/Arch/Esp32/Components/spi_flash/flashmem.cpp index 3e5f6bb3e9..43c6c19fcb 100644 --- a/Sming/Arch/Esp32/Components/spi_flash/flashmem.cpp +++ b/Sming/Arch/Esp32/Components/spi_flash/flashmem.cpp @@ -13,6 +13,7 @@ #include #include #include +#include /* * Physical <-> Virtual address mapping is handled in `$IDF_COMPONENTS/spi_flash/flash_mmap.c`. @@ -115,3 +116,12 @@ uint32_t flashmem_get_sector_of_address(uint32_t addr) { return flashmem_find_sector(addr, NULL, NULL); } + +uint32_t spi_flash_get_id(void) +{ + uint32_t id{0}; + if(esp_flash_read_id(esp_flash_default_chip, &id) != ESP_OK) { + id = 0; + } + return id; +} diff --git a/Sming/Arch/Esp32/Components/spi_flash/include/esp_spi_flash.h b/Sming/Arch/Esp32/Components/spi_flash/include/esp_spi_flash.h index 55cdd74859..aaa03d1370 100644 --- a/Sming/Arch/Esp32/Components/spi_flash/include/esp_spi_flash.h +++ b/Sming/Arch/Esp32/Components/spi_flash/include/esp_spi_flash.h @@ -149,6 +149,8 @@ uint32_t flashmem_get_sector_of_address(uint32_t addr); /** @} */ +uint32_t spi_flash_get_id(void); + #ifdef __cplusplus } #endif diff --git a/Sming/Arch/Esp32/Platform/AccessPointImpl.cpp b/Sming/Arch/Esp32/Platform/AccessPointImpl.cpp index a44cf2f6d9..33fe58ff40 100644 --- a/Sming/Arch/Esp32/Platform/AccessPointImpl.cpp +++ b/Sming/Arch/Esp32/Platform/AccessPointImpl.cpp @@ -16,8 +16,6 @@ static AccessPointImpl accessPoint; AccessPointClass& WifiAccessPoint = accessPoint; -static esp_netif_t* apNetworkInterface = nullptr; - void AccessPointImpl::enable(bool enabled, bool save) { wifi_mode_t mode; @@ -69,7 +67,7 @@ bool AccessPointImpl::isEnabled() const bool AccessPointImpl::config(const String& ssid, String password, WifiAuthMode mode, bool hidden, int channel, int beaconInterval) { - wifi_config_t config; + wifi_config_t config{}; if(ssid.length() >= sizeof(config.ap.ssid)) { return false; @@ -78,8 +76,8 @@ bool AccessPointImpl::config(const String& ssid, String password, WifiAuthMode m return false; } - memcpy(&config.ap.ssid, ssid.c_str(), ssid.length()); - memcpy(&config.ap.password, password.c_str(), password.length()); + memcpy(config.ap.ssid, ssid.c_str(), ssid.length()); + memcpy(config.ap.password, password.c_str(), password.length()); config.ap.ssid_len = ssid.length(); config.ap.ssid_hidden = hidden; config.ap.channel = channel; @@ -126,6 +124,9 @@ IpAddress AccessPointImpl::getNetworkGateway() const bool AccessPointImpl::setIP(IpAddress address) { + if(apNetworkInterface == nullptr) { + return false; + } esp_netif_dhcps_stop(apNetworkInterface); esp_netif_ip_info_t info; ESP_ERROR_CHECK(esp_netif_get_ip_info(apNetworkInterface, &info)); @@ -143,6 +144,11 @@ MacAddress AccessPointImpl::getMacAddress() const return addr; } +bool AccessPointImpl::setMacAddress(const MacAddress& addr) const +{ + return esp_wifi_set_mac(ESP_IF_WIFI_AP, &const_cast(addr)[0]); +} + String AccessPointImpl::getSSID() const { wifi_config_t config{}; diff --git a/Sming/Arch/Esp32/Platform/AccessPointImpl.h b/Sming/Arch/Esp32/Platform/AccessPointImpl.h index a060e84760..00f299cab1 100644 --- a/Sming/Arch/Esp32/Platform/AccessPointImpl.h +++ b/Sming/Arch/Esp32/Platform/AccessPointImpl.h @@ -13,6 +13,8 @@ #include #include +struct esp_netif_obj; + class AccessPointImpl : public AccessPointClass, protected ISystemReadyHandler { public: @@ -28,6 +30,7 @@ class AccessPointImpl : public AccessPointClass, protected ISystemReadyHandler IpAddress getIP() const override; bool setIP(IpAddress address) override; MacAddress getMacAddress() const override; + bool setMacAddress(const MacAddress& addr) const override; IpAddress getNetworkMask() const override; IpAddress getNetworkGateway() const override; IpAddress getNetworkBroadcast() const override; @@ -36,4 +39,7 @@ class AccessPointImpl : public AccessPointClass, protected ISystemReadyHandler protected: void onSystemReady() override; + +private: + esp_netif_obj* apNetworkInterface{nullptr}; }; diff --git a/Sming/Arch/Esp32/Platform/StationImpl.cpp b/Sming/Arch/Esp32/Platform/StationImpl.cpp index e82ef32dea..6780762c58 100644 --- a/Sming/Arch/Esp32/Platform/StationImpl.cpp +++ b/Sming/Arch/Esp32/Platform/StationImpl.cpp @@ -15,11 +15,27 @@ #include #include +#ifdef ENABLE_WPS +#include + +/* + * Information only required during WPS negotiation + */ +struct StationImpl::WpsConfig { + static constexpr unsigned timeoutMs{60000}; + static constexpr unsigned maxRetryAttempts{5}; + WPSConfigDelegate callback; + wifi_event_sta_wps_er_success_t creds; + uint8_t numRetries; + uint8_t credIndex; + bool ignoreDisconnects; +}; + +#endif + static StationImpl station; StationClass& WifiStation = station; -static esp_netif_t* stationNetworkInterface = nullptr; - class BssInfoImpl : public BssInfo { public: @@ -118,6 +134,9 @@ bool StationImpl::disconnect() bool StationImpl::isEnabledDHCP() const { + if(stationNetworkInterface == nullptr) { + return false; + } esp_netif_dhcp_status_t status; if(esp_netif_dhcps_get_status(stationNetworkInterface, &status) != ESP_OK) { return false; @@ -128,6 +147,9 @@ bool StationImpl::isEnabledDHCP() const void StationImpl::enableDHCP(bool enable) { + if(stationNetworkInterface == nullptr) { + return; + } if(enable) { esp_netif_dhcpc_start(stationNetworkInterface); } else { @@ -164,6 +186,11 @@ MacAddress StationImpl::getMacAddress() const return addr; } +bool StationImpl::setMacAddress(const MacAddress& addr) const +{ + return esp_wifi_set_mac(ESP_IF_WIFI_STA, &const_cast(addr)[0]); +} + IpAddress StationImpl::getNetworkBroadcast() const { esp_netif_ip_info_t info; @@ -187,6 +214,9 @@ IpAddress StationImpl::getNetworkGateway() const bool StationImpl::setIP(IpAddress address, IpAddress netmask, IpAddress gateway) { + if(stationNetworkInterface == nullptr) { + return false; + } disconnect(); enableDHCP(false); esp_netif_ip_info_t ipinfo; @@ -197,9 +227,9 @@ bool StationImpl::setIP(IpAddress address, IpAddress netmask, IpAddress gateway) ipinfo.netmask.addr = netmask; ipinfo.gw.addr = gateway; if(esp_netif_set_ip_info(stationNetworkInterface, &ipinfo) == ESP_OK) { - debugf("Station IP successfully updated"); + debug_i("Station IP successfully updated"); } else { - debugf("Station IP can't be updated"); + debug_e("Station IP can't be updated"); enableDHCP(true); } connect(); @@ -210,11 +240,11 @@ String StationImpl::getSSID() const { wifi_config_t config{}; if(esp_wifi_get_config(ESP_IF_WIFI_STA, &config) != ESP_OK) { - debugf("Can't read station configuration!"); + debug_e("Can't read station configuration!"); return nullptr; } auto ssid = reinterpret_cast(config.sta.ssid); - debugf("SSID: '%s'", ssid); + debug_d("SSID: '%s'", ssid); return ssid; } @@ -222,7 +252,7 @@ int8_t StationImpl::getRssi() const { wifi_ap_record_t info; ESP_ERROR_CHECK(esp_wifi_sta_get_ap_info(&info)); - debugf("Rssi: %d dBm", info.rssi); + debug_d("Rssi: %d dBm", info.rssi); return info.rssi; } @@ -230,10 +260,10 @@ uint8_t StationImpl::getChannel() const { wifi_config_t config; if(esp_wifi_get_config(ESP_IF_WIFI_STA, &config) != ESP_OK) { - debugf("Can't read station configuration!"); + debug_e("Can't read station configuration!"); return 0; } - debugf("Channel: %d CH", config.sta.channel); + debug_d("Channel: %d CH", config.sta.channel); return config.sta.channel; } @@ -241,11 +271,11 @@ String StationImpl::getPassword() const { wifi_config_t config{}; if(esp_wifi_get_config(ESP_IF_WIFI_STA, &config) != ESP_OK) { - debugf("Can't read station configuration!"); + debug_e("Can't read station configuration!"); return nullptr; } auto pwd = reinterpret_cast(config.sta.password); - debugf("Pass: '%s'", pwd); + debug_d("Pass: '%s'", pwd); return pwd; } @@ -274,7 +304,7 @@ bool StationImpl::startScan(ScanCompletedDelegate scanCompleted) ESP_ERROR_CHECK(esp_event_handler_register(WIFI_EVENT, WIFI_EVENT_STA_CONNECTED, connectHandler, NULL)); - debugf("startScan failed"); + debug_e("startScan failed"); } return true; @@ -298,9 +328,9 @@ void StationImpl::staticScanCompleted(wifi_event_sta_scan_done_t* event, uint8_t station.scanCompletedCallback(true, list); } - debugf("scan completed: %u found", list.count()); + debug_i("scan completed: %u found", list.count()); } else { - debugf("scan failed %u", status); + debug_e("scan failed %u", status); if(station.scanCompletedCallback) { station.scanCompletedCallback(false, list); } @@ -424,68 +454,141 @@ void StationImpl::smartConfigStop() #ifdef ENABLE_WPS -void StationImpl::internalWpsConfig(wps_cb_status status) +void StationImpl::wpsEventHandler(esp_event_base_t event_base, int32_t event_id, void* event_data) { - bool processInternal = true; - if(wpsConfigCallback) { - processInternal = wpsConfigCallback(WpsStatus(status)); + if(wpsConfig == nullptr) { + return; } - if(processInternal) { - switch(status) { - case WPS_CB_ST_SUCCESS: - debugf("wifi_wps_status_cb(): WPS_CB_ST_SUCCESS\n"); - wpsConfigStop(); - connect(); - break; - case WPS_CB_ST_FAILED: - debugf("wifi_wps_status_cb(): WPS_CB_ST_FAILED\n"); - wpsConfigStop(); - connect(); // try to reconnect with old config + + switch(event_id) { + case WIFI_EVENT_STA_DISCONNECTED: + debug_w("WIFI_EVENT_STA_DISCONNECTED"); + if(wpsConfig->ignoreDisconnects) { break; - case WPS_CB_ST_TIMEOUT: - debugf("wifi_wps_status_cb(): WPS_CB_ST_TIMEOUT\n"); - wpsConfigStop(); - connect(); // try to reconnect with old config + } + if(wpsConfig->numRetries < WpsConfig::maxRetryAttempts) { + esp_wifi_connect(); + ++wpsConfig->numRetries; break; - case WPS_CB_ST_WEP: - debugf("wifi_wps_status_cb(): WPS_CB_ST_WEP\n"); + } + + if(wpsConfigure(wpsConfig->credIndex + 1)) { + esp_wifi_connect(); break; - default: - debugf("wifi_wps_status_cb(): unknown wps_cb_status %d\n", status); + } + + debug_e("[WPS] Failed to connect!"); + if(wpsCallback(WpsStatus::Failed)) { + // try to reconnect with old config + wpsConfigStop(); + esp_wifi_connect(); + } + break; + + case WIFI_EVENT_STA_WPS_ER_SUCCESS: { + debug_i("WIFI_EVENT_STA_WPS_ER_SUCCESS"); + + if(!wpsCallback(WpsStatus::Success)) { + return; + } + + if(event_data != nullptr) { + /* If multiple AP credentials are received from WPS, connect with first one */ + wpsConfig->creds = *static_cast(event_data); + wpsConfigure(0); + } + /* + * If only one AP credential is received from WPS, there will be no event data and + * esp_wifi_set_config() is already called by WPS modules for backward compatibility + * with legacy apps. So directly attempt connection here. + */ + wpsConfigStop(); + esp_wifi_connect(); + break; + } + + case WIFI_EVENT_STA_WPS_ER_FAILED: { + debug_e("WIFI_EVENT_STA_WPS_ER_FAILED"); + if(wpsCallback(WpsStatus::Failed)) { + // Try to reconnect with old config + wpsConfigStop(); + esp_wifi_connect(); + } + break; + } + + case WIFI_EVENT_STA_WPS_ER_TIMEOUT: + debug_e("WIFI_EVENT_STA_WPS_ER_TIMEOUT"); + if(wpsCallback(WpsStatus::Timeout)) { + // Try to reconnect with old config wpsConfigStop(); - connect(); // try to reconnect with old config + esp_wifi_connect(); } + break; + + case WIFI_EVENT_STA_WPS_ER_PIN: + debug_e("WIFI_EVENT_STA_WPS_ER_PIN (not implemented)"); + break; + + default: + break; } } -bool StationImpl::wpsConfigStart(WPSConfigDelegate callback) +bool StationImpl::wpsConfigure(uint8_t credIndex) { - debugf("WPS start\n"); - wpsConfigCallback = callback; - wifi_station_disconnect(); - wifi_set_opmode_current(wifi_get_opmode() | STATION_MODE); - debugf("WPS stationmode activated\n"); - if(!wifi_wps_enable(WPS_TYPE_PBC)) { - debugf("Station::wpsConfigStart() : wps enable failed\n"); - return false; - } - if(!wifi_set_wps_cb([](int status) { station.internalWpsConfig(wps_cb_status(status)); })) { - debugf("Station::wpsConfigStart() : cb failed\n"); + wpsConfig->ignoreDisconnects = false; + if(credIndex >= wpsConfig->creds.ap_cred_cnt) { return false; } + wpsConfig->numRetries = 0; + wpsConfig->credIndex = credIndex; + auto& cred = wpsConfig->creds.ap_cred[credIndex]; + debug_i("Connecting to SSID: %s, Passphrase: %s", cred.ssid, cred.passphrase); + wifi_config_t cfg{}; + memcpy(cfg.sta.ssid, cred.ssid, sizeof(cred.ssid)); + memcpy(cfg.sta.password, cred.passphrase, sizeof(cred.passphrase)); + ESP_ERROR_CHECK(esp_wifi_set_config(WIFI_IF_STA, &cfg)); + return true; +} - if(!wifi_wps_start()) { - debugf("Station::wpsConfigStart() : wifi_wps_start() failed\n"); +bool StationImpl::wpsConfigStart(WPSConfigDelegate callback) +{ + if(wpsConfig != nullptr) { + debug_e("[WPS] Already in progress"); return false; } + + wpsConfig = new WpsConfig{}; + wpsConfig->callback = callback; + wpsConfig->ignoreDisconnects = true; + + debug_d("[WPS] wpsConfigStart()"); + + ESP_ERROR_CHECK(esp_event_handler_register(WIFI_EVENT, ESP_EVENT_ANY_ID, staticWpsEventHandler, this)); + + enable(true, false); + + connect(); + + esp_wps_config_t wps_config = WPS_CONFIG_INIT_DEFAULT(WPS_TYPE_PBC); + ESP_ERROR_CHECK(esp_wifi_wps_enable(&wps_config)); + ESP_ERROR_CHECK(esp_wifi_wps_start(WpsConfig::timeoutMs)); + return true; } +bool StationImpl::wpsCallback(WpsStatus status) +{ + return wpsConfig->callback ? wpsConfig->callback(status) : true; +} + void StationImpl::wpsConfigStop() { - if(!wifi_wps_disable()) { - debugf("Station::wpsConfigStop() : wifi_wps_disable() failed\n"); - } + ESP_ERROR_CHECK(esp_wifi_wps_disable()); + ESP_ERROR_CHECK(esp_event_handler_unregister(WIFI_EVENT, ESP_EVENT_ANY_ID, staticWpsEventHandler)); + delete wpsConfig; + wpsConfig = nullptr; } #endif // ENABLE_WPS diff --git a/Sming/Arch/Esp32/Platform/StationImpl.h b/Sming/Arch/Esp32/Platform/StationImpl.h index e785208713..6aa477391a 100644 --- a/Sming/Arch/Esp32/Platform/StationImpl.h +++ b/Sming/Arch/Esp32/Platform/StationImpl.h @@ -18,6 +18,8 @@ #include #endif +struct esp_netif_obj; + class StationImpl : public StationClass, protected ISystemReadyHandler { public: @@ -38,6 +40,7 @@ class StationImpl : public StationClass, protected ISystemReadyHandler String getHostname() const override; IpAddress getIP() const override; MacAddress getMacAddress() const override; + bool setMacAddress(const MacAddress& addr) const override; IpAddress getNetworkMask() const override; IpAddress getNetworkGateway() const override; IpAddress getNetworkBroadcast() const override; @@ -63,16 +66,28 @@ class StationImpl : public StationClass, protected ISystemReadyHandler private: static void staticScanCompleted(wifi_event_sta_scan_done_t* event, uint8_t status); +#ifdef ENABLE_WPS + static void staticWpsEventHandler(void* arg, esp_event_base_t event_base, int32_t event_id, void* event_data) + { + auto self = static_cast(arg); + self->wpsEventHandler(event_base, event_id, event_data); + } + void wpsEventHandler(esp_event_base_t event_base, int32_t event_id, void* event_data); + bool wpsCallback(WpsStatus status); + bool wpsConfigure(uint8_t credIndex); +#endif #ifdef ENABLE_SMART_CONFIG void internalSmartConfig(sc_status status, void* pdata); #endif -#ifdef ENABLE_WPS - void internalWpsConfig(wps_cb_status status); -#endif private: - bool runScan = false; + bool runScan{false}; +#ifdef ENABLE_WPS + struct WpsConfig; + WpsConfig* wpsConfig; +#endif #ifdef ENABLE_SMART_CONFIG std::unique_ptr smartConfigEventInfo; ///< Set during smart handling #endif + esp_netif_obj* stationNetworkInterface{nullptr}; }; diff --git a/Sming/Arch/Esp32/README.rst b/Sming/Arch/Esp32/README.rst index 6a39bb46d8..5c249ba8a9 100644 --- a/Sming/Arch/Esp32/README.rst +++ b/Sming/Arch/Esp32/README.rst @@ -15,15 +15,19 @@ Requirements ------------ In order to be able to compile for the ESP32 architecture you should have ESP-IDF v4.1 installed. -A detailed installation manual can be found in the `ESP-IDF documentation `__. +The Sming installers can do this for you - see :doc:`/getting-started/index`. + +You can find further details in the `ESP-IDF documentation `__. Building -------- -Make sure that the IDF_PATH environmental variable is set. Also make sure that the other ESP-IDF environmental variables are set. -For example on Linux this can be done using the following command:: +Make sure that the :envvar:`IDF_PATH` is set. +Also make sure that the other ESP-IDF environmental variables are set. + +In Linux this can be done using the following command:: - source $IDF_PATH/export.sh + source $SMING_HOME/Tools/export.sh Build the framework and application as usual, specifying :envvar:`SMING_ARCH` =Esp32. For example:: @@ -51,6 +55,13 @@ If you want to revert to using the default pre-compiled SDK then issue the follo make SMING_ARCH=Esp32 sdk-default +.. note:: + + If you have an ESP32-S2 device you'll need to change :envvar:`ESP_VARIANT`:: + + make ESP_VARIANT=esp32s2 + +See :component-esp32:`esp32` for further details. Components ---------- diff --git a/Sming/Arch/Esp32/Tools/ci/build.run.cmd b/Sming/Arch/Esp32/Tools/ci/build.run.cmd new file mode 100644 index 0000000000..49677d701a --- /dev/null +++ b/Sming/Arch/Esp32/Tools/ci/build.run.cmd @@ -0,0 +1,11 @@ +REM Esp32 build.run.cmd + +%MAKE_PARALLEL% Basic_Blink Basic_WiFi HttpServer_ConfigNetwork DEBUG_VERBOSE_LEVEL=3 STRICT=1 || goto :error +%MAKE_PARALLEL% Basic_Ssl ENABLE_SSL=Bearssl DEBUG_VERBOSE_LEVEL=3 STRICT=1 || goto :error + +goto :EOF + + +:error +echo Failed with error #%errorlevel%. +exit /b %errorlevel% diff --git a/Sming/Arch/Esp32/Tools/ci/build.run.sh b/Sming/Arch/Esp32/Tools/ci/build.run.sh new file mode 100755 index 0000000000..01209dfe4c --- /dev/null +++ b/Sming/Arch/Esp32/Tools/ci/build.run.sh @@ -0,0 +1,4 @@ +# Esp32 build.run.sh + +$MAKE_PARALLEL Basic_Blink Basic_WiFi HttpServer_ConfigNetwork DEBUG_VERBOSE_LEVEL=3 STRICT=1 +$MAKE_PARALLEL Basic_Ssl ENABLE_SSL=Bearssl DEBUG_VERBOSE_LEVEL=3 STRICT=1 diff --git a/Sming/Arch/Esp32/Tools/ci/build.setup.cmd b/Sming/Arch/Esp32/Tools/ci/build.setup.cmd new file mode 100644 index 0000000000..ea6e6a24cf --- /dev/null +++ b/Sming/Arch/Esp32/Tools/ci/build.setup.cmd @@ -0,0 +1 @@ +REM Esp32 build.setup.cmd diff --git a/Sming/Arch/Esp32/Tools/ci/build.setup.sh b/Sming/Arch/Esp32/Tools/ci/build.setup.sh new file mode 100755 index 0000000000..4aaa83f008 --- /dev/null +++ b/Sming/Arch/Esp32/Tools/ci/build.setup.sh @@ -0,0 +1 @@ +# Esp32 build.setup.sh diff --git a/Sming/Arch/Esp32/Tools/ci/install.cmd b/Sming/Arch/Esp32/Tools/ci/install.cmd new file mode 100644 index 0000000000..7007813206 --- /dev/null +++ b/Sming/Arch/Esp32/Tools/ci/install.cmd @@ -0,0 +1,15 @@ +REM Esp32 install.cmd + +if "%IDF_PATH%"=="" goto :EOF +if "%IDF_TOOLS_PATH%"=="" goto :EOF + +git clone -b release/v4.1 https://github.com/espressif/esp-idf.git %IDF_PATH% + +REM Espressif downloads very slow, fetch from SmingTools +mkdir %IDF_TOOLS_PATH% +set ESPTOOLS=esp32-tools-windows-4.1.7z +curl -LO %SMINGTOOLS%/%ESPTOOLS% +7z -o%IDF_TOOLS_PATH%\dist x %ESPTOOLS% + +python %IDF_PATH%\tools\idf_tools.py install +python -m pip install -r %IDF_PATH%\requirements.txt diff --git a/Sming/Arch/Esp32/Tools/install.sh b/Sming/Arch/Esp32/Tools/install.sh new file mode 100755 index 0000000000..6639c15ffc --- /dev/null +++ b/Sming/Arch/Esp32/Tools/install.sh @@ -0,0 +1,50 @@ +# Esp32 install.sh + +if [ -n "$IDF_PATH" ] && [ -n "$IDF_TOOLS_PATH" ]; then + +PACKAGES=(\ + bison \ + ccache \ + dfu-util \ + flex \ + gperf \ + ninja-build \ + ) + +case $DIST in + debian) + PACKAGES+=(\ + libffi-dev \ + libssl-dev \ + ) + ;; + + fedora) + PACKAGES+=(\ + libffi-devel \ + ) + ;; +esac + +$PKG_INSTALL ${PACKAGES[*]} + +if [ -d "$IDF_PATH" ]; then + printf "\n\n** Skipping ESP-IDF clone: '$IDF_PATH' exists\n\n" +else + git clone -b release/v4.1 https://github.com/espressif/esp-idf.git $IDF_PATH +fi + +# Espressif downloads very slow, fetch from SmingTools +mkdir -p $IDF_TOOLS_PATH +ESPTOOLS=esp32-tools-linux-4.1.zip +$WGET $SMINGTOOLS/$ESPTOOLS -O $DOWNLOADS/$ESPTOOLS +unzip $DOWNLOADS/$ESPTOOLS -d $IDF_TOOLS_PATH/dist + +python3 $IDF_PATH/tools/idf_tools.py install +python3 -m pip install -r $IDF_PATH/requirements.txt + +if [ -z "$KEEP_DOWNLOADS" ]; then + rm -rf $IDF_TOOLS_PATH/dist +fi + +fi diff --git a/Sming/Arch/Esp32/app.mk b/Sming/Arch/Esp32/app.mk index a78c572e86..a4f06d5efb 100644 --- a/Sming/Arch/Esp32/app.mk +++ b/Sming/Arch/Esp32/app.mk @@ -11,7 +11,7 @@ LDFLAGS += \ .PHONY: application -application: $(CUSTOM_TARGETS) $(TARGET_BIN) +application: $(TARGET_BIN) # $1 -> Linker script define LinkTarget @@ -42,52 +42,3 @@ $(TARGET_OUT): $(COMPONENTS_AR) $(TARGET_BIN): $(TARGET_OUT) $(Q) $(ESPTOOL_CMDLINE) elf2image --min-rev 0 --elf-sha256-offset 0xb0 $(flashimageoptions) -o $@ $< - -##@Flashing - -# Partitions -PARTITIONS_CSV ?= $(BUILD_BASE)/partitions.csv -PARTITIONS_BIN = $(FW_BASE)/partitions.bin - -CUSTOM_TARGETS += $(PARTITIONS_CSV) - -$(BUILD_BASE)/partitions.csv: | $(BUILD_BASE) - $(Q) cp $(SDK_PARTITION_PATH)/base.csv $@ - @echo "storage, data, spiffs, $(SPIFF_START_ADDR), $(SPIFF_SIZE)," >> $@ - -$(PARTITIONS_BIN): $(PARTITIONS_CSV) - $(Q) $(ESP32_PYTHON) $(SDK_COMPONENTS_PATH)/partition_table/gen_esp32part.py $< $@ - -.PHONY: partitions -partitions: $(PARTITIONS_BIN) ##Generate partitions table - - -FLASH_PARTITION_CHUNKS := 0x8000=$(PARTITIONS_BIN) - -# Application - -FLASH_APP_CHUNKS := 0x10000=$(TARGET_BIN) - -.PHONY: flashboot -flashboot: $(FLASH_BOOT_LOADER) ##Write just the Bootloader - $(call WriteFlash,$(FLASH_BOOT_CHUNKS)) - -.PHONY: flashconfig - -flashconfig: partitions kill_term ##Write partition config - $(call WriteFlash,$(FLASH_PARTITION_CHUNKS)) - -flashpartition: flashconfig - -.PHONY: flashapp -flashapp: all kill_term ##Write just the application image - $(call WriteFlash,$(FLASH_APP_CHUNKS)) - -.PHONY: flash -flash: all partitions kill_term ##Write the boot loader, application image, partition table and (if enabled) SPIFFS image - $(call WriteFlash,$(FLASH_BOOT_CHUNKS) $(FLASH_APP_CHUNKS) $(FLASH_PARTITION_CHUNKS) $(FLASH_SPIFFS_CHUNKS)) -ifeq ($(ENABLE_GDB), 1) - $(GDB_CMDLINE) -else - $(TERMINAL) -endif diff --git a/Sming/Arch/Esp32/standard.hw b/Sming/Arch/Esp32/standard.hw new file mode 100644 index 0000000000..96fd083214 --- /dev/null +++ b/Sming/Arch/Esp32/standard.hw @@ -0,0 +1,35 @@ +{ + "name": "Standard config with single ROM", + "comment": "Should work with any Esp32 variant", + "arch": "Esp32", + "partition_table_offset": "0x8000", + "devices": { + "spiFlash": { + "type": "flash", + "size": "4M", + "mode": "dio", + "speed": 40 + } + }, + "partitions": { + "phy_init": { + "address": "0x00f000", + "size": "0x1000", + "type": "data", + "subtype": "phy" + }, + "nvs": { + "address": "0x009000", + "size": "0x6000", + "type": "data", + "subtype": "nvs" + }, + "factory": { + "address": "0x010000", + "size": "0x180000", + "type": "app", + "subtype": "factory", + "filename": "$(TARGET_BIN)" + } + } +} diff --git a/Sming/Arch/Esp8266/Components/driver/uart.cpp b/Sming/Arch/Esp8266/Components/driver/uart.cpp index e6c50b94fc..22114d3818 100644 --- a/Sming/Arch/Esp8266/Components/driver/uart.cpp +++ b/Sming/Arch/Esp8266/Components/driver/uart.cpp @@ -910,9 +910,9 @@ void smg_uart_swap(smg_uart_t* uart, int tx_pin) bool smg_uart_set_tx(smg_uart_t* uart, int tx_pin) { if(uart != nullptr && uart->uart_nr == UART0 && smg_uart_tx_enabled(uart)) { - uart1_pin_restore(uart->tx_pin); + uart0_pin_restore(uart->tx_pin); uart->tx_pin = (tx_pin == 2) ? 2 : 1; - uart1_pin_select(uart->tx_pin); + uart0_pin_select(uart->tx_pin); return true; } diff --git a/Sming/Arch/Esp8266/Components/esp-open-lwip/component.mk b/Sming/Arch/Esp8266/Components/esp-open-lwip/component.mk index 0bc11ea6d4..e9a77126cc 100644 --- a/Sming/Arch/Esp8266/Components/esp-open-lwip/component.mk +++ b/Sming/Arch/Esp8266/Components/esp-open-lwip/component.mk @@ -25,12 +25,10 @@ endif COMPONENT_SUBMODULES := esp-open-lwip COMPONENT_SRCDIRS := COMPONENT_SRCFILES := \ - lwip/api/err.c \ lwip/core/def.c \ lwip/core/dhcp.c \ lwip/core/dns.c \ lwip/core/init.c \ - lwip/core/mdns.c \ lwip/core/mem.c \ lwip/core/memp.c \ lwip/core/netif.c \ @@ -63,7 +61,9 @@ ifeq ($(ENABLE_ESPCONN),1) espconn.c \ espconn_tcp.c \ espconn_udp.c \ - espconn_mdns.c) + espconn_mdns.c) \ + lwip/api/err.c \ + lwip/core/mdns.c else COMPONENT_SRCFILES += espconn_dummy.c endif diff --git a/Sming/Arch/Esp8266/Components/esp-open-lwip/esp-open-lwip.patch b/Sming/Arch/Esp8266/Components/esp-open-lwip/esp-open-lwip.patch index 5f3487a9e8..b5dc45183b 100644 --- a/Sming/Arch/Esp8266/Components/esp-open-lwip/esp-open-lwip.patch +++ b/Sming/Arch/Esp8266/Components/esp-open-lwip/esp-open-lwip.patch @@ -76,10 +76,10 @@ index 1bc584f..e4af916 100644 $(LIB): $(OBJS) $(AR) rcs $@ $^ diff --git a/include/arch/cc.h b/include/arch/cc.h -index ff03b30..5efcf13 100644 +index ff03b30..4a2b4d9 100644 --- a/include/arch/cc.h +++ b/include/arch/cc.h -@@ -38,8 +38,27 @@ +@@ -38,14 +38,31 @@ #include "c_types.h" #include "ets_sys.h" #include "osapi.h" @@ -106,8 +106,15 @@ index ff03b30..5efcf13 100644 + //#define LWIP_PROVIDE_ERRNO - #if (1) -@@ -56,6 +75,7 @@ typedef signed short s16_t; +-#if (1) ++#ifndef BYTE_ORDER + #define BYTE_ORDER LITTLE_ENDIAN +-#else +-#define BYTE_ORDER BIG_ENDIAN + #endif + + +@@ -56,6 +73,7 @@ typedef signed short s16_t; typedef unsigned long u32_t; typedef signed long s32_t; typedef unsigned long mem_ptr_t; @@ -115,7 +122,7 @@ index ff03b30..5efcf13 100644 #define S16_F "d" #define U16_F "d" -@@ -73,11 +93,12 @@ typedef unsigned long mem_ptr_t; +@@ -73,11 +91,12 @@ typedef unsigned long mem_ptr_t; #define PACK_STRUCT_BEGIN #define PACK_STRUCT_END @@ -405,21 +412,10 @@ index ae9ed50..46420df 100644 #endif diff --git a/include/lwip/mdns.h b/include/lwip/mdns.h -index 08db68a..93ca93d 100644 +index 08db68a..b4763ec 100644 --- a/include/lwip/mdns.h +++ b/include/lwip/mdns.h -@@ -32,8 +32,8 @@ - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - --#ifndef __LWIP_DNS_H__ --#define __LWIP_DNS_H__ -+#ifndef __LWIP_MDNS_H__ -+#define __LWIP_MDNS_H__ - - #include "lwip/opt.h" - -@@ -96,19 +96,20 @@ struct mdns_info { +@@ -96,16 +96,17 @@ struct mdns_info { char *txt_data[10]; }; #endif @@ -447,12 +443,6 @@ index 08db68a..93ca93d 100644 //void mdns_tmr(void); //void Delay(unsigned long ulSeconds); --#endif /* LWIP_DNS */ -+#endif /* LWIP_MDNS */ - --#endif /* __LWIP_DNS_H__ */ -+#endif /* __LWIP_MDNS_H__ */ - diff --git a/lwip/app/espconn_udp.c b/lwip/app/espconn_udp.c index 77ef471..0e4936a 100644 --- a/lwip/app/espconn_udp.c diff --git a/Sming/Arch/Esp8266/Components/esp8266/Tools/patch-phy-bin.py b/Sming/Arch/Esp8266/Components/esp8266/Tools/patch-phy-bin.py new file mode 100644 index 0000000000..8f38ce38aa --- /dev/null +++ b/Sming/Arch/Esp8266/Components/esp8266/Tools/patch-phy-bin.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +######################################################## +# +# Simple PHY binary patcher +# Author: Slavey Karadzhov +# +######################################################## +import os +import sys + +def usage(): + print("Usage: \n\t%s [offset]" % sys.argv[0]) + +if __name__ == "__main__": + if len(sys.argv) not in list(range(2,4)): + usage() + sys.exit(1) + + offset = 107 + if len(sys.argv) > 2: + offset = int(sys.argv[2]) + + with os.fdopen(os.open(sys.argv[1], os.O_RDWR | os.O_CREAT), 'rb+') as f: + f.seek(offset) + f.write(bytearray(b'\xff')); diff --git a/Sming/Arch/Esp8266/Components/esp8266/component.mk b/Sming/Arch/Esp8266/Components/esp8266/component.mk index dbbfcb4991..e3843a245f 100644 --- a/Sming/Arch/Esp8266/Components/esp8266/component.mk +++ b/Sming/Arch/Esp8266/Components/esp8266/component.mk @@ -4,19 +4,17 @@ COMPONENT_VARS := SDK_BASE SDK_BASE ?= $(COMPONENT_PATH)/ESP8266_NONOS_SDK SDK_BASE := $(call FixPath,$(SDK_BASE)) -FLASH_INIT_DATA = $(SDK_BASE)/bin/esp_init_data_default.bin -CUSTOM_TARGETS += $(FLASH_INIT_DATA) +DEBUG_VARS += FLASH_INIT_DATA FLASH_INIT_DATA_VCC +FLASH_INIT_DATA = $(SDK_BASE)/bin/esp_init_data_default.bin +FLASH_INIT_DATA_VCC = $(SDK_BASE)/bin/esp_init_data_vdd_default.bin -FLASH_INIT_CHUNKS += \ - $(call FlashOffset,0x5000)=$(BLANK_BIN) \ - $(call FlashOffset,0x4000)=$(FLASH_INIT_DATA) \ - $(call FlashOffset,0x2000)=$(BLANK_BIN) +CUSTOM_TARGETS += $(FLASH_INIT_DATA) $(FLASH_INIT_DATA_VCC) # => 'Internal' SDK - for SDK Version 3+ as submodule in Sming repository # SDK_BASE just needs to point into our repo as it's overridden with the correct submodule path # This provides backward-compatiblity, so $(SMING)/third-party/ESP8266_NONOS_SDK) still works -DEBUG_VARS += SDK_INTERNAL +DEBUG_VARS += SDK_INTERNAL SDK_BASE ifneq (,$(findstring $(SMING_HOME),$(SDK_BASE))) GLOBAL_CFLAGS += -DSDK_INTERNAL=1 SDK_INTERNAL := 1 @@ -29,6 +27,12 @@ else SDK_INTERNAL := 0 endif +PHY_TOOL := $(COMPONENT_PATH)/Tools/patch-phy-bin.py + +$(FLASH_INIT_DATA_VCC): $(FLASH_INIT_DATA) + $(Q) cp $< $@ + $(Q) $(PYTHON) $(PHY_TOOL) $@ + DEBUG_VARS += SDK_LIBDIR SDK_INCDIR SDK_LIBDIR := $(SDK_BASE)/lib SDK_INCDIR := $(SDK_BASE)/include @@ -63,7 +67,7 @@ COMPONENT_TARGETS += $(LIBCRYPTO_ORIG) # Make backup then modify original $(COMPONENT_RULE)$(LIBCRYPTO_ORIG): $(LIBCRYPTO) cp $^ $@ - ar -d $^ aes-internal-dec.o + $(AR) -d $^ aes-internal-dec.o # Define linker symbols EXTRA_LDFLAGS += -Wl,--just-symbols=$(COMPONENT_PATH)/ld/crypto.sym diff --git a/Sming/Arch/Esp8266/Components/esp8266/startup.cpp b/Sming/Arch/Esp8266/Components/esp8266/startup.cpp index f2adb9e490..ff62f084d2 100644 --- a/Sming/Arch/Esp8266/Components/esp8266/startup.cpp +++ b/Sming/Arch/Esp8266/Components/esp8266/startup.cpp @@ -14,6 +14,7 @@ #include #include #include +#include extern void init(); @@ -54,46 +55,8 @@ extern "C" void WEAK_ATTR user_rf_pre_init(void) extern "C" uint32 ICACHE_FLASH_ATTR WEAK_ATTR user_rf_cal_sector_set(void) { - enum flash_size_map size_map = system_get_flash_size_map(); - uint32 rf_cal_sec = 0; - - switch (int(size_map)) { - case FLASH_SIZE_2M: - rf_cal_sec = 64 - 5; - break; - - case FLASH_SIZE_4M_MAP_256_256: - rf_cal_sec = 128 - 5; - break; - - case FLASH_SIZE_8M_MAP_512_512: - rf_cal_sec = 256 - 5; - break; - - case FLASH_SIZE_16M_MAP_512_512: - case FLASH_SIZE_16M_MAP_1024_1024: - rf_cal_sec = 512 - 5; - break; - - case FLASH_SIZE_32M_MAP_512_512: - case FLASH_SIZE_32M_MAP_1024_1024: - rf_cal_sec = 1024 - 5; - break; - - case 8: // FLASH_SIZE_64M_MAP_1024_1024 - rf_cal_sec = 2048 - 5; - break; - - case 9: // FLASH_SIZE_128M_MAP_1024_1024 - rf_cal_sec = 4096 - 5; - break; - - default: - rf_cal_sec = 0; - break; - } - - return rf_cal_sec; + auto rfCal = *Storage::findPartition(Storage::Partition::SubType::Data::rfCal); + return rfCal.address(); } #ifdef SDK_INTERNAL @@ -104,25 +67,17 @@ extern "C" uint32 ICACHE_FLASH_ATTR WEAK_ATTR user_rf_cal_sector_set(void) extern "C" void ICACHE_FLASH_ATTR WEAK_ATTR user_pre_init(void) { - const uint32_t MAX_PROGRAM_SECTORS = 0x100000 / SPI_FLASH_SEC_SIZE; // 1MB addressable - - // WARNING: Sming supports SDK 3.0 with rBoot enabled apps ONLY! - const partition_type_t SYSTEM_PARTITION_RBOOT_CONFIG = static_cast(SYSTEM_PARTITION_CUSTOMER_BEGIN + 0); - const partition_type_t SYSTEM_PARTITION_PROGRAM = static_cast(SYSTEM_PARTITION_CUSTOMER_BEGIN + 1); + Storage::initialize(); - // Partitions offsets and sizes must be in sector multiples, so work in sectors - #define PARTITION_ITEM(_type, _start, _length) \ - {_type, (_start) * SPI_FLASH_SEC_SIZE, (_length) * SPI_FLASH_SEC_SIZE} + auto sysParam = *Storage::findPartition(Storage::Partition::SubType::Data::sysParam); + auto rfCal = *Storage::findPartition(Storage::Partition::SubType::Data::rfCal); + auto phy = *Storage::findPartition(Storage::Partition::SubType::Data::phy); - // Partitions in position order - uint32_t rfCalSector = user_rf_cal_sector_set(); static const partition_item_t partitions[] = { - PARTITION_ITEM(SYSTEM_PARTITION_BOOTLOADER, 0, 1), - PARTITION_ITEM(SYSTEM_PARTITION_RBOOT_CONFIG, 1, 1), - PARTITION_ITEM(SYSTEM_PARTITION_PROGRAM, 2, std::min(MAX_PROGRAM_SECTORS, rfCalSector) - 2), - PARTITION_ITEM(SYSTEM_PARTITION_RF_CAL, rfCalSector, 1), - PARTITION_ITEM(SYSTEM_PARTITION_PHY_DATA, rfCalSector + 1, 1), - PARTITION_ITEM(SYSTEM_PARTITION_SYSTEM_PARAMETER, rfCalSector + 2, 3), + {SYSTEM_PARTITION_BOOTLOADER, 0, SPI_FLASH_SEC_SIZE}, + {SYSTEM_PARTITION_PHY_DATA, phy.address(), phy.size()}, + {SYSTEM_PARTITION_SYSTEM_PARAMETER, sysParam.address(), sysParam.size()}, + {SYSTEM_PARTITION_RF_CAL, rfCal.address(), rfCal.size()}, }; enum flash_size_map sizeMap = system_get_flash_size_map(); @@ -134,7 +89,7 @@ extern "C" void ICACHE_FLASH_ATTR WEAK_ATTR user_pre_init(void) os_printf("partition[%u]: %u, 0x%08x, 0x%08x\n", i, part.type, part.addr, part.size); } if(sizeMap < FLASH_SIZE_8M_MAP_512_512) { - os_printf("** Note: SDK 3.0.1 requires SPI_SIZE >= 1M\n"); + os_printf("** Note: SDK 3.0.1 requires spiFlash size >= 1M\n"); } while(1) { // Cannot proceed diff --git a/Sming/Arch/Esp8266/Components/gdbstub/gdbhostio.cpp b/Sming/Arch/Esp8266/Components/gdbstub/gdbhostio.cpp index fc29b4a100..df984d9e4a 100644 --- a/Sming/Arch/Esp8266/Components/gdbstub/gdbhostio.cpp +++ b/Sming/Arch/Esp8266/Components/gdbstub/gdbhostio.cpp @@ -57,23 +57,23 @@ void ATTR_GDBEXTERNFN gdbHandleHostIo(char* commandBuffer, unsigned cmdLen) filename[len] = '\0'; ++data; // skip , unsigned flags = GdbPacket::readHexValue(data); - ++data; // Skip , + ++data; // Skip , unsigned mode = GdbPacket::readHexValue(data); // Skip mode (not used) (void)mode; FileOpenFlags openFlags; if((flags & 0xff) == O_RDWR) { - openFlags = eFO_ReadWrite; + openFlags = File::ReadWrite; } else if((flags & 0xff) == O_WRONLY) { - openFlags = eFO_WriteOnly; + openFlags = File::WriteOnly; } else { - openFlags = eFO_ReadOnly; + openFlags = File::ReadOnly; } if(flags & O_CREAT) { - openFlags = openFlags | eFO_CreateIfNotExist; + openFlags |= File::Create; } if(flags & O_TRUNC) { - openFlags = openFlags | eFO_Truncate; + openFlags |= File::Truncate; } int fd = fileOpen(filename, openFlags); diff --git a/Sming/Arch/Esp8266/Components/lwip2/lwip2 b/Sming/Arch/Esp8266/Components/lwip2/lwip2 index 5b73bb11d5..8bd8607b4e 160000 --- a/Sming/Arch/Esp8266/Components/lwip2/lwip2 +++ b/Sming/Arch/Esp8266/Components/lwip2/lwip2 @@ -1 +1 @@ -Subproject commit 5b73bb11d59285d49a6efeb20726186cc5cc2b74 +Subproject commit 8bd8607b4ed3b81cf656a72729aa70a09b5f4d6e diff --git a/Sming/Arch/Esp8266/Components/lwip2/lwip2.patch b/Sming/Arch/Esp8266/Components/lwip2/lwip2.patch index fbb74b7629..5032837ed2 100644 --- a/Sming/Arch/Esp8266/Components/lwip2/lwip2.patch +++ b/Sming/Arch/Esp8266/Components/lwip2/lwip2.patch @@ -162,44 +162,3 @@ index cfc10f8..8a32ef0 100644 #define IP2STR(ipaddr) ip4_addr1_16(ipaddr), \ ip4_addr2_16(ipaddr), \ - -diff --git a/glue-lwip/lwipopts.h b/glue-lwip/lwipopts.h -index fbb0bde..a84ac22 100644 ---- a/glue-lwip/lwipopts.h -+++ b/glue-lwip/lwipopts.h -@@ -1503,13 +1503,13 @@ - #define LWIP_NETIF_TX_SINGLE_PBUF 1 // MANDATORY FOR ESP8266 BLOBS !! - #endif /* LWIP_NETIF_TX_SINGLE_PBUF */ - -+#define LWIP_MDNS_RESPONDER 1 -+ - /** - * LWIP_NUM_NETIF_CLIENT_DATA: Number of clients that may store - * data in client_data member array of struct netif. - */ --#if !defined LWIP_NUM_NETIF_CLIENT_DATA || defined __DOXYGEN__ --#define LWIP_NUM_NETIF_CLIENT_DATA 0 --#endif -+#define LWIP_NUM_NETIF_CLIENT_DATA 2 - /** - * @} - */ - -diff --git a/makefiles/Makefile.lwip2 b/makefiles/Makefile.lwip2 -index 560e346..f46c4ed 100644 ---- a/makefiles/Makefile.lwip2 -+++ b/makefiles/Makefile.lwip2 -@@ -8,6 +8,7 @@ OBJ = \ - $(patsubst %.c,$(BUILD)/%.o,$(wildcard core/ipv6/*.c)) \ - $(patsubst %.c,$(BUILD)/%.o,$(wildcard api/*.c)) \ - $(patsubst %.c,$(BUILD)/%.o,$(wildcard apps/sntp/*.c)) \ -+ $(patsubst %.c,$(BUILD)/%.o,$(wildcard apps/mdns/*.c)) - - BUILD_INCLUDES = -I$(BUILD) -I$(SDK)/include -Iinclude -I../../glue -I../../glue-lwip - -@@ -27,4 +28,5 @@ include ../../makefiles/Makefile.rules - -include $(BUILD)/core/ipv6/*.d - -include $(BUILD)/api/*.d - -include $(BUILD)/apps/sntp/*.d -+-include $(BUILD)/apps/mdns/*.d - diff --git a/Sming/Arch/Esp8266/Platform/AccessPointImpl.cpp b/Sming/Arch/Esp8266/Platform/AccessPointImpl.cpp index 40d5484c86..93f0a266c6 100644 --- a/Sming/Arch/Esp8266/Platform/AccessPointImpl.cpp +++ b/Sming/Arch/Esp8266/Platform/AccessPointImpl.cpp @@ -153,6 +153,11 @@ MacAddress AccessPointImpl::getMacAddress() const } } +bool AccessPointImpl::setMacAddress(const MacAddress& addr) const +{ + return wifi_set_macaddr(SOFTAP_IF, &const_cast(addr)[0]); +} + String AccessPointImpl::getSSID() const { softap_config config = {0}; diff --git a/Sming/Arch/Esp8266/Platform/AccessPointImpl.h b/Sming/Arch/Esp8266/Platform/AccessPointImpl.h index ce285fff2f..53dce75022 100644 --- a/Sming/Arch/Esp8266/Platform/AccessPointImpl.h +++ b/Sming/Arch/Esp8266/Platform/AccessPointImpl.h @@ -28,6 +28,7 @@ class AccessPointImpl : public AccessPointClass, protected ISystemReadyHandler IpAddress getIP() const override; bool setIP(IpAddress address) override; MacAddress getMacAddress() const override; + bool setMacAddress(const MacAddress& addr) const override; IpAddress getNetworkMask() const override; IpAddress getNetworkGateway() const override; IpAddress getNetworkBroadcast() const override; diff --git a/Sming/Arch/Esp8266/Platform/StationImpl.cpp b/Sming/Arch/Esp8266/Platform/StationImpl.cpp index 627f8324b0..cbb7450312 100644 --- a/Sming/Arch/Esp8266/Platform/StationImpl.cpp +++ b/Sming/Arch/Esp8266/Platform/StationImpl.cpp @@ -172,6 +172,11 @@ MacAddress StationImpl::getMacAddress() const } } +bool StationImpl::setMacAddress(const MacAddress& addr) const +{ + return wifi_set_macaddr(STATION_IF, &const_cast(addr)[0]); +} + IpAddress StationImpl::getNetworkBroadcast() const { struct ip_info info = {0}; diff --git a/Sming/Arch/Esp8266/Platform/StationImpl.h b/Sming/Arch/Esp8266/Platform/StationImpl.h index 2e370d2f32..4614a6c179 100644 --- a/Sming/Arch/Esp8266/Platform/StationImpl.h +++ b/Sming/Arch/Esp8266/Platform/StationImpl.h @@ -38,6 +38,7 @@ class StationImpl : public StationClass, protected ISystemReadyHandler String getHostname() const override; IpAddress getIP() const override; MacAddress getMacAddress() const override; + bool setMacAddress(const MacAddress& addr) const override; IpAddress getNetworkMask() const override; IpAddress getNetworkGateway() const override; IpAddress getNetworkBroadcast() const override; diff --git a/Sming/Arch/Esp8266/README.rst b/Sming/Arch/Esp8266/README.rst index b97391a15c..a001a3c628 100644 --- a/Sming/Arch/Esp8266/README.rst +++ b/Sming/Arch/Esp8266/README.rst @@ -3,6 +3,7 @@ Sming Esp8266 Architecture Support building Sming for the Esp8266 architecture. +See also :doc:`/arch/esp8266/getting-started/eqt`. Build variables --------------- diff --git a/Sming/Arch/Esp8266/Tools/ci/build.run.cmd b/Sming/Arch/Esp8266/Tools/ci/build.run.cmd new file mode 100644 index 0000000000..830f5d6c46 --- /dev/null +++ b/Sming/Arch/Esp8266/Tools/ci/build.run.cmd @@ -0,0 +1,15 @@ +REM Esp8266 build.run.cmd + +make -C "%SMING_PROJECTS_DIR%\samples\HttpServer_FirmwareUpload" python-requirements +%MAKE_PARALLEL% samples || goto :error + +make clean samples-clean +%MAKE_PARALLEL% Basic_Blink ENABLE_CUSTOM_HEAP=1 DEBUG_VERBOSE_LEVEL=3 || goto :error +%MAKE_PARALLEL% HttpServer_ConfigNetwork ENABLE_CUSTOM_LWIP=2 STRICT=1 || goto :error + +goto :EOF + + +:error +echo Failed with error #%errorlevel%. +exit /b %errorlevel% diff --git a/Sming/Arch/Esp8266/Tools/ci/build.run.sh b/Sming/Arch/Esp8266/Tools/ci/build.run.sh new file mode 100755 index 0000000000..9ffe3edc5f --- /dev/null +++ b/Sming/Arch/Esp8266/Tools/ci/build.run.sh @@ -0,0 +1,11 @@ +# Esp8266 build.run.sh + +$MAKE_PARALLEL samples +make clean samples-clean +$MAKE_PARALLEL Basic_Blink ENABLE_CUSTOM_HEAP=1 DEBUG_VERBOSE_LEVEL=3 +$MAKE_PARALLEL HttpServer_ConfigNetwork ENABLE_CUSTOM_LWIP=2 STRICT=1 + +# Some samples (UPnP, for example) require more recent compiler +if [ "$BUILD_COMPILER" == "eqt" ]; then + $MAKE_PARALLEL component-samples +fi diff --git a/Sming/Arch/Esp8266/Tools/ci/build.setup.cmd b/Sming/Arch/Esp8266/Tools/ci/build.setup.cmd new file mode 100644 index 0000000000..f6f2238948 --- /dev/null +++ b/Sming/Arch/Esp8266/Tools/ci/build.setup.cmd @@ -0,0 +1 @@ +REM Esp8266 build.setup.cmd diff --git a/Sming/Arch/Esp8266/Tools/ci/build.setup.sh b/Sming/Arch/Esp8266/Tools/ci/build.setup.sh new file mode 100755 index 0000000000..941d5df444 --- /dev/null +++ b/Sming/Arch/Esp8266/Tools/ci/build.setup.sh @@ -0,0 +1,6 @@ +# Esp8266 build.setup.sh + +unset SPIFFY +unset ESPTOOL2 +unset SDK_BASE + diff --git a/Sming/Arch/Esp8266/Tools/ci/install.cmd b/Sming/Arch/Esp8266/Tools/ci/install.cmd new file mode 100644 index 0000000000..e7525a4b2a --- /dev/null +++ b/Sming/Arch/Esp8266/Tools/ci/install.cmd @@ -0,0 +1,11 @@ +REM Esp8266 install.cmd + +call :install "%UDK_ROOT%" esp-udk-win32.7z +call :install "%EQT_ROOT%" x86_64-w64-mingw32.xtensa-lx106-elf-e6a192b.201211.zip +goto :EOF + +:install +if "%1"=="" goto :EOF +mkdir %1 +curl -LO %SMINGTOOLS%/%2 +7z -o%1 x %2 diff --git a/Sming/Arch/Esp8266/Tools/docker/Dockerfile b/Sming/Arch/Esp8266/Tools/docker/Dockerfile deleted file mode 100644 index 98b53dfe02..0000000000 --- a/Sming/Arch/Esp8266/Tools/docker/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -# ------------------------------------ -# Fast dockerized development environment -# for the Sming Framework: https://github.com/SmingHub/Sming.git -# ------------------------------------ -FROM attachix/c9-esp8266-sdk:latest -MAINTAINER Slavey Karadzhov - -COPY assets/welcome.html /cloud9/plugins/c9.ide.welcome/welcome.html -COPY assets/welcome.js /cloud9/plugins/c9.ide.welcome/welcome.js - -RUN git clone https://github.com/SmingHub/Sming.git /workspace/Sming - -ENV SMING_HOME /workspace/Sming/Sming - -ENTRYPOINT /usr/bin/supervisord diff --git a/Sming/Arch/Esp8266/Tools/install.sh b/Sming/Arch/Esp8266/Tools/install.sh new file mode 100644 index 0000000000..039b999a6d --- /dev/null +++ b/Sming/Arch/Esp8266/Tools/install.sh @@ -0,0 +1,27 @@ +# Esp8266 install.sh + +# Old toolchain +if [ -n "$UDK_ROOT" ]; then + if [ -d "$UDK_ROOT" ]; then + printf "\n\n** Skipping Esp8266 tools installation: '$UDK_ROOT' exists\n\n" + else + TOOLCHAIN=esp-open-sdk-linux-x86_64.tar.gz + $WGET $SMINGTOOLS/$TOOLCHAIN -O $DOWNLOADS/$TOOLCHAIN + mkdir -p $UDK_ROOT + tar -zxf $DOWNLOADS/$TOOLCHAIN -C $UDK_ROOT --totals + mv $UDK_ROOT/esp-open-sdk/* $UDK_ROOT + rmdir $UDK_ROOT/esp-open-sdk + fi +fi + +# New toolchain +if [ -n "$EQT_ROOT" ]; then + if [ -d "$EQT_ROOT" ]; then + printf "\n\n** Skipping Esp8266 tools installation: '$EQT_ROOT' exists\n\n" + else + TOOLCHAIN=x86_64-linux-gnu.xtensa-lx106-elf-e6a192b.201211.tar.gz + $WGET $SMINGTOOLS/$TOOLCHAIN -O $DOWNLOADS/$TOOLCHAIN + mkdir -p $EQT_ROOT + tar -zxf $DOWNLOADS/$TOOLCHAIN -C $EQT_ROOT --totals + fi +fi diff --git a/Sming/Arch/Esp8266/app.mk b/Sming/Arch/Esp8266/app.mk index 88f45e491f..96ad91ec04 100644 --- a/Sming/Arch/Esp8266/app.mk +++ b/Sming/Arch/Esp8266/app.mk @@ -35,7 +35,7 @@ LDFLAGS += \ .PHONY: application -application: $(CUSTOM_TARGETS) $(FW_FILE_1) $(FW_FILE_2) +application: $(FW_FILE_1) $(FW_FILE_2) # $1 -> Linker script define LinkTarget @@ -71,24 +71,7 @@ $(TARGET_OUT_1): $(COMPONENTS_AR) $(LIBMAIN_DST) ##@Flashing -.PHONY: flashboot -flashboot: $(RBOOT_BIN) ##Write just the rBoot boot sector - $(call WriteFlash,$(FLASH_RBOOT_BOOT_CHUNKS)) - .PHONY: flashconfig flashconfig: kill_term ##Erase the rBoot config sector $(info Erasing rBoot config sector) $(call WriteFlash,$(FLASH_RBOOT_ERASE_CONFIG_CHUNKS)) - -.PHONY: flashapp -flashapp: all kill_term ##Write just the application image - $(call WriteFlash,$(FLASH_RBOOT_APP_CHUNKS)) - -.PHONY: flash -flash: all kill_term ##Write the rBoot boot sector, application image and (if enabled) SPIFFS image - $(call WriteFlash,$(FLASH_RBOOT_BOOT_CHUNKS) $(FLASH_RBOOT_APP_CHUNKS) $(FLASH_SPIFFS_CHUNKS)) -ifeq ($(ENABLE_GDB), 1) - $(GDB_CMDLINE) -else - $(TERMINAL) -endif diff --git a/Sming/Arch/Esp8266/options.json b/Sming/Arch/Esp8266/options.json new file mode 100644 index 0000000000..dfa35550eb --- /dev/null +++ b/Sming/Arch/Esp8266/options.json @@ -0,0 +1,28 @@ +{ + "vdd": { + "description": "Modify PHY configuration to support `system_get_vdd33()` function", + "partitions": { + "phy_init": { + "filename": "$(FLASH_INIT_DATA_VCC)" + } + } + }, + "alternate": { + "description": "ESP8266 layout with critical partitions at start of flash", + "partition_table_offset": "0x2000", + "partitions": { + "rf_cal": { + "address": "0x3000" + }, + "phy_init": { + "address": "0x4000" + }, + "sys_param": { + "address": "0x5000" + }, + "rom0": { + "address": "0x8000" + } + } + } +} \ No newline at end of file diff --git a/Sming/Arch/Esp8266/spiffs-two-roms.hw b/Sming/Arch/Esp8266/spiffs-two-roms.hw new file mode 100644 index 0000000000..93e4605b47 --- /dev/null +++ b/Sming/Arch/Esp8266/spiffs-two-roms.hw @@ -0,0 +1,16 @@ +{ + "name": "Two ROM slots with single SPIFFS", + "base_config": "spiffs", + "partitions": { + "rom0": { + "subtype": "ota_0" + }, + "rom1": { + "address": "0x108000", + "size": "992K", + "type": "app", + "subtype": "ota_1", + "filename": "$(RBOOT_ROM_1_BIN)" + } + } +} diff --git a/Sming/Arch/Esp8266/standard.hw b/Sming/Arch/Esp8266/standard.hw new file mode 100644 index 0000000000..37d6bd7d68 --- /dev/null +++ b/Sming/Arch/Esp8266/standard.hw @@ -0,0 +1,42 @@ +{ + "name": "Standard config with single ROM", + "comment": "Should work with any Esp8266 variant", + "arch": "Esp8266", + "partition_table_offset": "self.devices[0].size - 0x6000", + "devices": { + "spiFlash": { + "type": "flash", + "size": "1M", + "mode": "dio", + "speed": 40 + } + }, + "partitions": { + "rom0": { + "address": "0x002000", + "size": "992K", + "type": "app", + "subtype": "factory", + "filename": "$(RBOOT_ROM_0_BIN)" + }, + "rf_cal": { + "address": "self.device.size - 0x5000", + "size": "4K", + "type": "data", + "subtype": "rfcal" + }, + "phy_init": { + "address": "self.device.size - 0x4000", + "size": "4K", + "type": "data", + "subtype": "phy", + "filename": "$(FLASH_INIT_DATA)" + }, + "sys_param": { + "address": "self.device.size - 0x3000", + "size": "12K", + "type": "data", + "subtype": "sysparam" + } + } +} \ No newline at end of file diff --git a/Sming/Arch/Esp8266/two-rom-mode.hw b/Sming/Arch/Esp8266/two-rom-mode.hw new file mode 100644 index 0000000000..2be8a10eca --- /dev/null +++ b/Sming/Arch/Esp8266/two-rom-mode.hw @@ -0,0 +1,17 @@ +{ + "name": "Two ROM slots in 1MB of flash", + "base_config": "standard", + "partitions": { + "rom0": { + "subtype": "ota_0", + "size": "480K" + }, + "rom1": { + "address": "0x80000", + "size": "488K", + "type": "app", + "subtype": "ota_1", + "filename": "$(RBOOT_ROM_1_BIN)" + } + } +} diff --git a/Sming/Arch/Host/Components/driver/hw_timer.cpp b/Sming/Arch/Host/Components/driver/hw_timer.cpp index 2b6b04f638..6e8abc7172 100644 --- a/Sming/Arch/Host/Components/driver/hw_timer.cpp +++ b/Sming/Arch/Host/Components/driver/hw_timer.cpp @@ -168,7 +168,7 @@ void* CTimerThread::thread_routine() continue; // state changed } if(errno != ETIMEDOUT) { - hostmsg("Warning! Timer thread errno = %u", errno); + host_debug_w("Timer thread errno = %u", errno); break; } } diff --git a/Sming/Arch/Host/Components/driver/uart_server.cpp b/Sming/Arch/Host/Components/driver/uart_server.cpp index 5da69e90fa..08e9d4f74b 100644 --- a/Sming/Arch/Host/Components/driver/uart_server.cpp +++ b/Sming/Arch/Host/Components/driver/uart_server.cpp @@ -183,7 +183,7 @@ void CUartServer::terminate() { close(); join(); - hostmsg("UART%u server destroyed", uart_nr); + host_debug_i("UART%u server destroyed", uart_nr); } void CUartServer::onNotify(smg_uart_t* uart, smg_uart_notify_code_t code) @@ -271,7 +271,7 @@ int CUartServer::serviceWrite() do { int sent = socket->send(data, avail); if(sent < 0) { - hostmsg("Uart send returned %d", sent); + host_debug_w("Uart send returned %d", sent); result = sent; break; } @@ -295,11 +295,11 @@ void* CUartServer::thread_routine() auto port = portBase + uart_nr; CSockAddr addr(nullptr, port); if(!listen(addr, 1)) { - hostmsg("Listen %s failed", addr.text().c_str()); + host_debug_e("Listen %s failed", addr.text().c_str()); return nullptr; } - hostmsg("UART%u server listening on port %u", uart_nr, port); + host_debug_i("UART%u server listening on port %u", uart_nr, port); while(active()) { socket = try_connect(); @@ -308,7 +308,7 @@ void* CUartServer::thread_routine() continue; } - hostmsg("Uart #%u socket open", uart_nr); + host_debug_i("Uart #%u socket open", uart_nr); while(socket->active()) { if(txsem.timedwait(IDLE_SLEEP_MS)) { @@ -335,7 +335,7 @@ void* CUartServer::thread_routine() } socket->close(); - hostmsg("Uart #%u socket closed", uart_nr); + host_debug_i("Uart #%u socket closed", uart_nr); } return nullptr; diff --git a/Sming/Arch/Host/Components/esp_hal/tasks.cpp b/Sming/Arch/Host/Components/esp_hal/tasks.cpp index eec99b5bb7..064fe7e860 100644 --- a/Sming/Arch/Host/Components/esp_hal/tasks.cpp +++ b/Sming/Arch/Host/Components/esp_hal/tasks.cpp @@ -55,12 +55,12 @@ const uint8_t HOST_TASK_PRIO = USER_TASK_PRIO_MAX; bool system_os_task(os_task_t callback, uint8_t prio, os_event_t* events, uint8_t qlen) { if(prio >= USER_TASK_PRIO_MAX) { - hostmsg("Invalid priority %u", prio); + host_debug_e("Invalid priority %u", prio); return false; } auto& queue = task_queues[prio]; if(queue != nullptr) { - hostmsg("Queue %u already initialised", prio); + host_debug_e("Queue %u already initialised", prio); return false; } @@ -71,12 +71,12 @@ bool system_os_task(os_task_t callback, uint8_t prio, os_event_t* events, uint8_ bool system_os_post(uint8_t prio, os_signal_t sig, os_param_t par) { if(prio >= USER_TASK_PRIO_MAX) { - hostmsg("Invalid priority %u", prio); + host_debug_e("Invalid priority %u", prio); return false; } auto& queue = task_queues[prio]; if(queue == nullptr) { - hostmsg("Task queue %u not initialised", prio); + host_debug_e("Task queue %u not initialised", prio); return false; } diff --git a/Sming/Arch/Host/Components/gdbstub/component.mk b/Sming/Arch/Host/Components/gdbstub/component.mk index d2f160a6f7..46fd3ee841 100644 --- a/Sming/Arch/Host/Components/gdbstub/component.mk +++ b/Sming/Arch/Host/Components/gdbstub/component.mk @@ -3,4 +3,4 @@ DEBUG_VARS += GDBSTUB_DIR GDBSTUB_DIR := $(COMPONENT_PATH) CACHE_VARS += GDB_CMDLINE -GDB_CMDLINE = trap '' INT; $(GDB) -x $(GDBSTUB_DIR)/gdbcmds --args $(TARGET_OUT_0) $(CLI_TARGET_OPTIONS) --pause +GDB_CMDLINE = trap '' INT; $(GDB) -x $(GDBSTUB_DIR)/gdbcmds --args $(TARGET_OUT_0) $(CLI_TARGET_OPTIONS) --pause -- $(HOST_PARAMETERS) diff --git a/Sming/Arch/Host/Components/hostlib/hostmsg.c b/Sming/Arch/Host/Components/hostlib/hostmsg.c index 6076a9fefe..c4be9a91c4 100644 --- a/Sming/Arch/Host/Components/hostlib/hostmsg.c +++ b/Sming/Arch/Host/Components/hostlib/hostmsg.c @@ -22,6 +22,8 @@ #include #include "hostmsg.h" +int host_debug_level = 2; + /* * e.g. from "void a::sub(int)" we just want "a::sub" */ diff --git a/Sming/Arch/Host/Components/hostlib/hostmsg.h b/Sming/Arch/Host/Components/hostlib/hostmsg.h index 1fd6537dc4..149c62335c 100644 --- a/Sming/Arch/Host/Components/hostlib/hostmsg.h +++ b/Sming/Arch/Host/Components/hostlib/hostmsg.h @@ -36,6 +36,22 @@ void host_puts(const char* str); #define hostmsg(fmt, ...) host_printfp(fmt "\n", __func__, ##__VA_ARGS__) #endif +/** + * @brief Emit message only if host_debug_level >= level + */ +#define host_debug(level, fmt, ...) \ + do { \ + if(host_debug_level >= (level)) { \ + host_printf(fmt "\n", ##__VA_ARGS__); \ + } \ + } while(0) + +#define host_debug_e(fmt, ...) host_debug(0, "Error! " fmt, ##__VA_ARGS__) +#define host_debug_w(fmt, ...) host_debug(1, "Warning! " fmt, ##__VA_ARGS__) +#define host_debug_i(fmt, ...) host_debug(2, fmt, ##__VA_ARGS__) + +extern int host_debug_level; + #ifdef __cplusplus } #endif diff --git a/Sming/Arch/Host/Components/hostlib/include/hostlib/hostlib.h b/Sming/Arch/Host/Components/hostlib/include/hostlib/hostlib.h index c2e52a10f5..b30b0cec12 100644 --- a/Sming/Arch/Host/Components/hostlib/include/hostlib/hostlib.h +++ b/Sming/Arch/Host/Components/hostlib/include/hostlib/hostlib.h @@ -19,7 +19,7 @@ #pragma once -// Required for sleep(), ftruncate(), probably others +// Required for sleep(), probably others #undef _POSIX_C_SOURCE #define _POSIX_C_SOURCE 200112L @@ -35,10 +35,6 @@ #include #include -#ifndef O_BINARY -#define O_BINARY 0 -#endif - #ifndef ARRAY_SIZE #define ARRAY_SIZE(x) (int)(sizeof(x) / sizeof((x)[0])) #endif diff --git a/Sming/Arch/Host/Components/hostlib/options.cpp b/Sming/Arch/Host/Components/hostlib/options.cpp index b46908e07c..bc40b88e12 100644 --- a/Sming/Arch/Host/Components/hostlib/options.cpp +++ b/Sming/Arch/Host/Components/hostlib/options.cpp @@ -92,3 +92,8 @@ option_tag_t get_option(int argc, char* argv[], const char*& arg) arg = optarg; return option_tag_t(option_index); } + +int get_first_non_option() +{ + return optind; +} diff --git a/Sming/Arch/Host/Components/hostlib/options.h b/Sming/Arch/Host/Components/hostlib/options.h index 85a31f75b8..a7371993b6 100644 --- a/Sming/Arch/Host/Components/hostlib/options.h +++ b/Sming/Arch/Host/Components/hostlib/options.h @@ -48,7 +48,9 @@ XX(flashsize, required_argument, "Change default flash size if file doesn't exist", "SIZE", \ "Size of flash in bytes (e.g. 512K, 524288, 0x80000)", nullptr) \ XX(initonly, no_argument, "Initialise only, do not start Sming", nullptr, nullptr, nullptr) \ - XX(nonet, no_argument, "Skip network initialisation", nullptr, nullptr, nullptr) + XX(nonet, no_argument, "Skip network initialisation", nullptr, nullptr, nullptr) \ + XX(debug, required_argument, "Set debug verbosity", "LEVEL", "Maximum debug message level to print", \ + "0 = errors only, 1 = +warnings, 2 = +info\0") enum option_tag_t { #define XX(tag, has_arg, desc, argname, arghelp, examples) opt_##tag, @@ -58,4 +60,5 @@ enum option_tag_t { }; option_tag_t get_option(int argc, char* argv[], const char*& arg); +int get_first_non_option(); void print_help(); diff --git a/Sming/Arch/Host/Components/hostlib/sockets.cpp b/Sming/Arch/Host/Components/hostlib/sockets.cpp index 41c4a3fbb5..5be1025302 100644 --- a/Sming/Arch/Host/Components/hostlib/sockets.cpp +++ b/Sming/Arch/Host/Components/hostlib/sockets.cpp @@ -147,20 +147,20 @@ bool CSocket::create() // creation of the socket m_fd = ::socket(AF_INET, m_type, 0); if(m_fd <= 0) { - hostmsg("%s", socket_strerror().c_str()); + host_debug_e("%s", socket_strerror().c_str()); return false; } int reuse = 1; if(setsockopt(m_fd, SOL_SOCKET, SO_REUSEADDR, (sock_ptr_t)&reuse, sizeof(reuse)) < 0) { - hostmsg("REUSEADDR: %s", socket_strerror().c_str()); + host_debug_e("REUSEADDR: %s", socket_strerror().c_str()); close(); return false; } #ifdef SO_REUSEPORT if(setsockopt(m_fd, SOL_SOCKET, SO_REUSEPORT, (sock_ptr_t)&reuse, sizeof(reuse)) < 0) { - hostmsg("REUSEPORT: %s", socket_strerror().c_str()); + host_debug_e("REUSEPORT: %s", socket_strerror().c_str()); close(); return false; } @@ -212,7 +212,7 @@ void CSocket::close() return; } - hostmsg("%s", addr().text().c_str()); + host_debug_i("%s", addr().text().c_str()); socket_close(m_fd); m_fd = 0; @@ -375,7 +375,7 @@ CSocket* CSocketList::recv(void* buf, size_t& n) #endif if(errno != EPIPE) { // Broken pipe - hostmsg("%s", socket_strerror().c_str()); + host_debug_e("%s", socket_strerror().c_str()); } skt->close(); } diff --git a/Sming/Arch/Host/Components/hostlib/startup.cpp b/Sming/Arch/Host/Components/hostlib/startup.cpp index 330c6c44f7..f33bba2f5b 100644 --- a/Sming/Arch/Host/Components/hostlib/startup.cpp +++ b/Sming/Arch/Host/Components/hostlib/startup.cpp @@ -32,6 +32,7 @@ #include #include #include "include/hostlib/CommandLine.h" +#include #include #include @@ -50,19 +51,19 @@ static void cleanup() CUartServer::shutdown(); sockets_finalise(); host_lwip_shutdown(); - hostmsg("Goodbye!"); + host_debug_i("Goodbye!"); } void host_exit(int code) { static unsigned exit_count; - hostmsg("returning %d", code); + host_debug_i("returning %d", code); exitCode = code; done = true; if(exit_count++) { - hostmsg("Forcing exit"); + host_debug_w("Forcing exit"); exit(exitCode); } } @@ -107,8 +108,6 @@ int main(int argc, char* argv[]) { trap_exceptions(); - host_printf("\nWelcome to the Sming Host emulator\n\n"); - static struct { int pause; int exitpause; @@ -196,11 +195,18 @@ int main(int argc, char* argv[]) config.enable_network = false; break; + case opt_debug: + host_debug_level = atoi(arg); + break; + default:; } } - commandLine.parse(argc - optind, &argv[optind]); + host_debug_i("\nWelcome to the Sming Host emulator\n\n"); + + auto i = get_first_non_option(); + commandLine.parse(argc - i, &argv[i]); if(!host_flashmem_init(config.flash)) { return 1; @@ -211,8 +217,10 @@ int main(int argc, char* argv[]) atexit(cleanup); if(config.initonly) { - hostmsg("Initialise-only requested"); + host_debug_i("Initialise-only requested"); } else { + Storage::initialize(); + CThread::startup(); hw_timer_init(); @@ -229,13 +237,13 @@ int main(int argc, char* argv[]) host_wifi_lwip_init_complete(); } } else { - hostmsg("Network initialisation skipped as requested"); + host_debug_i("Network initialisation skipped as requested"); } - hostmsg("If required, you may start terminal application(s) now"); + host_debug_i("If required, you may start terminal application(s) now"); pause(config.pause); - hostmsg(">> Starting Sming <<\n"); + host_debug_i(">> Starting Sming <<\n"); System.initialize(); @@ -253,7 +261,7 @@ int main(int argc, char* argv[]) system_soft_wdt_feed(); } - hostmsg(">> Normal Exit <<\n"); + host_debug_i(">> Normal Exit <<\n"); } pause(config.exitpause); diff --git a/Sming/Arch/Host/Components/lwip/Linux/CMakeLists.txt b/Sming/Arch/Host/Components/lwip/Linux/CMakeLists.txt index 7e24c7a386..98f0637927 100644 --- a/Sming/Arch/Host/Components/lwip/Linux/CMakeLists.txt +++ b/Sming/Arch/Host/Components/lwip/Linux/CMakeLists.txt @@ -31,7 +31,6 @@ add_library(lwip ${lwipcore4_SRCS} ${lwipcore6_SRCS} ${lwipnetif_SRCS} - ${lwipmdns_SRCS} ${lwipcontribportunix_SRCS} ${lwipcontribportunixnetifs_SRCS} ) diff --git a/Sming/Arch/Host/Components/lwip/Linux/host_lwip.c b/Sming/Arch/Host/Components/lwip/Linux/host_lwip.c index 5de4cf3891..9c84f4fe9a 100644 --- a/Sming/Arch/Host/Components/lwip/Linux/host_lwip.c +++ b/Sming/Arch/Host/Components/lwip/Linux/host_lwip.c @@ -75,7 +75,7 @@ static bool getifaddr(const char* ifname, struct net_config* netcfg) { struct ifaddrs* list; if(getifaddrs(&list) < 0) { - hostmsg("getifaddrs: %s", strerror(errno)); + host_debug_e("getifaddrs: %s", strerror(errno)); return false; } @@ -111,26 +111,26 @@ static bool getifaddr(const char* ifname, struct net_config* netcfg) bool host_lwip_init(const struct lwip_param* param) { - hostmsg("%s", "Initialising LWIP"); + host_debug_i("%s", "Initialising LWIP"); struct net_config netcfg = {0}; if(!getifaddr(param->ifname, &netcfg)) { if(param->ifname == NULL) { - hostmsg("%s", "No compatible interface found"); + host_debug_e("%s", "No compatible interface found"); } else { - hostmsg("Interface '%s' not found", param->ifname); + host_debug_e("Interface '%s' not found", param->ifname); } return false; } if(param->gateway != NULL && ip4addr_aton(param->gateway, &netcfg.gw) != 1) { - hostmsg("Failed to parse provided Gateway address '%s'", param->gateway); + host_debug_e("Failed to parse provided Gateway address '%s'", param->gateway); return false; } if(param->netmask != NULL && ip4addr_aton(param->netmask, &netcfg.netmask) != 1) { - hostmsg("Failed to parse provided Network Mask '%s'", param->netmask); + host_debug_e("Failed to parse provided Network Mask '%s'", param->netmask); return false; } @@ -139,7 +139,7 @@ bool host_lwip_init(const struct lwip_param* param) IP4_ADDR(&netcfg.ipaddr, (uint32_t)ip4_addr1(&netcfg.gw), (uint32_t)ip4_addr2(&netcfg.gw), (uint32_t)ip4_addr3(&netcfg.gw), 10U); } else if(ip4addr_aton(param->ipaddr, &netcfg.ipaddr) != 1) { - hostmsg("Failed to parse provided IP address '%s'", param->ipaddr); + host_debug_e("Failed to parse provided IP address '%s'", param->ipaddr); return false; } @@ -149,7 +149,8 @@ bool host_lwip_init(const struct lwip_param* param) ip4addr_ntoa_r(&netcfg.netmask, nm_str, sizeof(nm_str)); char gw_str[IP4ADDR_STRLEN_MAX]; ip4addr_ntoa_r(&netcfg.gw, gw_str, sizeof(gw_str)); - hostmsg("Using interface '%s', gateway = %s, netmask = %s; using ip = %s", netcfg.ifname, gw_str, nm_str, ip_str); + host_debug_i("Using interface '%s', gateway = %s, netmask = %s; using ip = %s", netcfg.ifname, gw_str, nm_str, + ip_str); setenv("PRECONFIGURED_TAPIF", netcfg.ifname, true); @@ -158,8 +159,8 @@ bool host_lwip_init(const struct lwip_param* param) netif_add(&netif, &netcfg.ipaddr, &netcfg.netmask, &netcfg.gw, NULL, tapif_init, ethernet_input); getMacAddress(netcfg.ifname, netif.hwaddr); - hostmsg("MAC: %02x:%02x:%02x:%02x:%02x:%02x", netif.hwaddr[0], netif.hwaddr[1], netif.hwaddr[2], netif.hwaddr[3], - netif.hwaddr[4], netif.hwaddr[5]); + host_debug_i("MAC: %02x:%02x:%02x:%02x:%02x:%02x", netif.hwaddr[0], netif.hwaddr[1], netif.hwaddr[2], + netif.hwaddr[3], netif.hwaddr[4], netif.hwaddr[5]); netif_set_default(&netif); @@ -170,6 +171,7 @@ void host_lwip_service(void) { /* poll netif, pass packet to lwIP */ tapif_select(&netif); + netif_poll(&netif); sys_check_timeouts(); } diff --git a/Sming/Arch/Host/Components/lwip/Windows/CMakeLists.txt b/Sming/Arch/Host/Components/lwip/Windows/CMakeLists.txt index 6d4173cfac..0a9cfd1226 100644 --- a/Sming/Arch/Host/Components/lwip/Windows/CMakeLists.txt +++ b/Sming/Arch/Host/Components/lwip/Windows/CMakeLists.txt @@ -46,7 +46,6 @@ add_library(lwip ${lwipcore4_SRCS} ${lwipcore6_SRCS} ${lwipnetif_SRCS} - ${lwipmdns_SRCS} ) add_dependencies(lwip npcap) diff --git a/Sming/Arch/Host/Components/lwip/Windows/host_lwip.c b/Sming/Arch/Host/Components/lwip/Windows/host_lwip.c index 404e049739..4c9751d1df 100644 --- a/Sming/Arch/Host/Components/lwip/Windows/host_lwip.c +++ b/Sming/Arch/Host/Components/lwip/Windows/host_lwip.c @@ -84,7 +84,7 @@ static bool find_adapter(const struct lwip_param* param, struct net_config* netc pcap_if_t* alldevs; char errbuf[PCAP_ERRBUF_SIZE + 1]; if(pcap_findalldevs(&alldevs, errbuf) < 0) { - hostmsg("Error in pcap_findalldevs: %s", errbuf); + host_debug_e("Error in pcap_findalldevs: %s", errbuf); return false; } @@ -163,7 +163,7 @@ static bool find_adapter(const struct lwip_param* param, struct net_config* netc bool host_lwip_init(const struct lwip_param* param) { - hostmsg("%s", "Initialising LWIP"); + host_debug_i("%s", "Initialising LWIP"); if(!npcap_init()) { return false; @@ -172,17 +172,17 @@ bool host_lwip_init(const struct lwip_param* param) struct net_config netcfg = {0}; if(param->ipaddr != NULL && ip4addr_aton(param->ipaddr, &netcfg.ipaddr) != 1) { - hostmsg("Failed to parse IP address '%s'", param->ipaddr); + host_debug_e("Failed to parse IP address '%s'", param->ipaddr); return false; } if(param->netmask != NULL && ip4addr_aton(param->netmask, &netcfg.netmask) != 1) { - hostmsg("Failed to parse Network Mask '%s'", param->netmask); + host_debug_e("Failed to parse Network Mask '%s'", param->netmask); return false; } if(param->gateway != NULL && ip4addr_aton(param->gateway, &netcfg.gw) != 1) { - hostmsg("Failed to parse Gateway address '%s'", param->gateway); + host_debug_e("Failed to parse Gateway address '%s'", param->gateway); return false; } @@ -196,7 +196,7 @@ bool host_lwip_init(const struct lwip_param* param) ip4addr_ntoa_r(&netcfg.netmask, nm_str, sizeof(nm_str)); char gw_str[IP4ADDR_STRLEN_MAX]; ip4addr_ntoa_r(&netcfg.gw, gw_str, sizeof(gw_str)); - hostmsg("gateway = %s, netmask = %s; using ip = %s", gw_str, nm_str, ip_str); + host_debug_i("gateway = %s, netmask = %s; using ip = %s", gw_str, nm_str, ip_str); // Even though we're running as NO_SYS, stuff like crypt needs initialising sys_init(); @@ -206,8 +206,8 @@ bool host_lwip_init(const struct lwip_param* param) netif_add(&netif, &netcfg.ipaddr, &netcfg.netmask, &netcfg.gw, state, pcapif_init, ethernet_input); netif_set_default(&netif); - hostmsg("MAC: %02x:%02x:%02x:%02x:%02x:%02x", netif.hwaddr[0], netif.hwaddr[1], netif.hwaddr[2], netif.hwaddr[3], - netif.hwaddr[4], netif.hwaddr[5]); + host_debug_i("MAC: %02x:%02x:%02x:%02x:%02x:%02x", netif.hwaddr[0], netif.hwaddr[1], netif.hwaddr[2], + netif.hwaddr[3], netif.hwaddr[4], netif.hwaddr[5]); return true; } @@ -216,6 +216,7 @@ void host_lwip_service(void) { /* check for packets and link status*/ pcapif_poll(&netif); + netif_poll(&netif); sys_check_timeouts(); } diff --git a/Sming/Arch/Host/Components/lwip/lwipopts.h b/Sming/Arch/Host/Components/lwip/lwipopts.h index 6a66da4e4e..229eea94b4 100644 --- a/Sming/Arch/Host/Components/lwip/lwipopts.h +++ b/Sming/Arch/Host/Components/lwip/lwipopts.h @@ -322,8 +322,6 @@ * transport. */ #define LWIP_DNS LWIP_UDP -#define LWIP_MDNS_RESPONDER LWIP_UDP -#define LWIP_NUM_NETIF_CLIENT_DATA LWIP_MDNS_RESPONDER /* --------------------------------- @@ -382,6 +380,9 @@ */ #define LWIP_HAVE_LOOPIF 0 +// Enable loopback on interfaces +#define LWIP_NETIF_LOOPBACK 1 + /* ---------------------------------------------- ---------- Sequential layer options ---------- diff --git a/Sming/Arch/Host/Components/spi_flash/flashmem.cpp b/Sming/Arch/Host/Components/spi_flash/flashmem.cpp index 8489fe1ac5..ae479fbcba 100644 --- a/Sming/Arch/Host/Components/spi_flash/flashmem.cpp +++ b/Sming/Arch/Host/Components/spi_flash/flashmem.cpp @@ -21,23 +21,26 @@ #include "flashmem.h" #include #include +#include -static int flashFile = -1; -static size_t flashFileSize = 0x400000U; -static char flashFileName[256]; -static const char defaultFlashFileName[] = "flash.bin"; +namespace +{ +IFS::File flashFile(&IFS::Host::getFileSystem()); +size_t flashFileSize{0x400000U}; +char flashFileName[256]; +const char defaultFlashFileName[]{"flash.bin"}; // Top bit of flash address is set to indicate it's actually program memory -static constexpr uint32_t FLASHMEM_REAL_BIT = 0x80000000U; -static constexpr uint32_t FLASHMEM_REAL_MASK = ~FLASHMEM_REAL_BIT; +constexpr uint32_t FLASHMEM_REAL_BIT{0x80000000U}; +constexpr uint32_t FLASHMEM_REAL_MASK{~FLASHMEM_REAL_BIT}; -#define SPI_FLASH_SEC_SIZE 4096 +} // namespace #define CHECK_ALIGNMENT(_x) assert(((uint32_t)(_x)&0x00000003) == 0) #define CHECK_RANGE(_addr, _size) \ if((_addr) + (_size) > flashFileSize) { \ - hostmsg("addr = 0x%08x, size = 0x%08x", _addr, _size); \ + host_debug_e("addr = 0x%08x, size = 0x%08x", _addr, _size); \ return false; \ } @@ -55,32 +58,30 @@ bool host_flashmem_init(FlashmemConfig& config) config.filename = flashFileName; } - flashFile = open(flashFileName, O_CREAT | O_RDWR | O_BINARY, 0644); - if(flashFile < 0) { - hostmsg("Error opening \"%s\"", flashFileName); + if(!flashFile.open(flashFileName, IFS::File::Create | IFS::File::ReadWrite)) { + host_debug_e("Error opening \"%s\"", flashFileName); return false; } - int res = lseek(flashFile, 0, SEEK_END); + int res = flashFile.seek(0, SeekOrigin::End); if(res < 0) { - hostmsg("Error seeking \"%s\"", flashFileName); - close(flashFile); - flashFile = -1; + host_debug_e("Error seeking \"%s\": %s", flashFileName, flashFile.getErrorString(res).c_str()); + flashFile.close(); return false; } if(res == 0) { size_t size = config.createSize ?: flashFileSize; - res = lseek(flashFile, size, SEEK_SET); + res = flashFile.seek(size, SeekOrigin::Start); if(res != int(size)) { - hostmsg("Error seeking beyond end of file \"%s\"", flashFileName); - } else if(ftruncate(flashFile, size) < 0) { - hostmsg("Error truncating \"%s\" to %u bytes", flashFileName, size); + host_debug_e("Error seeking beyond end of file \"%s\"", flashFileName); + } else if(!flashFile.truncate(size)) { + host_debug_e("Error truncating \"%s\" to %u bytes", flashFileName, size); } else { - hostmsg("Created blank \"%s\", %u bytes", flashFileName, size); + host_debug_i("Created blank \"%s\", %u bytes", flashFileName, size); } } else { - hostmsg("Opened \"%s\", size = 0x%08x", flashFileName, res); + host_debug_i("Opened \"%s\", size = 0x%08x", flashFileName, res); } flashFileSize = res; @@ -91,27 +92,38 @@ bool host_flashmem_init(FlashmemConfig& config) void host_flashmem_cleanup() { - close(flashFile); - flashFile = -1; - hostmsg("Closed \"%s\"", flashFileName); + flashFile.close(); + host_debug_i("Closed \"%s\"", flashFileName); } static int readFlashFile(uint32_t offset, void* buffer, size_t count) { - if(flashFile < 0) { + if(!flashFile) { return -1; } - int res = lseek(flashFile, offset, SEEK_SET); - return (res < 0) ? res : read(flashFile, buffer, count); + int res = flashFile.seek(offset, SeekOrigin::Start); + if(res >= 0) { + res = flashFile.read(buffer, count); + } + if(res < 0) { + debug_w("readFlashFile(0x%08x, %u) failed: %s", offset, count, flashFile.getErrorString(res).c_str()); + } + return res; } static int writeFlashFile(uint32_t offset, const void* data, size_t count) { - if(flashFile < 0) { + if(!flashFile) { return -1; } - int res = lseek(flashFile, offset, SEEK_SET); - return (res < 0) ? res : write(flashFile, data, count); + int res = flashFile.seek(offset, SeekOrigin::Start); + if(res >= 0) { + res = flashFile.write(data, count); + } + if(res < 0) { + debug_w("writeFlashFile(0x%08x, %u) failed: %s", offset, count, flashFile.getErrorString(res).c_str()); + } + return res; } SPIFlashInfo flashmem_get_info() diff --git a/Sming/Arch/Host/Components/vflash/README.rst b/Sming/Arch/Host/Components/vflash/README.rst index f94b6416f1..5fa2f06fa0 100644 --- a/Sming/Arch/Host/Components/vflash/README.rst +++ b/Sming/Arch/Host/Components/vflash/README.rst @@ -19,9 +19,9 @@ The following options are interpreted and used to provide command-line parameter This defaults to a combination of the above variables, but you can override if necessary. -The size of the flash memory is set via :envvar:`SPI_SIZE`. +The size of the flash memory is set via :ref:`hardware_config`. -See :component-esp8266:`esptool` for details and other applicable variables. +See :component:`esptool` for details and other applicable variables. Build targets diff --git a/Sming/Arch/Host/Components/vflash/component.mk b/Sming/Arch/Host/Components/vflash/component.mk index 10dc7b47cc..98410de609 100644 --- a/Sming/Arch/Host/Components/vflash/component.mk +++ b/Sming/Arch/Host/Components/vflash/component.mk @@ -7,32 +7,38 @@ DD := dd CACHE_VARS += FLASH_BIN FLASH_BIN ?= $(FW_BASE)/flash.bin -CONFIG_VARS += SPI_SIZE -SPI_SIZE ?= 4M +DEBUG_VARS += SPI_SIZE +SPI_SIZE = $(STORAGE_DEVICE_spiFlash_SIZE) # Options to add when running emulator CACHE_VARS += HOST_FLASH_OPTIONS HOST_FLASH_OPTIONS ?= --flashfile=$(FLASH_BIN) --flashsize=$(SPI_SIZE) CLI_TARGET_OPTIONS += $(HOST_FLASH_OPTIONS) -# Write data to flash -# $1 -> Start offset -# $2 -> File containing data to write -define WriteFlashChunk - $(info WriteFlash $1 -> $2) - $(Q) if [ ! -f $(FLASH_BIN) ]; then \ - $(EraseFlash); \ - fi - $(Q) $(DD) if=$2 of=$(FLASH_BIN) obs=1 seek=$$(($1)) conv=notrunc -endef +# Virtual flasher tool +VFLASH := $(PYTHON) $(COMPONENT_PATH)/vflash.py $(FLASH_BIN) $(STORAGE_DEVICE_spiFlash_SIZE_BYTES) # Write one or more chunks to flash # $1 -> List of `Offset=File` chunks define WriteFlash - $(foreach c,$1,$(call WriteFlashChunk,$(word 1,$(subst =, ,$c)),$(word 2,$(subst =, ,$c)))) + $(if $1,$(Q) $(VFLASH) write-chunks $1) +endef + +# Read flash memory into file +# $1 -> `Offset,Size` chunk +# $2 -> Output filename +define ReadFlash + $(info ReadFlash $1,$2) + $(Q) $(VFLASH) read-chunks $1=$2 +endef + +# Erase a region of Flash +# $1 -> List of `Offset,Size` chunks +define EraseFlashRegion + $(Q) $(VFLASH) fill-regions $1 endef # Reset/create flash backing file define EraseFlash - $(DD) if=/dev/zero ibs=1 count=$(SPI_SIZE) | tr "\000" "\377" > $(FLASH_BIN) -endef + $(Q) $(VFLASH) erase +endef diff --git a/Sming/Arch/Host/Components/vflash/vflash.py b/Sming/Arch/Host/Components/vflash/vflash.py new file mode 100644 index 0000000000..cf91295cf5 --- /dev/null +++ b/Sming/Arch/Host/Components/vflash/vflash.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# +# Virtual flash tool to support operations on flash backing file for Sming Host. +# +# Copyright 2021 mikee47 +# +# This file is part of the Sming Framework Project +# +# This library is free software: you can redistribute it and/or modify it under the terms of the +# GNU General Public License as published by the Free Software Foundation, version 3 or later. +# +# This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. +# + +import os, argparse + +# Interpret argument such as 0x1234 +def auto_int(x): + return int(x, 0) + +def fill_region(offset, length, value): + print(" fill_region(0x%08x, 0x%04x, 0x%02x)" % (offset, length, value)) + flashImage.seek(offset, os.SEEK_SET) + tmp = bytearray([value] * length) + flashImage.write(tmp) + +def erase(args): + fill_region(0, args.imagesize, 0xff) + +def erase_region(args): + fill_region(args.offset, args.size, 0xff) + +# Offset,size +def erase_regions(args): + for s in args.chunks: + offset, size = s.split(',') + fill_region(int(offset, 0), int(size, 0), 0xff) + +# Offset,size=value +def fill_regions(args): + for s in args.chunks: + offset, tmp = s.split(',') + x = tmp.split('=') + if len(x) == 1: + size = x[0] + value = '0xff' + else: + size, value = x + fill_region(int(offset, 0), int(size, 0), int(value,0)) + +def write_chunk(offset, filename): + print(" write_chunk(0x%08x, '%s')" % (offset, filename)) + flashImage.seek(offset, os.SEEK_SET) + data = open(filename, "rb").read() + len = flashImage.write(data) + print(" - wrote 0x%04x bytes" % len) + +# Offset=filename +def write_chunks(args): + for s in args.chunks: + offset, file = s.split('=') + write_chunk(int(offset, 0), file) + + +def read_chunk(offset, size, filename): + print(" read_chunk(0x%08x, 0x%04x, '%s')" % (offset, size, filename)) + flashImage.seek(offset, os.SEEK_SET) + data = flashImage.read(size) + open(filename, "wb").write(data) + +# Offset,size=filename +def read_chunks(args): + for s in args.chunks: + offset, x = s.split(',') + size, filename = x.split('=') + read_chunk(int(offset, 0), int(size, 0), filename) + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description='Virtual flash tool') + parser.add_argument('filename', help='Flash image filename') + parser.add_argument('imagesize', type=auto_int, help='Flash image size') + parser.set_defaults(func=None) + subparsers = parser.add_subparsers() + + fill_regions_parser = subparsers.add_parser('fill-regions', help='Fill region(s) of flash') + fill_regions_parser.add_argument('chunks', nargs='+', help='List of "offset,size[=value]" chunks, default value is 0xff') + fill_regions_parser.set_defaults(func=fill_regions) + + erase_parser = subparsers.add_parser('erase', help='Erase entire flash') + erase_parser.set_defaults(func=erase) + + write_chunks_parser = subparsers.add_parser('write-chunks', help='Write chunks to flash') + write_chunks_parser.add_argument('chunks', nargs='+', help='List of "offset=file" chunks') + write_chunks_parser.set_defaults(func=write_chunks) + + read_chunks_parser = subparsers.add_parser('read-chunks', help='Read chunks from flash') + read_chunks_parser.add_argument('chunks', nargs='+', help='List of "offset,size=file" chunks') + read_chunks_parser.set_defaults(func=read_chunks) + + args = parser.parse_args() + + flashImage = open(args.filename, "rb+" if os.path.isfile(args.filename) else "wb+") + + print("vflash('%s')" % args.filename) + + # Pad flash image file if it's not big enough + flashImage.seek(0, os.SEEK_END) + imgsize = flashImage.tell() + if imgsize < args.imagesize: + fill_region(imgsize, args.imagesize - imgsize, 0xff) + + # Invoke any user-provided functions (optional) + fn = args.func + if fn is None: + parser.print_usage() + else: + fn(args) + + flashImage.close() diff --git a/Sming/Arch/Host/Core/Data/Stream/HostFileStream.cpp b/Sming/Arch/Host/Core/Data/Stream/HostFileStream.cpp index 3f1eb40952..70783b5703 100644 --- a/Sming/Arch/Host/Core/Data/Stream/HostFileStream.cpp +++ b/Sming/Arch/Host/Core/Data/Stream/HostFileStream.cpp @@ -4,174 +4,12 @@ * http://github.com/SmingHub/Sming * All files of the Sming Core are provided under the LGPL v3 license. * - * HostFileStream.h + * HostFileStream.cpp * ****/ -// Required for ftruncate() -#define _POSIX_C_SOURCE 200112L - #include "HostFileStream.h" -#include -#include -#include - -#ifndef O_BINARY -#define O_BINARY 0 -#endif - -#define CHECK(res) check(res, __FUNCTION__) - -static int mapFlags(FileOpenFlags flags) -{ - int f = 0; - if(flags & eFO_Append) { - f |= O_APPEND; - } - if(flags & eFO_CreateIfNotExist) { - f |= O_CREAT; - } - if(flags & eFO_ReadOnly) { - f |= O_RDONLY; - } - if(flags & eFO_Truncate) { - f |= O_TRUNC; - } - if(flags & eFO_WriteOnly) { - f |= O_WRONLY; - } - return f; -} - -bool HostFileStream::check(int res, const char* func) -{ - if(res >= 0) { - return true; - } - - lastError = errno; - - debug_e("check(%s) = %d", func, lastError); - - return false; -} - -bool HostFileStream::open(const String& fileName, FileOpenFlags openFlags) -{ - close(); - - handle = ::open(fileName.c_str(), O_BINARY | mapFlags(openFlags), 0644); - if(!CHECK(handle)) { - debug_e("Failed to open '%s'", fileName.c_str()); - return false; - } - - int len = ::lseek(handle, 0, SEEK_END); - if(len < 0) { - len = 0; - } else if(openFlags & eFO_Append) { - pos = len; - } else { - ::lseek(handle, 0, SEEK_SET); - } - size = len; - - debug_i("Opened file '%s', %u bytes", fileName.c_str(), size); - filename = fileName; - return true; -} - -void HostFileStream::close() -{ - if(!isValid()) { - return; - } - - ::close(handle); - handle = -1; - size = 0; - pos = 0; - filename = nullptr; - lastError = 0; -} - -size_t HostFileStream::write(const uint8_t* buffer, size_t size) -{ - if(!isValid()) { - return 0; - } - - ::lseek(handle, 0, SEEK_END); - - int res = ::write(handle, buffer, size); - if(!CHECK(res)) { - return 0; - } - pos += res; - if(pos > this->size) { - this->size = pos; - } - return res; -} - -uint16_t HostFileStream::readMemoryBlock(char* data, int bufSize) -{ - if(!isValid()) { - return 0; - } - - int res = ::lseek(handle, pos, SEEK_SET); - if(!CHECK(res)) { - return 0; - } - - res = ::read(handle, data, bufSize); - if(!CHECK(res)) { - return 0; - } - - return res; -} - -int HostFileStream::seekFrom(int offset, SeekOrigin origin) -{ - if(!isValid()) { - return -1; - } - - if(origin == SeekOrigin::Current) { - origin = SeekOrigin::Start; - offset += pos; - } - int newPos = ::lseek(handle, offset, int(origin)); - if(!CHECK(newPos)) { - return -1; - } - - pos = newPos; - if(newPos > int(size)) { - size = newPos; - } - - return true; -} - -bool HostFileStream::isFinished() -{ - return pos == size; -} -bool HostFileStream::truncate(size_t newSize) +HostFileStream::HostFileStream() : IFS::FileStream(&IFS::Host::getFileSystem()) { - if(!isValid()) { - return false; - } - if(::ftruncate(handle, newSize) < 0) { - return false; - } - size = newSize; - if(pos > newSize) { - pos = newSize; - } - return true; } diff --git a/Sming/Arch/Host/Core/Data/Stream/HostFileStream.h b/Sming/Arch/Host/Core/Data/Stream/HostFileStream.h index 8532cc05ed..381e955812 100644 --- a/Sming/Arch/Host/Core/Data/Stream/HostFileStream.h +++ b/Sming/Arch/Host/Core/Data/Stream/HostFileStream.h @@ -10,127 +10,19 @@ #pragma once -#include -#include +#include /** * @brief Host File stream class * @ingroup stream data - * - * @{ */ - -class HostFileStream : public ReadWriteStream +class HostFileStream : public IFS::FileStream { public: - HostFileStream() - { - } + HostFileStream(); - /** @brief Create a file stream - * @param fileName Name of file to open - * @param openFlags - */ - HostFileStream(const String& fileName, FileOpenFlags openFlags = eFO_ReadOnly) + HostFileStream(const String& fileName, IFS::OpenFlags openFlags = IFS::OpenFlag::Read) : HostFileStream() { open(fileName, openFlags); } - - ~HostFileStream() - { - close(); - } - - /** @brief Open a file and attach this stream object to it - * @param fileName - * @param openFlags - * @retval bool true on success, false on error - * @note call getLastError() to determine cause of failure - */ - bool open(const String& fileName, FileOpenFlags openFlags = eFO_ReadOnly); - - /** @brief Close file - */ - void close(); - - StreamType getStreamType() const override - { - return eSST_File; - } - - size_t write(const uint8_t* buffer, size_t size) override; - - uint16_t readMemoryBlock(char* data, int bufSize) override; - - int seekFrom(int offset, SeekOrigin origin) override; - - bool isFinished() override; - - String getName() const override - { - return filename; - } - - bool isValid() const override - { - return handle >= 0; - } - - /** @brief Get the offset of cursor from beginning of data - * @retval size_t Cursor offset - */ - size_t getPos() const - { - return pos; - } - - /** @brief Get the total file size - * @retval size_t File size - */ - size_t getSize() const - { - return size; - } - - /** @brief Return the maximum bytes available to read, from current position - * @retval int -1 is returned when the size cannot be determined - */ - int available() override - { - return size - pos; - } - - /** @brief determine if an error occurred during operation - * @retval int filesystem error code - */ - int getLastError() - { - return lastError; - } - - /** @brief Reduce the file size - * @param newSize - * @retval bool true on success - */ - bool truncate(size_t newSize); - - /** @brief Truncate file at current position - * @retval bool true on success - */ - bool truncate() - { - return truncate(pos); - } - -private: - bool check(int res, const char* func); - -private: - int handle = -1; - String filename; - size_t pos = 0; - size_t size = 0; - int lastError = 0; }; - -/** @} */ diff --git a/Sming/Arch/Host/Platform/AccessPointImpl.cpp b/Sming/Arch/Host/Platform/AccessPointImpl.cpp index edc9c043f4..77e52c94a6 100644 --- a/Sming/Arch/Host/Platform/AccessPointImpl.cpp +++ b/Sming/Arch/Host/Platform/AccessPointImpl.cpp @@ -58,6 +58,12 @@ MacAddress AccessPointImpl::getMacAddress() const return MACADDR_NONE; } +bool AccessPointImpl::setMacAddress(const MacAddress& addr) const +{ + debug_w("[HOST] setMacAddress not implemented"); + return false; +} + String AccessPointImpl::getSSID() const { return nullptr; diff --git a/Sming/Arch/Host/Platform/AccessPointImpl.h b/Sming/Arch/Host/Platform/AccessPointImpl.h index ff2f0a315d..c45b324102 100644 --- a/Sming/Arch/Host/Platform/AccessPointImpl.h +++ b/Sming/Arch/Host/Platform/AccessPointImpl.h @@ -22,6 +22,7 @@ class AccessPointImpl : public AccessPointClass IpAddress getIP() const override; bool setIP(IpAddress address) override; MacAddress getMacAddress() const override; + bool setMacAddress(const MacAddress& addr) const override; IpAddress getNetworkMask() const override; IpAddress getNetworkGateway() const override; IpAddress getNetworkBroadcast() const override; diff --git a/Sming/Arch/Host/Platform/StationImpl.cpp b/Sming/Arch/Host/Platform/StationImpl.cpp index aff69e4352..4128401682 100644 --- a/Sming/Arch/Host/Platform/StationImpl.cpp +++ b/Sming/Arch/Host/Platform/StationImpl.cpp @@ -124,10 +124,10 @@ void StationImpl::statusCallback(netif* nif) if(changed_flags & NETIF_FLAG_UP) { assert(currentAp != nullptr); if(nif->flags & NETIF_FLAG_UP) { - host_printf("IF_UP, AP: %s\n", currentAp->ssid); + host_debug_i("IF_UP, AP: %s", currentAp->ssid); wifiEventsImpl.stationConnected(*currentAp); } else { - host_printf("IF_DOWN, AP: %s\n", currentAp->ssid); + host_debug_i("IF_DOWN, AP: %s", currentAp->ssid); wifiEventsImpl.stationDisconnected(*currentAp, WIFI_DISCONNECT_REASON_CONNECTION_FAIL); } } @@ -138,12 +138,12 @@ void StationImpl::statusCallback(netif* nif) ipaddr = nif->ip_addr; netmask = nif->netmask; gateway = nif->gw; - host_printf("IP_CHANGE, ip: %s, netmask: %s, gateway: %s\n", ipaddr.toString().c_str(), - netmask.toString().c_str(), gateway.toString().c_str()); + host_debug_i("IP_CHANGE, ip: %s, netmask: %s, gateway: %s", ipaddr.toString().c_str(), + netmask.toString().c_str(), gateway.toString().c_str()); wifiEventsImpl.stationGotIp(ipaddr, netmask, gateway); } } else { - hostmsg("No IP address"); + host_debug_w("No IP address"); connectionStatus = eSCS_ConnectionFailed; } } @@ -257,6 +257,12 @@ MacAddress StationImpl::getMacAddress() const } } +bool StationImpl::setMacAddress(const MacAddress& addr) const +{ + debug_w("[HOST] setMacAddress not implemented"); + return false; +} + IpAddress StationImpl::getNetworkBroadcast() const { return ipaddr | ~netmask; diff --git a/Sming/Arch/Host/Platform/StationImpl.h b/Sming/Arch/Host/Platform/StationImpl.h index 1db69b0f67..6523f4e0b0 100644 --- a/Sming/Arch/Host/Platform/StationImpl.h +++ b/Sming/Arch/Host/Platform/StationImpl.h @@ -46,6 +46,7 @@ class StationImpl : public StationClass String getHostname() const override; IpAddress getIP() const override; MacAddress getMacAddress() const override; + bool setMacAddress(const MacAddress& addr) const override; IpAddress getNetworkMask() const override; IpAddress getNetworkGateway() const override; IpAddress getNetworkBroadcast() const override; diff --git a/Sming/Arch/Host/README.rst b/Sming/Arch/Host/README.rst index 7e20eca7b6..96867e170f 100644 --- a/Sming/Arch/Host/README.rst +++ b/Sming/Arch/Host/README.rst @@ -26,7 +26,7 @@ For Linux, you may require the ``gcc-multilib`` and ``g++-multilib`` packages to build 32-bit executables on a 64-bit OS. For Windows, make sure your ``MinGW`` distro is up to date. -See :doc:`/arch/esp8266/getting-started/windows-manual` for further details. +See :doc:`/getting-started/windows` for further details. Building -------- @@ -58,12 +58,3 @@ Components Components/sming-arch/index Components/*/index - - -todo ----- - -* Add passthrough support for real serial ports to permit connection of physical devices. -* Consider how this mechanism might be used to support emulation of other devices (SPI, I2C, etc). -* Development platforms with SPI or I2C (e.g. Raspberry Pi) could be supported. -* Are there any generic device emulators available? For example, to simulate specific types of SPI slave. diff --git a/Sming/Arch/Host/Tools/ci/build.run.cmd b/Sming/Arch/Host/Tools/ci/build.run.cmd new file mode 100644 index 0000000000..55174dc850 --- /dev/null +++ b/Sming/Arch/Host/Tools/ci/build.run.cmd @@ -0,0 +1,13 @@ +REM Host build.run.cmd + +REM Build a couple of basic applications +%MAKE_PARALLEL% Basic_Serial Basic_ProgMem Basic_IFS STRICT=1 V=1 || goto :error + +REM Run basic tests +%MAKE_PARALLEL% tests || goto :error + +goto :EOF + +:error +echo Failed with error #%errorlevel%. +exit /b %errorlevel% diff --git a/Sming/Arch/Host/Tools/ci/build.run.sh b/Sming/Arch/Host/Tools/ci/build.run.sh new file mode 100755 index 0000000000..ada35bb8fb --- /dev/null +++ b/Sming/Arch/Host/Tools/ci/build.run.sh @@ -0,0 +1,19 @@ +# Host build.run.sh + +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" + +if [[ $CHECK_SCA -eq 1 ]]; then + $DIR/coverity-scan.sh +else + $MAKE_PARALLEL Basic_Blink Basic_DateTime Basic_Delegates Basic_Interrupts Basic_ProgMem Basic_Serial Basic_Servo Basic_Ssl LiveDebug DEBUG_VERBOSE_LEVEL=3 +fi + +# Build and run tests +export SMING_TARGET_OPTIONS='--flashfile=$(FLASH_BIN) --flashsize=$(SPI_SIZE)' +$MAKE_PARALLEL tests diff --git a/Sming/Arch/Host/Tools/ci/build.setup.cmd b/Sming/Arch/Host/Tools/ci/build.setup.cmd new file mode 100644 index 0000000000..27b7b02d62 --- /dev/null +++ b/Sming/Arch/Host/Tools/ci/build.setup.cmd @@ -0,0 +1,10 @@ +REM Host build.setup.cmd + +REM Build documentation +make -C %SMING_HOME% docs || goto :error + +goto :EOF + +:error +echo Failed with error #%errorlevel%. +exit /b %errorlevel% diff --git a/Sming/Arch/Host/Tools/ci/build.setup.sh b/Sming/Arch/Host/Tools/ci/build.setup.sh new file mode 100755 index 0000000000..a2370b039b --- /dev/null +++ b/Sming/Arch/Host/Tools/ci/build.setup.sh @@ -0,0 +1,28 @@ +# Host build.setup.sh + +# Check coding style +make cs +DIFFS=$(git diff) +if [ -n "$DIFFS" ]; then + echo "!!! Coding Style issues Found!!!" + echo " Run: 'make cs' to fix them. " + echo "$DIFFS" + exit 1 +fi + +# Make deployment keys, etc. available +set +x +if [ -n "$SMING_SECRET" ]; then + openssl aes-256-cbc -d -a -iter 100 -in $CI_BUILD_DIR/Tools/ci/secrets.sh.enc -out /tmp/secrets.sh -pass pass:$SMING_SECRET + source /tmp/secrets.sh + unset SMING_SECRET +fi +set -x + +# Setup networking +sudo ip tuntap add dev tap0 mode tap user $(whoami) +sudo ip a a dev tap0 192.168.13.1/24 +sudo ip link set tap0 up + +# Build documentation +make -C $SMING_HOME docs diff --git a/Sming/Arch/Host/Tools/ci/coverity-scan.sh b/Sming/Arch/Host/Tools/ci/coverity-scan.sh new file mode 100755 index 0000000000..984660af48 --- /dev/null +++ b/Sming/Arch/Host/Tools/ci/coverity-scan.sh @@ -0,0 +1,99 @@ +#!/bin/bash + +set -e + +COVERITY_SCAN_PROJECT_NAME=SmingHub/Sming +COVERITY_SCAN_NOTIFICATION_EMAIL="slaff@attachix.com" +COVERITY_SCAN_BUILD_COMMAND="$MAKE_PARALLEL Basic_Blink Basic_DateTime Basic_Delegates Basic_Interrupts Basic_ProgMem Basic_Serial Basic_Servo Basic_Ssl HttpServer_WebSockets SMING_ARCH=Host DEBUG_VERBOSE_LEVEL=3" + +set +x +source /tmp/secrets.sh + +# Environment check +[ -z "$COVERITY_SCAN_PROJECT_NAME" ] && echo "ERROR: COVERITY_SCAN_PROJECT_NAME must be set" && exit 1 +[ -z "$COVERITY_SCAN_NOTIFICATION_EMAIL" ] && echo "ERROR: COVERITY_SCAN_NOTIFICATION_EMAIL must be set" && exit 1 +[ -z "$COVERITY_SCAN_BUILD_COMMAND" ] && echo "ERROR: COVERITY_SCAN_BUILD_COMMAND must be set" && exit 1 +[ -z "$COVERITY_SCAN_TOKEN" ] && echo "ERROR: COVERITY_SCAN_TOKEN must be set" && exit 1 + +PLATFORM=$(uname) +TOOL_ARCHIVE=/tmp/cov-analysis-${PLATFORM}.tgz +TOOL_URL=https://scan.coverity.com/download/${PLATFORM} +TOOL_BASE=/tmp/coverity-scan-analysis +UPLOAD_URL="https://scan.coverity.com/builds" +SCAN_URL="https://scan.coverity.com" + +# Do not run on pull requests +if [ -n "${CI_PULL_REQUEST}" ]; then + echo -e "\033[33;1mINFO: Skipping Coverity Analysis: branch is a pull request.\033[0m" + exit 0 +fi + +# Verify upload is permitted +permit=true +AUTH_RES=$(curl -s --form project="$COVERITY_SCAN_PROJECT_NAME" --form token="$COVERITY_SCAN_TOKEN" $SCAN_URL/api/upload_permitted) +if [ "$AUTH_RES" = "Access denied" ]; then + echo -e "\033[33;1mCoverity Scan API access denied. Check COVERITY_SCAN_PROJECT_NAME and COVERITY_SCAN_TOKEN.\033[0m" + exit 1 +else + AUTH=$(echo $AUTH_RES | ruby -e "require 'rubygems'; require 'json'; puts JSON[STDIN.read]['upload_permitted']") + if [ "$AUTH" = "true" ]; then + echo -e "\033[33;1mCoverity Scan analysis authorized per quota.\033[0m" + else + WHEN=$(echo $AUTH_RES | ruby -e "require 'rubygems'; require 'json'; puts JSON[STDIN.read]['next_upload_permitted_at']") + echo -e "\033[33;1mOops!Coverity Scan analysis engine NOT authorized until $WHEN.\033[0m" + permit=false + fi +fi + +if [ "$permit" = true ]; then +if [ ! -d $TOOL_BASE ]; then + # Download Coverity Scan Analysis Tool + if [ ! -e $TOOL_ARCHIVE ]; then + echo -e "\033[33;1mDownloading Coverity Scan Analysis Tool...\033[0m" + wget -nv -O $TOOL_ARCHIVE $TOOL_URL --post-data "project=$COVERITY_SCAN_PROJECT_NAME&token=$COVERITY_SCAN_TOKEN" + fi + + # Extract Coverity Scan Analysis Tool + echo -e "\033[33;1mExtracting Coverity Scan Analysis Tool...\033[0m" + mkdir -p $TOOL_BASE + CURRENT_DIR=$(pwd) + cd $TOOL_BASE + tar xzf $TOOL_ARCHIVE + cd $CURRENT_DIR +fi + +TOOL_DIR=$(find $TOOL_BASE -type d -name 'cov-analysis*') +export PATH=$TOOL_DIR/bin:$PATH + +# Build +echo -e "\033[33;1mRunning Coverity Scan Analysis Tool...\033[0m" +COV_BUILD_OPTIONS="" +#COV_BUILD_OPTIONS="--return-emit-failures 8 --parse-error-threshold 85" +RESULTS_DIR="cov-int" +eval "${COVERITY_SCAN_BUILD_COMMAND_PREPEND}" +COVERITY_UNSUPPORTED=1 cov-build --dir $RESULTS_DIR $COV_BUILD_OPTIONS $COVERITY_SCAN_BUILD_COMMAND +cov-import-scm --dir $RESULTS_DIR --scm git --log $RESULTS_DIR/scm_log.txt 2>&1 + +# Upload results +echo -e "\033[33;1mTarring Coverity Scan Analysis results...\033[0m" +RESULTS_ARCHIVE=analysis-results.tgz +tar czf $RESULTS_ARCHIVE $RESULTS_DIR +SHA=$(git rev-parse --short HEAD) + +echo -e "\033[33;1mUploading Coverity Scan Analysis results...\033[0m" +response=$(curl \ + --silent --write-out "\n%{http_code}\n" \ + --form project=$COVERITY_SCAN_PROJECT_NAME \ + --form token=$COVERITY_SCAN_TOKEN \ + --form email=$COVERITY_SCAN_NOTIFICATION_EMAIL \ + --form file=@$RESULTS_ARCHIVE \ + --form version=$SHA \ + --form description="CI build" \ + $UPLOAD_URL) +status_code=$(echo "$response" | sed -n '$p') +if [ "$status_code" -ge "400" ]; then + TEXT=$(echo "$response" | sed '$d') + echo -e "\033[33;1mCoverity Scan upload failed: $TEXT. Status code: $status_code.\033[0m" + exit 1 +fi +fi diff --git a/Sming/Arch/Host/Tools/ci/install.cmd b/Sming/Arch/Host/Tools/ci/install.cmd new file mode 100644 index 0000000000..556cb4dd2f --- /dev/null +++ b/Sming/Arch/Host/Tools/ci/install.cmd @@ -0,0 +1,14 @@ +REM Host install.cmd + +call :install "c:\tools\doxygen" doxygen-1.9.1.windows.bin.zip +call :install "c:\tools" stable_windows_10_msbuild_Release_Win32_graphviz-2.46.1-win32.zip + +python -m pip install -r %SMING_HOME%/../docs/requirements.txt + +goto :EOF + +:install +if "%~1"=="" goto :EOF +mkdir %1 +curl -LO %SMINGTOOLS%/%2 +7z -o%1 -y x %2 diff --git a/Sming/Arch/Host/Tools/install.sh b/Sming/Arch/Host/Tools/install.sh new file mode 100755 index 0000000000..4b21178d6b --- /dev/null +++ b/Sming/Arch/Host/Tools/install.sh @@ -0,0 +1,8 @@ +# Host install.sh + +# Required by deployment script +if [ -n "$APPVEYOR" ]; then + $PKG_INSTALL \ + jq \ + xmlstarlet +fi diff --git a/Sming/Arch/Host/Tools/setup-network-linux.sh b/Sming/Arch/Host/Tools/setup-network-linux.sh index 1d2585241a..7f5b7c2df8 100755 --- a/Sming/Arch/Host/Tools/setup-network-linux.sh +++ b/Sming/Arch/Host/Tools/setup-network-linux.sh @@ -17,7 +17,7 @@ fi # Create a tap0 interface with IP network 192.168.13.1/24 sudo ip tuntap add dev tap0 mode tap user $(whoami) sudo ip a a dev tap0 192.168.13.1/24 -sudo ifconfig tap0 up +sudo ip link set tap0 up # The following lines are needed if you plan to access Internet sudo sysctl net.ipv4.ip_forward=1 diff --git a/Sming/Arch/Host/Tools/travis/build.run.sh b/Sming/Arch/Host/Tools/travis/build.run.sh index 2bfa88cfa0..1b24170fc9 100755 --- a/Sming/Arch/Host/Tools/travis/build.run.sh +++ b/Sming/Arch/Host/Tools/travis/build.run.sh @@ -23,4 +23,5 @@ $MAKE_PARALLEL tests mv $SMING_PROJECTS_DIR/samples .. mv $SMING_PROJECTS_DIR/tests .. unset SMING_PROJECTS_DIR +$SMING_HOME/../Tools/install.sh doc make docs V=1 diff --git a/Sming/Arch/Host/Tools/travis/coverity-scan.sh b/Sming/Arch/Host/Tools/travis/coverity-scan.sh index 11a1248da8..2f74499e34 100755 --- a/Sming/Arch/Host/Tools/travis/coverity-scan.sh +++ b/Sming/Arch/Host/Tools/travis/coverity-scan.sh @@ -4,7 +4,7 @@ set -e COVERITY_SCAN_PROJECT_NAME=${TRAVIS_REPO_SLUG} COVERITY_SCAN_NOTIFICATION_EMAIL="slaff@attachix.com" -COVERITY_SCAN_BUILD_COMMAND="$MAKE_PARALLEL Basic_Blink Basic_DateTime Basic_Delegates Basic_Interrupts Basic_ProgMem Basic_Serial Basic_Servo Basic_Ssl HttpServer_FirmwareUpload SMING_ARCH=Host DEBUG_VERBOSE_LEVEL=3" +COVERITY_SCAN_BUILD_COMMAND="$MAKE_PARALLEL dist-clean Basic_Blink Basic_DateTime Basic_Delegates Basic_Interrupts Basic_ProgMem Basic_Serial Basic_Servo Basic_Ssl HttpServer_WebSockets SMING_ARCH=Host DEBUG_VERBOSE_LEVEL=3" # Environment check [ -z "$COVERITY_SCAN_PROJECT_NAME" ] && echo "ERROR: COVERITY_SCAN_PROJECT_NAME must be set" && exit 1 @@ -63,6 +63,7 @@ TOOL_DIR=$(find $TOOL_BASE -type d -name 'cov-analysis*') export PATH=$TOOL_DIR/bin:$PATH # Build + echo -e "\033[33;1mRunning Coverity Scan Analysis Tool...\033[0m" COV_BUILD_OPTIONS="" #COV_BUILD_OPTIONS="--return-emit-failures 8 --parse-error-threshold 85" diff --git a/Sming/Arch/Host/Tools/travis/install.sh b/Sming/Arch/Host/Tools/travis/install.sh index 5db995b801..fd27cb5ba9 100755 --- a/Sming/Arch/Host/Tools/travis/install.sh +++ b/Sming/Arch/Host/Tools/travis/install.sh @@ -2,5 +2,8 @@ set -ex # exit with nonzero exit code if anything fails sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-6.0 100 -python3 -m pip install --upgrade pip -python3 -m pip install -r $TRAVIS_BUILD_DIR/docs/requirements.txt +python3 -m pip install -q --upgrade pip + +if [ "$BUILD_DOCS" == "1" ]; then + python3 -m pip install -q -r $TRAVIS_BUILD_DIR/docs/requirements.txt +fi diff --git a/Sming/Arch/Host/app.mk b/Sming/Arch/Host/app.mk index 8e26c5a03f..3d895d08ff 100644 --- a/Sming/Arch/Host/app.mk +++ b/Sming/Arch/Host/app.mk @@ -15,7 +15,7 @@ TARGET_OUT_0 := $(FW_BASE)/$(APP_NAME)$(TOOL_EXT) # Target definitions .PHONY: application -application: $(CUSTOM_TARGETS) $(TARGET_OUT_0) +application: $(TARGET_OUT_0) $(TARGET_OUT_0): $(COMPONENTS_AR) $(info $(notdir $(PROJECT_DIR)): Linking $@) @@ -29,7 +29,7 @@ $(TARGET_OUT_0): $(COMPONENTS_AR) valgrind: all ##Run the application under valgrind to detect memory issues. Requires `valgrind` to be installed on the host system. $(Q) valgrind --track-origins=yes --leak-check=full \ $(foreach id,$(ENABLE_HOST_UARTID),echo '$(call RunHostTerminal,$(id))' >> $@;) \ - $(TARGET_OUT_0) $(CLI_TARGET_OPTIONS) + $(TARGET_OUT_0) $(CLI_TARGET_OPTIONS) -- $(HOST_PARAMETERS) RUN_SCRIPT := $(FW_BASE)/run.sh @@ -44,5 +44,9 @@ $(RUN_SCRIPT):: echo '$(TARGET_OUT_0) $(CLI_TARGET_OPTIONS) -- $(HOST_PARAMETERS)' >> $@; \ chmod a+x $@ -.PHONY: flash -flash: all flashfs ##Write all images to (virtual) flash +##@Flashing + +.PHONY: flashconfig +flashconfig: kill_term ##Erase the rBoot config sector + $(info Erasing rBoot config sector) + $(call WriteFlash,$(FLASH_RBOOT_ERASE_CONFIG_CHUNKS)) diff --git a/Sming/Arch/Host/build.mk b/Sming/Arch/Host/build.mk index 5700a06743..db1582468f 100644 --- a/Sming/Arch/Host/build.mk +++ b/Sming/Arch/Host/build.mk @@ -21,7 +21,8 @@ GCC_UPGRADE_URL := https://sming.readthedocs.io/en/latest/arch/host/host-emulato CPPFLAGS += \ -m32 \ - -Wno-deprecated-declarations + -Wno-deprecated-declarations \ + -D_FILE_OFFSET_BITS=64 # => Tools MEMANALYZER = size diff --git a/Sming/Arch/Host/spiffs-two-roms.hw b/Sming/Arch/Host/spiffs-two-roms.hw new file mode 100644 index 0000000000..80128f0191 --- /dev/null +++ b/Sming/Arch/Host/spiffs-two-roms.hw @@ -0,0 +1,13 @@ +{ + "name": "Two ROM slots with single SPIFFS", + "base_config": "spiffs", + "partitions": { + "rom1": { + "address": "0x108000", + "size": "992K", + "type": "app", + "subtype": "ota_1", + "filename": "$(BLANK_BIN)" + } + } +} diff --git a/Sming/Arch/Host/standard.hw b/Sming/Arch/Host/standard.hw new file mode 100644 index 0000000000..7572825a4c --- /dev/null +++ b/Sming/Arch/Host/standard.hw @@ -0,0 +1,20 @@ +{ + "name": "Standard config with single ROM", + "arch": "Host", + "partition_table_offset": "0x2000", + "devices": { + "spiFlash": { + "type": "flash", + "size": "4M" + } + }, + "partitions": { + "rom0": { + "address": "0x008000", + "size": "992K", + "type": "app", + "subtype": "factory", + "filename": "$(BLANK_BIN)" + } + } +} diff --git a/Sming/Components/FlashString b/Sming/Components/FlashString index 4d0934f499..e0e95dbe53 160000 --- a/Sming/Components/FlashString +++ b/Sming/Components/FlashString @@ -1 +1 @@ -Subproject commit 4d0934f499d9d2a583d1de62c65ce5266f4ec82f +Subproject commit e0e95dbe5399c15a4637eee9dbf46a7cb212ef36 diff --git a/Sming/Components/IFS b/Sming/Components/IFS new file mode 160000 index 0000000000..b31da3aa3e --- /dev/null +++ b/Sming/Components/IFS @@ -0,0 +1 @@ +Subproject commit b31da3aa3ec5b786bbea1967ed31a8e6ff931676 diff --git a/Sming/Components/Storage/README.rst b/Sming/Components/Storage/README.rst new file mode 100644 index 0000000000..0ab3666e67 --- /dev/null +++ b/Sming/Components/Storage/README.rst @@ -0,0 +1,387 @@ +Storage Management +================== + +.. highlight:: bash + +This Component provides support for using storage devices in a structured way by partitioning +them into areas for specific uses. +Partitions may can contain information such as: + +- Application (firmware) images +- Filesystem(s) +- Configuration/calibration/parameter data +- Custom flash storage areas + +A single partition table is located on the main flash device, :cpp:var:`Storage::spiFlash`, +and defines all partitions with a unique name and associated +:cpp:enum:`Storage::Partition::Type` / :cpp:type:`Storage::Partition::SubType`. + + + +.. _hardware_config: + +Hardware configuration +---------------------- + +Each project has an associated ``Hardware configuration``, specified by the :envvar:`HWCONFIG` setting: +this is a JSON file with a ``.hw`` extension. + +For user convenience, the configuration file may contain comments however these are stripped before +processing. + +The build system locates the file by searching, in order: + +- ``{PROJECT_DIR}`` the root project directory +- ``{SMING_HOME}/Arch/{SMING_ARCH}`` +- ``{SMING_HOME}`` + +Each architecture provides a ``standard`` configuration which defines such things as the +partition table location and standard system partitions. Other configurations inherit +from this by providing a ``base_config`` value. + +You can list the available configs like this:: + + make hwconfig-list + +This also shows the file path should you wish to view or edit it. + +To select and view the resulting configuration, do this:: + + make hwconfig HWCONFIG=spiffs + +or, to show the partition map:: + + make map HWCONFIG=spiffs + +.. note:: + + You can set :envvar:`HWCONFIG` in your project's ``component.mk`` file, however as with other + configuration variables it will be overridden by the cached value set on the command line. + + For example, if you want to change from ``standard`` to ``standard-4m`` for your project, + first add this line to your component.mk file:: + + HWCONFIG := standard-4m + + Then either run ``make HWCONFIG=standard-4m`` or ``make config-clean``. + +.. _hwconfig_options: + +Hardware configuration options +------------------------------ + +Commonly used settings can be stored in an option library for easier use. +The library files are named ``options.json`` and located in the place as .hw files. + +For example, we can do this:: + + make HWCONFIG=standard HWCONFIG_OPTS=4m,spiffs + +This loads the 'standard' profile then merges the fragments found in the option library with the given names. +This is how the ``standard-4m`` profile is constructed. + +If using this approach, remember to updated your project's ``component.mk`` with the desired settings, +and verify the layout is correct using ``make map``. + + +OTA updates +----------- + +When planning OTA updates please check that the displayed partition map corresponds to your project. +For example, the partition table requires a free sector so must not overlap other partitions. + +Your OTA update process must include a step to write the partition table to the correct location. + +It is not necessary to update the bootloader. See :component:`rboot` for further information. + + +Custom configurations +--------------------- + +To customise the hardware configuration for a project, for example 'my_project': + +1. Create a new configuration file in your project root, such as ``my_project.hw``: + + .. code-block:: json + + { + "name": "My project config", + "base_config": "spiffs", + "options": ["vdd"] + } + + You can use any available configuration as the base_config. + Option fragments can be pulled in as shown. See :ref:`hwconfig_options`. + + +2. If required, modify any inherited settings: + + .. code-block:: json + + { + "name": "My config", + "base_config": "standard", + "devices": { + "spiFlash": { + "speed": 80, + "mode": "qio", + "size": "2M" + } + }, + "partitions": { + "rom0": { + "address": "0x10000", + "size": "0x80000" + } + } + } + + This will adjust flash parameters (previously via SPI_SPEED, SPI_MODE and SPI_SIZE), + and the location/size of the primary application partition. + +3. Add any additional partitions: + + .. code-block:: json + + { + "name": "My config", + "base_config": "standard-4m", + "partitions": { + "rom0": { + "address": "0x10000", + "size": "0x80000" + }, + "spiffs1": { + "address": "0x00280000", + "size": "256K", + "type": "data", + "subtype": "spiffs", + "filename": "$(FW_BASE)/spiffs1_rom.bin", + "build": { + "target": "spiffsgen", + "files": "files/spiffs1" + } + } + } + } + + This adds a second SPIFFS partition, and instructs the build system to generate + an image file for it using the files in the project's ``files/spiffs1`` directory. + +4. Select the new configuration and re-build the project:: + + make HWCONFIG=my_project + + You should also add this to your project's ``component.mk`` file:: + + HWCONFIG := my_project + +5. Program your device:: + + make flash + + This will flash everything: bootloader, partition table and all defined partitions (those with a ``filename`` entry). + + +.. note:: + + The build system isn't smart enough to track dependencies for partition build targets. + + To rebuild these manually type:: + + make partbuild + + These will be removed when ``make clean`` is run, but you can also clean them separately thus:: + + make part-clean + + +Partition maps +-------------- + +This is a concise view of your flash partitions. Display it like this:: + + make map + +For the :sample:`Basic_Storage` sample application, we get this: + +.. code-block:: text + + Basic_Storage: Invoking 'map' for Esp8266 (debug) architecture + Partition map: + Device Start End Size Type SubType Name Filename + ---------------- ---------- ---------- ---------- -------- -------- ---------------- ------------ + spiFlash 0x00000000 0x00001fff 8K Boot Sector + spiFlash 0x00002000 0x00002fff 4K Partition Table + spiFlash 0x00003000 0x00003fff 4K data phy phy_init $(FLASH_INIT_DATA) + spiFlash 0x00004000 0x00007fff 16K data sysparam sys_param + spiFlash 0x00008000 0x000fffff 992K app factory rom0 $(RBOOT_ROM_0_BIN) + spiFlash 0x00100000 0x001effff 960K (unused) + spiFlash 0x001f0000 0x001f3fff 16K user 0 user0 user0.bin + spiFlash 0x001f4000 0x001f7fff 16K user 1 user1 + spiFlash 0x001f8000 0x001fffff 32K (unused) + spiFlash 0x00200000 0x0027ffff 512K data spiffs spiffs0 $(SPIFF_BIN_OUT) + spiFlash 0x00280000 0x002bffff 256K data spiffs spiffs1 $(FW_BASE)/spiffs1_rom.bin + spiFlash 0x002c0000 0x002fffff 256K data spiffs spiffs2 $(FW_BASE)/spiffs2_rom.bin + spiFlash 0x00300000 0x003fffff 1M (unused) + +For comparison, here's the output for Esp32: + +.. code-block:: text + + Basic_Storage: Invoking 'map' for Esp32 (debug) architecture + Partition map: + Device Start End Size Type SubType Name Filename + ---------------- ---------- ---------- ---------- -------- -------- ---------------- ------------ + spiFlash 0x00000000 0x00007fff 32K Boot Sector + spiFlash 0x00008000 0x00008fff 4K Partition Table + spiFlash 0x00009000 0x0000efff 24K data nvs nvs + spiFlash 0x0000f000 0x0000ffff 4K data phy phy_init + spiFlash 0x00010000 0x001fffff 1984K app factory factory $(TARGET_BIN) + spiFlash 0x001f0000 0x001f3fff 16K user 0 user0 user0.bin + spiFlash 0x001f4000 0x001f7fff 16K user 1 user1 + spiFlash 0x001f8000 0x001fffff 32K (unused) + spiFlash 0x00200000 0x0027ffff 512K data spiffs spiffs0 $(SPIFF_BIN_OUT) + spiFlash 0x00280000 0x002bffff 256K data spiffs spiffs1 $(FW_BASE)/spiffs1_rom.bin + spiFlash 0x002c0000 0x002fffff 256K data spiffs spiffs2 $(FW_BASE)/spiffs2_rom.bin + spiFlash 0x00300000 0x003fffff 1M (unused) + + +To compare this with the partition map programmed into a device, do this:: + + make readmap map + + +JSON validation +--------------- + +When the binary partition table is built or updated, the configuration is first +validated against a schema :source:`Sming/Components/Storage/schema.json`. + +This complements the checks performed by the ``hwconfig`` tool. + +You can run the validation manually like this:: + + make hwconfig-validate + +See `JSON Schema `__ for details about JSON schemas. + + +Configuration +------------- + +.. envvar:: HWCONFIG + + default: standard + + Set this to the hardware configuration to use for your project. + + Default configurations: + + standard + Base profile with 1MB flash size which should work on all device variants. + Located in the ``Sming/Arch/{SMING_ARCH}`` directory. + + standard-4m + Overrides ``standard`` to set 4Mbyte flash size + + spiffs + Adds a single SPIFFS partition. See :component:`spiffs`. + + Other configurations may be available, depending on architecture. + You can see these by running ``make hwconfig-list``. + + For example, to select ``spiffs`` add the following line to your project:: + + HWCONFIG := spiffs + + You will also need to run ``make HWCONFIG=spiffs`` to change the cached value + (or ``make config-clean`` to reset everything). + + +.. envvar:: HWCONFIG_OPTS + + Set this to adjust the hardware profile using option fragments. See :ref:`hwconfig_options`. + + +Binary partition table +---------------------- + +Sming uses the same binary partition table structure as ESP-IDF, located immediately after the boot sector. +However, it is organised slighly differently to allow partitions to be registered for multiple storage devices. + +Entries are fixed 32-byte structures, :cpp:class:`Storage::esp_partition_info_t`, organised as follows: + +- The first entry is always a ``storage`` type defining the main :cpp:var:`spiFlash` device. +- This is followed by regular partition entries sorted in ascending address order. + There may be gaps between the partitions. +- The partition table md5sum entry is inserted as normal +- If any external devices are defined: + - A SMING_EXTENSION entry, which the esp32 bootloader interprets as the end of the partition table. + - The next entry is a ``storage`` type for the ``external`` device. + - This is followed by regular partition entries as before. + - A second md5sum entry is inserted for the entire partition table thus far +- The end of the partition table is identified by an empty sector (i.e. all bytes 0xFF). + + + +Partition API +------------- + +This is a C++ interface. Some examples:: + + Storage::Partition part = Storage::findPartition("spiffs0"); // Find by name + if(part) { + debugf("Partition '%s' found", part.name().c_str()); + } else { + debugf("Partition NOT found"); + } + + // Enumerate all partitions + for(auto it = Storage::findPartition(); it; ++it) { + auto part = *it; + debugf("Found '%s' at 0x%08x, size 0x%08x", part.name().c_str(), part.address(), part.size()); + } + + // Enumerate all SPIFFS partitions + for(auto it = Storage::findPartition(Partition::SubType::Data::spiffs; it; it++) { + debugf("Found '%s' at 0x%08x, size 0x%08x", it->name().c_str(), it->address(), it->size()); + } + + +A :cpp:class:`Storage::Partition` object is just a wrapper and can be freely copied around. +It defines methods which should be used to read/write/erase the partition contents. + +Each partition has an associated :cpp:class:`Storage::Device`. +This is usually :cpp:var:`Storage::spiFlash` for the main flash device. + +Other devices must be registed via :cpp:func:`Storage::PartitionTable::registerStorageDevice`. + +You can query partition entries from a Storage object directly, for example:: + + #include + + for(auto part: Storage::spiFlash->partitions()) { + debugf("Found '%s' at 0x%08x, size 0x%08x", part.name().c_str(), part.address(), part.size()); + } + + +External Storage +---------------- + +If your design has additional fixed storage devices, such as SPI RAM, flash or EEPROM, +you can take advantage of the partition API to manage them as follows: + +- Implement a class to manage the storage, inheriting from :cpp:class:`Storage::Device`. +- Create a custom hardware configuration for your project and add a ``devices`` entry + describing your storage device, plus partition entries: the ``device`` field identifies + which device these entries relate to. +- Create an instance of your custom device and make a call to :cpp:func:`Storage::registerDevice` + in your ``init()`` function (or elsewhere if more appropriate). + + +API +--- + +.. doxygennamespace:: Storage + :members: diff --git a/Sming/Components/Storage/Tools/README.rst b/Sming/Components/Storage/Tools/README.rst new file mode 100644 index 0000000000..04d56eacff --- /dev/null +++ b/Sming/Components/Storage/Tools/README.rst @@ -0,0 +1,6 @@ +Partition tools +=============== + +hwconfig.py + Supports parsing and creation of hardware configuration files which include partition information. + diff --git a/Sming/Components/Storage/Tools/hwconfig/common.py b/Sming/Components/Storage/Tools/hwconfig/common.py new file mode 100644 index 0000000000..89d09efa87 --- /dev/null +++ b/Sming/Components/Storage/Tools/hwconfig/common.py @@ -0,0 +1,113 @@ +# +# Common functions and definitions +# + +import sys, json, platform + +quiet = False + +def status(msg): + """ Print status message to stderr """ + if not quiet: + critical(msg) + + +def critical(msg): + """ Print critical message to stderr """ + sys.stderr.write(msg) + sys.stderr.write('\n') + + +def fixpath(path): + """ Paths in Windows can get a little weird """ + if len(path) > 2 and path[1] != ':' and platform.system() == 'Windows' and path[2] == '/': + return path[1] + ':' + path[2:] + return path + + +def parse_int(v, keywords=None): + """Generic parser for integer fields - int(x,0) with provision for + k/m/K/M suffixes and 'keyword' value lookup. + """ + if not isinstance(v, str): + return v + if keywords is None or len(keywords) == 0: + try: + for letter, multiplier in [("k", 1024), ("m", 1024 * 1024), ("g", 1024 * 1024 * 1024)]: + if v.lower().endswith(letter): + return parse_int(v[:-1], keywords) * multiplier + return int(v, 0) + except ValueError: + raise InputError("Invalid field value %s" % v) + try: + return keywords[v.lower()] + except KeyError: + raise InputError("Value '%s' is not valid. Known keywords: %s" % (v, ", ".join(keywords))) + + +def stringnum(s): + """Return number if s contains only digits, otherwise return the string + """ + return int(s) if s.isdigit() else s + + +def addr_format(a): + return "0x%08x" % a + + +def size_format(a): + if a == 0: + return '0' + for (val, suffix) in [(0x40000000, "G"), (0x100000, "M"), (0x400, "K")]: + if a % val == 0: + return "%d%s" % (a // val, suffix) + return "0x%08x" % a + + +def size_frac_str(a): + KB = 1024 + MB = KB * 1024 + GB = MB * 1024 + if a >= GB: + div = GB + unit = 'G' + elif a >= MB: + div = MB + unit = 'M' + else: + div = KB + unit = 'K' + if a % div == 0: + return "%u%s" % (a // div, unit) + else: + return "%.2f%s" % (a / div, unit) + + +def quote(v): + return '"' + v + '"' + + +def contains_whitespace(s): + return ''.join(s.split()) != s + + +def to_json(obj): + return json.dumps(obj, indent=4) + + +def lookup_keyword(t, keywords): + for k, v in keywords.items(): + if t == v: + return k + return "%d" % t + + +class InputError(RuntimeError): + def __init__(self, e): + super(InputError, self).__init__(e) + + +class ValidationError(InputError): + def __init__(self, obj, message): + super(ValidationError, self).__init__("%s.%s '%s' invalid: %s" % (type(obj).__module__, type(obj).__name__, obj.name, message)) + self.obj = obj diff --git a/Sming/Components/Storage/Tools/hwconfig/config.py b/Sming/Components/Storage/Tools/hwconfig/config.py new file mode 100644 index 0000000000..ff40d62ead --- /dev/null +++ b/Sming/Components/Storage/Tools/hwconfig/config.py @@ -0,0 +1,183 @@ +# +# Configuration object +# + +import os, partition, storage, copy +from rjsmin import jsmin +from common import * +from builtins import classmethod + +HW_EXT = '.hw' + +def get_config_dirs(): + s = os.environ['HWCONFIG_DIRS'] + dirs = s.replace(' ', ' ').split(' ') + return dirs + +def load_option_library(): + library = {} + dirs = get_config_dirs() + for d in dirs: + filename = fixpath(d) + '/options.json' + if os.path.exists(filename): + with open(filename) as f: + data = json.loads(jsmin(f.read())) + library.update(data) + return library + +def get_config_list(): + list = {} + dirs = get_config_dirs() + for d in reversed(dirs): + for f in os.listdir(fixpath(d)): + if f.endswith(HW_EXT): + n = os.path.splitext(f)[0] + list[n] = d + '/' + f + return list + +def find_config(name): + dirs = get_config_dirs() + for d in dirs: + path = fixpath(d) + '/' + name + HW_EXT + if os.path.exists(path): + return path + raise InputError("Config '%s' not found" % name) + +class Config(object): + def __init__(self): + self.partitions = partition.Table() + self.devices = storage.List() + self.depends = [] + self.options = [] + self.option_library = load_option_library() + self.base_config = None + + def __str__(self): + return "'%s' for %s" % (self.name, self.arch) + + @classmethod + def from_name(cls, name): + """Create configuration given its name and resolve options + """ + config = Config() + config.load(name) + options = os.environ.get('HWCONFIG_OPTS', '').replace(' ', '') + if options != '': + config.parse_options(options.split(',')) + config.resolve_expressions() + config.partitions.sort() + return config + + @classmethod + def from_json(cls, json, options = []): + config = Config() + config.parse_dict(copy.deepcopy(json)) + config.parse_options(options) + config.resolve_expressions() + config.partitions.sort() + return config + + def load(self, name): + """Load a configuration recursively + """ + filename = find_config(name) + self.depends.append(filename) + with open(filename) as f: + data = json.loads(jsmin(f.read())) + self.parse_dict(data) + + def parse_options(self, options): + """Apply any specified options, each option is applied only once + """ + for option in options: + if option in self.options: + continue + self.options.append(option) + data = self.option_library.get(option) + if data is None: + raise InputError("Option '%s' undefined" % option) + # Don't modify library entries + temp = copy.deepcopy(data) + temp.pop('description', None) + self.parse_dict(temp) + + def resolve_expressions(self): + self.partitions.offset = eval(str(self.partitions.offset)) + for p in self.partitions: + p.resolve_expressions() + + def parse_dict(self, data): + base_config = data.pop('base_config', None) + if self.base_config is None: + self.base_config = base_config + if base_config is not None: + self.load(base_config) + + self.parse_options(data.pop('options', [])) + + # We'll process partitions after other settings + partitions = data.pop('partitions', None) + + for k, v in data.items(): + if k == 'arch': + self.arch = v + elif k == 'partition_table_offset': + self.partitions.offset = v + elif k == 'devices': + self.devices.parse_dict(v) + elif k != 'name' and k != 'comment': + raise InputError("Unknown config key '%s'" % k) + + self.name = data.get('name', '') + self.comment = data.get('comment', '') + + if not partitions is None: + self.partitions.parse_dict(partitions, self.devices) + + def dict(self): + res = {} + + res['name'] = self.name + if hasattr(self, 'comment'): + res['comment'] = self.comment + res['arch'] = self.arch; + res['options'] = self.options + res['partition_table_offset'] = self.partitions.offset_str() + res['devices'] = self.devices.dict() + res['partitions'] = self.partitions.dict() + return res + + def to_json(self): + return to_json(self.dict()) + + def buildVars(self): + dict = {} + + dict['SMING_ARCH_HW'] = self.arch + dict['PARTITION_TABLE_OFFSET'] = self.partitions.offset_str() + dict['PARTITION_TABLE_LENGTH'] = "0x%04x" % partition.MAX_PARTITION_LENGTH + dict['SPIFLASH_PARTITION_NAMES'] = " ".join(p.name for p in filter(lambda p: p.device == self.devices[0], self.partitions)) + dict['HWCONFIG_DEPENDS'] = " ".join(self.depends) + dict.update(self.devices.buildVars()) + dict.update(self.partitions.buildVars()) + + res = "# Generated from hardware configuration '%s'\r\n" % self.name + for k, v in dict.items(): + res += "%s = %s\r\n" % (k, v) + return res + + def verify(self, secure): + self.devices.verify() + self.partitions.verify(self.arch, self.devices[0], secure) + + def map(self): + return partition.Map(self.partitions, self.devices) + + @classmethod + def from_binary(cls, b): + res = Config() + res.name = 'from binary' + res.arch = os.environ.get('SMING_ARCH', 'Unknown') + res.partitions.offset = 0 + res.partitions.parse_binary(b, res.devices) + return res diff --git a/Sming/Components/Storage/Tools/hwconfig/editor.py b/Sming/Components/Storage/Tools/hwconfig/editor.py new file mode 100644 index 0000000000..3a66b3cfdf --- /dev/null +++ b/Sming/Components/Storage/Tools/hwconfig/editor.py @@ -0,0 +1,1070 @@ +import argparse, os, partition, configparser, string +from common import * +from config import * +import tkinter as tk +from tkinter import ttk, filedialog, messagebox, font +from collections import OrderedDict + +app_name = 'Sming Hardware Profile Editor' + +def read_property(obj, name): + """Read an object property, preferring string representation + """ + value = getattr(obj, name + '_str', None) + return getattr(obj, name, None) if value is None else value() + + +def get_dict_value(dict, key, default): + """Read dictionary value, creating one if it doesn't exist + """ + if not key in dict: + dict[key] = default + return dict[key] + + +def load_config_vars(filename): + out_base = os.environ['OUT_BASE'] + path = out_base + '/' + filename + if not os.path.exists(path): + return {} + parser = configparser.ConfigParser() + parser.optionxform = str # preserve case + with open(path) as f: + data = "[config]\n" + f.read() + parser.read_string(data) + return parser['config'] + + +def checkProfilePath(filename): + filename = os.path.realpath(filename) + if filename.startswith(os.getcwd()): + return True + + messagebox.showerror( + 'Invalid profile path', + 'Must be in working project directory where `make hwedit-config` was run') + return False + + +def get_id(obj): + """Get string identifier for a device or partition object + """ + if isinstance(obj, partition.Entry): + if obj.is_unused(): + return obj.device.name + '/' + str(obj.address) + return obj.name + +def resolve_id(config, id): + """Get corresponding device or partition object given the ID value provided from get_id + """ + elem = id.split('/') + if len(elem) == 2: + dev = config.devices.find_by_name(elem[0]) + return config.map().find_by_address(dev, elem[1]) + dev = config.devices.find_by_name(id) + if dev is not None: + return dev + part = config.map().find_by_name(id) + if part is not None: + return part + +def json_loads(s): + return json.loads(jsmin(s), object_pairs_hook=OrderedDict) + +class Field: + """Manages widget and associated variable + """ + def __init__(self, var, widget): + self.var = var + self.widget = widget + + def get_value(self): + return str(self.var.get()) + + def is_disabled(self): + try: + return str(self.widget.cget('state')) == 'disabled' + except Exception: + return False + + +class EditState(dict): + """Manage details of Config/Device/Partition editing using dictionary of Field objects + """ + def __init__(self, editor, objectType, dictName, obj, enumDict): + super().__init__(self) + self.editor = editor + self.objectType = objectType + self.dictName = dictName + self.enumDict = enumDict + self.obj = obj + self.name = obj.name + self.schema = editor.schema[objectType] + self.allow_delete = False + baseConfig = self.editor.getBaseConfig() + optionBaseConfig = self.editor.getOptionBaseConfig() + if objectType == 'Config': + self.is_inherited = False + elif objectType == 'Device': + self.is_inherited = optionBaseConfig.devices.find_by_name(self.name) is not None + self.allow_delete = not self.is_inherited + elif objectType == 'Partition': + if obj.is_unused(): + self.name = 'New Partition' + self.is_inherited = False + else: + self.is_inherited = optionBaseConfig.map().find_by_name(self.name) is not None + self.allow_delete = not self.is_inherited + else: + raise InputError('Unsupported objectType') + + if objectType == 'Config': + self.base_obj = baseConfig + self.obj_dict = editor.json + else: + self.base_obj = getattr(baseConfig, dictName).find_by_name(self.name) + self.obj_dict = editor.json.get(dictName, {}).get(self.name, {}) + + self.init() + + def init(self): + f = self.editor.editFrame + for c in f.winfo_children(): + c.destroy() + + label = ttk.Label(f, text=self.schema['title'], font='heading') + label.pack() + desc = self.schema.get('description') + if desc is not None: + label = ttk.Label(f, text=self.schema['description']) + label.pack() + + f = self.controlFrame = ttk.Frame(self.editor.editFrame) + f.pack(side=tk.TOP) + self.array = {} # dictionary for array element variables + self.row = 0 + keys = self.schema['properties'].keys() + if not 'name' in keys: + self.addControl('name') + for k in keys: + if k != 'devices' and k != 'partitions': + self.addControl(k) + f = ttk.Frame(self.editor.editFrame) + f.pack(side=tk.BOTTOM) + btn = ttk.Button(f, text='Apply', command=lambda *args: self.apply()) + btn.grid(row=0, column=0) + btn = ttk.Button(f, text='Undo', command=lambda *args: self.init()) + btn.grid(row=0, column=1) + if self.allow_delete: + btn = ttk.Button(f, text='Delete', command=lambda *args: self.delete()) + btn.grid(row=0, column=2) + + def addControl(self, fieldName): + schema = self.get_property(fieldName) + fieldType = schema.get('type') + frame = self.controlFrame + disabled = False + if fieldName == 'name': + value = self.name + else: + value = self.obj_dict.get(fieldName, self.obj.dict().get(fieldName)) + if fieldName == 'device': + disabled = True + try: + if fieldType == 'object': + value = '' if value is None else json.dumps(value) + elif fieldType == 'array': + value = [] if value is None else json.dumps(value) + except Exception as err: + critical(str(err)) + var = tk.StringVar(value=value) + + if fieldType == 'boolean': + c = ttk.Checkbutton(frame, text=fieldName, variable=var) + else: + l = tk.Label(frame, text=fieldName) + l.grid(row=self.row, column=0, sticky=tk.W) + values = self.enumDict.get(fieldName, schema.get('enum')) + if values is None: + c = tk.Entry(frame, width=64, textvariable=var) + elif fieldType == 'array': + c = ttk.Frame(frame) + def array_changed(fieldName, key, var): + values = set(json_loads(var.get())) + if self.array[fieldName][key].get(): + values.add(key) + else: + values.discard(key) + var.set(json.dumps(list(values))) + elements = self.array[fieldName] = {} + for k, v in values.items(): + elements[k] = tk.BooleanVar(value=k in getattr(self.obj, fieldName)) + btn = tk.Checkbutton(c, text = k + ': ' + v, + command=lambda *args, fieldName=fieldName, key=k, var=var: array_changed(fieldName, key, var), + variable=elements[k]) + btn.grid(sticky=tk.W) + base = getattr(self.base_obj, fieldName) + if base is not None and k in base: + btn.configure(state='disabled') + else: + c = ttk.Combobox(frame, values=values, textvariable=var) + if fieldName == 'subtype': + def set_subtype_values(): + t = self['type'].get_value() + t = partition.TYPES[t] + values = partition.SUBTYPES.get(t, []) + c.configure(values=list(values)) + c.configure(postcommand=set_subtype_values) + self[fieldName] = Field(var, c) + + if fieldName == 'name': + # Name is read-only for inherited devices/partitions + if self.is_inherited: + disabled = self.is_inherited + # Internal 'partitions' are generally not editable, but make an exception to allow + # creation of new partitions (on an 'unused' type) or changing the partition table offset + if self.objectType == 'Partition' and self.is_inherited and self.obj.is_internal() and not self.obj.is_unused(): + if not (self.obj.is_internal(partition.INTERNAL_PARTITION_TABLE) and fieldName == 'address'): + disabled = True + if disabled: + c.configure(state='disabled') + c.grid(row=self.row, column=1, sticky=tk.EW) + self.row += 1 + return c + + def apply(self): + # Fetch base JSON for comparison + baseConfig = self.editor.getBaseConfig() + json_config = copy.deepcopy(self.editor.json) + if self.objectType == 'Config': + base = baseConfig + json_dict = None + obj = json_config + else: + base = getattr(baseConfig, self.dictName).find_by_name(self.name) + json_dict = get_dict_value(json_config, self.dictName, {}) + obj = get_dict_value(json_dict, self.name, {}) + if self.objectType == 'Partition' and self.obj.is_unused(): + obj['device'] = self.obj.device.name + base = {} if base is None else base.dict() + new_name = None + try: + for k, f in self.items(): + if f.is_disabled(): + continue + value = f.get_value() + schema = self.get_property(k) + fieldType = schema.get('type') + if k == 'name' and json_dict is not None: + value = value.strip() + if value != self.name: + if value in self.editor.config.map(): + raise InputError("Name '%s' already used" % value) + old = json_dict.pop(self.name) + obj = json_dict[value] = old + new_name = value + # If renaming a device, then all partitions must be updated + if self.objectType == 'Device': + for n, p in json_config.get('partitions', {}).items(): + if p.get('device') == self.name: + p['device'] = new_name + elif k == 'address' and self.objectType == 'Partition' and self.obj.is_internal(partition.INTERNAL_PARTITION_TABLE): + json_config['partition_table_offset'] = value + elif value == '' and k != 'filename': # TODO mark 'allow empty' values in schema somehow + if k in obj: + del obj[k] + elif fieldType == 'object' or fieldType == 'array': + obj[k] = {} if value == '' else json_loads(value) + elif fieldType == 'boolean': + obj[k] = (value != '0') + elif value.isdigit() and 'integer' in fieldType: + obj[k] = int(value) + else: + obj[k] = value + if k in base and obj.get(k) == base[k]: + del obj[k] + + if len(obj) == 0: + del json_dict[self.name] + if len(json_dict) == 0: + del json_config[self.dictName] + + + if self.editor.verify_config(json_config): + self.editor.set_json(json_config) + if new_name is not None: + self.name = new_name + if self.objectType != 'Config': + self.editor.selected = self.name + self.editor.reload() + except Exception as err: + self.editor.user_error(err) + raise err + + def delete(self): + json_dict = self.editor.json[self.dictName] + # If deleting a device, then delete all partitions first + if self.objectType == 'Device': + devparts = [] + partitions = self.editor.json.get('partitions', {}) + for n, p in partitions.items(): + if p.get('device') == self.name: + devparts.append(n) + for n in devparts: + partitions.pop(n) + self.editor.selected = self.editor.config.devices[0].name + else: + self.editor.selected = self.obj.device.name + del json_dict[self.name] + if len(json_dict) == 0: + del self.editor.json[self.dictName] + self.editor.reload() + + def get_property(self, name): + if name == 'name': + return {'type': 'text'} + else: + return self.schema['properties'][name] + + def nameChanged(self): + return self.name != self['name'].get_value() + + +class Rect: + def __init__(self, x=0, y=0, width=0, height=0): + self.x = x + self.y = y + self.x2 = x + width + self.y2 = y + height + + def inflate(self, x, y): + self.x -= x + self.y -= y + self.x2 += x + self.y2 += y + + def getWidth(self): + return self.x2 - self.x + + def setWidth(self, w): + self.x2 = self.x + w + + def getHeight(self): + return self.y2 - self.y + + def setHeight(self, h): + self.y2 = self.y + h + + def pos(self, anchor=tk.NW): + if anchor == tk.N: + return ((self.x + self.x2) / 2, self.y) + if anchor == tk.NE: + return (self.x2, self.y) + if anchor == tk.E: + return (self.x2, (self.y + self.y2) / 2) + if anchor == tk.SE: + return (self.x2, self.y2) + if anchor == tk.S: + return ((self.x + self.x2) / 2, self.y2) + if anchor == tk.SW: + return (self.x, self.y2) + if anchor == tk.W: + return (self.x, (self.y + self.y2) / 2) + if anchor == tk.CENTER: + return ((self.x + self.x2) / 2, (self.y + self.y2) / 2) + # Default and NW + return (self.x, self.y) + + def bounds(self): + return (self.x, self.y, self.x2, self.y2) + + +class TkMap(tk.Frame): + def __init__(self, parent, editor): + super().__init__(parent) + self.pack(fill=tk.BOTH) + self.editor = editor + self.device = None + self.selected_id = None + self.bytesPerPixel = 32 + self.maxPartitionDrawSize = 0x4000 + canvas = self.canvas = tk.Canvas(self, height=150, xscrollincrement=1) + canvas.pack(side=tk.TOP, expand=True, fill=tk.BOTH) + self.autoZoom = tk.BooleanVar(value=True) + btn = tk.Checkbutton(self, text='Auto Zoom', variable=self.autoZoom, + command=lambda *args: self.update()) + btn.pack(side=tk.LEFT) + s = ttk.Scrollbar(self, orient=tk.HORIZONTAL, command=canvas.xview) + s.pack(side=tk.BOTTOM, fill=tk.X) + canvas['xscrollcommand'] = s.set + # Windows + canvas.bind('', self.onMouseWheel) + # Linux + canvas.bind('<4>', self.onMouseWheel) + canvas.bind('<5>', self.onMouseWheel) + + def clear(self): + self.canvas.delete('all') + self.items = {} + + def update(self): + # Margins to separate drawn rectangular regions + M = 5 + M_HEADING = 16 + M_OUTER = 20 + + # Pixels to draw map row + ROW_HEIGHT = 80 + # Text line spacing + LINE_SPACE = 16 + # How far ticks extend below map + TICK_LENGTH = 10 + + self.clear() + + if self.device is None: + return + + device = self.device + canvas = self.canvas + partitions = list(filter(lambda p: p.device == self.device, self.editor.config.map())) + + # Minimum width (in pixels) to draw a partition + minPartitionWidth = M + + if self.autoZoom.get(): + self.bind('', lambda x: self.update()) + maxPartitionDrawSize = 0xffffffffffffffff + # Calculate draw width + w = self.winfo_width() - (M_OUTER + M) * 2 - M * len(partitions) + bytesPerPixel = self.device.size // w + minSize = minPartitionWidth * bytesPerPixel + # Now re-check to ensure all partitions have a sensible minimum drawn size + def is_small(p): + return p.size < minSize + extra = sum(minSize - p.size for p in filter(is_small, partitions)) + bytesPerPixel = (self.device.size + extra) // w + else: + self.unbind('') + bytesPerPixel = self.bytesPerPixel + maxPartitionDrawSize = self.maxPartitionDrawSize + + # Partitions smaller than this will be expanded + minPartitionDrawSize = minPartitionWidth * bytesPerPixel + + labelFont = font.Font(family='fixed', size=8) + labelFontBold = font.Font(family='fixed', size=8, weight='bold') + headingFont = font.Font(family='courier', size=10, weight='bold') + + # Draw device title + s = "[%s]" % device.name + if device.size != 0: + s += ': %s' % size_frac_str(device.size) + if device.type != 0: + s += ' (%s)' % device.type_str() + canvas.create_text(self.winfo_width() / 2, 0, anchor=tk.N, text=s, font=headingFont, fill='blue') + + # When auto-zoom is disabled, drawing the map linearly is unhelpful, + # it's too spread-out and navigation is difficult. + # So, fix a limit for the drawn size of each partition (in bytes) + # Marker ticks will be drawn according to this scale + drawsize = 0 # Equivalent size (in bytes) for the partition + x_device_end = 0 # Determines final x co-ordinate for end of device memory + + MIN_TICK_SPACING = 100 + def draw_tick(x, addr): + canvas.create_line(x, r.y2 + M, x, r.y2 + M + TICK_LENGTH, fill='black', width=3) + canvas.create_text(x, r.y + ROW_HEIGHT + M + TICK_LENGTH, + anchor=tk.N, + text = size_frac_str(addr), + state='disabled', + font=labelFontBold) + + # Track current partition area + r_prev = Rect() + r_prev.y = M_HEADING + M_OUTER + r_prev.x = M_OUTER + part_prev = None + + device_item = canvas.create_rectangle(r_prev.bounds(), fill='lightgray', outline='black') + canvas.tag_bind(device_item, '', lambda x: self.editor.editDevice(self.device)) + self.items[get_id(self.device)] = device_item + + for p in partitions: + # Starting x co-ordinate for this partition depends on scale of previous partition + r = copy.copy(r_prev) + r.setHeight(ROW_HEIGHT) + if part_prev is not None: + r.x += M + drawsize * (p.address - part_prev.address) / part_prev.size / bytesPerPixel + + # Determine actual size to draw this partition (in bytes) + drawsize = min(maxPartitionDrawSize, max(minPartitionDrawSize, p.size)) + r.setWidth(drawsize / bytesPerPixel) + # Identify where end of device memory actually is in case partitions exceed this boundary + if x_device_end == 0 and p.end() >= device.size - 1: + sz = device.size - p.address + x_device_end = r.x + (drawsize * sz / p.size / bytesPerPixel) + + # Draw tick marks + div = device.size // 16 + addr = p.address - (p.address % div) + x_next_tick = 0 + while addr <= p.end(): + if addr >= p.address: + x = r.x + (drawsize * (addr - p.address) / p.size / bytesPerPixel) + if x >= x_next_tick: + draw_tick(x, addr) + x_next_tick = x + MIN_TICK_SPACING + addr += div + + # Outer rectangle for partition + r2 = copy.copy(r) + # r2.inflate(-M, -M) + color = 'lightgray' if p.is_unused() else 'white' + item = canvas.create_rectangle(r2.bounds(), fill=color, outline=color) + canvas.tag_bind(item, "", lambda event, part=p: self.editor.editPartition(part)) + # Used region + used = self.editor.get_used(p) + if used.size != 0: + r3 = copy.copy(r2) + r3.inflate(-1, -1) + if used.size > p.size: + color = 'lightpink' + else: + r3.setWidth(used.size * r3.getWidth() / p.size) + color = 'lightblue' + canvas.create_rectangle(r3.bounds(), fill=color, outline=color, state='disabled') + item = canvas.create_rectangle(r2.bounds(), outline='blue', state='disabled') + self.items[get_id(p)] = item + + # Internal text + r2.inflate(-M, -M) + def createText(r, anchor, text, font): + if font.measure(text) < r.getWidth(): + canvas.create_text(r.pos(tk.N if anchor == tk.CENTER else tk.NW), anchor=anchor, text=text, state='disabled', font=font) + createText(r2, tk.NW, p.address_str(), labelFont) + r2.y += LINE_SPACE + r2.y += LINE_SPACE + createText(r2, tk.CENTER, p.name, labelFontBold) + r2.y += LINE_SPACE + createText(r2, tk.CENTER, size_frac_str(p.size), labelFontBold) + + part_prev = p + r_prev = r + + r2 = Rect(M_OUTER, M_HEADING + M_OUTER) + r2.x2 = x_device_end + r2.y2 = r_prev.y2 + r2.inflate(M, M) + self.canvas.coords(device_item, r2.bounds()) + if x_device_end >= r_prev.x: + draw_tick(x_device_end, device.size) + self.scroll_width = r_prev.x2 + M_OUTER + canvas.config(scrollregion=(0, 0, self.scroll_width, 0)) + + # Highlight the selected item (if any) + item = self.items.get(self.selected_id) + if item is not None: + self.canvas.itemconfigure(item, width=3) + + def onMouseWheel(self, event): + if self.autoZoom.get(): + return + shift = (event.state & 0x01) != 0 + control = (event.state & 0x04) != 0 + if event.num == 5: + delta = -120 + elif event.num == 4: + delta = 120 + else: + delta = event.delta + # Adjust draw size + if shift and not control: + ds = self.maxPartitionDrawSize + if delta < 0: + ds >>= 1 + else: + ds <<= 1 + if ds >= 0x1000 and ds <= 0x100000: + self.maxPartitionDrawSize = ds + self.update() + if control and not shift: + # Zoom + bpp = self.bytesPerPixel + if delta < 0: + bpp <<= 1 + else: + bpp >>= 1 + if bpp >= 4 and bpp <= 0x10000: + self.bytesPerPixel = bpp + self.update() + if not (shift or control): + self.canvas.xview_scroll(delta, tk.UNITS) + + def set_device(self, dev): + if self.device != dev: + self.device = dev + self.update() + + def select(self): + id = self.editor.selected + if id == self.selected_id: + return + item = self.items.get(self.selected_id) + if item is not None: + self.canvas.itemconfigure(item, width=1) + item = self.items.get(id) + if item is None: + return + self.canvas.itemconfigure(item, width=3) + self.selected_id = id + # Ensure item is visible + pos = self.canvas.coords(item) + id_x1 = pos[0] / self.scroll_width + id_x2 = pos[2] / self.scroll_width + xview = self.canvas.xview() + if id_x1 > xview[0] and id_x1 < xview[1]: + return + if id_x2 > xview[0] and id_x2 < xview[1]: + return + if xview[0] >= id_x1 and xview[1] <= id_x2: + return + self.canvas.xview_moveto(id_x1) + + +class TkTree(tk.Frame): + def __init__(self, parent, editor): + super().__init__(parent) + self.pack(fill=tk.BOTH) + self.editor = editor + s = ttk.Style() + s.configure('Treeview', font='TkFixedFont') + s.configure('Treeview.Heading', font='TkFixedFont', padding=4) + self.init() + + def init(self): + self.headings = [ + ("#0", "Partition"), + ("type", "Type"), + ("subtype", "Subtype"), + ("start", "Start"), + ("end", "End"), + ("size", "Size"), + ("used", "Used"), + ("unused", "Unused"), + ("filename", "Image Filename"), + ] + + cols = [h[0] for h in self.headings[1:]] + tree = self.tree = ttk.Treeview(self, selectmode=tk.BROWSE, columns=cols) + tree.pack(side=tk.LEFT, expand=True, fill=tk.BOTH) + + s = ttk.Scrollbar(self, orient=tk.VERTICAL, command=tree.yview) + s.pack(side=tk.RIGHT, fill=tk.Y) + tree['yscrollcommand'] = s.set + + for (k, v) in self.headings: + tree.heading(k, text=v, anchor=tk.W) + + def select(*args): + id = tree.focus() + if id == '': + return + obj = resolve_id(self.editor.config, id) + if isinstance(obj, partition.Entry): + self.editor.editPartition(obj) + elif isinstance(obj, storage.Device): + self.editor.editDevice(obj) + tree.bind('<>', select) + + def clear(self): + for c in self.tree.get_children(): + self.tree.delete(c) + + def update(self): + config = self.editor.config + tree = self.tree + fnt = font.Font(font='TkFixedFont') + + self.clear() + + columnWidths = [0] * len(self.headings) + for i, (c, v) in enumerate(self.headings): + columnWidths[i] = fnt.measure(v) + + def addItem(parent, id, values): + text = [] + for i, (c, v) in enumerate(self.headings): + v = values.get(c, '') + if i != 0: + text.append(v) + columnWidths[i] = max(columnWidths[i], fnt.measure(v)) + tree.insert(parent, 'end', id, open=True, text=values['#0'], values=text) + + # Devices are our root nodes + map = config.map() + for dev in config.devices: + def is_used(p): + return p.device == dev and not p.is_unused() + used = sum(p.size for p in filter(is_used, map)) + values = { + "#0": dev.name, + "start": addr_format(0), + "end": addr_format(dev.size - 1), + "size": size_frac_str(dev.size), + "used": size_frac_str(used), + "unused": size_frac_str(dev.size - used), + "type": dev.type_str() + } + addItem('', get_id(dev), values) + + # Partitions are children + for p in config.map(): + used = self.editor.get_used(p) + values = { + "#0": p.name, + "start": p.address_str(), + "end": p.end_str(), + "size": size_frac_str(p.size), + "used": used.text, + "type": p.type_str(), + "subtype": p.subtype_str(), + "filename": p.filename + } + if used.path != '': + values['unused'] = size_frac_str(p.size - used.size) + addItem(p.device.name, get_id(p), values) + + # Auto-size columns + columnWidths[0] += 32 # Allow for indented child items + for i, w in enumerate(columnWidths): + w += 16 + tree.column(self.headings[i][0], stretch=False, width=w, minwidth=w) + + self.select() + + + def select(self): + id = self.editor.selected + tree = self.tree + if tree.exists(id) and tree.focus() != id: + tree.focus(id) + tree.selection_set(id) + + +class Schema(dict): + def __init__(self, filename): + with open(filename) as f: + self.schema = json.load(f, object_pairs_hook=OrderedDict) + + def __getitem__(self, name): + return self.schema['definitions'][name] + + +class Editor: + def __init__(self, root): + root.title(app_name) + self.main = root + self.edit = None + self.selected = '' + self.config_vars = load_config_vars('config.mk') + self.config_vars.update(load_config_vars('debug.mk')) + self.initialise() + + def initialise(self): + self.main.option_add('*tearOff', False) + + self.schema = Schema(os.environ['HWCONFIG_SCHEMA']) + + hwFilter = [('Hardware Profiles', '*' + HW_EXT)] + + # Menus + def fileNew(): + self.reset() + self.reload() + self.editDevice(self.config.devices[0]) + + def fileOpen(): + filename = filedialog.askopenfilename( + title='Select profile ' + HW_EXT + ' file', + filetypes=hwFilter, + initialdir=os.getcwd()) + if filename != '' and checkProfilePath(filename): + self.loadConfig(filename) + + def fileSave(): + filename = self.json['name'] + filename = filedialog.asksaveasfilename( + title='Save profile to file', + filetypes=hwFilter, + initialfile=filename, + initialdir=os.getcwd()) + if filename != '' and checkProfilePath(filename): + ext = os.path.splitext(filename)[1] + if ext != HW_EXT: + filename += HW_EXT + with open(filename, "w") as f: + json.dump(self.json, f, indent=4) + + # Toolbar + toolbar = ttk.Frame(self.main) + toolbar.pack(side=tk.TOP, fill=tk.X) + btn = ttk.Button(toolbar, text="New", command=fileNew) + btn.grid(row=0, column=1) + btn = ttk.Button(toolbar, text="Open...", command=fileOpen) + btn.grid(row=0, column=2) + btn = ttk.Button(toolbar, text="Save...", command=fileSave) + btn.grid(row=0, column=3) + sep = ttk.Separator(toolbar, orient=tk.VERTICAL) + sep.grid(row=0, column=4, sticky=tk.NS) + btn = ttk.Button(toolbar, text='Edit Config', command=self.editConfig) + btn.grid(row=0, column=5) + btn = ttk.Button(toolbar, text='Add Device', command=self.addDevice) + btn.grid(row=0, column=6) + + + # Group main controls into areas which can be re-sized by the user + pwin = ttk.PanedWindow(self.main, orient=tk.VERTICAL) + pwin.pack(side=tk.TOP, expand=True, fill=tk.BOTH) + + self.map = TkMap(pwin, self) + pwin.add(self.map) + self.tree = TkTree(pwin, self) + pwin.add(self.tree) + + + # Lower pane + lower = ttk.Frame(pwin) + pwin.add(lower) + + # Edit frame + frame = ttk.LabelFrame(lower, text='Edit') + frame.pack(anchor=tk.NW, side=tk.LEFT, fill=tk.BOTH) + canvas = self.editCanvas = tk.Canvas(frame, highlightthickness=0) + canvas.pack(side=tk.LEFT, expand=True, fill=tk.BOTH) + self.editFrame = ttk.Frame(canvas) + canvas.create_window(0, 0, window=self.editFrame, anchor=tk.NW) + s = self.editScroll = ttk.Scrollbar(frame, orient=tk.VERTICAL, command=canvas.yview) + s.pack(side=tk.RIGHT, fill=tk.Y) + canvas['yscrollcommand'] = s.set + def configure(event): + canvas.config(scrollregion=(0, 0, 0, self.editFrame.winfo_height())) + canvas.config(width = self.editFrame.winfo_width()) + self.editFrame.bind('', configure) + + + # JSON editor + frame = self.jsonFrame = ttk.LabelFrame(lower, text='JSON') + frame.pack(anchor=tk.NW, side=tk.LEFT, expand=True, fill=tk.BOTH) + def apply(*args): + try: + json_config = json_loads(self.jsonEditor.get('1.0', 'end')) + if self.verify_config(json_config): + self.set_json(json_config) + self.updateWindowTitle() + self.reload() + except Exception as err: + self.user_error(err) + def undo(*args): + self.jsonEditor.replace('1.0', 'end', to_json(self.json)) + f = ttk.Frame(frame) + f.pack(anchor=tk.S, side=tk.BOTTOM) + btn = ttk.Button(f, text="Apply", command=apply) + btn.grid(row=0, column=0) + btn = ttk.Button(f, text="Undo", command=undo) + btn.grid(row=0, column=1) + self.jsonEditor = tk.Text(frame, height=14) + self.jsonEditor.pack(anchor=tk.N, side=tk.LEFT, expand=True, fill=tk.BOTH) + s = ttk.Scrollbar(frame, orient=tk.VERTICAL, command=self.jsonEditor.yview) + s.pack(anchor=tk.NE, side=tk.RIGHT, fill=tk.Y) + self.jsonEditor['yscrollcommand'] = s.set + + # Status box + self.status = tk.StringVar() + status = ttk.Label(self.main, textvariable=self.status) + status.pack(side=tk.BOTTOM, fill=tk.X) + + self.reset() + + def user_error(self, err): + self.status.set(err) + messagebox.showerror(type(err).__name__, err) + + def getBaseConfig(self): + """Load the base configuration + """ + return Config.from_json(self.json_base_config) + + def getOptionBaseConfig(self): + """Load the base configuration with currently selected options applied + """ + return Config.from_json(self.json_base_config, self.json.get('options', [])) + + def loadConfig(self, filename): + self.reset() + # If this is a core profile, don't edit it but create a new profile based on it + if filename.startswith(os.environ['SMING_HOME']): + config_name = os.path.splitext(os.path.basename(filename))[0] + self.json['base_config'] = config_name + else: + with open(filename) as f: + json_config = json_loads(f.read()) + + options = get_dict_value(self.json, 'options', []) + for opt in os.environ.get('HWCONFIG_OPTS', '').replace(' ', '').split(): + if not opt in options: + options.append(opt) + + self.set_json(json_config) + + self.reload() + self.updateWindowTitle() + self.editDevice(self.config.devices[0]) + + def updateWindowTitle(self): + name = self.json.get('name', None) + if name is None: + name = '(new)' + else: + name = '"' + name + '"' + self.main.title(self.config.arch + ' ' + name + ' - ' + app_name) + + def verify_config(self, json_config): + try: + Config.from_json(json_config).verify(False) + return True + except Exception as err: + self.user_error(err) + return False + + def set_json(self, json_config): + # Keep output order consistent + self.json = {} + for k in self.schema['Config']['properties'].keys(): + if k in json_config: + self.json[k] = json_config[k] + + def reset(self): + self.tree.clear() + self.map.clear() + self.status.set('') + self.json = {"name": "New Profile"} + self.json['base_config'] = 'standard' + self.reload() + self.updateWindowTitle() + + def resolve_path(self, path): + tmp = str(path) + while True: + tmp = tmp.replace('(', '{') + tmp = tmp.replace(')', '}') + new_path = string.Template(tmp).substitute(self.config_vars) + if new_path == tmp: + return new_path + tmp = new_path + + class Used: + def __init__(self): + self.text = '' + self.size = 0 + self.path = '' + + def get_used(self, part): + used = self.Used() + if part.filename != '': + try: + used.path = self.resolve_path(part.filename) + if os.path.exists(used.path): + used.size = os.path.getsize(used.path) + used.text = size_frac_str(used.size) + else: + used.text = '(not found)' + except KeyError as err: + used.text = str(err) + ' undefined' + return used + + def reload(self): + with open(find_config(self.json['base_config'])) as f: + self.json_base_config = json_loads(f.read()) + + self.jsonEditor.replace('1.0', 'end', to_json(self.json)) + try: + config = Config.from_json(self.json) + except Exception as err: + self.status.set(err) + return + + self.status.set('') + self.config = config + + self.map.update() + self.tree.update() + + def select(self, obj): + self.selected = get_id(obj) + if isinstance(obj, partition.Entry): + self.map.set_device(obj.device) + else: + self.map.set_device(obj) + self.map.select() + self.tree.select() + + def is_editing(self, obj): + return getattr(self.edit, 'obj', None) == obj + + def editConfig(self): + if self.is_editing(self.config): + return + enumDict = {} + enumDict['base_config'] = list(get_config_list().keys()) + optionlib = load_option_library() + options = {} + for k, v in optionlib.items(): + options[k] = v['description'] + enumDict['options'] = options + self.edit = EditState(self, 'Config', None, self.config, enumDict) + + def addDevice(self): + dev = storage.Device('New device') + self.editDevice(dev) + + def editDevice(self, dev): + if isinstance(dev, str): + dev = self.config.devices.find_by_name(dev) + if self.is_editing(dev): + return + self.select(dev) + enumDict = {} + enumDict['type'] = list((storage.TYPES).keys()) + self.edit = EditState(self, 'Device', 'devices', dev, enumDict) + + def editPartition(self, part): + if isinstance(part, str): + part = self.config.partitions.find_by_name(part) + if self.is_editing(part): + return + self.select(part) + enumDict = {} + enumDict['type'] = list((partition.TYPES).keys() - ['storage', 'internal']) + enumDict['subtype'] = [] + self.edit = EditState(self, 'Partition', 'partitions', part, enumDict) + + +def main(): + parser = argparse.ArgumentParser(description='Sming hardware profile editor') + parser.add_argument('input', help='Name of existing hardware profile') + args = parser.parse_args() + + root = tk.Tk() + editor = Editor(root) + editor.loadConfig(find_config(args.input)) + root.mainloop() + + +if __name__ == '__main__': + try: + print("TCL version %s" % tk.TclVersion) + main() + except InputError as e: + print(e, file=sys.stderr) + sys.exit(2) diff --git a/Sming/Components/Storage/Tools/hwconfig/hwconfig.py b/Sming/Components/Storage/Tools/hwconfig/hwconfig.py new file mode 100644 index 0000000000..1aff6773af --- /dev/null +++ b/Sming/Components/Storage/Tools/hwconfig/hwconfig.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# +# Sming hardware configuration tool +# + +import common, argparse, os, partition +from common import * +from config import Config + +def openOutput(path): + if path == '-': + try: + stdout_binary = sys.stdout.buffer # Python 3 + except AttributeError: + stdout_binary = sys.stdout + return stdout_binary + status("Writing to '%s'" % path) + output_dir = os.path.abspath(os.path.dirname(path)) + os.makedirs(output_dir, exist_ok=True) + return open(path, 'wb') + + +def handle_validate(args, config, part): + # Validate resulting hardware configuration against schema + try: + from jsonschema import Draft7Validator + inst = json.loads(config.to_json()) + with open(os.environ['HWCONFIG_SCHEMA']) as f: + schema = json.load(f) + v = Draft7Validator(schema) + errors = sorted(v.iter_errors(inst), key=lambda e: e.path) + if errors != []: + for e in errors: + critical("%s @ %s" % (e.message, e.path)) + sys.exit(3) + except ImportError as err: + critical("\n** WARNING! %s: Cannot validate '%s', please run `make python-requirements **\n\n" % (str(err), args.input)) + + +def handle_flashcheck(args, config, part): + # Expect list of chunks, such as "0x100000=/out/Esp8266/debug/firmware/spiff_rom.bin 0x200000=custom.bin" + list = args.expr.split() + if len(list) == 0: + raise InputError("No chunks to flash!") + for e in list: + addr, filename = e.split('=') + addr = int(addr, 0) + part = config.partitions.find_by_address(config.devices[0], addr) + if part is None: + raise InputError("No partition contains address 0x%08x" % addr) + if part.address != addr: + raise InputError("Address 0x%08x is within partition '%s', not at start (0x%08x)" % (addr, part.name, part.address)) + filesize = os.path.getsize(filename) + if filesize > part.size: + raise InputError("File '%s' is 0x%08x bytes, too big for partition '%s' (0x%08x bytes)" % (os.path.basename(filename), filesize, part.name, part.size)) + + +def handle_partgen(args, config, part): + # Generate partition table binary + if not args.no_verify: + status("Verifying partition table...") + config.verify(args.secure) + return config.partitions.to_binary(config.devices) + + +def handle_expr(args, config, part): + # Evaluate expression against configuration data + return str(eval(args.expr)).encode() + + +def main(): + parser = argparse.ArgumentParser(description='Sming hardware configuration utility') + + parser.add_argument('--no-verify', help="Don't verify partition table fields", action='store_true') + parser.add_argument('--quiet', '-q', help="Don't print non-critical status messages to stderr", action='store_true') + parser.add_argument('--secure', help="Require app partitions to be suitable for secure boot", action='store_true') + parser.add_argument('--part', help="Name of partition to operate on") + parser.add_argument('command', help='Action to perform', choices=['partgen', 'expr', 'validate', 'flashcheck']) + parser.add_argument('input', help='Name of hardware configuration or path to binary partition table') + parser.add_argument('output', help='Path to output file. Will use stdout if omitted.', nargs='?', default='-') + parser.add_argument('expr', help='Expression to evaluate', nargs='?', default=None) + + args = parser.parse_args() + + common.quiet = args.quiet + + output = None + input_is_binary = False + if os.path.exists(args.input): + inputData = open(args.input, "rb").read() + input_is_binary = inputData[0:2] == partition.Entry.MAGIC_BYTES + if input_is_binary: + config = Config.from_binary(inputData) + else: + raise InputError("File '%s' not recognised as partition table" % args.input) + else: + config = Config.from_name(args.input) + partitions = config.partitions + + # Locate any supplied partition by name + part = None + if args.part is not None: + part = partitions.find_by_name(args.part) + if part is None: + return + + output = globals()['handle_' + args.command](args, config, part) + + if output is not None: + openOutput(args.output).write(output) + + +if __name__ == '__main__': + try: + main() + except InputError as e: + print("** ERROR! %s" % e, file=sys.stderr) + sys.exit(2) diff --git a/Sming/Components/Storage/Tools/hwconfig/partition.py b/Sming/Components/Storage/Tools/hwconfig/partition.py new file mode 100644 index 0000000000..7551823109 --- /dev/null +++ b/Sming/Components/Storage/Tools/hwconfig/partition.py @@ -0,0 +1,601 @@ +#!/usr/bin/env python3 +# +# From Espressif gen_esp32part.py +# +# Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct, hashlib, storage, binascii, copy +from common import * + +MAX_PARTITION_LENGTH = 0xC00 # 3K for partition data (96 entries) leaves 1K in a 4K sector for signature +MD5_PARTITION_BEGIN = b"\xEB\xEB" + b"\xFF" * 14 # The first 2 bytes are like magic numbers for MD5 sum +FLASH_SECTOR_SIZE = 0x1000 +PARTITION_TABLE_SIZE = 0x1000 # Size of partition table +PARTITION_ENTRY_SIZE = 32 +PARTITION_NAME_SIZE = 16 + +MIN_PARTITION_SUBTYPE_APP_OTA = 0x10 +NUM_PARTITION_SUBTYPE_APP_OTA = 16 + +__version__ = '1.2' + +APP_TYPE = 0x00 +DATA_TYPE = 0x01 +STORAGE_TYPE = 0x02 # Reference to storage device +USER_TYPE = 0x40 # First user-defined type + +# Used for internal management, e.g. in maps +INTERNAL_TYPE = 0xff +INTERNAL_BOOT_SECTOR = 0x00 +INTERNAL_PARTITION_TABLE = 0x01 +INTERNAL_UNUSED = 0x02 + +# Partition start alignment +ALIGNMENT = { + "Esp32": { + APP_TYPE: 0x10000, + }, + "Esp8266": { + APP_TYPE: 0x1000, + }, + "Host": { + APP_TYPE: 0x1000, + } +} + +TYPES = { + "app": APP_TYPE, + "data": DATA_TYPE, + "storage": STORAGE_TYPE, + "user": USER_TYPE, + "internal": INTERNAL_TYPE, +} + +# Keep this map in sync with esp_partition_subtype_t enum in esp_partition.h +SUBTYPES = { + APP_TYPE: { + "factory": 0x00, + "ota_0": 0x10, + "ota_1": 0x11, + "ota_2": 0x12, + "ota_3": 0x13, + "ota_4": 0x14, + "ota_5": 0x15, + "ota_6": 0x16, + "ota_7": 0x17, + "ota_8": 0x18, + "ota_9": 0x19, + "ota_10": 0x1a, + "ota_11": 0x1b, + "ota_12": 0x1c, + "ota_13": 0x1d, + "ota_14": 0x1e, + "ota_15": 0x1f, + "test": 0x20, + "internal": 0xff, + }, + DATA_TYPE: { + "ota": 0x00, + "phy": 0x01, + "nvs": 0x02, + "coredump": 0x03, + "nvs_keys": 0x04, + "efuse": 0x05, + "sysparam": 0x40, + "rfcal": 0x41, + "esphttpd": 0x80, + "fat": 0x81, + "spiffs": 0x82, + "fwfs": 0xf1, + }, + STORAGE_TYPE: storage.TYPES, + INTERNAL_TYPE: { + "boot": INTERNAL_BOOT_SECTOR, + "pt": INTERNAL_PARTITION_TABLE, + "unused": INTERNAL_UNUSED, + } +} + + +def parse_type(value): + if isinstance(value, str): + if value == "": + raise InputError("Field 'type' can't be left empty.") + return parse_int(value, TYPES) + return value + + +def parse_subtype(ptype, value): + if isinstance(value, str): + if value == "": + return 0 # default + return parse_int(value, SUBTYPES.get(ptype, {})) + return value + + +class Table(list): + + def __init__(self): + super().__init__(self) + + def parse_dict(self, data, devices): + partnames = [] + for name, entry in data.items(): + if name in partnames: + raise InputError("Duplicate partition '%s'" % name) + partnames += name + part = self.find_by_name(name) + if part is None: + part = Entry(devices[0], name) + self.append(part) + part.parse_dict(entry, devices) + + def sort(self): + # Ensure spiFlash partitions are first in the list + def get_key(p): + key = p.device.name + p.address_str() + if p.device.name == 'spiFlash': + key = ' ' + key + return key + super().sort(key=get_key) + + def dict(self): + res = {} + for entry in self: + res[entry.name] = entry.dict() + return res + + def to_json(self): + return to_json(self.dict()); + + def to_csv(self): + res = "Device Start End Size Type SubType Name Filename\n" + res += "---------------- ---------- ---------- ---------- -------- -------- ---------------- ------------\n" + for p in self: + res += "%-16s %10s %10s %10s %-8s %-8s %-16s %s\n" \ + % (p.device.name, p.address_str(), p.end_str(), p.size_str(), p.type_str(), p.subtype_str(), p.name, p.filename) + return res + + def offset_str(self): + return addr_format(self.offset) + + def buildVars(self): + dict = {} + dict['PARTITION_NAMES'] = " ".join(p.name for p in self) + buildparts = [p for p in self if p.build is not None] + dict['PARTITIONS_WITH_TARGETS'] = " ".join(p.name for p in buildparts) + dict['PARTITION_BUILD_TARGETS'] = " ".join(p.filename for p in buildparts) + + for p in self: + dict.update(p.buildVars()) + return dict + + def __getitem__(self, item): + """Allow partition table access by name or index + """ + if isinstance(item, str): + p = self.find_by_name(item) + if p is None: + raise ValueError("No partition entry named '%s'" % item) + return p + return super().__getitem__(item) + + def find_by_type(self, ptype, subtype): + """Return a partition by type & subtype, returns None if not found + """ + # convert ptype & subtypes names (if supplied this way) to integer values + try: + ptype = TYPES[ptype] + except KeyError: + try: + ptype = int(ptype, 0) + except TypeError: + pass + try: + subtype = SUBTYPES[int(ptype)][subtype] + except KeyError: + try: + subtype = int(subtype, 0) + except TypeError: + pass + + for p in self: + if p.type == ptype and p.subtype == subtype: + yield p + return + + def find_by_name(self, name): + for p in self: + if p.name == name: + return p + return None + + def find_by_address(self, device, addr): + if isinstance(addr, str): + addr = eval(addr) + for p in self: + if p.device == device and p.contains(addr): + return p + return None + + def verify(self, arch, spiFlash, secure): + """Verify partition layout + """ + # verify each partition individually + for p in self: + p.verify(arch, secure) + + if self.offset % FLASH_SECTOR_SIZE != 0: + raise InputError("Partition table offset not aligned to flash sector") + + p = self.find_by_address(spiFlash, self.offset) + if p is None: + p = self.find_by_address(spiFlash, self.offset + PARTITION_TABLE_SIZE - 1) + if not p is None: + raise InputError("Partition table conflict with '%s'" % p.name) + + # check on duplicate name + names = [p.name for p in self] + duplicates = set(n for n in names if names.count(n) > 1) + + # print sorted duplicate partitions by name + if len(duplicates) != 0: + print("A list of partitions that have the same name:") + for p in sorted(self, key=lambda x:x.name): + if len(duplicates.intersection([p.name])) != 0: + print("%s" % (p.to_csv())) + raise InputError("Partition names must be unique") + + # check for overlaps + if arch == 'Esp32': + minPartitionAddress = self.offset + PARTITION_TABLE_SIZE + else: + minPartitionAddress = 0x00002000 + dev = None + last = None + for p in self: + if p.device != dev: + last = None + dev = p.device + if dev != spiFlash: + minPartitionAddress = 0 + if dev == self[0].device and p.address < minPartitionAddress: + raise InputError("Partition '%s' @ %s-%s must be located after @ %s" \ + % (p.name, p.address_str(), p.end_str(), addr_format(minPartitionAddress))) + if last is not None and p.address <= last.end(): + raise InputError("Partition '%s' @ %s-%s overlaps '%s' @ %s-%s" \ + % (p.name, p.address_str(), p.end_str(), \ + last.name, last.address_str(), last.end_str())) + last = p + + def parse_binary(self, b, devices): + """Construct partition table object from binary image + """ + dev = None + md5 = hashlib.md5() + for o in range(0, len(b), PARTITION_ENTRY_SIZE): + data = b[o:o + PARTITION_ENTRY_SIZE] + if len(data) != PARTITION_ENTRY_SIZE: + raise InputError("Partition table length must be a multiple of %d bytes" % PARTITION_ENTRY_SIZE) + if data == b'\xFF' * PARTITION_ENTRY_SIZE: + return # got end marker + + if data[:2] == MD5_PARTITION_BEGIN[:2]: # check only the magic number part + if data[16:] == md5.digest(): + md5.update(data) + continue # the next iteration will check for the end marker + else: + raise InputError("MD5 checksums don't match! (computed: 0x%s, parsed: 0x%s)" % (md5.hexdigest(), binascii.hexlify(data[16:]))) + + md5.update(data) + + if data[:2] == b"\xff\xff": # Pseudo-end marker to keep esp32 bootloader happy + continue + + e = Entry.from_binary(data) + if e.type == STORAGE_TYPE: + dev = storage.Device(e.name, e.subtype, e.size) + devices.append(dev) + else: + e.device = dev + self.append(e) + raise InputError("Partition table is missing an end-of-table marker") + + def to_binary(self, devices): + """Create binary image of partition table + """ + dev_count = 0 + dev = None + result = b"" + for e in self: + if e.device != dev: + if dev_count == 1: + result += MD5_PARTITION_BEGIN + hashlib.md5(result).digest() + # esp32 bootloader will see this as end of partition table + result += struct.pack(Entry.STRUCT_FORMAT, + b"\xff\xff", + 0xff, 0xff, + 0, 0, + b"SMING EXTENSIONS", + 0) + dev = e.device + s = Entry(dev, dev.name, 0, dev.size, STORAGE_TYPE, dev.type) + result += s.to_binary() + dev_count += 1 + result += e.to_binary() + + result += MD5_PARTITION_BEGIN + hashlib.md5(result).digest() + + if len(result) >= MAX_PARTITION_LENGTH: + raise InputError("Binary partition table length (%d) longer than max" % len(result)) + result += b"\xFF" * (MAX_PARTITION_LENGTH - len(result)) # pad the sector, for signing + return result + + +class Entry(object): + MAGIC_BYTES = b"\xAA\x50" + + # dictionary maps flag name (as used in CSV flags list, property name) + # to bit set in flags words in binary format + FLAGS = { + "encrypted": 0, + "readonly": 31, + } + + def __init__(self, device=None, name="", address=None, size=None, ptype=None, subtype=None): + self.device = device + self.name = name + self.address = address + self.size = size + self.type = parse_type(ptype) + self.subtype = parse_subtype(self.type, subtype) + self.readonly = False + self.encrypted = False + self.filename = '' + self.build = None + + + def parse_dict(self, data, devices): + """Construct a partition object from JSON definition + """ + try: + # Sort out type information first + v = data.pop('type', None) + if not v is None: + self.type = parse_type(v) + v = data.pop('subtype', None) + if not v is None: + self.subtype = parse_subtype(self.type, v) + + for k, v in data.items(): + if k == 'device': + self.device = devices.find_by_name(v) + elif k == 'address': + self.address = v + elif k == 'size': + self.size = parse_int(v) + elif k == 'filename': + self.filename = v + elif k == 'build': + if self.build is None: + self.build = v + else: + self.build.update(v) + else: + setattr(self, k, v) + except InputError as e: + raise InputError("Error in partition entry '%s': %s" % (self.name, e)) + + def resolve_expressions(self): + self.address = eval(str(self.address)) + + def dict(self): + res = {} + for k, v in self.__dict__.items(): + if k == 'device': + res[k] = v.name + elif k == 'address': + res[k] = self.address_str() + elif k == 'size': + res[k] = self.size_str() + elif k == 'type': + res[k] = stringnum(self.type_str()) + elif k == 'subtype': + res[k] = stringnum(self.subtype_str()) + elif v is not None and k != 'name': + res[k] = v + return res + + def buildVars(self): + res = {} + + dict = self.dict() + dict['size_bytes'] = "0x%x" % self.size + dict.pop('build', None) + for k, v in dict.items(): + k = "PARTITION_%s_%s" % (self.name, k.upper()) + res[k] = int(v) if type(v) is bool else v + + return res + + def to_json(self): + return to_json(self.dict()); + + def end(self): + return self.address + self.size - 1 + + def end_str(self): + return addr_format(self.end()) + + def address_str(self): + return addr_format(self.address) + + def size_str(self): + return size_format(self.size) + + def contains(self, addr): + return addr >= self.address and addr <= self.end() + + def type_str(self): + return "" if self.type == INTERNAL_TYPE else lookup_keyword(self.type, TYPES) + + def type_is(self, t): + return self.type_str() == t if isinstance(t, str) else self.type == t + + def subtype_str(self): + return "" if self.type == INTERNAL_TYPE else lookup_keyword(self.subtype, SUBTYPES.get(self.type, {})) + + def subtype_is(self, subtype): + return self.subtype_str() == subtype if isinstance(subtype, str) else self.subtype == subtype + + def is_internal(self, subtype = None): + return (self.type == INTERNAL_TYPE) and (subtype is None or self.subtype == subtype) + + def is_unused(self): + return self.is_internal(INTERNAL_UNUSED) + + def __eq__(self, other): + if isinstance(other, str): + return self.name == other + else: + return self.name == other.name and self.type == other.type \ + and self.subtype == other.subtype and self.address == other.address \ + and self.size == other.size + + def __repr__(self): + + def maybe_hex(x): + return "0x%x" % x if x is not None else "None" + + return "Entry('%s', 0x%x, 0x%x, %s, %s)" % (self.name, self.type, self.subtype or 0, + maybe_hex(self.address), maybe_hex(self.size)) + + def __str__(self): + return "Part '%s' %s/%s @ 0x%x size 0x%x" % (self.name, self.type_str(), self.subtype_str(), self.address or -1, self.size or -1) + + def alignment(self, arch): + return ALIGNMENT[arch].get(self.type, 4) + + def verify(self, arch, secure): + if self.type is None: + raise ValidationError(self, "Type field is not set") + if self.subtype is None: + raise ValidationError(self, "Subtype field is not set") + if self.address is None: + raise ValidationError(self, "Address field is not set") + align = self.alignment(arch) + if self.address % align: + raise ValidationError(self, "Offset 0x%x is not aligned to 0x%x" % (self.address, align)) + if self.size is None or self.size == 0: + raise ValidationError(self, "Size field is not set") + if self.size % align and secure: + raise ValidationError(self, "Size 0x%x is not aligned to 0x%x" % (self.size, align)) + if self.address >= self.device.size: + raise ValidationError(self, "Offset 0x%x exceeds device size 0x%x" % (self.address, self.device.size)) + if self.end() >= self.device.size: + raise ValidationError(self, "End 0x%x exceeds device size 0x%x" % (self.end(), self.device.size)) + + if self.name == '': + raise ValidationError(self, "Name not specified") + if len(self.name) > PARTITION_NAME_SIZE: + raise ValidationError(self, "Name too long, max. %u chars" % PARTITION_NAME_SIZE) + if contains_whitespace(self.name): + raise ValidationError(self, "Name may not contain whitespace") + if self.name in TYPES and TYPES.get(self.name, "") != self.type: + raise ValidationError(self, "Name is a partition type, but does not match this partition's type (%s)" % (self.type)) + all_subtype_names = [] + for names in (t.keys() for t in SUBTYPES.values()): + all_subtype_names += names + if self.name in all_subtype_names and SUBTYPES.get(self.type, {}).get(self.name, "") != self.subtype: + raise ValidationError(self, "Name is a partition subtype, but does not match this partition's type/subtype (%s/%s)" % (self.type_str(), self.subtype_str())) + + + STRUCT_FORMAT = b"<2sBBLL16sL" + + @classmethod + def from_binary(cls, b): + if len(b) != PARTITION_ENTRY_SIZE: + raise InputError("Partition entry size incorrect. Expected %d bytes, got %d." % (PARTITION_ENTRY_SIZE, len(b))) + res = cls() + (magic, res.type, res.subtype, res.address, + res.size, res.name, flags) = struct.unpack(cls.STRUCT_FORMAT, b) + if b"\x00" in res.name: # strip null byte padding from name string + res.name = res.name[:res.name.index(b"\x00")] + res.name = res.name.decode() + if magic != cls.MAGIC_BYTES: + raise InputError("Invalid magic bytes (%r) for partition definition" % magic) + for flag, bit in cls.FLAGS.items(): + if flags & (1 << bit): + setattr(res, flag, True) + flags &= ~(1 << bit) + if flags != 0: + critical("WARNING: Partition definition had unknown flag(s) 0x%08x. Newer binary format?" % flags) + return res + + def to_binary(self): + flags = 0 + for key in self.FLAGS: + if getattr(self, key): + flags |= (1 << self.FLAGS[key]) + return struct.pack(self.STRUCT_FORMAT, + self.MAGIC_BYTES, + self.type, self.subtype, + self.address, self.size, + self.name.encode(), + flags) + + +class Map(Table): + """Contiguous map of flash memory + """ + def __init__(self, table, devices): + def add(table, device, name, address, size, subtype): + entry = Entry(device, name, address, size, INTERNAL_TYPE, subtype) + table.append(entry) + return entry + + def add_unused(table, device, address, last_end): + if address > last_end + 1: + add(table, device, '(unused)', last_end + 1, address - last_end - 1, INTERNAL_UNUSED) + + device = devices[0] + + # Take copy of source partitions and add internal ones to appear in the map + partitions = copy.copy(table) + if table.offset != 0: + add(partitions, device, 'Boot Sector', 0, min(table.offset, partitions[0].address), INTERNAL_BOOT_SECTOR) + add(partitions, device, 'Partition Table', table.offset, PARTITION_TABLE_SIZE, INTERNAL_PARTITION_TABLE) + + # Devices with no defined partitions + pdevs = set(p.device for p in partitions) + for dev in devices: + if not dev in pdevs: + add_unused(partitions, dev, dev.size, -1) + + partitions.sort() + + last = None + for p in partitions: + if last is not None: + if p.device != last.device: + add_unused(self, device, last.device.size, last.end()) + device = p.device + add_unused(self, device, p.address, -1) + elif p.address > last.end() + 1: + add_unused(self, device, p.address, last.end()) + self.append(p) + last = p + + add_unused(self, device, device.size, last.end()) diff --git a/Sming/Components/Storage/Tools/hwconfig/rjsmin.py b/Sming/Components/Storage/Tools/hwconfig/rjsmin.py new file mode 100644 index 0000000000..68b65aeef0 --- /dev/null +++ b/Sming/Components/Storage/Tools/hwconfig/rjsmin.py @@ -0,0 +1,543 @@ +#!/usr/bin/env python +# -*- coding: ascii -*- +u""" +===================== + Javascript Minifier +===================== + +rJSmin is a javascript minifier written in python. + +The minifier is based on the semantics of `jsmin.c by Douglas Crockford`_\\. + +:Copyright: + + Copyright 2011 - 2019 + Andr\xe9 Malo or his licensors, as applicable + +:License: + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +The module is a re-implementation aiming for speed, so it can be used at +runtime (rather than during a preprocessing step). Usually it produces the +same results as the original ``jsmin.c``. It differs in the following ways: + +- there is no error detection: unterminated string, regex and comment + literals are treated as regular javascript code and minified as such. +- Control characters inside string and regex literals are left untouched; they + are not converted to spaces (nor to \\n) +- Newline characters are not allowed inside string and regex literals, except + for line continuations in string literals (ECMA-5). +- "return /regex/" is recognized correctly. +- More characters are allowed before regexes. +- Line terminators after regex literals are handled more sensibly +- "+ +" and "- -" sequences are not collapsed to '++' or '--' +- Newlines before ! operators are removed more sensibly +- (Unnested) template literals are supported (ECMA-6) +- Comments starting with an exclamation mark (``!``) can be kept optionally +- rJSmin does not handle streams, but only complete strings. (However, the + module provides a "streamy" interface). + +Since most parts of the logic are handled by the regex engine it's way faster +than the original python port of ``jsmin.c`` by Baruch Even. The speed factor +varies between about 6 and 55 depending on input and python version (it gets +faster the more compressed the input already is). Compared to the +speed-refactored python port by Dave St.Germain the performance gain is less +dramatic but still between 3 and 50 (for huge inputs). See the docs/BENCHMARKS +file for details. + +rjsmin.c is a reimplementation of rjsmin.py in C and speeds it up even more. + +Supported python versions are 2.7 and 3.4+. + +.. _jsmin.c by Douglas Crockford: + http://www.crockford.com/javascript/jsmin.c +""" +__author__ = u"Andr\xe9 Malo" +__docformat__ = "restructuredtext en" +__license__ = "Apache License, Version 2.0" +__version__ = '1.1.0' +__all__ = ['jsmin'] + +import functools as _ft +import re as _re + + +def _make_jsmin(python_only=False): + """ + Generate JS minifier based on `jsmin.c by Douglas Crockford`_ + + .. _jsmin.c by Douglas Crockford: + http://www.crockford.com/javascript/jsmin.c + + :Parameters: + `python_only` : ``bool`` + Use only the python variant. If true, the c extension is not even + tried to be loaded. + + :Return: Minifier + :Rtype: ``callable`` + """ + # pylint: disable = unused-variable + # pylint: disable = too-many-locals + + if not python_only: + try: + import _rjsmin + except ImportError: + pass + else: + # Ensure that the C version is in sync + # https://github.com/ndparker/rjsmin/issues/11 + if getattr(_rjsmin, '__version__', None) == __version__: + return _rjsmin.jsmin + try: + xrange + except NameError: + xrange = range # pylint: disable = redefined-builtin + + space_chars = r'[\000-\011\013\014\016-\040]' + + line_comment = r'(?://[^\r\n]*)' + space_comment = r'(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/)' + space_comment_nobang = r'(?:/\*(?!!)[^*]*\*+(?:[^/*][^*]*\*+)*/)' + bang_comment = r'(?:/\*![^*]*\*+(?:[^/*][^*]*\*+)*/)' + + string1 = r"(?:'[^'\\\r\n]*(?:\\(?:[^\r\n]|\r?\n|\r)[^'\\\r\n]*)*')" + string1 = string1.replace("'", r'\047') # portability + string2 = r'(?:"[^"\\\r\n]*(?:\\(?:[^\r\n]|\r?\n|\r)[^"\\\r\n]*)*")' + string3 = r'(?:`[^`\\]*(?:\\(?:[^\r\n]|\r?\n|\r)[^`\\]*)*`)' + string3 = string3.replace('`', r'\140') # portability + strings = r'(?:%s|%s|%s)' % (string1, string2, string3) + + charclass = r'(?:\[[^\\\]\r\n]*(?:\\[^\r\n][^\\\]\r\n]*)*\])' + nospecial = r'[^/\\\[\r\n]' + regex = r'(?:/(?![\r\n/*])%s*(?:(?:\\[^\r\n]|%s)%s*)*/)' % ( + nospecial, charclass, nospecial + ) + space = r'(?:%s|%s)' % (space_chars, space_comment) + newline = r'(?:%s?[\r\n])' % line_comment + + def fix_charclass(result): + """ Fixup string of chars to fit into a regex char class """ + pos = result.find('-') + if pos >= 0: + result = r'%s%s-' % (result[:pos], result[pos + 1:]) + + def sequentize(string): + """ + Notate consecutive characters as sequence + + (1-4 instead of 1234) + """ + first, last, result = None, None, [] + for char in map(ord, string): + if last is None: + first = last = char + elif last + 1 == char: + last = char + else: + result.append((first, last)) + first = last = char + if last is not None: + result.append((first, last)) + return ''.join(['%s%s%s' % ( + chr(first), + last > first + 1 and '-' or '', + last != first and chr(last) or '' + ) for first, last in result]) # noqa + + return _re.sub( + r"([\000-\040'`])", # ' and ` for better portability + lambda m: '\\%03o' % ord(m.group(1)), ( + sequentize(result) + .replace('\\', '\\\\') + .replace('[', '\\[') + .replace(']', '\\]') + ) + ) + + def id_literal_(what): + """ Make id_literal like char class """ + match = _re.compile(what).match + result = ''.join([ + chr(c) for c in xrange(127) if not match(chr(c)) + ]) + return '[^%s]' % fix_charclass(result) + + def not_id_literal_(keep): + """ Make negated id_literal like char class """ + match = _re.compile(id_literal_(keep)).match + result = ''.join([ + chr(c) for c in xrange(127) if not match(chr(c)) + ]) + return r'[%s]' % fix_charclass(result) + + not_id_literal = not_id_literal_(r'[a-zA-Z0-9_$]') + preregex1 = r'[(,=:\[!&|?{};\r\n+*-]' + preregex2 = r'%(not_id_literal)sreturn' % locals() + + id_literal = id_literal_(r'[a-zA-Z0-9_$]') + id_literal_open = id_literal_(r'[a-zA-Z0-9_${\[(!+-]') + id_literal_close = id_literal_(r'[a-zA-Z0-9_$}\])"\047\140+-]') + post_regex_off = id_literal_(r'[^\000-\040}\])?:|,;.&=+-]') + + dull = r'[^\047"\140/\000-\040]' + + space_sub_simple = _re.compile(( + # noqa pylint: disable = bad-continuation + + r'(%(dull)s+)' # 0 + r'|(%(strings)s%(dull)s*)' # 1 + r'|(?<=%(preregex1)s)' + r'%(space)s*(?:%(newline)s%(space)s*)*' + r'(%(regex)s)' # 2 + r'(%(space)s*(?:%(newline)s%(space)s*)+' # 3 + r'(?=%(post_regex_off)s))?' + r'|(?<=%(preregex2)s)' + r'%(space)s*(?:(%(newline)s)%(space)s*)*' # 4 + r'(%(regex)s)' # 5 + r'(%(space)s*(?:%(newline)s%(space)s*)+' # 6 + r'(?=%(post_regex_off)s))?' + r'|(?<=%(id_literal_close)s)' + r'%(space)s*(?:(%(newline)s)%(space)s*)+' # 7 + r'(?=%(id_literal_open)s)' + r'|(?<=%(id_literal)s)(%(space)s)+(?=%(id_literal)s)' # 8 + r'|(?<=\+)(%(space)s)+(?=\+)' # 9 + r'|(?<=-)(%(space)s)+(?=-)' # 10 + r'|%(space)s+' + r'|(?:%(newline)s%(space)s*)+' + ) % locals()).sub + + # print(space_sub_simple.__self__.pattern) + + def space_subber_simple(match): + """ Substitution callback """ + # pylint: disable = too-many-return-statements + + groups = match.groups() + if groups[0]: + return groups[0] + elif groups[1]: + return groups[1] + elif groups[2]: + if groups[3]: + return groups[2] + '\n' + return groups[2] + elif groups[5]: + return "%s%s%s" % ( + groups[4] and '\n' or '', + groups[5], + groups[6] and '\n' or '', + ) + elif groups[7]: + return '\n' + elif groups[8] or groups[9] or groups[10]: + return ' ' + else: + return '' + + space_sub_banged = _re.compile(( + # noqa pylint: disable = bad-continuation + + r'(%(dull)s+)' # 0 + r'|(%(strings)s%(dull)s*)' # 1 + r'|(?<=%(preregex1)s)' + r'(%(space)s*(?:%(newline)s%(space)s*)*)' # 2 + r'(%(regex)s)' # 3 + r'(%(space)s*(?:%(newline)s%(space)s*)+' # 4 + r'(?=%(post_regex_off)s))?' + r'|(?<=%(preregex2)s)' + r'(%(space)s*(?:(%(newline)s)%(space)s*)*)' # 5, 6 + r'(%(regex)s)' # 7 + r'(%(space)s*(?:%(newline)s%(space)s*)+' # 8 + r'(?=%(post_regex_off)s))?' + r'|(?<=%(id_literal_close)s)' + r'(%(space)s*(?:%(newline)s%(space)s*)+)' # 9 + r'(?=%(id_literal_open)s)' + r'|(?<=%(id_literal)s)(%(space)s+)(?=%(id_literal)s)' # 10 + r'|(?<=\+)(%(space)s+)(?=\+)' # 11 + r'|(?<=-)(%(space)s+)(?=-)' # 12 + r'|(%(space)s+)' # 13 + r'|((?:%(newline)s%(space)s*)+)' # 14 + ) % locals()).sub + + # print(space_sub_banged.__self__.pattern) + + keep = _re.compile(( + r'%(space_chars)s+|%(space_comment_nobang)s+|%(newline)s+' + r'|(%(bang_comment)s+)' + ) % locals()).sub + keeper = lambda m: m.groups()[0] or '' + + # print(keep.__self__.pattern) + + def space_subber_banged(match): + """ Substitution callback """ + # pylint: disable = too-many-return-statements + + groups = match.groups() + if groups[0]: + return groups[0] + elif groups[1]: + return groups[1] + elif groups[3]: + return "%s%s%s%s" % ( + keep(keeper, groups[2]), + groups[3], + keep(keeper, groups[4] or ''), + groups[4] and '\n' or '', + ) + elif groups[7]: + return "%s%s%s%s%s" % ( + keep(keeper, groups[5]), + groups[6] and '\n' or '', + groups[7], + keep(keeper, groups[8] or ''), + groups[8] and '\n' or '', + ) + elif groups[9]: + return keep(keeper, groups[9]) + '\n' + elif groups[10] or groups[11] or groups[12]: + return keep(keeper, groups[10] or groups[11] or groups[12]) or ' ' + else: + return keep(keeper, groups[13] or groups[14]) + + banged = _ft.partial(space_sub_banged, space_subber_banged) + simple = _ft.partial(space_sub_simple, space_subber_simple) + + def jsmin(script, keep_bang_comments=False): + r""" + Minify javascript based on `jsmin.c by Douglas Crockford`_\. + + Instead of parsing the stream char by char, it uses a regular + expression approach which minifies the whole script with one big + substitution regex. + + .. _jsmin.c by Douglas Crockford: + http://www.crockford.com/javascript/jsmin.c + + :Parameters: + `script` : ``str`` + Script to minify + + `keep_bang_comments` : ``bool`` + Keep comments starting with an exclamation mark? (``/*!...*/``) + + :Return: Minified script + :Rtype: ``str`` + """ + # pylint: disable = redefined-outer-name + + is_bytes, script = _as_str(script) + script = (banged if keep_bang_comments else simple)( + '\n%s\n' % script + ).strip() + if is_bytes: + return script.encode('latin-1') + return script + + return jsmin + +jsmin = _make_jsmin() + + +def _as_str(script): + """ Make sure the script is a text string """ + is_bytes = False + if str is bytes: + if not isinstance(script, basestring): # noqa pylint: disable = undefined-variable + raise TypeError("Unexpected type") + elif isinstance(script, (bytes, bytearray)): + is_bytes = True + script = script.decode('latin-1') + elif not isinstance(script, str): + raise TypeError("Unexpected type") + + return is_bytes, script + + +def jsmin_for_posers(script, keep_bang_comments=False): + r""" + Minify javascript based on `jsmin.c by Douglas Crockford`_\. + + Instead of parsing the stream char by char, it uses a regular + expression approach which minifies the whole script with one big + substitution regex. + + .. _jsmin.c by Douglas Crockford: + http://www.crockford.com/javascript/jsmin.c + + :Warning: This function is the digest of a _make_jsmin() call. It just + utilizes the resulting regexes. It's here for fun and may + vanish any time. Use the `jsmin` function instead. + + :Parameters: + `script` : ``str`` + Script to minify + + `keep_bang_comments` : ``bool`` + Keep comments starting with an exclamation mark? (``/*!...*/``) + + :Return: Minified script + :Rtype: ``str`` + """ + if not keep_bang_comments: + rex = ( + r'([^\047"\140/\000-\040]+)|((?:(?:\047[^\047\\\r\n]*(?:\\(?:[^' + r'\r\n]|\r?\n|\r)[^\047\\\r\n]*)*\047)|(?:"[^"\\\r\n]*(?:\\(?:[^' + r'\r\n]|\r?\n|\r)[^"\\\r\n]*)*")|(?:\140[^\140\\]*(?:\\(?:[^\r\n' + r']|\r?\n|\r)[^\140\\]*)*\140))[^\047"\140/\000-\040]*)|(?<=[(,=' + r':\[!&|?{};\r\n+*-])(?:[\000-\011\013\014\016-\040]|(?:/\*[^*]*' + r'\*+(?:[^/*][^*]*\*+)*/))*(?:(?:(?://[^\r\n]*)?[\r\n])(?:[\000-' + r'\011\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/))*)*(' + r'(?:/(?![\r\n/*])[^/\\\[\r\n]*(?:(?:\\[^\r\n]|(?:\[[^\\\]\r\n]*' + r'(?:\\[^\r\n][^\\\]\r\n]*)*\]))[^/\\\[\r\n]*)*/))((?:[\000-\011' + r'\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/))*(?:(?:(' + r'?://[^\r\n]*)?[\r\n])(?:[\000-\011\013\014\016-\040]|(?:/\*[^*' + r']*\*+(?:[^/*][^*]*\*+)*/))*)+(?=[^\000-\040&)+,.:;=?\]|}-]))?|' + r'(?<=[\000-#%-,./:-@\[-^\140{-~-]return)(?:[\000-\011\013\014\0' + r'16-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/))*(?:((?:(?://[^\r' + r'\n]*)?[\r\n]))(?:[\000-\011\013\014\016-\040]|(?:/\*[^*]*\*+(?' + r':[^/*][^*]*\*+)*/))*)*((?:/(?![\r\n/*])[^/\\\[\r\n]*(?:(?:\\[^' + r'\r\n]|(?:\[[^\\\]\r\n]*(?:\\[^\r\n][^\\\]\r\n]*)*\]))[^/\\\[\r' + r'\n]*)*/))((?:[\000-\011\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/' + r'*][^*]*\*+)*/))*(?:(?:(?://[^\r\n]*)?[\r\n])(?:[\000-\011\013' + r'\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/))*)+(?=[^\000' + r'-\040&)+,.:;=?\]|}-]))?|(?<=[^\000-!#%&(*,./:-@\[\\^{|~])(?:[' + r'\000-\011\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/)' + r')*(?:((?:(?://[^\r\n]*)?[\r\n]))(?:[\000-\011\013\014\016-\040' + r']|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/))*)+(?=[^\000-\040"#%-\047' + r')*,./:-@\\-^\140|-~])|(?<=[^\000-#%-,./:-@\[-^\140{-~-])((?:[' + r'\000-\011\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/)' + r'))+(?=[^\000-#%-,./:-@\[-^\140{-~-])|(?<=\+)((?:[\000-\011\013' + r'\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/)))+(?=\+)|(?<' + r'=-)((?:[\000-\011\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]' + r'*\*+)*/)))+(?=-)|(?:[\000-\011\013\014\016-\040]|(?:/\*[^*]*\*' + r'+(?:[^/*][^*]*\*+)*/))+|(?:(?:(?://[^\r\n]*)?[\r\n])(?:[\000-' + r'\011\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/))*)+' + ) + + def subber(match): + """ Substitution callback """ + groups = match.groups() + return ( + groups[0] or + groups[1] or + (groups[3] and (groups[2] + '\n')) or + groups[2] or + (groups[5] and "%s%s%s" % ( + groups[4] and '\n' or '', + groups[5], + groups[6] and '\n' or '', + )) or + (groups[7] and '\n') or + (groups[8] and ' ') or + (groups[9] and ' ') or + (groups[10] and ' ') or + '' + ) + else: + rex = ( + r'([^\047"\140/\000-\040]+)|((?:(?:\047[^\047\\\r\n]*(?:\\(?:[^' + r'\r\n]|\r?\n|\r)[^\047\\\r\n]*)*\047)|(?:"[^"\\\r\n]*(?:\\(?:[^' + r'\r\n]|\r?\n|\r)[^"\\\r\n]*)*")|(?:\140[^\140\\]*(?:\\(?:[^\r\n' + r']|\r?\n|\r)[^\140\\]*)*\140))[^\047"\140/\000-\040]*)|(?<=[(,=' + r':\[!&|?{};\r\n+*-])((?:[\000-\011\013\014\016-\040]|(?:/\*[^*]' + r'*\*+(?:[^/*][^*]*\*+)*/))*(?:(?:(?://[^\r\n]*)?[\r\n])(?:[\000' + r'-\011\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/))*)*' + r')((?:/(?![\r\n/*])[^/\\\[\r\n]*(?:(?:\\[^\r\n]|(?:\[[^\\\]\r\n' + r']*(?:\\[^\r\n][^\\\]\r\n]*)*\]))[^/\\\[\r\n]*)*/))((?:[\000-\0' + r'11\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/))*(?:(?' + r':(?://[^\r\n]*)?[\r\n])(?:[\000-\011\013\014\016-\040]|(?:/\*[' + r'^*]*\*+(?:[^/*][^*]*\*+)*/))*)+(?=[^\000-\040&)+,.:;=?\]|}-]))' + r'?|(?<=[\000-#%-,./:-@\[-^\140{-~-]return)((?:[\000-\011\013\01' + r'4\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/))*(?:((?:(?://[^' + r'\r\n]*)?[\r\n]))(?:[\000-\011\013\014\016-\040]|(?:/\*[^*]*\*+' + r'(?:[^/*][^*]*\*+)*/))*)*)((?:/(?![\r\n/*])[^/\\\[\r\n]*(?:(?:' + r'\\[^\r\n]|(?:\[[^\\\]\r\n]*(?:\\[^\r\n][^\\\]\r\n]*)*\]))[^/' + r'\\\[\r\n]*)*/))((?:[\000-\011\013\014\016-\040]|(?:/\*[^*]*\*+' + r'(?:[^/*][^*]*\*+)*/))*(?:(?:(?://[^\r\n]*)?[\r\n])(?:[\000-\01' + r'1\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/))*)+(?=[' + r'^\000-\040&)+,.:;=?\]|}-]))?|(?<=[^\000-!#%&(*,./:-@\[\\^{|~])' + r'((?:[\000-\011\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*' + r'+)*/))*(?:(?:(?://[^\r\n]*)?[\r\n])(?:[\000-\011\013\014\016-' + r'\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/))*)+)(?=[^\000-\040"#%' + r'-\047)*,./:-@\\-^\140|-~])|(?<=[^\000-#%-,./:-@\[-^\140{-~-])(' + r'(?:[\000-\011\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+' + r')*/))+)(?=[^\000-#%-,./:-@\[-^\140{-~-])|(?<=\+)((?:[\000-\011' + r'\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/))+)(?=\+)' + r'|(?<=-)((?:[\000-\011\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*]' + r'[^*]*\*+)*/))+)(?=-)|((?:[\000-\011\013\014\016-\040]|(?:/\*[^' + r'*]*\*+(?:[^/*][^*]*\*+)*/))+)|((?:(?:(?://[^\r\n]*)?[\r\n])(?:' + r'[\000-\011\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/' + r'))*)+)' + ) + + keep = _re.compile( + r'[\000-\011\013\014\016-\040]+|(?:/\*(?!!)[^*]*\*+(?:[^/*][^*]*' + r'\*+)*/)+|(?:(?://[^\r\n]*)?[\r\n])+|((?:/\*![^*]*\*+(?:[^/*][^' + r'*]*\*+)*/)+)' + ).sub + keeper = lambda m: m.groups()[0] or '' + + def subber(match): + """ Substitution callback """ + groups = match.groups() + return ( + groups[0] or + groups[1] or + groups[3] and "%s%s%s%s" % ( + keep(keeper, groups[2]), + groups[3], + keep(keeper, groups[4] or ''), + groups[4] and '\n' or '', + ) or + groups[7] and "%s%s%s%s%s" % ( + keep(keeper, groups[5]), + groups[6] and '\n' or '', + groups[7], + keep(keeper, groups[8] or ''), + groups[8] and '\n' or '', + ) or + groups[9] and (keep(keeper, groups[9]) + '\n') or + groups[10] and (keep(keeper, groups[10]) or ' ') or + groups[11] and (keep(keeper, groups[11]) or ' ') or + groups[12] and (keep(keeper, groups[12]) or ' ') or + keep(keeper, groups[13] or groups[14]) + ) + + is_bytes, script = _as_str(script) + script = _re.sub(rex, subber, '\n%s\n' % script).strip() + if is_bytes: + return script.encode('latin-1') + return script + + +if __name__ == '__main__': + def main(): + """ Main """ + import sys as _sys + + argv = _sys.argv[1:] + keep_bang_comments = '-b' in argv or '-bp' in argv or '-pb' in argv + if '-p' in argv or '-bp' in argv or '-pb' in argv: + xjsmin = _make_jsmin(python_only=True) + else: + xjsmin = jsmin + + _sys.stdout.write(xjsmin( + _sys.stdin.read(), keep_bang_comments=keep_bang_comments + )) + + main() diff --git a/Sming/Components/Storage/Tools/hwconfig/storage.py b/Sming/Components/Storage/Tools/hwconfig/storage.py new file mode 100644 index 0000000000..c72c0fddda --- /dev/null +++ b/Sming/Components/Storage/Tools/hwconfig/storage.py @@ -0,0 +1,130 @@ +# +# Storage devices +# + +from common import * +import partition + +TYPES = { + "unknown": 0x00, + "flash": 0x01, + "spiram": 0x02, + "sdcard": 0x03, + "disk": 0x04, + "file": 0x05, +} + + +def parse_type(value): + if value == "": + raise InputError("Field 'type' can't be left empty.") + return parse_int(value, TYPES) + + +class List(list): + def parse_dict(self, data): + for name, v in data.items(): + if contains_whitespace(name): + raise InputError("Device names may not contain spaces '%s'" % name) + dev = self.find_by_name(name) + if dev is None: + dev = Device(name) + self.append(dev) + dev.parse_dict(v) + + def dict(self): + res = {} + for dev in self: + res[dev.name] = dev.dict() + return res + + def to_json(self): + return to_json(self.dict()) + + def buildVars(self): + dict = {} + dict['STORAGE_DEVICE_NAMES'] = " ".join(p.name for p in self) + for p in self: + dict.update(p.buildVars()) + return dict + + def __getitem__(self, item): + if isinstance(item, str): + d = self.find_by_name(item) + if d is None: + raise ValueError("No device named '%s'" % item) + return d + return super().__getitem__(item) + + def find_by_name(self, name): + for d in self: + if d.name == name: + return d + return None + + def verify(self): + for dev in self: + dev.verify() + +class Device(object): + def __init__(self, name, stype = 0, size = 0): + self.name = name + self.type = parse_type(stype) + self.size = parse_int(size) + + def parse_dict(self, data): + for k, v in data.items(): + if k == 'type': + self.type = parse_int(v, TYPES) + elif k == 'size': + self.size = parse_int(v) + elif k == 'mode': + self.mode = v + elif k == 'speed': + self.speed = v + else: + raise InputError("Unknown storage field '%s'" % k) + + def dict(self): + res = {} + + # Some fields are optional + def tryAdd(k): + if hasattr(self, k): + res[k] = getattr(self, k) + + res['type'] = self.type_str() + res['size'] = self.size_str() + tryAdd('mode') + tryAdd('speed') + return res + + def to_json(self): + return to_json(self.dict()) + + def buildVars(self): + res = {} + + dict = self.dict() + dict['size_bytes'] = "0x%x" % self.size + for k, v in dict.items(): + k = "STORAGE_DEVICE_%s_%s" % (self.name, k.upper()) + res[k] = v + + return res + + def type_str(self): + return lookup_keyword(self.type, TYPES) + + def size_str(self): + return size_format(self.size) + + def verify(self): + if self.size == 0: + raise ValidationError(self, "Size field is not set") + if self.type is None: + raise ValidationError(self, "Type field is not set") + if self.name == '': + raise ValidationError(self, "Name not specified") + if len(self.name) > partition.PARTITION_NAME_SIZE: + raise ValidationError(self, "Name too long, max. %u chars" % partition.PARTITION_NAME_SIZE) diff --git a/Sming/Components/esptool/blank.bin b/Sming/Components/Storage/blank.bin similarity index 100% rename from Sming/Components/esptool/blank.bin rename to Sming/Components/Storage/blank.bin diff --git a/Sming/Components/Storage/component.mk b/Sming/Components/Storage/component.mk new file mode 100644 index 0000000000..dd7680fb48 --- /dev/null +++ b/Sming/Components/Storage/component.mk @@ -0,0 +1,229 @@ +COMPONENT_INCDIRS := src/include +COMPONENT_SRCDIRS := src +COMPONENT_DOXYGEN_INPUT := src/include + +CONFIG_VARS += HWCONFIG HWCONFIG_OPTS +ifndef HWCONFIG +override HWCONFIG := standard +$(info Using configuration '$(HWCONFIG)') +endif + +# Directories to search for hardware config +HWCONFIG_DIRS := $(PROJECT_DIR) $(COMPONENT_SEARCH_DIRS) $(ARCH_BASE) $(SMING_HOME) + +# List of all hardware configurations +ALL_HWCONFIG_PATHS := $(foreach d,$(HWCONFIG_DIRS),$(wildcard $d/*.hw)) +ALL_HWCONFIG := $(sort $(notdir $(ALL_HWCONFIG_PATHS))) +ALL_HWCONFIG := $(ALL_HWCONFIG:.hw=) + +# Path to selected hardware config file +HWCONFIG_PATH := $(firstword $(foreach d,$(HWCONFIG_DIRS),$(wildcard $d/$(HWCONFIG).hw))) + +ifeq (,$(wildcard $(HWCONFIG_PATH))) +ifeq (,$(MAKE_CLEAN)) +$(info $(HWCONFIG_DIRS)) +$(eval $(call PrintVariable,ALL_HWCONFIG,Available configurations)) +$(error Hardware configuration '$(HWCONFIG)' not found) +endif +endif + +PARTITION_PATH := $(COMPONENT_PATH) +PARTITION_TOOLS := $(PARTITION_PATH)/Tools +HWCONFIG_SCHEMA := $(PARTITION_PATH)/schema.json +HWCONFIG_VARS := \ + SMING_HOME \ + OUT_BASE \ + HWCONFIG_DIRS \ + HWCONFIG_OPTS \ + HWCONFIG_SCHEMA +HWCONFIG_EXPORTS := $(foreach v,$(HWCONFIG_VARS),$v="$($v)") +HWCONFIG_CMDLINE := $(PYTHON) $(PARTITION_TOOLS)/hwconfig +HWCONFIG_TOOL := $(HWCONFIG_EXPORTS) $(HWCONFIG_CMDLINE)/hwconfig.py + +HWCONFIG_EDITOR := $(HWCONFIG_EXPORTS) $(HWCONFIG_CMDLINE)/editor.py $(HWCONFIG) + +# When using WSL without an X server available, use native Windows python +ifdef WSL_ROOT +ifndef DISPLAY +space := +space += +WSLENV := $(WSLENV)$(subst $(space),,$(foreach v,$(HWCONFIG_VARS),::$v)) +HWCONFIG_EDITOR := $(HWCONFIG_EXPORTS) powershell.exe -Command "$(HWCONFIG_CMDLINE)/editor.py $(HWCONFIG)" +endif +endif + +HWCONFIG_MK := $(PROJECT_DIR)/$(OUT_BASE)/hwconfig.mk +ifneq (,$(MAKE_DOCS)$(MAKE_CLEAN)) +-include $(HWCONFIG_MK) +else +# Generate build variables from hardware configuration +$(shell $(HWCONFIG_TOOL) --quiet expr $(HWCONFIG) $(HWCONFIG_MK) "config.buildVars()") +include $(HWCONFIG_MK) +ifeq ($(SMING_ARCH_HW),) +$(error Hardware configuration error) +else ifneq ($(SMING_ARCH),$(SMING_ARCH_HW)) +$(error Hardware configuration is for arch $(SMING_ARCH_HW), does not match SMING_ARCH ($(SMING_ARCH))) +endif +COMPONENT_CXXFLAGS := -DPARTITION_TABLE_OFFSET=$(PARTITION_TABLE_OFFSET) +COMPONENT_CPPFLAGS := -DPARTITION_TABLE_OFFSET=$(PARTITION_TABLE_OFFSET) + +# Function to evaluate expression against config +HWEXPR := $(HWCONFIG_TOOL) $(if $(PART),--part "$(PART)") expr $(HWCONFIG) - + +define HwExpr +$(shell $(HWEXPR) "$1") +endef + + +##@Configuration + +.PHONY: map +map: $(HWCONFIG_PATH) ##Show partition map + @echo "Partition map: $(HWCONFIG)" + $(Q) $(HWEXPR) "'options: %s\n%s' % (', '.join(config.options), config.map().to_csv())" + @echo + +.PHONY: hwexpr +hwexpr: $(HWCONFIG_PATH) ##Evaluate expression against hardware configuration (use EXPR= and optionally PART=) + $(Q) $(HWEXPR) "$(EXPR)" + +.PHONY: hwconfig +hwconfig: $(HWCONFIG_PATH) ##Show current hardware configuration + @echo + $(Q) $(HWEXPR) "config.to_json()" + @echo +ifneq ($(V),) + @echo "$(HWCONFIG): $(foreach c,$(HWCONFIG_DEPENDS),\n $c)" +endif + +.PHONY: hwconfig-list +hwconfig-list: ##List available hardware configurations + @echo "Available configurations: $(foreach c,$(ALL_HWCONFIG),\n $(if $(subst $c,,$(HWCONFIG)), ,*) $(shell printf "%-25s" "$c") $(filter %/$c.hw,$(ALL_HWCONFIG_PATHS)))" + @echo + +.PHONY: hwconfig-options +hwconfig-options: ##List available hardware configuration options + @echo + @echo "Available options (use with HWCONFIG_OPTS or in custom profile):" + $(Q) $(HWEXPR) "''.join((' %-10s %s\n' % (k, v['description']) for k, v in config.option_library.items()))" + @echo + +.PHONY: hwconfig-validate +hwconfig-validate: $(HWCONFIG_PATH) ##Validate current hardware configuration + @echo "Validating hardware config '$(HWCONFIG)'" + $(Q) $(HWCONFIG_TOOL) validate $(HWCONFIG) - + +.PHONY: hwconfig-edit +hwconfig-edit: $(HWCONFIG_PATH) ##Open profile editor + $(Q) $(HWCONFIG_EDITOR) + +##@Building + +# The partition table +PARTITIONS_BIN := $(FW_BASE)/partitions.bin +CUSTOM_TARGETS += partmap-build + +.PHONY: partmap-build +partmap-build: + $(Q) $(MAKE) --no-print-directory hwconfig-validate + $(Q) $(HWCONFIG_TOOL) partgen $(HWCONFIG) $(PARTITIONS_BIN) + + +# Create build target for a partition +# $1 -> Partition name +define PartitionTarget +$(PARTITION_$1_FILENAME): + $$(Q) $$(MAKE) --no-print-directory $$(shell $$(HWCONFIG_TOOL) --part $1 expr $$(HWCONFIG) - "part.build['target']") PART=$1 +CUSTOM_TARGETS += $(PARTITION_$1_FILENAME) +endef + +# Must be invoked from project.mk after all Components have been processed +# This allows partition definitions to include variables which may not yet be defined +define PartitionCreateTargets +$(foreach p,$(PARTITIONS_WITH_TARGETS),$(eval $(call PartitionTarget,$p))) +endef + +.PHONY: buildpart +buildpart: ##Rebuild all partition targets + $(Q) if [ "$(PARTITION_BUILD_TARGETS)" -eq "" ]; then \ + echo "No partitions have build targets"; \ + else \ + rm -f $(PARTITION_BUILD_TARGETS); \ + $(MAKE) $(PARTITION_BUILD_TARGETS); \ + fi + +##@Flashing + +# Get chunks for a list of partitions, ignore any without filenames +# $1 -> List of partition names +define PartChunks +$(foreach p,$1,$(if $(PARTITION_$p_FILENAME),$(PARTITION_$p_ADDRESS)=$(PARTITION_$p_FILENAME))) +endef + +# Get regions for list of partitions +# $1 -> List of partition names +define PartRegions +$(foreach p,$1,$(PARTITION_$p_ADDRESS),$(PARTITION_$p_SIZE_BYTES)) +endef + +# One flash sector of 0xFF +DEBUG_VARS += BLANK_BIN +BLANK_BIN := $(PARTITION_PATH)/blank.bin + +# Just the application chunks +SPIFLASH_APP_PARTITION_NAMES := $(foreach p,$(SPIFLASH_PARTITION_NAMES),$(if $(filter app,$(PARTITION_$p_TYPE)),$p)) +FLASH_APP_CHUNKS = $(call PartChunks,$(SPIFLASH_APP_PARTITION_NAMES)) +# Partition map chunk +FLASH_MAP_CHUNK := $(PARTITION_TABLE_OFFSET)=$(PARTITIONS_BIN) +# All partitions with image files +FLASH_PARTITION_CHUNKS = $(call PartChunks,$(SPIFLASH_PARTITION_NAMES)) + +# User-selected partition(s) +FLASH_PART_CHUNKS = $(call PartChunks,$(PART)) +FLASH_PART_REGION = $(call PartRegions,$(PART)) + +# Where to store read partition map file +READMAP_BIN := $(OUT_BASE)/partition-table.read.bin +PARTITION_TABLE_REGION = $(PARTITION_TABLE_OFFSET),$(PARTITION_TABLE_LENGTH) + +.PHONY: readmap +readmap:##Read partition map from device + $(Q) $(call ReadFlash,$(PARTITION_TABLE_REGION),$(READMAP_BIN)) + @echo + @echo "Partition map read from device:" + $(Q) $(HWCONFIG_TOOL) expr $(READMAP_BIN) - "config.map().to_csv()" + @echo + +# Run sanity checks against a list of partitions before flashing commences +# $1 -> List of partition names +define CheckPartitionChunks +$(HWCONFIG_TOOL) flashcheck $(HWCONFIG) - "$1" +endef + +.PHONY: flashpart +flashpart: all kill_term ##Flash a specific partition, set PART=name + $(Q) $(call CheckPartitionChunks,$(FLASH_PART_CHUNKS)) + $(call WriteFlash,$(FLASH_PART_CHUNKS)) + +.PHONY: erasepart +erasepart: kill_term ##Erase a specific partition, set PART=name + $(call EraseFlashRegion,$(FLASH_PART_REGION)) + +.PHONY: readpart +readpart: kill_term ##Read partition from device, set PART=name + $(call ReadFlash,$(FLASH_PART_REGION),$(OUT_BASE)/$(PART).read.bin) + +.PHONY: flashmap +flashmap: partmap-build kill_term ##Write partition table to device + $(call WriteFlash,$(FLASH_MAP_CHUNK)) + + +endif # MAKE_DOCS + +##@Cleaning + +clean: part-clean + +.PHONY: part-clean +part-clean: ##Clean partition targets + $(Q) rm -f $(PARTITION_BUILD_TARGETS) diff --git a/Sming/Components/Storage/notes.rst b/Sming/Components/Storage/notes.rst new file mode 100644 index 0000000000..3b4765aa06 --- /dev/null +++ b/Sming/Components/Storage/notes.rst @@ -0,0 +1,69 @@ +Storage Partition notes +======================= + +External devices +---------------- + +Support for 'external' storage devices is implemented by marking groups of entries +with a ``storage`` type partition entry. This is intended to support designs where the +memory devices are fixed (such as additional SPI RAM or flash chips) rather than +removable (such as SD cards). When these devices are registered with the Storage subsystem, +the primary partition table is scanned for related entries. + +Devices may contain their own partition table, but such tables are not loaded automatically. +This is done by calling :cpp:func:`Storage::Device::loadPartitions`. +The application must also consider how the partition table will be initialised on external +devices as this is not handled by the build system. + + + +But it also has some disadvantages: + +- Adding/removing storage devices dynamically does not fit well. + For example, when an SD Card is inserted the partition layout must be established + dynamically. + +The main purpose of the partition table is to manage the system's fixed storage. +Mostly that will just be the main flash device, but could also be 'external' devices +such as SPI RAM or flash. + +Devices can be dynamically registered/un-registered to the partition table: existing Partition objects remain valid but calls will fail. + +Dynamic partitions have additional issues. + +Partition identification +------------------------ + +Two devices may have identical partition names so these would need +to be qualified, e.g. ``spiFlash/spiffs0``. Without qualification the first matching +name would be used. + +Alternatively, each :cpp:class:`Storage::Device` object could manage its own partitions, +rather than having them in a single table. The existing layout scheme is fine, we can +just extend it so that devices can support their own partitions. + +Device serial numbers should be exposed so that devices can be uniquely identified +by applications. + +Device partitions +----------------- + +A global :cpp:class:`Storage::DeviceManager` object replaces ``partitionTable``. + +Storage each device manages its own partitions, therefore spiFlash loads its own partition table. + + + +Partitions in devices ? +----------------------- + +JSON may be more logically arranged so that partitions are defined within each device. + +This implies that partition_table_offset is also defined within the device, +so adding this entry to external devices allows definition of a partition table there also. +The application now only has to handle how to write any supplementary tables to external devices. + +Problem: Way too complicated. + +So, if we want external partition tables how to do that? +Leave for another time. \ No newline at end of file diff --git a/Sming/Components/Storage/requirements.txt b/Sming/Components/Storage/requirements.txt new file mode 100644 index 0000000000..d89304b1a8 --- /dev/null +++ b/Sming/Components/Storage/requirements.txt @@ -0,0 +1 @@ +jsonschema diff --git a/Sming/Components/Storage/schema.json b/Sming/Components/Storage/schema.json new file mode 100644 index 0000000000..5e05c9ff10 --- /dev/null +++ b/Sming/Components/Storage/schema.json @@ -0,0 +1,189 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/Config", + "definitions": { + "Config": { + "title": "Hardware Configuration", + "description": "Defines memory devices and partitions for a specific hardware implementation", + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "comment": { + "type": "string" + }, + "arch": { + "type": "string", + "title": "Target architecture", + "description": "Defined *only* in the base 'standard' spec" + }, + "base_config": { + "type": "string", + "title": "Base configuration", + "description": "Inherit a previously-defined configuration" + }, + "options": { + "type": "array", + "title": "List of option fragments", + "description": "Import additional settings from option libraries" + }, + "partition_table_offset": { + "type": [ + "string", + "integer" + ], + "description": "Location of partition table in spiFlash. Simple address or python expression to be evaluated." + }, + "devices": { + "type": "object", + "$ref": "#/definitions/Devices" + }, + "partitions": { + "type": "object", + "$ref": "#/definitions/Partitions" + } + }, + "required": [ + "name", + "arch", + "partition_table_offset", + "devices", + "partitions" + ] + }, + "Devices": { + "title": "Devices", + "type": "object", + "additionalProperties": false, + "properties": { + "spiFlash": { + "$ref": "#/definitions/Device", + "description": "Main flash memory device" + } + }, + "patternProperties": { + "^[A-Za-z_][A-Za-z0-9_]*$": { + "type": "object", + "$ref": "#/definitions/Device" + } + }, + "required": [ + "spiFlash" + ] + }, + "Device": { + "title": "Storage device definition", + "type": "object", + "additionalProperties": false, + "properties": { + "size": { + "type": "string" + }, + "type": { + "type": "string" + }, + "mode": { + "type": "string", + "enum": [ + "qio", + "qout", + "dio", + "dout" + ] + }, + "speed": { + "type": "integer" + } + }, + "required": [ + "size", + "type" + ] + }, + "Partitions": { + "title": "Partitions", + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[A-Za-z_][A-Za-z0-9_]*$": { + "type": "object", + "$ref": "#/definitions/Partition" + } + } + }, + "Partition": { + "title": "Partition definition", + "type": "object", + "additionalProperties": false, + "properties": { + "device": { + "type": "string", + "description": "ID of device this partition relates to" + }, + "address": { + "type": [ + "string", + "integer" + ], + "description": "Starting address for partition. Simple address or python expression to be evaluated." + }, + "size": { + "type": [ + "string", + "integer" + ] + }, + "type": { + "type": [ + "string", + "integer" + ] + }, + "subtype": { + "type": [ + "string", + "integer" + ] + }, + "readonly": { + "type": "boolean" + }, + "encrypted": { + "type": "boolean" + }, + "filename": { + "type": "string", + "description": "Location of file to write to this partition" + }, + "build": { + "type": "object", + "$ref": "#/definitions/Build", + "description": "If present, used to build 'filename'" + } + }, + "required": [ + "address", + "size", + "type", + "subtype" + ] + }, + "Build": { + "title": "Build specification", + "description": "Additional properties as required by build target", + "type": "object", + "additionalProperties": true, + "properties": { + "target": { + "type": "string", + "description": "Makefile target for this build" + } + }, + "required": [ + "target" + ] + } + } +} \ No newline at end of file diff --git a/Sming/Components/Storage/src/CustomDevice.cpp b/Sming/Components/Storage/src/CustomDevice.cpp new file mode 100644 index 0000000000..a960528961 --- /dev/null +++ b/Sming/Components/Storage/src/CustomDevice.cpp @@ -0,0 +1,42 @@ +/* + * CustomDevice.cpp + */ + +#include "include/Storage/CustomDevice.h" +#include + +namespace Storage +{ +namespace +{ +static constexpr size_t maxPartitions{16}; ///< Hard limit on partition table size + +class Partitions : public PartitionTable +{ +public: + Partition add(const Partition::Info& info) + { + if(!mEntries) { + mEntries.reset(new Partition::Info[maxPartitions]); + } else + assert(mCount < maxPartitions); + + auto i = mCount++; + mEntries.get()[i] = info; + return operator[](i); + } +}; + +} // namespace + +Partition CustomDevice::createPartition(const Partition::Info& info) +{ + if(mPartitions.count() >= maxPartitions) { + debug_e("Partition table is full for '%s'", getName().c_str()); + return Partition{}; + } + + return reinterpret_cast(mPartitions).add(info); +} + +} // namespace Storage diff --git a/Sming/Components/Storage/src/Device.cpp b/Sming/Components/Storage/src/Device.cpp new file mode 100644 index 0000000000..127396828a --- /dev/null +++ b/Sming/Components/Storage/src/Device.cpp @@ -0,0 +1,112 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * Device.h - external storage device API + * + ****/ + +#include "include/Storage.h" +#include "include/Storage/Device.h" +#include "include/Storage/partition_info.h" +#include +#include + +namespace +{ +#define XX(type, value, desc) DEFINE_FSTR_LOCAL(typestr_##type, #type) +STORAGE_TYPE_MAP(XX) +#undef XX + +#define XX(type, value, desc) &typestr_##type, +DEFINE_FSTR_VECTOR_LOCAL(typeStrings, FlashString, STORAGE_TYPE_MAP(XX)) +#undef XX + +#define XX(type, value, desc) DEFINE_FSTR_LOCAL(long_typestr_##type, desc) +STORAGE_TYPE_MAP(XX) +#undef XX + +#define XX(type, value, desc) &long_typestr_##type, +DEFINE_FSTR_VECTOR_LOCAL(longTypeStrings, FlashString, STORAGE_TYPE_MAP(XX)) +#undef XX + +} // namespace + +String toString(Storage::Device::Type type) +{ + return typeStrings[unsigned(type)]; +} + +String toLongString(Storage::Device::Type type) +{ + return longTypeStrings[unsigned(type)]; +} + +namespace Storage +{ +Device::~Device() +{ + unRegisterDevice(this); +} + +bool Device::loadPartitions(Device& source, uint32_t tableOffset) +{ + constexpr size_t maxEntries = ESP_PARTITION_TABLE_MAX_LEN / sizeof(esp_partition_info_t); + esp_partition_info_t buffer[maxEntries]; + if(!source.read(tableOffset, buffer, sizeof(buffer))) { + debug_e("[Partition] Failed to read partition table at offset 0x%08x", tableOffset); + return false; + } + + if(buffer[0].type != Partition::Type::storage) { + debug_e("[Partition] Bad partition table for device '%s' @ 0x%08x", source.getName().c_str(), tableOffset); + return false; + } + + String devname = getName(); + for(unsigned i = 0; i < maxEntries; ++i) { + auto entry = &buffer[i]; + if(entry->magic != ESP_PARTITION_MAGIC) { + continue; + } + if(entry->type != Partition::Type::storage) { + continue; + } + + auto len = devname.length(); + if(len > Partition::nameSize) { + continue; + } + if(strncmp(entry->name, devname.c_str(), len) != 0) { + continue; + } + + if(entry->subtype != uint8_t(getType())) { + debug_w("[Device] '%s' type mismatch, '%s' in partition table but device reports '%s'", getName().c_str(), + toString(Device::Type(entry->subtype)).c_str(), toString(getType()).c_str()); + } + if(entry->size != getSize()) { + debug_w("[Device] '%s' size mismatch, 0x%08x in partition table but device reports 0x%08x", + getName().c_str(), entry->size, getSize()); + } + + // Skip the storage entry, not required + ++entry; + ++i; + unsigned count{0}; + while(i < maxEntries && buffer[i].magic == ESP_PARTITION_MAGIC) { + ++i; + ++count; + } + + mPartitions.load(entry, count); + return true; + } + + // No partitions found + return false; +} + +} // namespace Storage diff --git a/Sming/Components/Storage/src/Iterator.cpp b/Sming/Components/Storage/src/Iterator.cpp new file mode 100644 index 0000000000..e7a4e2a2ab --- /dev/null +++ b/Sming/Components/Storage/src/Iterator.cpp @@ -0,0 +1,67 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * Iterator.cpp + * + ****/ + +#include "include/Storage/Iterator.h" +#include "include/Storage/SpiFlash.h" + +namespace Storage +{ +Iterator::Iterator(Device& device, uint8_t partitionIndex) + : mSearch{&device, Partition::Type::any, Partition::SubType::any}, mDevice(&device), mPos(partitionIndex) + +{ + if(partitionIndex >= device.partitions().count()) { + mDevice = nullptr; + mPos = afterEnd; + } +} + +Iterator::Iterator(Partition::Type type, uint8_t subtype) : mSearch{nullptr, type, subtype} +{ + mDevice = spiFlash; + next(); +} + +bool Iterator::next() +{ + while(mDevice != nullptr) { + while(uint8_t(++mPos) < mDevice->partitions().count()) { + auto entry = mDevice->partitions()[mPos]; + + if(mSearch.type != Partition::Type::any && mSearch.type != entry.type()) { + continue; + } + + if(mSearch.subType != Partition::SubType::any && mSearch.subType != entry.subType()) { + continue; + } + + return true; + } + + mPos = afterEnd; + if(mSearch.device != nullptr) { + mDevice = nullptr; + break; + } + + mDevice = mDevice->getNext(); + mPos = beforeStart; + } + + return false; +} + +Partition Iterator::operator*() const +{ + return mDevice ? mDevice->partitions()[mPos] : Partition{}; +} + +} // namespace Storage diff --git a/Sming/Components/Storage/src/Partition.cpp b/Sming/Components/Storage/src/Partition.cpp new file mode 100644 index 0000000000..d66c128fbb --- /dev/null +++ b/Sming/Components/Storage/src/Partition.cpp @@ -0,0 +1,251 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * Partition.cpp - Partition support for all architectures + * + ****/ + +#include "include/Storage/Partition.h" +#include "include/Storage/Device.h" +#include +#include + +using namespace Storage; + +namespace +{ +/* APP type strings */ + +#define XX(subtype, value, desc) DEFINE_FSTR_LOCAL(app_subTypeStr_##subtype, #subtype) +PARTITION_APP_SUBTYPE_MAP(XX) +#undef XX + +#define XX(subtype, value, desc) {Partition::SubType::App::subtype, &app_subTypeStr_##subtype}, +DEFINE_FSTR_MAP_LOCAL(appSubTypeStrings, Partition::SubType::App, FlashString, PARTITION_APP_SUBTYPE_MAP(XX)) +#undef XX + +#define XX(subtype, value, desc) DEFINE_FSTR_LOCAL(app_subTypeStr_long_##subtype, desc) +PARTITION_APP_SUBTYPE_MAP(XX) +#undef XX + +#define XX(subtype, value, desc) {Partition::SubType::App::subtype, &app_subTypeStr_long_##subtype}, +DEFINE_FSTR_MAP_LOCAL(longAppSubTypeStrings, Partition::SubType::App, FlashString, PARTITION_APP_SUBTYPE_MAP(XX)) +#undef XX + +/* DATA subtype strings */ + +#define XX(subtype, value, desc) DEFINE_FSTR_LOCAL(data_subTypeStr_##subtype, #subtype) +PARTITION_DATA_SUBTYPE_MAP(XX) +#undef XX + +#define XX(subtype, value, desc) {Partition::SubType::Data::subtype, &data_subTypeStr_##subtype}, +DEFINE_FSTR_MAP_LOCAL(dataSubTypeStrings, Partition::SubType::Data, FlashString, PARTITION_DATA_SUBTYPE_MAP(XX)) +#undef XX + +#define XX(subtype, value, desc) DEFINE_FSTR_LOCAL(data_subTypeStr_long_##subtype, desc) +PARTITION_DATA_SUBTYPE_MAP(XX) +#undef XX + +#define XX(subtype, value, desc) {Partition::SubType::Data::subtype, &data_subTypeStr_long_##subtype}, +DEFINE_FSTR_MAP_LOCAL(longDataSubTypeStrings, Partition::SubType::Data, FlashString, PARTITION_DATA_SUBTYPE_MAP(XX)) +#undef XX + +} // namespace + +String toString(Partition::Type type, uint8_t subType) +{ + String s; + switch(type) { + case Partition::Type::app: + s = F("app/"); + if(auto v = appSubTypeStrings[Partition::SubType::App(subType)]) { + s += String(v); + } else { + s += subType; + } + break; + case Partition::Type::data: + s = F("data/"); + if(auto v = dataSubTypeStrings[Partition::SubType::Data(subType)]) { + s += String(v); + } else { + s += subType; + } + break; + + case Partition::Type::storage: + s = F("storage/"); + if(auto v = toString(Device::Type(subType))) { + s += v; + } else { + s += subType; + } + break; + + default: + s = unsigned(type); + s += '.'; + s += subType; + } + return s; +} + +String toLongString(Partition::Type type, uint8_t subType) +{ + switch(type) { + case Partition::Type::app: + if(auto v = longAppSubTypeStrings[Partition::SubType::App(subType)]) { + return F("App: ") + String(v); + } + break; + case Partition::Type::data: + if(auto v = longDataSubTypeStrings[Partition::SubType::Data(subType)]) { + return F("Data: ") + String(v); + } + break; + + case Partition::Type::storage: + if(auto s = toLongString(Device::Type(subType))) { + return F("Storage: ") + s; + } + break; + + default:; + // Unknown + } + + return toString(type, subType); +} + +namespace Storage +{ +String Partition::typeString() const +{ + return toString(type(), subType()); +} + +String Partition::longTypeString() const +{ + return toLongString(type(), subType()); +} + +bool Partition::verify(Partition::Type type, uint8_t subtype) const +{ + if(mPart == nullptr) { + debug_e("[Partition] invalid"); + return false; + } + + if(type != mPart->type) { + debug_e("[Partition] type mismatch, expected %u got %u", unsigned(type), unsigned(mPart->type)); + return false; + } + + if(mPart->subtype != subtype) { + debug_e("[Partition] subtype mismatch, expected %u got %u", subtype, mPart->subtype); + return false; + } + + return true; +} + +bool Partition::getDeviceAddress(uint32_t& address, size_t size) const +{ + if(mDevice == nullptr || mPart == nullptr) { + debug_e("[Partition] Invalid"); + return false; + } + + if(address >= mPart->size || (address + size - 1) >= mPart->size) { + debug_e("[Partition] Invalid range, address: 0x%08x, size: 0x%08x", address, size); + return false; + } + + // Storage partitions refer directly to the underlying device + if(type() != Partition::Type::storage) { + address += mPart->offset; + } + + return true; +} + +String Partition::getDeviceName() const +{ + return mDevice ? mDevice->getName() : nullptr; +} + +size_t Partition::getBlockSize() const +{ + return mDevice ? mDevice->getBlockSize() : 0; +} + +bool Partition::allowRead() +{ + if(mDevice == nullptr || mPart == nullptr) { + debug_e("[Partition] Invalid"); + return false; + } + + return true; +} + +bool Partition::allowWrite() +{ + if(!allowRead()) { + return false; + } + + if(mPart->flags[Flag::readOnly]) { + debug_e("[Partition] %s is read-only", mPart ? mPart->name.c_str() : "?"); + return false; + } + + return true; +} + +bool Partition::read(size_t offset, void* dst, size_t size) +{ + if(!allowRead()) { + return false; + } + + uint32_t addr = offset; + if(!getDeviceAddress(addr, size)) { + return false; + } + + return mDevice ? mDevice->read(addr, dst, size) : false; +} + +bool Partition::write(size_t offset, const void* src, size_t size) +{ + if(!allowWrite()) { + return false; + } + + uint32_t addr = offset; + if(!getDeviceAddress(addr, size)) { + return false; + } + + return mDevice ? mDevice->write(addr, src, size) : false; +} + +bool Partition::erase_range(size_t offset, size_t size) +{ + if(!allowWrite()) { + return false; + } + + uint32_t addr = offset; + if(!getDeviceAddress(addr, size)) { + return false; + } + + return mDevice ? mDevice->erase_range(addr, size) : false; +} + +} // namespace Storage diff --git a/Sming/Components/Storage/src/PartitionStream.cpp b/Sming/Components/Storage/src/PartitionStream.cpp new file mode 100644 index 0000000000..86ad9b037c --- /dev/null +++ b/Sming/Components/Storage/src/PartitionStream.cpp @@ -0,0 +1,61 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * PartitionStream.cpp + * + ****/ + +#include "include/Storage/PartitionStream.h" + +namespace Storage +{ +uint16_t PartitionStream::readMemoryBlock(char* data, int bufSize) +{ + int len = std::min(bufSize, available()); + return partition.read(startOffset + readPos, data, len) ? len : 0; +} + +int PartitionStream::seekFrom(int offset, SeekOrigin origin) +{ + size_t newPos; + switch(origin) { + case SeekOrigin::Start: + newPos = 0; + break; + case SeekOrigin::Current: + newPos = readPos + offset; + break; + case SeekOrigin::End: + newPos = size + offset; + break; + default: + return -1; + } + + if(newPos > size) { + return -1; + } + + readPos = newPos; + return readPos; +} + +size_t PartitionStream::write(const uint8_t* data, size_t length) +{ + auto len = std::min(size - writePos, length); + if(len != 0) { + if(!partition.write(startOffset + writePos, data, len)) { + len = 0; + } else { + writePos += len; + } + } + + // Return amount actually written + return len; +} + +} // namespace Storage diff --git a/Sming/Components/Storage/src/PartitionTable.cpp b/Sming/Components/Storage/src/PartitionTable.cpp new file mode 100644 index 0000000000..793cd061bf --- /dev/null +++ b/Sming/Components/Storage/src/PartitionTable.cpp @@ -0,0 +1,37 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * PartitionTable.cpp + * + ****/ + +#include "include/Storage/PartitionTable.h" +#include "include/Storage/partition_info.h" +#include + +namespace Storage +{ +void PartitionTable::load(const esp_partition_info_t* entry, unsigned count) +{ + if(count == 0) { + mEntries.reset(); + mCount = count; + return; + } + + mCount = count; + mEntries.reset(new Partition::Info[count]); + for(unsigned i = 0; i < count; ++i) { + auto& e = entry[i]; + // name may not be zero-terminated + char name[Partition::nameSize + 1]; + memcpy(name, e.name, Partition::nameSize); + name[Partition::nameSize] = '\0'; + mEntries.get()[i] = Partition::Info{name, e.type, e.subtype, e.offset, e.size, e.flags}; + } +} + +} // namespace Storage diff --git a/Sming/Components/Storage/src/ProgMem.cpp b/Sming/Components/Storage/src/ProgMem.cpp new file mode 100644 index 0000000000..8d95f594cc --- /dev/null +++ b/Sming/Components/Storage/src/ProgMem.cpp @@ -0,0 +1,35 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * ProgMem.cpp + * + ****/ + +#include "include/Storage/ProgMem.h" +#include + +namespace Storage +{ +ProgMem progMem; + +bool ProgMem::read(uint32_t address, void* dst, size_t size) +{ + size_t readCount = flashmem_read(dst, address, size); + return readCount == size; +} + +Partition ProgMem::createPartition(const String& name, const void* flashPtr, size_t size, Partition::Type type, + uint8_t subtype) +{ + auto addr = flashmem_get_address(flashPtr); + if(addr == 0) { + return Partition{}; + } + + return createPartition(name, type, subtype, addr, size, Partition::Flag::readOnly); +} + +} // namespace Storage diff --git a/Sming/Components/Storage/src/SpiFlash.cpp b/Sming/Components/Storage/src/SpiFlash.cpp new file mode 100644 index 0000000000..ee62586b80 --- /dev/null +++ b/Sming/Components/Storage/src/SpiFlash.cpp @@ -0,0 +1,71 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * SpiFlash.cpp + * + ****/ + +#include "include/Storage/SpiFlash.h" +#include "include/Storage/partition_info.h" +#include + +namespace Storage +{ +DEFINE_FSTR(FS_SPIFLASH, "spiFlash") +SpiFlash* spiFlash; + +String SpiFlash::getName() const +{ + return FS_SPIFLASH; +} + +uint32_t SpiFlash::getId() const +{ + return spi_flash_get_id(); +} + +size_t SpiFlash::getBlockSize() const +{ + return SPI_FLASH_SEC_SIZE; +} + +size_t SpiFlash::getSize() const +{ + return flashmem_get_size_bytes(); +} + +bool SpiFlash::read(uint32_t address, void* dst, size_t size) +{ + size_t readCount = flashmem_read(dst, address, size); + return readCount == size; +} + +bool SpiFlash::write(uint32_t address, const void* src, size_t size) +{ + size_t writeCount = flashmem_write(src, address, size); + return writeCount == size; +} + +bool SpiFlash::erase_range(uint32_t address, size_t size) +{ + if(address % SPI_FLASH_SEC_SIZE != 0 || size % SPI_FLASH_SEC_SIZE != 0) { + debug_e("[Partition] erase address/size misaligned: 0x%08x / 0x%08x", address, size); + return false; + } + + auto sec = address / SPI_FLASH_SEC_SIZE; + auto end = (address + size) / SPI_FLASH_SEC_SIZE; + while(sec < end) { + if(!flashmem_erase_sector(sec)) { + return false; + } + ++sec; + } + + return true; +} + +} // namespace Storage diff --git a/Sming/Components/Storage/src/Storage.cpp b/Sming/Components/Storage/src/Storage.cpp new file mode 100644 index 0000000000..98b6c8ed18 --- /dev/null +++ b/Sming/Components/Storage/src/Storage.cpp @@ -0,0 +1,75 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * DeviceManager.cpp + * + ****/ + +#include "include/Storage.h" +#include "include/Storage/SpiFlash.h" +#include + +namespace Storage +{ +void initialize() +{ + if(spiFlash == nullptr) { + spiFlash = new SpiFlash; + registerDevice(spiFlash); + spiFlash->loadPartitions(PARTITION_TABLE_OFFSET); + } +} + +const Device::List getDevices() +{ + return Device::List(spiFlash); +} + +bool registerDevice(Device* device) +{ + if(device == nullptr) { + return false; + } + auto devname = device->getName(); + + Device::List devices(spiFlash); + auto it = std::find(devices.begin(), devices.end(), devname); + if(!it) { + devices.add(device); + device->loadPartitions(*spiFlash, PARTITION_TABLE_OFFSET); + debug_i("[Storage] Device '%s' registered", devname.c_str()); + } else if(*it != *device) { + debug_e("[Storage] Another device is already registered with name '%s'", devname.c_str()); + return false; + } + + return true; +} + +bool unRegisterDevice(Device* device) +{ + return Device::List(spiFlash).remove(device); +} + +Device* findDevice(const String& name) +{ + Device::List devices(spiFlash); + return std::find(devices.begin(), devices.end(), name); +} + +Partition findPartition(const String& name) +{ + for(auto& dev : getDevices()) { + auto part = dev.partitions().find(name); + if(part) { + return part; + } + } + + return Partition{}; +} + +} // namespace Storage diff --git a/Sming/Components/Storage/src/SysMem.cpp b/Sming/Components/Storage/src/SysMem.cpp new file mode 100644 index 0000000000..5281259f62 --- /dev/null +++ b/Sming/Components/Storage/src/SysMem.cpp @@ -0,0 +1,25 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * SysMem.cpp + * + ****/ + +#include "include/Storage/SysMem.h" +#include + +namespace Storage +{ +SysMem sysMem; + +Partition SysMem::createPartition(const String& name, const FSTR::ObjectBase& fstr, Partition::Type type, + uint8_t subtype) +{ + return createPartition(name, type, subtype, reinterpret_cast(fstr.data()), fstr.size(), + Partition::Flag::readOnly); +} + +} // namespace Storage diff --git a/Sming/Components/Storage/src/include/Storage.h b/Sming/Components/Storage/src/include/Storage.h new file mode 100644 index 0000000000..81977f4d85 --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage.h @@ -0,0 +1,62 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * Storage.h + * + ****/ +#pragma once + +#include "Storage/Device.h" + +namespace Storage +{ +/** + * @brief Called early in the startup phase + */ +void initialize(); + +/** + * @brief Get read-only reference to device list + */ +const Device::List getDevices(); + +/** + * @brief Register a storage device + * @retval bool true on success, false if another device already registered with same name + */ +bool registerDevice(Device* device); + +/** + * @brief Unregister a storage device + * + * Use extreme care: behaviour is unpredictable if partitions are in use + */ +bool unRegisterDevice(Device* device); + +/** + * @brief Find a registered device + */ +Device* findDevice(const String& name); + +/** + * @brief Find the first partition matching the given name + */ +Partition findPartition(const String& name); + +/** + * @brief Find partitions of the given type + */ +inline Iterator findPartition(Partition::Type type = Partition::Type::any, uint8_t subType = Partition::SubType::any) +{ + return Iterator(type, subType); +} + +template Iterator findPartition(T subType) +{ + return Iterator(Partition::Type(T::partitionType), uint8_t(subType)); +} + +} // namespace Storage diff --git a/Sming/Components/Storage/src/include/Storage/CustomDevice.h b/Sming/Components/Storage/src/include/Storage/CustomDevice.h new file mode 100644 index 0000000000..82222f2667 --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage/CustomDevice.h @@ -0,0 +1,35 @@ +/* + * CustomDevice.h + */ + +#pragma once + +#include "Device.h" + +namespace Storage +{ +/** + * @brief Class to support dynamic partitions + * + * Call `createPartition` to add partitions up to a maximum of 16 entries. + */ +class CustomDevice : public Device +{ +public: + Partition createPartition(const Partition::Info& info); + + Partition createPartition(const String& name, Partition::Type type, uint8_t subtype, uint32_t offset, size_t size, + Partition::Flags flags = 0) + { + return createPartition(Partition::Info{name, type, subtype, offset, size, flags}); + } + + template + Partition createPartition(const String& name, SubType subtype, uint32_t offset, size_t size, + Partition::Flags flags = 0) + { + return createPartition(name, Partition::Type(SubType::partitionType), uint8_t(subtype), offset, size, flags); + } +}; + +} // namespace Storage diff --git a/Sming/Components/Storage/src/include/Storage/Device.h b/Sming/Components/Storage/src/include/Storage/Device.h new file mode 100644 index 0000000000..3b7ca0bb89 --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage/Device.h @@ -0,0 +1,149 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * Device.h - external storage device API + * + ****/ +#pragma once + +#include +#include +#include "PartitionTable.h" + +#define STORAGE_TYPE_MAP(XX) \ + XX(unknown, 0x00, "Other storage device") \ + XX(flash, 0x01, "SPI flash") \ + XX(spiram, 0x02, "SPI RAM") \ + XX(sdcard, 0x03, "SD Card") \ + XX(disk, 0x04, "Physical disk") \ + XX(file, 0x05, "Backing file on separate filesystem") \ + XX(sysmem, 0x06, "System Memory") + +namespace Storage +{ +class SpiFlash; + +/** + * @brief Represents a storage device (e.g. flash memory) + */ +class Device : public LinkedObjectTemplate +{ +public: + using List = LinkedObjectListTemplate; + using OwnedList = OwnedLinkedObjectListTemplate; + + /** + * @brief Storage type + */ + enum class Type : uint8_t { +#define XX(type, value, desc) type = value, + STORAGE_TYPE_MAP(XX) +#undef XX + }; + + Device() : mPartitions(*this) + { + } + + ~Device(); + + bool operator==(const String& name) const + { + return getName() == name; + } + + PartitionTable& partitions() + { + return mPartitions; + } + + const PartitionTable& partitions() const + { + return mPartitions; + } + + /** + * @brief Load partition table entries + * @tableOffset Location of partition table to read + * @retval bool true on success, false on failure + */ + bool loadPartitions(uint32_t tableOffset) + { + return loadPartitions(*this, tableOffset); + } + + /** + * @brief Load partition table entries from another table + * @param source Device to load entries from + * @tableOffset Location of partition table to read + * @retval bool true on success, false on failure + */ + bool loadPartitions(Device& source, uint32_t tableOffset); + + /** + * @brief Obtain unique device name + */ + virtual String getName() const = 0; + + /** + * @brief Obtain device ID + * @retval uint32_t typically flash chip ID + */ + virtual uint32_t getId() const + { + return 0; + } + + /** + * @brief Obtain smallest allocation unit for erase operations + */ + virtual size_t getBlockSize() const = 0; + + /** + * @brief Obtain addressable size of this device + * @retval size_t Must be at least as large as the value declared in the partition table + */ + virtual size_t getSize() const = 0; + + /** + * @brief Obtain device type + */ + virtual Type getType() const = 0; + + /** + * @brief Read data from the storage device + * @param address Where to start reading + * @param dst Buffer to store data + * @param size Size of data to be read, in bytes. + * @retval bool true on success, false on error + */ + virtual bool read(uint32_t address, void* dst, size_t size) = 0; + + /** + * @brief Write data to the storage device + * @param address Where to start writing + * @param src Data to write + * @param size Size of data to be written, in bytes. + * @retval bool true on success, false on error + */ + virtual bool write(uint32_t address, const void* src, size_t size) = 0; + + /** + * @brief Erase a region of storage in preparation for writing + * @param address Where to start erasing + * @param size Size of region to erase, in bytes + * @retval bool true on success, false on error + */ + virtual bool erase_range(uint32_t address, size_t size) = 0; + +protected: + PartitionTable mPartitions; +}; + +} // namespace Storage + +String toString(Storage::Device::Type type); +String toLongString(Storage::Device::Type type); diff --git a/Sming/Components/Storage/src/include/Storage/Iterator.h b/Sming/Components/Storage/src/include/Storage/Iterator.h new file mode 100644 index 0000000000..dcd3b7ca88 --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage/Iterator.h @@ -0,0 +1,78 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * Iterator.h + * + ****/ +#pragma once + +#include "Partition.h" + +namespace Storage +{ +class Device; + +class Iterator : public std::iterator +{ +public: + Iterator(Device& device, uint8_t partitionIndex); + + Iterator(Device& device, Partition::Type type, uint8_t subtype) : mSearch{&device, type, subtype} + { + mDevice = &device; + next(); + } + + Iterator(Partition::Type type, uint8_t subtype); + + explicit operator bool() const + { + return (mDevice != nullptr) && (mPos > beforeStart) && (mPos < afterEnd); + } + + Iterator operator++(int) + { + auto result = *this; + next(); + return result; + } + + Iterator& operator++() + { + next(); + return *this; + } + + bool operator==(const Iterator& other) const + { + return (mDevice == other.mDevice) && (mPos == other.mPos); + } + + bool operator!=(const Iterator& other) const + { + return !operator==(other); + } + + Partition operator*() const; + +private: + static constexpr int8_t beforeStart{-1}; + static constexpr int8_t afterEnd{0x7f}; + + bool seek(uint8_t pos); + bool next(); + + struct Search { + Device* device; + Partition::Type type; + uint8_t subType; + }; + Search mSearch{}; + Device* mDevice{nullptr}; + int8_t mPos{beforeStart}; +}; + +} // namespace Storage diff --git a/Sming/Components/Storage/src/include/Storage/Partition.h b/Sming/Components/Storage/src/include/Storage/Partition.h new file mode 100644 index 0000000000..9b5badeee1 --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage/Partition.h @@ -0,0 +1,388 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * Partition.h - C++ wrapper for universal partition table support + * + * Original license for IDF code: + * + * Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * + ****/ +#pragma once + +#include +#include +#include +#include + +#define PARTITION_APP_SUBTYPE_MAP(XX) \ + XX(factory, 0x00, "Factory application") \ + XX(ota0, 0x10, "OTA #0") \ + XX(ota1, 0x11, "OTA #1") \ + XX(ota2, 0x12, "OTA #2") \ + XX(ota3, 0x13, "OTA #3") \ + XX(ota4, 0x14, "OTA #4") \ + XX(ota5, 0x15, "OTA #5") \ + XX(ota6, 0x16, "OTA #6") \ + XX(ota7, 0x17, "OTA #7") \ + XX(ota8, 0x18, "OTA #8") \ + XX(ota9, 0x19, "OTA #9") \ + XX(ota10, 0x1a, "OTA #10") \ + XX(ota11, 0x1b, "OTA #11") \ + XX(ota12, 0x1c, "OTA #12") \ + XX(ota13, 0x1d, "OTA #13") \ + XX(ota14, 0x1e, "OTA #14") \ + XX(ota15, 0x1f, "OTA #15") \ + XX(test, 0x20, "Test application") + +#define PARTITION_DATA_SUBTYPE_MAP(XX) \ + XX(ota, 0x00, "OTA selection") \ + XX(phy, 0x01, "PHY init data") \ + XX(nvs, 0x02, "NVS") \ + XX(coreDump, 0x03, "Core Dump data") \ + XX(nvsKeys, 0x04, "NVS key information") \ + XX(eFuseEm, 0x05, "eFuse emulation") \ + XX(sysParam, 0x40, "System Parameters") \ + XX(rfCal, 0x41, "RF Calibration") \ + XX(espHttpd, 0x80, "ESPHTTPD") \ + XX(fat, 0x81, "FAT") \ + XX(spiffs, 0x82, "SPIFFS") \ + XX(fwfs, 0xF1, "FWFS") + +namespace Storage +{ +class Device; +class PartitionTable; +struct esp_partition_info_t; + +/** + * @brief Represents a flash partition + */ +class Partition +{ +public: + enum class Type : uint8_t { + app = 0x00, + data = 0x01, + storage = 0x02, + userMin = 0x40, + userMax = 0xFE, + invalid = 0xff, + any = 0xff, + }; + + struct SubType { + static constexpr uint8_t any{0xff}; + static constexpr uint8_t invalid{0xff}; + + /** + * @brief Application partition type + */ + enum class App : uint8_t { + partitionType = uint8_t(Type::app), +#define XX(type, value, desc) type = value, + PARTITION_APP_SUBTYPE_MAP(XX) +#undef XX + ota_min = ota0, + ota_max = ota15, + any = 0xff + }; + + /** + * @brief Data partition type + */ + enum class Data : uint8_t { + partitionType = uint8_t(Type::data), +#define XX(subtype, value, desc) subtype = value, + PARTITION_DATA_SUBTYPE_MAP(XX) +#undef XX + any = 0xff + }; + }; + + enum class Flag { + encrypted = 0, + readOnly = 31, ///< Write/erase prohibited + }; + + static constexpr size_t nameSize{16}; + using Name = char[nameSize]; + using Flags = BitSet; + + /** + * @brief Partition information + */ + struct Info { + CString name; + uint32_t offset{0}; + uint32_t size{0}; + Type type{Type::invalid}; + uint8_t subtype{SubType::invalid}; + Flags flags; + + Info() + { + } + + Info(const String& name, Type type, uint8_t subtype, uint32_t offset, uint32_t size, Flags flags) + : name(name), offset(offset), size(size), type(type), subtype(subtype), flags(flags) + { + } + }; + + Partition() + { + } + + Partition(const Partition& other) : mDevice(other.mDevice), mPart(other.mPart) + { + } + + Partition(Device& device, const Info& info) : mDevice(&device), mPart(&info) + { + } + + /** + * @name Confirm partition is of the expected type + * @param type Expected partition type + * @param subtype Expected partition sub-type + * @retval bool true if type is OK, false if not. + * Logs debug messages on failure. + * @{ + */ + bool verify(Type type, uint8_t subtype) const; + + bool verify(uint8_t type, uint8_t subtype) const + { + return verify(Type(type), subtype); + } + + template bool verify(T subType) const + { + return verify(Type(T::partitionType), uint8_t(subType)); + } + + /** @} */ + + /** + * @brief Convenience function to get SubType value for the i-th OTA partition + */ + static inline SubType::App apptypeOta(uint8_t i) + { + auto subtype = SubType::App(uint8_t(SubType::App::ota_min) + i); + assert(subtype >= SubType::App::ota_min && subtype <= SubType::App::ota_max); + return subtype; + } + + explicit operator bool() const + { + return mDevice != nullptr && mPart != nullptr; + } + + /** + * @brief Read data from the partition + * @param offset Where to start reading, relative to start of partition + * @param dst Buffer to store data + * @param size Size of data to be read, in bytes. + * @retval bool true on success, false on error + */ + bool read(size_t offset, void* dst, size_t size); + + template typename std::enable_if::value, bool>::type read(size_t offset, T& value) + { + return read(offset, &value, sizeof(value)); + } + + /** + * @brief Write data to the partition + * @param offset Where to start writing, relative to start of partition + * @param src Data to write + * @param size Size of data to be written, in bytes. + * @retval bool true on success, false on error + * @note Flash region must be erased first + */ + bool write(size_t offset, const void* src, size_t size); + + /** + * @brief Erase part of the partition + * @param offset Where to start erasing, relative to start of partition + * @param size Size of region to erase, in bytes + * @retval bool true on success, false on error + * @note Both offset and size must be aligned to flash sector size (4Kbytes) + */ + bool erase_range(size_t offset, size_t size); + + /** + * @brief Obtain partition type + */ + Partition::Type type() const + { + return mPart ? Partition::Type(mPart->type) : Type::invalid; + } + + /** + * @brief Obtain partition sub-type + */ + uint8_t subType() const + { + return mPart ? mPart->subtype : SubType::invalid; + } + + /** + * @brief Obtain partition starting address + * @param uint32_t Device address + */ + uint32_t address() const + { + return (mPart && mPart->type != Partition::Type::storage) ? mPart->offset : 0; + } + + /** + * @brief Obtain address of last byte in this this partition + * @param uint32_t Device address + */ + uint32_t lastAddress() const + { + return mPart ? (mPart->offset + mPart->size - 1) : 0; + } + + /** + * @brief Obtain partition size + * @retval uint32_t Size in bytes + */ + uint32_t size() const + { + return mPart ? mPart->size : 0; + } + + /** + * @brief Get partition name + */ + String name() const + { + return mPart ? mPart->name.c_str() : nullptr; + } + + /** + * @brief Get partition flags + */ + Flags flags() const + { + return mPart ? mPart->flags : 0; + } + + /** + * @brief Check state of partition `encrypted` flag + */ + bool isEncrypted() const + { + return flags()[Flag::encrypted]; + } + + /** + * @brief Check state of partition `readOnly` flag + */ + bool isReadOnly() const + { + return mPart ? mPart->flags[Flag::readOnly] : true; + } + + /** + * @name Get partition type expressed as a string + * @{ + */ + String typeString() const; + String longTypeString() const; + /** @} */ + + /** + * @brief Get corresponding storage device address for a given partition offset + * @param address IN: Zero-based offset within partition, OUT: Device address + * @param size Size of data to be accessed + * @retval bool true on success, false on failure + * Fails if the given offset/size combination is out of range, or the partition is undefined. + */ + bool getDeviceAddress(uint32_t& address, size_t size) const; + + /** + * @brief Get name of storage device for this partition + * @retval String + */ + String getDeviceName() const; + + /** + * @brief Get storage device containing this partition + * @retval Device* null if device isn't registered + */ + Device* getDevice() const + { + return mDevice; + } + + /** + * @brief Determine if given address contained within this partition + */ + bool contains(uint32_t addr) const + { + return mPart ? (addr >= mPart->offset && addr <= lastAddress()) : false; + } + + bool operator==(const Partition& other) const + { + return this == &other; + } + + bool operator==(const char* name) const + { + return mPart ? mPart->name.equals(name) : false; + } + + bool operator==(const String& name) const + { + return mPart ? mPart->name.equals(name) : false; + } + + /** + * @brief Obtain smallest allocation unit for erase operations + */ + size_t getBlockSize() const; + +protected: + Device* mDevice{nullptr}; + const Info* mPart{nullptr}; + +private: + bool allowRead(); + bool allowWrite(); +}; + +} // namespace Storage + +String toString(Storage::Partition::Type type, uint8_t subType); +String toLongString(Storage::Partition::Type type, uint8_t subType); + +template typename std::enable_if::type toString(E subType) +{ + return toString(Storage::Partition::Type(E::partitionType), uint8_t(subType)); +} + +template typename std::enable_if::type toLongString(E subType) +{ + return toLongString(Storage::Partition::Type(E::partitionType), uint8_t(subType)); +} diff --git a/Sming/Components/Storage/src/include/Storage/PartitionStream.h b/Sming/Components/Storage/src/include/Storage/PartitionStream.h new file mode 100644 index 0000000000..93debaeaff --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage/PartitionStream.h @@ -0,0 +1,61 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * PartitionStream.h + * + ****/ + +#pragma once + +#include +#include "Partition.h" + +namespace Storage +{ +/** + * @brief Stream operating directory on a Storage partition + * + * To support write operations, the target region must be erased first. + * + * @ingroup stream + */ +class PartitionStream : public ReadWriteStream +{ +public: + PartitionStream(Partition partition, uint32_t offset, size_t size) + : partition(partition), startOffset(offset), size(size) + { + } + + PartitionStream(Partition partition) : partition(partition), startOffset(0), size(partition.size()) + { + } + + int available() override + { + return size - readPos; + } + + uint16_t readMemoryBlock(char* data, int bufSize) override; + + int seekFrom(int offset, SeekOrigin origin) override; + + size_t write(const uint8_t* buffer, size_t size) override; + + bool isFinished() override + { + return available() <= 0; + } + +private: + Partition partition; + uint32_t startOffset; + size_t size; + uint32_t writePos{0}; + uint32_t readPos{0}; +}; + +} // namespace Storage diff --git a/Sming/Components/Storage/src/include/Storage/PartitionTable.h b/Sming/Components/Storage/src/include/Storage/PartitionTable.h new file mode 100644 index 0000000000..5acfc135bd --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage/PartitionTable.h @@ -0,0 +1,112 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * PartitionTable.h + * + ****/ +#pragma once + +#include "Partition.h" +#include "Iterator.h" + +namespace Storage +{ +// Used Partition table entries cached in RAM, initialised on first request +class PartitionTable +{ +public: + PartitionTable(Device& device) : mDevice(device) + { + } + + /** + * @name Partition search + * @{ + * + * @brief Find partitions based on one or more parameters + * @param type Partition type + * @param subtype Partition sub-type + * @retval Iterator Forward-iterator for matching partitions + */ + Iterator find(Partition::Type type = Partition::Type::any, uint8_t subType = Partition::SubType::any) const + { + return Iterator(mDevice, type, subType); + } + + template Iterator find(T subType) const + { + return find(Partition::Type(T::partitionType), uint8_t(subType)); + } + + /** @} */ + + /** + * @brief Find partition by name + * @param Name Name to search for, case-sensitive + * @retval Partition + * + * Names are unique so at most only one match + */ + Partition find(const String& name) const + { + return *std::find(begin(), end(), name); + } + + /** + * @brief Find partition containing the given address + * @param address Address to search for + * @retval Partition + */ + Partition find(uint32_t address) const + { + return *std::find_if(begin(), end(), [address](Partition part) { return part.contains(address); }); + } + + /** + * @brief Find the n'th OTA partition + */ + Partition findOta(uint8_t index) + { + using App = Partition::SubType::App; + auto subtype = App(uint8_t(App::ota0) + index); + return (subtype >= App::ota_min && subtype <= App::ota_max) ? *find(subtype) : Partition{}; + } + + Iterator begin() const + { + return Iterator(mDevice, 0); + } + + Iterator end() const + { + return Iterator(mDevice, mCount); + } + + uint8_t count() const + { + return mCount; + } + + Device& device() const + { + return mDevice; + } + + Partition operator[](unsigned index) const + { + return (index < mCount) ? Partition(mDevice, mEntries.get()[index]) : Partition(); + } + +protected: + friend Device; + void load(const esp_partition_info_t* entry, unsigned count); + + Device& mDevice; + std::unique_ptr mEntries; + uint8_t mCount{0}; +}; + +} // namespace Storage diff --git a/Sming/Components/Storage/src/include/Storage/ProgMem.h b/Sming/Components/Storage/src/include/Storage/ProgMem.h new file mode 100644 index 0000000000..d8758244d0 --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage/ProgMem.h @@ -0,0 +1,89 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * ProgMem.h + * + ****/ +#pragma once + +#include "CustomDevice.h" + +namespace Storage +{ +/** + * @brief Storage device to access PROGMEM using flash API + */ +class ProgMem : public CustomDevice +{ +public: + String getName() const override + { + return F("ProgMem"); + } + + size_t getBlockSize() const override + { + return sizeof(uint32_t); + } + + size_t getSize() const override + { + return 0x80000000; + } + + Type getType() const override + { + return Type::flash; + } + + bool read(uint32_t address, void* dst, size_t size) override; + + bool write(uint32_t address, const void* src, size_t size) override + { + return false; + } + + bool erase_range(uint32_t address, size_t size) override + { + return false; + } + + using CustomDevice::createPartition; + + /** + * @brief Create partition for PROGMEM data access + * @param name Name for partition + * @param flashPtr PROGMEM pointer + * @param size Size of PROGMEM data + * @param type Partition type + * @param subtype Partition sub-type + * @retval Partition Invalid if data is not progmem + */ + Partition createPartition(const String& name, const void* flashPtr, size_t size, Partition::Type type, + uint8_t subtype); + + template Partition createPartition(const String& name, const void* flashPtr, size_t size, T subType) + { + return createPartition(name, flashPtr, size, Partition::Type(T::partitionType), uint8_t(subType)); + } + + /** + * @brief Create partition for FlashString data access + */ + Partition createPartition(const String& name, const FSTR::ObjectBase& fstr, Partition::Type type, uint8_t subtype) + { + return createPartition(name, fstr.data(), fstr.size(), type, subtype); + } + + template Partition createPartition(const String& name, const FSTR::ObjectBase& fstr, T subType) + { + return createPartition(name, fstr, Partition::Type(T::partitionType), uint8_t(subType)); + } +}; + +extern ProgMem progMem; + +} // namespace Storage diff --git a/Sming/Components/Storage/src/include/Storage/SpiFlash.h b/Sming/Components/Storage/src/include/Storage/SpiFlash.h new file mode 100644 index 0000000000..c6ac39119a --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage/SpiFlash.h @@ -0,0 +1,40 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * SpiFlash.h + * + ****/ +#pragma once + +#include "Device.h" + +namespace Storage +{ +extern SpiFlash* spiFlash; + +/** + * @brief Main flash storage device + */ +class SpiFlash : public Device +{ +public: + String getName() const override; + size_t getBlockSize() const override; + size_t getSize() const override; + + Type getType() const override + { + return Type::flash; + } + + uint32_t getId() const; + + bool read(uint32_t address, void* dst, size_t size) override; + bool write(uint32_t address, const void* src, size_t size) override; + bool erase_range(uint32_t address, size_t size) override; +}; + +} // namespace Storage diff --git a/Sming/Components/Storage/src/include/Storage/StreamDevice.h b/Sming/Components/Storage/src/include/Storage/StreamDevice.h new file mode 100644 index 0000000000..18ca2d64cf --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage/StreamDevice.h @@ -0,0 +1,67 @@ +/* + * StreamDevice.h + */ + +#include "CustomDevice.h" +#include + +namespace Storage +{ +/** + * @brief Read-only partition on a stream object + * @note Writes not possible as streams always append data, cannot do random writes + */ +class StreamDevice : public CustomDevice +{ +public: + StreamDevice(IDataSourceStream* stream, size_t size) : CustomDevice(nameOf(stream), size), mStream(stream) + { + } + + StreamDevice(IDataSourceStream* stream) : StreamDevice(stream, size_t(stream->available())) + { + } + + static String nameOf(IDataSourceStream* stream) + { + String s; + if(stream != nullptr) { + s = stream->getName(); + } + if(!s) { + s = F("stream_") + String(uint32_t(stream), HEX); + } + return s; + } + + Type getType() const override + { + return Type::stream; + } + + bool read(uint32_t address, void* buffer, size_t len) override + { + if(mStream == nullptr) { + return false; + } + if(mStream->seekFrom(address, SeekOrigin::Start) != int(address)) { + return false; + } + return mStream->readBytes(static_cast(buffer), len) == len; + } + + bool write(uint32_t address, const void* data, size_t len) override + { + return false; + } + + bool erase_range(uint32_t address, size_t len) override + { + return false; + } + +private: + std::unique_ptr mStream; +}; + +} // namespace Storage diff --git a/Sming/Components/Storage/src/include/Storage/SysMem.h b/Sming/Components/Storage/src/include/Storage/SysMem.h new file mode 100644 index 0000000000..9f49a3fa3e --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage/SysMem.h @@ -0,0 +1,88 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * SysMem.h + * + ****/ + +#pragma once + +#include "CustomDevice.h" + +namespace Storage +{ +/** + * @brief Storage device to access system memory, e.g. RAM + */ +class SysMem : public CustomDevice +{ +public: + String getName() const override + { + return F("SysMem"); + } + + size_t getBlockSize() const override + { + return sizeof(uint32_t); + } + + size_t getSize() const override + { + return 0x80000000; + } + + Type getType() const override + { + return Type::sysmem; + } + + bool read(uint32_t address, void* buffer, size_t len) override + { + if(isFlashPtr(reinterpret_cast(address))) { + memcpy_P(buffer, reinterpret_cast(address), len); + } else { + memcpy(buffer, reinterpret_cast(address), len); + } + return true; + } + + bool write(uint32_t address, const void* data, size_t len) override + { + if(isFlashPtr(reinterpret_cast(address))) { + return false; + } + + memcpy(reinterpret_cast(address), data, len); + return true; + } + + bool erase_range(uint32_t address, size_t len) override + { + if(isFlashPtr(reinterpret_cast(address))) { + return false; + } + + memset(&address, 0xFF, len); + return true; + } + + using CustomDevice::createPartition; + + /** + * @brief Create partition for FlashString data access + */ + Partition createPartition(const String& name, const FSTR::ObjectBase& fstr, Partition::Type type, uint8_t subtype); + + template Partition createPartition(const String& name, const FSTR::ObjectBase& fstr, T subType) + { + return createPartition(name, fstr, Partition::Type(T::partitionType), uint8_t(subType)); + } +}; + +extern SysMem sysMem; + +} // namespace Storage diff --git a/Sming/Components/Storage/src/include/Storage/partition_info.h b/Sming/Components/Storage/src/include/Storage/partition_info.h new file mode 100644 index 0000000000..a4b54c7f62 --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage/partition_info.h @@ -0,0 +1,32 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * partition_info.h + * + ****/ + +#include "Partition.h" + +namespace Storage +{ +/** + * @brief Internal structure describing the binary layout of a partition table entry. + */ +struct esp_partition_info_t { + uint16_t magic; ///< Fixed value to identify valid entry, appears as 0xFFFF at end of table + Partition::Type type; ///< Main type of partition + uint8_t subtype; ///< Sub-type for partition (interpretation dependent upon type) + uint32_t offset; ///< Start offset + uint32_t size; ///< Size of partition in bytes + Storage::Partition::Name name; ///< Unique identifer for entry + Storage::Partition::Flags flags; ///< Various option flags +}; + +constexpr uint16_t ESP_PARTITION_MAGIC{0x50AA}; ///< Identifies a valid partition +constexpr uint16_t ESP_PARTITION_MAGIC_MD5{0xEBEB}; ///< Identifies an MD5 hash block +constexpr size_t ESP_PARTITION_TABLE_MAX_LEN{0xC00}; // Maximum length of partition table data + +} // namespace Storage diff --git a/Sming/Components/bearssl-esp8266/bearssl b/Sming/Components/bearssl-esp8266/bearssl index 5c771bed8b..c0b69dfb83 160000 --- a/Sming/Components/bearssl-esp8266/bearssl +++ b/Sming/Components/bearssl-esp8266/bearssl @@ -1 +1 @@ -Subproject commit 5c771bed8b11b968e2f75f26b9b00de51f2e9333 +Subproject commit c0b69dfb837f0027180c72f13f7c90cfafb83c16 diff --git a/Sming/Components/esptool/README.rst b/Sming/Components/esptool/README.rst index df11b055fa..43acb7c806 100644 --- a/Sming/Components/esptool/README.rst +++ b/Sming/Components/esptool/README.rst @@ -8,18 +8,26 @@ Options .. envvar:: SPI_SPEED + [read-only] Set by :ref:`hardware_config`. + Clock speed for flash memory (20, 26, 40 or 80). Default is 40. .. envvar:: SPI_MODE - Flash memory operating mode (quot, dio, dout, qio). Default is qio. + [read-only] Set by :ref:`hardware_config`. + Flash memory operating mode (quot, dio, dout, qio). Default is qio. .. envvar:: SPI_SIZE + [read-only] Set by :ref:`hardware_config`. + Size of flash memory chip (256KB, 512KB, 1MB, 2MB, 4MB). Default is 512K bytes. + The default hardware profile ``standard`` sets this to 1MB. + You can set ``HWCONFIG=standard-4m`` to increase it or create a custom :ref:`hardware_config` for your project. + .. envvar:: ESPTOOL diff --git a/Sming/Components/esptool/component.mk b/Sming/Components/esptool/component.mk index c2f8d6d22c..2a4cf0bbc0 100644 --- a/Sming/Components/esptool/component.mk +++ b/Sming/Components/esptool/component.mk @@ -1,44 +1,11 @@ COMPONENT_LIBNAME := -CONFIG_VARS += SPI_SPEED SPI_MODE SPI_SIZE +DEBUG_VARS += SPI_SPEED SPI_MODE SPI_SIZE +SPI_SPEED = $(STORAGE_DEVICE_spiFlash_SPEED) +SPI_MODE = $(STORAGE_DEVICE_spiFlash_MODE) +SPI_SIZE = $(STORAGE_DEVICE_spiFlash_SIZE) -# SPI_SPEED = 40, 26, 20, 80 -SPI_SPEED ?= 40 -# SPI_MODE: qio, qout, dio, dout -SPI_MODE ?= dio -# SPI_SIZE: 512K, 256K, 1M, 2M, 4M -SPI_SIZE ?= 1M - -ifeq ($(SPI_SPEED), 26) - flashimageoptions := -ff 26m -else ifeq ($(SPI_SPEED), 20) - flashimageoptions := -ff 20m -else ifeq ($(SPI_SPEED), 80) - flashimageoptions := -ff 80m -else - flashimageoptions := -ff 40m -endif - -ifeq ($(SPI_MODE), qout) - flashimageoptions += -fm qout -else ifeq ($(SPI_MODE), dio) - flashimageoptions += -fm dio -else ifeq ($(SPI_MODE), dout) - flashimageoptions += -fm dout -else - flashimageoptions += -fm qio -endif - -# Calculate parameters from SPI_SIZE value (esptool will check validity) -ifeq ($(SPI_SIZE),detect) -flashimageoptions += -fs detect -else -flashimageoptions += -fs $(SPI_SIZE)B -endif -FLASH_SIZE := $(subst M,*1024K,$(SPI_SIZE)) -FLASH_SIZE := $(subst K,*1024,$(FLASH_SIZE)) -FlashOffset = $$(($(FLASH_SIZE)-$1)) -BLANK_BIN := $(COMPONENT_PATH)/blank.bin +flashimageoptions += -fs $(SPI_SIZE)B -ff $(SPI_SPEED)m -fm $(SPI_MODE) # Default COM port and speed used for flashing CACHE_VARS += COM_PORT_ESPTOOL COM_SPEED_ESPTOOL @@ -74,6 +41,8 @@ else ESPTOOL_EXECUTE = $(ESPTOOL_CMDLINE) $1 endif +comma := , + # Write file contents to Flash # $1 -> List of `Offset=File` chunks define WriteFlash @@ -83,6 +52,21 @@ define WriteFlash ) endef +# Read flash memory into file +# $1 -> `Offset,Size` chunk +# $2 -> Output filename +define ReadFlash + $(info ReadFlash $1,$2) + $(call ESPTOOL_EXECUTE,read_flash $(subst $(comma), ,$1) $2) +endef + +# Erase a region of Flash +# $1 -> Offset,Size +define EraseFlashRegion + $(info EraseFlashRegion $1) + $(call ESPTOOL_EXECUTE,erase_region $(subst $(comma), ,$1)) +endef + # Erase flash memory contents define EraseFlash $(call ESPTOOL_EXECUTE,erase_flash) diff --git a/Sming/Components/mdns/README.rst b/Sming/Components/mdns/README.rst deleted file mode 100644 index b5714dd685..0000000000 --- a/Sming/Components/mdns/README.rst +++ /dev/null @@ -1,53 +0,0 @@ -mDNS: Multicast Domain Name System -================================== - -.. highlight:: c++ - -https://en.wikipedia.org/wiki/Multicast_DNS - -Sming provides the :cpp:class:`mDNS::Responder` class to allow applications -to advertise themselves on the local network. - -Issuing discovery requests is not currently supported. - - -Using ------ - -1. Add ``COMPONENT_DEPENDS += mdns`` to your application componenent.mk file. -2. Add these lines to your application:: - - #include - - static mDNS::Responder responder; - - // Call when IP address has been obtained - void startmDNS() - { - responder.begin(F("myhostname"); - } - -This will advertise the device as ``myhostname.local``. - -To provide a custom service, implement a :cpp:class:`mDNS::Service` class -and call :cpp:func:`mDNS::Responder::addService`. - -See the :sample:`UdpServer_mDNS` sample application. - - -Testing -------- - -For linux, you can use `avahi `__ -to perform mDNS queries and confirm output is as expected: - -.. code-block:: bash - - sudo apt install avahi - avahi-browse --all -r - - -API Documentation ------------------ - -.. doxygennamespace:: mDNS diff --git a/Sming/Components/mdns/component.mk b/Sming/Components/mdns/component.mk deleted file mode 100644 index f07f849a3f..0000000000 --- a/Sming/Components/mdns/component.mk +++ /dev/null @@ -1,14 +0,0 @@ -COMPONENT_SRCDIRS := src -COMPONENT_INCDIRS := include - -COMPONENT_DOXYGEN_INPUT := include - -COMPONENT_VARS += ENABLE_CUSTOM_LWIP - -# If using Espressif LWIP... -ifeq ($(ENABLE_CUSTOM_LWIP),0) -ENABLE_ESPCONN := 1 -GLOBAL_CFLAGS += -DENABLE_ESPCONN -endif - -GLOBAL_CFLAGS += -DENABLE_CUSTOM_LWIP=$(ENABLE_CUSTOM_LWIP) diff --git a/Sming/Components/mdns/include/Network/Mdns/Responder.h b/Sming/Components/mdns/include/Network/Mdns/Responder.h deleted file mode 100644 index c1afb8773a..0000000000 --- a/Sming/Components/mdns/include/Network/Mdns/Responder.h +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include "Service.h" - -namespace mDNS -{ -class Responder -{ -public: - /** - * @brief Initialise the responder - * @param hostname - * @retval bool true on success - */ - bool begin(const String& hostname); - - /** - * @brief Stop the responder - * - * Must reinitialise the stack again to restart - */ - void end(); - - /** - * @brief Add a service object - * - * Responder doesn't own the object, must remain in scope - * - * @param svc Service object - * @retval bool true on success - */ - bool addService(Service& svc); - - /** - * @brief Restart a running stack - * - * Call when IP address changes, for example. - */ - bool restart(); - -private: -#if LWIP_VERSION_MAJOR == 1 - String hostname; - Service* service = nullptr; -#endif -}; - -} // namespace mDNS diff --git a/Sming/Components/mdns/include/Network/Mdns/Service.h b/Sming/Components/mdns/include/Network/Mdns/Service.h deleted file mode 100644 index baa485ed4b..0000000000 --- a/Sming/Components/mdns/include/Network/Mdns/Service.h +++ /dev/null @@ -1,72 +0,0 @@ -#pragma once - -#include -#include -#include - -#if ENABLE_CUSTOM_LWIP == 2 -#ifdef ARCH_ESP8266 -#define MDNS_LWIP 0x0200 -#else -#define MDNS_LWIP 0x0202 -#endif -#elif ENABLE_CUSTOM_LWIP == 1 -#define MDNS_LWIP 0x0101 -#else -#define MDNS_LWIP 0x0100 -#endif - -namespace mDNS -{ -class Responder; - -class Service -{ -public: - enum class Protocol { - Udp /* = DNSSD_PROTO_UCP */, - Tcp /* = DNSSD_PROTO_TCP */, - }; - - /** - * @brief Basic service information - * - * Defaults fine for most cases, just need to set name. - */ - struct Info { - String name = "Sming"; - String type = "http"; - Protocol protocol = Protocol::Tcp; - uint16_t port = 80; - }; - - /** - * @brief Override to obtain service information - */ - virtual Info getInfo() = 0; - - /** - * @brief Override to obtain txt items - * - * LWIP2 calls this each time a TXT reply is created - * Other implementations call it via begin(). - * - * @retval CStringArray List of txt items, e.g. name=value pairs - */ - virtual CStringArray getTxt() - { - return nullptr; - } - -private: - friend class Responder; - -#if MDNS_LWIP >= 0x0200 - int8_t id = -1; -#else - String type; - CStringArray txt; -#endif -}; - -} // namespace mDNS diff --git a/Sming/Components/mdns/src/Responder.cpp b/Sming/Components/mdns/src/Responder.cpp deleted file mode 100644 index de00cf2159..0000000000 --- a/Sming/Components/mdns/src/Responder.cpp +++ /dev/null @@ -1,215 +0,0 @@ -#include -#include - -#if MDNS_LWIP >= 0x0200 - -extern "C" { -#ifndef LWIP_MDNS_RESPONDER -#define LWIP_MDNS_RESPONDER 1 -#endif -#include -} - -#ifdef ARCH_ESP8266 -extern struct netif* netif_default_LWIP2; -#define netif_default netif_default_LWIP2 -#endif - -#else - -extern "C" { -#if MDNS_LWIP > 0x0100 -#include -#endif - -// Sets the service name -void mdns_set_name(const char* name); -} - -#include - -#endif - -#define MDNS_TTL_DEFAULT 120 - -namespace mDNS -{ -bool Responder::begin(const String& hostname) -{ -#if MDNS_LWIP >= 0x0200 - - mdns_resp_init(); - -#if MDNS_LWIP >= 0x0202 - err_t err = mdns_resp_add_netif(netif_default, hostname.c_str()); - if(err == ERR_OK) { - System.queueCallback(InterruptCallback([]() { mdns_resp_announce(netif_default); })); - } -#else - err_t err = mdns_resp_add_netif(netif_default, hostname.c_str(), MDNS_TTL_DEFAULT); -#endif - - if(err != ERR_OK) { - debug_e("mdns_resp_add_netif failed, err = %d", err); - return false; - } - -#else - - this->hostname = hostname; - - mdns_info mi{}; - mi.host_name = this->hostname.begin(); - mi.server_name = const_cast("Sming"); - mi.ipAddr = WifiStation.getIP(); - -#ifdef ENABLE_ESPCONN - espconn_mdns_init(&mi); -#else - mdns_init(&mi); -#endif - -#endif - - debug_i("MDNS initialised for '%s'", hostname.c_str()); - return true; -} - -bool Responder::addService(Service& svc) -{ - auto si = svc.getInfo(); - debug_i("MDNS addService '%s'", si.name.c_str()); - -#if MDNS_LWIP >= 0x0200 - - if(svc.id >= 0) { - debug_e("MDNS: Service already in use"); - return false; - } - - auto getSrvTxt = [](mdns_service* service, void* userdata) { - auto txt = static_cast(userdata)->getTxt(); - err_t res = 0; - for(auto s : txt) { - res |= mdns_resp_add_service_txtitem(service, s, strlen(s)); - } - debug_i("mdns add service: res = %d", res); - }; - - String serviceType = String('_') + si.type; -#if MDNS_LWIP >= 0x0202 - auto id = mdns_resp_add_service(netif_default, si.name.c_str(), serviceType.c_str(), mdns_sd_proto(si.protocol), - si.port, getSrvTxt, &svc); -#else - auto id = mdns_resp_add_service(netif_default, si.name.c_str(), serviceType.c_str(), mdns_sd_proto(si.protocol), - si.port, MDNS_TTL_DEFAULT, getSrvTxt, &svc); -#endif - if(id < 0) { - debug_w("mdns_resp_add_service failed, err = %d", id); - return false; - } - - debug_i("mdns_resp_add_service returned %d", id); - - svc.id = id; - -#else - - // Service already registered? - if(service != nullptr) { - debug_e("MDNS: Service already registered"); - return false; - } - - // Initialise text - auto txt = svc.getTxt(); - if(txt.count() > ARRAY_SIZE(mdns_info::txt_data)) { - debug_e("Too many MDNS TXT items"); - return false; - } - - // Have to tear down then re-initialise -#ifdef ENABLE_ESPCONN - espconn_mdns_close(); -#else - mdns_close(); -#endif - - service = &svc; - - svc.type = std::move(si.type); - svc.txt = std::move(txt); - - mdns_info mi{}; - mi.host_name = hostname.begin(); - mi.server_name = svc.type.begin(); - mi.ipAddr = WifiStation.getIP(); - mi.server_port = si.port; - - unsigned i = 0; - for(auto s : svc.txt) { - mi.txt_data[i++] = const_cast(s); - } - -#ifdef ENABLE_ESPCONN - espconn_mdns_init(&mi); -#else - mdns_init(&mi); -#endif - - // We can now set the service name - mdns_set_name(si.name.c_str()); -#endif - - return true; -} - -void Responder::end() -{ -#if MDNS_LWIP >= 0x0200 - - mdns_resp_remove_netif(netif_default); - -#else - -#ifdef ENABLE_ESPCONN - espconn_mdns_close(); -#else - mdns_close(); -#endif - - if(service != nullptr) { - service->type = nullptr; - service->txt = nullptr; - service = nullptr; - } - -#endif - - debug_i("MDNS stopped"); -} - -bool Responder::restart() -{ - debug_i("MDNS restarting..."); - -#if MDNS_LWIP >= 0x0202 - - mdns_resp_restart(netif_default); - return true; - -#elif MDNS_LWIP >= 0x0200 - - mdns_resp_netif_settings_changed(netif_default); - return true; - -#else - - auto svc = service; - end(); - return svc ? addService(*svc) : begin(hostname); - -#endif -} - -} // namespace mDNS diff --git a/Sming/Components/mqtt-codec b/Sming/Components/mqtt-codec index 172f792ee2..3767dfce41 160000 --- a/Sming/Components/mqtt-codec +++ b/Sming/Components/mqtt-codec @@ -1 +1 @@ -Subproject commit 172f792ee2b90b6f13631390886c52ec2f4b6362 +Subproject commit 3767dfce416ebb0ea42903a4bbd93f5675be6cdb diff --git a/Sming/Components/rboot/.patches/rboot.patch b/Sming/Components/rboot/.patches/rboot.patch deleted file mode 100644 index ee164e45ac..0000000000 --- a/Sming/Components/rboot/.patches/rboot.patch +++ /dev/null @@ -1,389 +0,0 @@ -diff --git a/Makefile b/Makefile -index 0b43474..5176f8f 100644 ---- a/Makefile -+++ b/Makefile -@@ -58,6 +58,19 @@ endif - ifeq ($(RBOOT_IROM_CHKSUM),1) - CFLAGS += -DBOOT_IROM_CHKSUM - endif -+ifneq ($(RBOOT_ROM0_ADDR),) -+ CFLAGS += -DBOOT_ROM0_ADDR=$(RBOOT_ROM0_ADDR) -+endif -+ifneq ($(RBOOT_ROM1_ADDR),) -+ CFLAGS += -DBOOT_ROM1_ADDR=$(RBOOT_ROM1_ADDR) -+endif -+ifneq ($(RBOOT_ROM2_ADDR),) -+ CFLAGS += -DBOOT_ROM2_ADDR=$(RBOOT_ROM2_ADDR) -+endif -+ifeq ($(RBOOT_SILENT),1) -+ CFLAGS += -DBOOT_SILENT=$(RBOOT_SILENT) -+endif -+ - ifneq ($(RBOOT_EXTRA_INCDIR),) - CFLAGS += $(addprefix -I,$(RBOOT_EXTRA_INCDIR)) - endif -@@ -75,6 +88,10 @@ else ifeq ($(SPI_SIZE), 2Mb) - E2_OPTS += -2048b - else ifeq ($(SPI_SIZE), 4M) - E2_OPTS += -4096 -+else ifeq ($(SPI_SIZE), 8M) -+ E2_OPTS += -8192 -+else ifeq ($(SPI_SIZE), 16M) -+ E2_OPTS += -16384 - endif - ifeq ($(SPI_MODE), qio) - E2_OPTS += -qio -diff --git a/rboot.c b/rboot.c -index d622f97..5e254fa 100644 ---- a/rboot.c -+++ b/rboot.c -@@ -13,6 +13,12 @@ - #define UART_CLK_FREQ (26000000 * 2) - #endif - -+#ifndef BOOT_SILENT -+#define echof(fmt, ...) ets_printf(fmt, ##__VA_ARGS__) -+#else -+#define echof(fmt, ...) -+#endif -+ - static uint32_t check_image(uint32_t readpos) { - - uint8_t buffer[BUFFER_SIZE]; -@@ -257,8 +263,30 @@ static uint8_t calc_chksum(uint8_t *start, uint8_t *end) { - // created on first boot or in case of corruption - static uint8_t default_config(rboot_config *romconf, uint32_t flashsize) { - romconf->count = 2; -+#ifdef BOOT_ROM0_ADDR -+ romconf->roms[0] = BOOT_ROM0_ADDR; -+#else - romconf->roms[0] = SECTOR_SIZE * (BOOT_CONFIG_SECTOR + 1); -+#endif -+ -+#ifdef BOOT_ROM1_ADDR -+ romconf->roms[1] = BOOT_ROM1_ADDR; -+#else - romconf->roms[1] = (flashsize / 2) + (SECTOR_SIZE * (BOOT_CONFIG_SECTOR + 1)); -+#endif -+ -+#if defined(BOOT_BIG_FLASH) && defined(BOOT_GPIO_ENABLED) -+ if(flashsize > 0x200000) { -+ romconf->count += 1; -+#ifdef BOOT_ROM2_ADDR -+ romconf->roms[2] = BOOT_ROM2_ADDR; -+#else -+ romconf->roms[2] = 0x310000; -+#endif -+ romconf->gpio_rom = 2; -+} -+#endif -+ - #ifdef BOOT_GPIO_ENABLED - romconf->mode = MODE_GPIO_ROM; - #endif -@@ -306,100 +334,100 @@ uint32_t NOINLINE find_image(void) { - ets_delay_us(BOOT_DELAY_MICROS); - #endif - -- ets_printf("\r\nrBoot v1.4.2 - richardaburton@gmail.com\r\n"); -+ echof("\r\nrBoot v1.4.2 - richardaburton@gmail.com\r\n"); - - // read rom header - SPIRead(0, header, sizeof(rom_header)); - - // print and get flash size -- ets_printf("Flash Size: "); -+ echof("Flash Size: "); - flag = header->flags2 >> 4; - if (flag == 0) { -- ets_printf("4 Mbit\r\n"); -+ echof("4 Mbit\r\n"); - flashsize = 0x80000; - } else if (flag == 1) { -- ets_printf("2 Mbit\r\n"); -+ echof("2 Mbit\r\n"); - flashsize = 0x40000; - } else if (flag == 2) { -- ets_printf("8 Mbit\r\n"); -+ echof("8 Mbit\r\n"); - flashsize = 0x100000; - } else if (flag == 3 || flag == 5) { -- ets_printf("16 Mbit\r\n"); -+ echof("16 Mbit\r\n"); - #ifdef BOOT_BIG_FLASH - flashsize = 0x200000; - #else - flashsize = 0x100000; // limit to 8Mbit - #endif - } else if (flag == 4 || flag == 6) { -- ets_printf("32 Mbit\r\n"); -+ echof("32 Mbit\r\n"); - #ifdef BOOT_BIG_FLASH - flashsize = 0x400000; - #else - flashsize = 0x100000; // limit to 8Mbit - #endif - } else if (flag == 8) { -- ets_printf("64 Mbit\r\n"); -+ echof("64 Mbit\r\n"); - #ifdef BOOT_BIG_FLASH - flashsize = 0x800000; - #else - flashsize = 0x100000; // limit to 8Mbit - #endif - } else if (flag == 9) { -- ets_printf("128 Mbit\r\n"); -+ echof("128 Mbit\r\n"); - #ifdef BOOT_BIG_FLASH - flashsize = 0x1000000; - #else - flashsize = 0x100000; // limit to 8Mbit - #endif - } else { -- ets_printf("unknown\r\n"); -+ echof("unknown\r\n"); - // assume at least 4mbit - flashsize = 0x80000; - } - - // print spi mode -- ets_printf("Flash Mode: "); -+ echof("Flash Mode: "); - if (header->flags1 == 0) { -- ets_printf("QIO\r\n"); -+ echof("QIO\r\n"); - } else if (header->flags1 == 1) { -- ets_printf("QOUT\r\n"); -+ echof("QOUT\r\n"); - } else if (header->flags1 == 2) { -- ets_printf("DIO\r\n"); -+ echof("DIO\r\n"); - } else if (header->flags1 == 3) { -- ets_printf("DOUT\r\n"); -+ echof("DOUT\r\n"); - } else { -- ets_printf("unknown\r\n"); -+ echof("unknown\r\n"); - } - - // print spi speed -- ets_printf("Flash Speed: "); -+ echof("Flash Speed: "); - flag = header->flags2 & 0x0f; -- if (flag == 0) ets_printf("40 MHz\r\n"); -- else if (flag == 1) ets_printf("26.7 MHz\r\n"); -- else if (flag == 2) ets_printf("20 MHz\r\n"); -- else if (flag == 0x0f) ets_printf("80 MHz\r\n"); -- else ets_printf("unknown\r\n"); -+ if (flag == 0) echof("40 MHz\r\n"); -+ else if (flag == 1) echof("26.7 MHz\r\n"); -+ else if (flag == 2) echof("20 MHz\r\n"); -+ else if (flag == 0x0f) echof("80 MHz\r\n"); -+ else echof("unknown\r\n"); - - // print enabled options - #ifdef BOOT_BIG_FLASH -- ets_printf("rBoot Option: Big flash\r\n"); -+ echof("rBoot Option: Big flash\r\n"); - #endif - #ifdef BOOT_CONFIG_CHKSUM -- ets_printf("rBoot Option: Config chksum\r\n"); -+ echof("rBoot Option: Config chksum\r\n"); - #endif - #ifdef BOOT_GPIO_ENABLED -- ets_printf("rBoot Option: GPIO rom mode (%d)\r\n", BOOT_GPIO_NUM); -+ echof("rBoot Option: GPIO rom mode (%d)\r\n", BOOT_GPIO_NUM); - #endif - #ifdef BOOT_GPIO_SKIP_ENABLED -- ets_printf("rBoot Option: GPIO skip mode (%d)\r\n", BOOT_GPIO_NUM); -+ echof("rBoot Option: GPIO skip mode (%d)\r\n", BOOT_GPIO_NUM); - #endif - #ifdef BOOT_RTC_ENABLED -- ets_printf("rBoot Option: RTC data\r\n"); -+ echof("rBoot Option: RTC data\r\n"); - #endif - #ifdef BOOT_IROM_CHKSUM -- ets_printf("rBoot Option: irom chksum\r\n"); -+ echof("rBoot Option: irom chksum\r\n"); - #endif -- ets_printf("\r\n"); -+ echof("\r\n"); - - // read boot config - SPIRead(BOOT_CONFIG_SECTOR * SECTOR_SIZE, buffer, SECTOR_SIZE); -@@ -410,7 +438,7 @@ uint32_t NOINLINE find_image(void) { - #endif - ) { - // create a default config for a standard 2 rom setup -- ets_printf("Writing default boot config.\r\n"); -+ echof("Writing default boot config.\r\n"); - ets_memset(romconf, 0x00, sizeof(rboot_config)); - romconf->magic = BOOT_CONFIG_MAGIC; - romconf->version = BOOT_CONFIG_VERSION; -@@ -433,10 +461,10 @@ uint32_t NOINLINE find_image(void) { - - if (rtc.next_mode & MODE_TEMP_ROM) { - if (rtc.temp_rom >= romconf->count) { -- ets_printf("Invalid temp rom selected.\r\n"); -+ echof("Invalid temp rom selected.\r\n"); - return 0; - } -- ets_printf("Booting temp rom.\r\n"); -+ echof("Booting temp rom.\r\n"); - temp_boot = 1; - romToBoot = rtc.temp_rom; - } -@@ -447,10 +475,10 @@ uint32_t NOINLINE find_image(void) { - if (perform_gpio_boot(romconf)) { - #if defined(BOOT_GPIO_ENABLED) - if (romconf->gpio_rom >= romconf->count) { -- ets_printf("Invalid GPIO rom selected.\r\n"); -+ echof("Invalid GPIO rom selected.\r\n"); - return 0; - } -- ets_printf("Booting GPIO-selected rom.\r\n"); -+ echof("Booting GPIO-selected rom.\r\n"); - romToBoot = romconf->gpio_rom; - gpio_boot = 1; - #elif defined(BOOT_GPIO_SKIP_ENABLED) -@@ -462,7 +490,7 @@ uint32_t NOINLINE find_image(void) { - #endif - updateConfig = 1; - if (romconf->mode & MODE_GPIO_ERASES_SDKCONFIG) { -- ets_printf("Erasing SDK config sectors before booting.\r\n"); -+ echof("Erasing SDK config sectors before booting.\r\n"); - for (sec = 1; sec < 5; sec++) { - SPIEraseSector((flashsize / SECTOR_SIZE) - sec); - } -@@ -474,7 +502,7 @@ uint32_t NOINLINE find_image(void) { - // gpio/temp boots will have already validated this - if (romconf->current_rom >= romconf->count) { - // if invalid rom selected try rom 0 -- ets_printf("Invalid rom selected, defaulting to 0.\r\n"); -+ echof("Invalid rom selected, defaulting to 0.\r\n"); - romToBoot = 0; - romconf->current_rom = 0; - updateConfig = 1; -@@ -486,14 +514,14 @@ uint32_t NOINLINE find_image(void) { - #ifdef BOOT_GPIO_ENABLED - if (gpio_boot && loadAddr == 0) { - // don't switch to backup for gpio-selected rom -- ets_printf("GPIO boot rom (%d) is bad.\r\n", romToBoot); -+ echof("GPIO boot rom (%d) is bad.\r\n", romToBoot); - return 0; - } - #endif - #ifdef BOOT_RTC_ENABLED - if (temp_boot && loadAddr == 0) { - // don't switch to backup for temp rom -- ets_printf("Temp boot rom (%d) is bad.\r\n", romToBoot); -+ echof("Temp boot rom (%d) is bad.\r\n", romToBoot); - // make sure rtc temp boot mode doesn't persist - rtc.next_mode = MODE_STANDARD; - rtc.chksum = calc_chksum((uint8_t*)&rtc, (uint8_t*)&rtc.chksum); -@@ -504,7 +532,7 @@ uint32_t NOINLINE find_image(void) { - - // check we have a good rom - while (loadAddr == 0) { -- ets_printf("Rom %d at %x is bad.\r\n", romToBoot, romconf->roms[romToBoot]); -+ echof("Rom %d at %x is bad.\r\n", romToBoot, romconf->roms[romToBoot]); - // for normal mode try each previous rom - // until we find a good one or run out - updateConfig = 1; -@@ -512,7 +540,7 @@ uint32_t NOINLINE find_image(void) { - if (romToBoot < 0) romToBoot = romconf->count - 1; - if (romToBoot == romconf->current_rom) { - // tried them all and all are bad! -- ets_printf("No good rom available.\r\n"); -+ echof("No good rom available.\r\n"); - return 0; - } - loadAddr = check_image(romconf->roms[romToBoot]); -@@ -543,7 +571,7 @@ uint32_t NOINLINE find_image(void) { - system_rtc_mem(RBOOT_RTC_ADDR, &rtc, sizeof(rboot_rtc_data), RBOOT_RTC_WRITE); - #endif - -- ets_printf("Booting rom %d at %x, load addr %x.\r\n", romToBoot, romconf->roms[romToBoot], loadAddr); -+ echof("Booting rom %d at %x, load addr %x.\r\n", romToBoot, romconf->roms[romToBoot], loadAddr); - // copy the loader to top of iram - ets_memcpy((void*)_text_addr, _text_data, _text_len); - // return address to load from -diff --git a/appcode/rboot-api.c b/appcode/rboot-api.c -index eb4d028..490a763 100644 ---- a/appcode/rboot-api.c -+++ b/appcode/rboot-api.c -@@ -9,7 +9,7 @@ - #include - // c_types.h needed for spi_flash.h - #include --#include -+#include - - #include "rboot-api.h" - -@@ -33,7 +33,7 @@ static uint8_t calc_chksum(uint8_t *start, uint8_t *end) { - // get the rboot config - rboot_config ICACHE_FLASH_ATTR rboot_get_config(void) { - rboot_config conf; -- spi_flash_read(BOOT_CONFIG_SECTOR * SECTOR_SIZE, (uint32_t*)&conf, sizeof(rboot_config)); -+ flashmem_read(&conf, BOOT_CONFIG_SECTOR * SECTOR_SIZE, sizeof(rboot_config)); - return conf; - } - -@@ -43,7 +43,7 @@ rboot_config ICACHE_FLASH_ATTR rboot_get_config(void) { - // updates checksum automatically (if enabled) - bool ICACHE_FLASH_ATTR rboot_set_config(rboot_config *conf) { - uint8_t *buffer; -- buffer = (uint8_t*)pvPortMalloc(SECTOR_SIZE, 0, 0); -+ buffer = (uint8_t*)os_malloc(SECTOR_SIZE); - if (!buffer) { - //os_printf("No ram!\r\n"); - return false; -@@ -53,12 +53,12 @@ bool ICACHE_FLASH_ATTR rboot_set_config(rboot_config *conf) { - conf->chksum = calc_chksum((uint8_t*)conf, (uint8_t*)&conf->chksum); - #endif - -- spi_flash_read(BOOT_CONFIG_SECTOR * SECTOR_SIZE, (uint32_t*)((void*)buffer), SECTOR_SIZE); -+ flashmem_read(buffer, BOOT_CONFIG_SECTOR * SECTOR_SIZE, SECTOR_SIZE); - memcpy(buffer, conf, sizeof(rboot_config)); -- spi_flash_erase_sector(BOOT_CONFIG_SECTOR); -- spi_flash_write(BOOT_CONFIG_SECTOR * SECTOR_SIZE, (uint32_t*)((void*)buffer), SECTOR_SIZE); -+ flashmem_erase_sector(BOOT_CONFIG_SECTOR); -+ flashmem_write(buffer, BOOT_CONFIG_SECTOR * SECTOR_SIZE, SECTOR_SIZE); - -- vPortFree(buffer, 0, 0); -+ os_free(buffer); - return true; - } - -@@ -114,7 +114,7 @@ bool ICACHE_FLASH_ATTR rboot_write_flash(rboot_write_status *status, uint8_t *da - } - - // get a buffer -- buffer = (uint8_t *)pvPortMalloc(len + status->extra_count, 0, 0); -+ buffer = (uint8_t *)os_malloc(len + status->extra_count); - if (!buffer) { - //os_printf("No ram!\r\n"); - return false; -@@ -141,18 +141,18 @@ bool ICACHE_FLASH_ATTR rboot_write_flash(rboot_write_status *status, uint8_t *da - lastsect = ((status->start_addr + len) - 1) / SECTOR_SIZE; - while (lastsect > status->last_sector_erased) { - status->last_sector_erased++; -- spi_flash_erase_sector(status->last_sector_erased); -+ flashmem_erase_sector(status->last_sector_erased); - } - - // write current chunk - //os_printf("write addr: 0x%08x, len: 0x%04x\r\n", status->start_addr, len); -- if (spi_flash_write(status->start_addr, (uint32_t *)((void*)buffer), len) == SPI_FLASH_RESULT_OK) { -+ if (flashmem_write(buffer, status->start_addr, len) == len) { - ret = true; - status->start_addr += len; - } - //} - -- vPortFree(buffer, 0, 0); -+ os_free(buffer); - return ret; - } - diff --git a/Sming/Components/rboot/.patches/rboot/README.rst b/Sming/Components/rboot/.patches/rboot/README.rst deleted file mode 100644 index e365cdb1f0..0000000000 --- a/Sming/Components/rboot/.patches/rboot/README.rst +++ /dev/null @@ -1,349 +0,0 @@ -================================================== -rBoot - An open source boot loader for the ESP8266 -================================================== - -by Richard A Burton, richardaburton@gmail.com -http://richard.burtons.org/ - -rBoot is designed to be a flexible open source boot loader, a -replacement for the binary blob supplied with the SDK. It has the -following advantages over the Espressif loader: - -- Open source (written in C). -- Supports up to 256 roms. -- Roms can be variable size. -- Able to test multiple roms to find a valid backup (without - resetting). -- Flash layout can be changed on the fly (with care and appropriately - linked rom images). -- GPIO support for rom selection. -- Wastes no stack space (SDK boot loader uses 144 bytes). -- Documented config structure to allow easy editing from user code. -- Can validate .irom0.text section with checksum. -- Temporary next-boot rom selection. - -Limitations -=========== - -The ESP8266 can only map 8Mbits (1MB) of flash to memory, but which -8Mbits to map is selectable. This allows individual roms to be up to 1MB -in size, so long as they do not straddle an 8Mbit boundary on the flash. -This means you could have four 1MB roms or 8 512KB roms on a 32Mbit -flash (such as on the ESP-12), or a combination. Note, however, that you -could not have, for example, a 512KB rom followed immediately by a 1MB -rom because the 2nd rom would then straddle an 8MBit boundary. By -default support for using more than the first 8Mbit of the flash is -disabled, because it requires several steps to get it working. See below -for instructions. - -Building -======== - -A Makefile is included, which should work with the gcc xtensa cross -compiler. There are two source files, the first is compiled and included -as data in the second. When run this code is copied to memory and -executed (there is a good reason for this, see my blog for an -explanation). The make file will handle this for you, but you’ll need my -esptool2 (see github). - -To use the Makefile set ``SDK_BASE`` to point to the root of the -Espressif SDK and either set ``XTENSA_BINDIR`` to the gcc xtensa bin -directory or include it in your ``PATH``. These can be set as -environment variables or by editing the Makefile. - -Two small assembler stub functions allow the bootloader to launch the -user code without reserving any space on the stack (while the SDK boot -loader uses 144 bytes). This compiles fine with GCC, but if you use -another compiler and it will not compile/work for you then uncomment the -``#define BOOT_NO_ASM`` in ``rboot.h`` to use a C version of these -functions (this uses 32 bytes). - -Tested with SDK v2.2 and GCC v4.8.5. - -Installation -============ - -Simply write rboot.bin to the first sector of the flash. Remember to set -your flash size correctly with your chosen flash tool (e.g. for -esptool.py use the ``-fs`` option). When run rBoot will create it’s own -config at the start of sector two for a simple two rom system. You can -can then write your two roms to flash addresses ``0x2000`` and (half -chip size + ``0x2000``). E.g. for 8Mbit flash: -``esptool.py write_flash -fs 8m 0x0000 rboot.bin 0x2000 user1.bin 0x82000 user2.bin`` - -Note: your device may need other options specified. E.g. The nodemcu -devkit v1.0 (commonly, but incorrectly, sold as v2) also needs the -``-fm dio`` option. - -For more interesting rom layouts you’ll need to write an rBoot config -sector manually, see next step. - -The two testload bin files can be flashed in place of normal user roms -for testing rBoot. You do not need these for normal use. - -rBoot Config -============ - -:: - - typedef struct { - uint8_t magic; // our magic - uint8_t version; // config struct version - uint8_t mode; // boot loader mode - uint8_t current_rom; // currently selected rom - uint8_t gpio_rom; // rom to use for gpio boot - uint8_t count; // number of roms in use - uint8_t unused[2]; // padding - uint32_t roms[MAX_ROMS]; // flash addresses of the roms - #ifdef BOOT_CONFIG_CHKSUM - uint8_t chksum; // boot config chksum - #endif - } rboot_config; - -Write a config structure as above to address ``0x1000`` on the flash. If -you want more than 4 roms (default) just increase MAX_ROMS when you -compile rBoot. Think about how you intend to layout your flash before -you start! Rom addresses must be sector aligned i.e start on a multiple -of 4096. - -- ``magic`` should have value ``0xe1`` (defined as - ``BOOT_CONFIG_MAGIC``). -- ``version`` is used in case the config structure changes after - deployment. It is defined as ``0x01`` (``BOOT_CONFIG_VERSION``). I - don’t intend to increase this, but you should if you choose to - reflash the bootloader after deployment and the config structure has - changed. -- ``mode`` can be ``0x00`` (``MODE_STANDARD``) or ``0x01`` - (``MODE_GPIO_ROM``). See below for an explanation of - ``MODE_GPIO_ROM``. There is also an optional extra mode flag ``0x04`` - (``MODE_GPIO_ERASES_SDKCONFIG``), see below for details. -- ``current_rom`` is the rom to boot, numbered ``0`` to ``count-1``. -- ``gpio_rom`` is the rom to boot when the GPIO is triggered at boot. -- ``count`` is the number of roms available (may be less than - ``MAX_ROMS``, but not more). -- ``unused[2]`` is padding so the ``uint32_t`` rom addresses are 4 - bytes aligned. -- ``roms`` is the array of flash address for the roms. The default - generated config will contain two entries: ``0x00002000`` and - ``0x00082000``. -- ``chksum`` (if enabled, not by deafult) should be the xor of ``0xef`` - followed by each of the bytes of the config structure up to (but - obviously not including) the chksum byte itself. - -Default config -============== - -A default config sector will be created on boot if one does not exists, -or if an existing config is corrupted, and the default rom will be set -to rom 0. If you want to have a very customised config for which the -default would not be suitable, you can override the implementation in -the ``rboot.h`` header file. See the comments and example code in -``rboot.h`` for more information. - -GPIO boot mode -============== - -.. envvar:: RBOOT_GPIO_ENABLED - -If rBoot is compiled with ``BOOT_GPIO_ENABLED`` set in ``rboot.h`` (or -``RBOOT_GPIO_ENABLED`` set in the Makefile), then GPIO boot -functionality will be included in the rBoot binary. The feature can then -be enabled by setting the rboot_config ``mode`` field to -``MODE_GPIO_ROM``. You must also set ``gpio_rom`` in the config to -indicate which rom to boot when the GPIO is activated at boot. - -If the GPIO input pin reads high at boot then rBoot will start the -currently selected normal or temp rom (as appropriate). However if the -GPIO is pulled low then the rom indicated in config option ``gpio_rom`` -is started instead. - -The default GPIO is 16, but this can be overriden in the Makefile -(``RBOOT_GPIO_NUMBER``) or ``rboot.h`` (``BOOT_GPIO_NUM``). If GPIOs -other than 16 are used, the internal pullup resistor is enabled before -the pin is read and disabled immediately afterwards. For pins that -default on reset to configuration other than GPIO input, the pin mode is -changed to input when reading but changed back before rboot continues. - -After a GPIO boot the ``current_rom`` field will be updated in the -config, so the GPIO booted rom should change this again if required. - -GPIO boot skip mode -=================== - -.. envvar:: RBOOT_GPIO_SKIP_ENABLED - -If rBoot is compiled with ``BOOT_GPIO_SKIP_ENABLED`` set in ``rboot.h`` -(or ``RBOOT_GPIO_SKIP_ENABLED`` set in the Makefile), then a GPIO can be -used to skip to the next rom at boot. The feature must then be enabled -by setting the rboot_config ‘mode’ field to ``MODE_GPIO_SKIP``. This -means you do not need to have a dedicated GPIO boot rom. If you have a -rom that is technically good (valid checksum, etc.) but has operational -problems, e.g. wifi doesn’t work or it crashes on boot, rBoot will not -be able to detect that and switch rom automatically. In this scenario -rebooting the device while pulling the GPIO low will force rBoot to skip -this rom and try the next one instead. In a simple two rom setup this -simply toggles booting of the other rom. - -``RBOOT_GPIO_SKIP_ENABLED`` and ``RBOOT_GPIO_ENABLED`` cannot be used at -the same time. ``BOOT_GPIO_NUM`` is used to select the GPIO pin, as with -``RBOOT_GPIO_ENABLED``. - -Erasing SDK configuration on GPIO boot (rom or skip mode) -========================================================= - -If you set the ``MODE_GPIO_ERASES_SDKCONFIG`` flag in the configuration -like this: ``conf.mode = MODE_GPIO_ROM|MODE_GPIO_ERASES_SDKCONFIG``; -then a GPIO boot will also the erase the Espressif SDK persistent -settings store in the final 16KB of flash. This includes removing -calibration constants, saved SSIDs, etc. - -Note that ``MODE_GPIO_ERASES_SDKCONFIG`` is a flag, so it has to be set -as well as ``MODE_GPIO_ROM`` to take effect. - -Linking user code -================= - -Each rom will need to be linked with an appropriate linker file, -specifying where it will reside on the flash. If you are only flashing -one rom to multiple places on the flash it must be linked multiple times -to produce the set of rom images. This is the same as with the SDK -loader. - -Because there are endless possibilities for layout with this loader I -don’t supply sample linker files. Instead I’ll tell you how to make -them. - -For each rom slot on the flash take a copy of the ``eagle.app.v6.ld`` -linker script from the sdk. You then need to modify just one line in it -for each rom: -``irom0_0_seg : org = 0x40240000, len = 0x3C000`` - -Change the org address to be ``0x40200000`` (base memory mapped location -of the flash) + flash address + ``0x10`` (offset of data after the -header). The logical place for your first rom is the third sector, -address ``0x2000``. ``0x40200000 + 0x2000 + 0x10 = 0x40202010`` If you -use the default generated config the loader will expect to find the -second rom at flash address half-chip-size + ``0x2000`` -(e.g. ``0x82000`` on an 8MBit flash) so the ``irom0_0_seg`` should be: -``0x40200000 + 0x82000 + 0x10 = 0x40282010`` Due to the limitation of -mapped flash (max 8MBit) if you use a larger chip and do not have big -flash support enabled the second rom in the default config will still be -placed at ``0x082000``, not truly half-chip-size + ``0x2000``. Ideally -you should also adjust the len to help detect over sized sections at -link time, but more important is the overall size of the rom which you -need to ensure fits in the space you have allocated for it in your flash -layout plan. - -Then simply compile and link as you would normally for OTA updates with -the SDK boot loader, except using the linker scripts you’ve just -prepared rather than the ones supplied with the SDK. Remember when -building roms to create them as ‘new’ type roms (for use with SDK boot -loader v1.2+). Or if using my esptool2 use the ``-boot2`` option. Note: -the test loads included with rBoot are built with ``-boot0`` because -they do not contain a ``.irom0.text`` section (and so the value of -``irom0_0_seg`` in the linker file is irrelevant to them) but ‘normal’ -user apps always do. - -irom checksum -============= - -The SDK boot loader checksum only covers sections loaded into ram (data -and some code). Most of the SDK and user code remains on the flash and -that is not included in the checksum. This means you could attempt to -boot a corrupt rom and, because it looks ok to the boot loader, there -will be no attempt to switch to a backup rom. rBoot improves on this by -allowing the ``.irom0.text`` section to be included in the checksum. To -enable this uncomment ``#define BOOT_IROM_CHKSUM`` in ``rboot.h`` and -build your roms with esptool2 using the ``-iromchksum`` option. - -.. _big_flash_support: - -Big flash support -================= - -This only needs to be enabled if you wish to be able to memory map more -than the first 8MBit of the flash. Note you can still only map 8Mbit at -a time. Use this if you want to have multiple 1MB roms, or more smaller -roms than will fit in 8Mbits. If you have a large flash but only need, -for example, two 512KB roms you do not need to enable this mode. - -Support in rBoot is enabled by uncommenting the -``#define BOOT_BIG_FLASH`` in ``rboot.h``. - -Thinking about your linker files is either simpler or more complicated, -depending on your usage of the flash. If you intend to use multiple 1MB -roms you will only need one linker file and you only need to link once -for OTA updates. Although when you perform an OTA update the rom will be -written to a different position on the flash, each 8Mbit of flash is -mapped (separately) to ``0x40200000``. So when any given rom is run the -code will appear at the same place in memory regardless of where it is -on the flash. Your base address for the linker would be ``0x40202010``. -(Actually all but the first rom could base at ``0x40200010`` (because -they don’t need to leave space for rBoot and config) but then you’re -just making it more complicated again!) - -If you wanted eight 512KB roms you would need two linker files - one for -the first half of any given 8Mbits of flash and another for the second -half. Just remember you are really laying out within a single 8MBit -area, which can then be replicated multiple times on the flash. - -Now the clever bit - rBoot needs to hijack the memory mapping code to -select which 8Mbits gets mapped. There is no API for this, but we can -override the SDK function. First we need to slightly modify the SDK -library ``libmain.a``, like so: - -:: - - xtensa-lx106-elf-objcopy -W Cache_Read_Enable_New libmain.a libmain2.a - -This produces a version of libmain with a ‘weakened’ -``Cache_Read_Enable_New`` function, which we can then override with our -own. Modify your Makefile to link against the library ``main2`` instead -of ``main``. - -Next add ``rboot-bigflash.c`` (from the ``appcode`` directory) & -``rboot.h`` to your project - this adds the replacement -``Cache_Read_Enable_New`` to your code. - -Getting gcc to apply the override correctly can be slightly tricky (I’m -not sure why, it shouldn’t be). One option is to add -``-u Cache_Read_Enable_New`` to your ``LD_FLAGS`` and change the order -of objects on the LD command so your ``objects/.a`` file is before the -libraries. Another way that seems easier was to -``#include rboot-bigflash.c`` into the main .c file, rather than -compiling it to a separate object file. I can’t make any sense of that, -but I suggest you uncomment the message in the ``Cache_Read_Enable_New`` -function when you first build with it, to make sure you are getting your -version into the rom. - -Now when rBoot starts your rom, the SDK code linked in it that normally -performs the memory mapping will delegate part of that task to rBoot -code (linked in your rom, not in rBoot itself) to choose which part of -the flash to map. - -Temporary boot option and rBoot<–>app communication -=================================================== - -.. envvar:: RBOOT_RTC_ENABLED - -To enable communication between rBoot and your app you should enable the -``BOOT_RTC_ENABLED`` option in ``rboot.h``. rBoot will then use the RTC -data area to pass a structure with boot information which can be read by -the app. This will allow the app to determine the boot mode (normal, -temporary or GPIO) and the booted rom (even if it is a tempoary boot). -Your app can also update this structure to communicate with rBoot when -the device is next rebooted, e.g. to instruct it to temporarily boot a -different rom to the one saved in the config. See the api documentation -and/or the rBoot sample project for more details. Note: the message -“don’t use rtc mem data”, commonly seen on startup, comes from the sdk -and is not related to this rBoot feature. - -Integration into other frameworks -================================= - -If you wish to integrate rBoot into a development framework (e.g. Sming) -you can set the define ``RBOOT_INTEGRATION`` and at compile time the -file ``rboot-integration.h`` will be included into the source. This -should allow you to set some platform specific options without having to -modify the source of rBoot which makes it easier to integrate and -maintain. diff --git a/Sming/Components/rboot/README.rst b/Sming/Components/rboot/README.rst index df119b1787..95ddeb0203 100644 --- a/Sming/Components/rboot/README.rst +++ b/Sming/Components/rboot/README.rst @@ -7,7 +7,26 @@ Introduction rBoot is a second-stage bootloader that allows booting application images from several pre-configured flash memory addresses, called "slots". Sming supports up to three slots. -Sming uses rBoot exclusively because of its flexibility, reliability and ease of use. +.. note:: + + With Sming 4.3 partitions are used to manage flash storage. + A "slot" refers to a specific application partition, typically ``rom0``, ``rom1`` or ``rom2``. + + The location or size of these partitions is determined by the :ref:`hardware_config`. + + The bootloader has been modified to use the partition table as reference, identifying slots + by the partition sub-type. + + Where systems are to be updated Over the Air (OTA) at least two application partitions are required. + The bootloader identifies these by their partition subtype: ``slot #0`` -> ``App/Ota_0``, ``slot #1`` -> ``App/Ota_1``, etc. + + Fixed applications without OTA capability use a single application image. + This must be the ``App/Factory`` partition type, and corresponds to ``slot #0``. + + At startup, the bootloader will use the partition table to locate the application image. + It will also ensure that the ROM slot information in the boot configuration is consistent, + and update it if necessary. + .. attention:: @@ -19,38 +38,40 @@ Sming uses rBoot exclusively because of its flexibility, reliability and ease of Slot 0 ------ -This is the default slot which is always used. +This is the default slot (``rom0``, the primary application partition) which is always used. .. envvar:: RBOOT_ROM0_ADDR - default: 0x2000. + [read-only] - This is the start address for slot 0. The default is sector 3, immediately after rBoot and its configuration data. + This is the start address for slot 0. - Except for the use case described in `Slot2`_ below, you should not need to set this value. + Except for the use case described in `Slot2`_ below, you should not need to change this. Slot 1 ------ .. envvar:: RBOOT_ROM1_ADDR - default: disabled + [read-only] default: disabled - The start address of slot 1. If you don't need it, leave unconfigured (empty). + The start address of slot 1. If your application includes any kind of Over-the-Air (OTA) firmware update functionality, you will need a second memory slot to store the received update image while the update routines execute from the first slot. +.. note:: + + The ``spiffs-two-roms`` configuration can be used for this purpose. + Upon successful completion of the update, the second slot is activated, such that on next reset rBoot boots into the uploaded application. While now running from slot 1, the next update will be stored to slot 0 again, i.e. the roles of slot 0 and slot 1 are flipped with every update. -For devices with more than 1MB of flash memory, it is advisable to choose an address with the same -offset within its 1MB block as :envvar:`RBOOT_ROM0_ADDR`, e.g. 0x102000 for the default -slot 0 address (0x2000). This way, the same application image can be used for both -slots. See :ref:`single_vs_dual` for further details. +For devices with more than 1MB of flash memory, it is advisable to choose an address +for ``rom1`` with the same offset within its 1MB block as ``rom0``. .. _Slot2: @@ -69,23 +90,61 @@ To enable slot 2, set these values: .. envvar:: RBOOT_ROM2_ADDR - Address for slot 2 + [read-only] + + Address for slot 2. You must create a custom :ref:`hardware_config` for your project + with a definition for ``rom2``. + + .. code-block:: json + + { + ... + "partitions": { + "rom2": { + "address": "0x100000", + "size": "512K", + "type": "app", + "subtype": "ota_1" + } + } + } + +.. note:: + + At present, this will only configure rBoot. + Sming will not create an application image for slot 2. + + You can, however, use a second Sming project to build a recovery application image as follows: + + - Create a new Sming project for your recovery application. This will be a simple + single-slot project. Create a new :ref:`hardware_config` and configure the + ``rom0`` start address and size to the same as the ``rom2`` partition of the main project. + + option (a) + + - Build and flash the recovery project as usual by typing ``make flash``. This will + install the recovery ROM (into slot 2 of the main project) and a temporary + bootloader, which will be overwritten in the next step. + + - Go back to your main project. Build and flash it with ``make flash``. This will + install the main application (into slot 0) and the final bootloader. You are + now all set for booting into the recovery image if the need arises. + + option (b) + + - Run a normal ``make`` for your recovery project + + - Locate the firmware image file, typically ``out/Esp8266/release/firmware/rom0.bin`` + (adjust accordingly if using a debug build). + Copy this image file as ``rom2.bin`` into your main project directory. -Note that this will only configure rBoot. -Sming will not create an application image for slot 2. -You can, however, use a second Sming project to build a recovery application image as follows: + - Add an additional property to the ``rom2`` partition entry in your main project: -1. Create a new Sming project for your recovery application. This will be a simple - single-slot project. Set :envvar:`RBOOT_ROM0_ADDR` of the recovery project to the value - of :envvar:`RBOOT_ROM2_ADDR` of the main project. + .. code-block:: json -2. Build and flash the recovery project as usual by typing ``make flash``. This will - install the recovery ROM (into slot 2 of the main project) and a temporary - bootloader, which will be overwritten in the next step. + "filename": "rom2.bin" -3. Go back to your main project. Build and flash it with ``make flash``. This will - install the main application (into slot 0) and the final bootloader. You are - now all set for booting into the recovery image if the need arises. + When you run ``make flash`` in this will get written along with the other partitions. Automatically derived settings ------------------------------ @@ -122,8 +181,9 @@ the same address offsets within their 1MB blocks, i.e. ``(RBOOT_ROM0_ADDR & 0xFF Consequently, for such configurations, the Sming build system generates only one ROM image. In all other cases, two distinct application images must be linked with different addresses -for the 'irom0_0_seg' memory region. The Sming build system takes care of all the details, -including linker script generation. +for the 'irom0_0_seg' memory region. +You should use the ``two-rom-mode`` :ref:`hardware_config` for this. +The Sming build system will handle everything else, including linker script generation. .. envvar:: RBOOT_TWO_ROMS diff --git a/Sming/Components/rboot/appcode/rboot-overrides.c b/Sming/Components/rboot/appcode/rboot-overrides.c deleted file mode 100644 index 04132ad0cf..0000000000 --- a/Sming/Components/rboot/appcode/rboot-overrides.c +++ /dev/null @@ -1,43 +0,0 @@ -/**** - * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. - * Created 2015 by Skurydin Alexey - * http://github.com/SmingHub/Sming - * All files of the Sming Core are provided under the LGPL v3 license. - * - * rboot-overrides.c - * - ****/ - -#include "spiffs_sming.h" -#include - -/* - * rBoot uses different spiffs organization and we need to override this method - * during application compile time in order to make automatic - * mounting with `spiffs_mount()` work as expected. - */ -spiffs_config spiffs_get_storage_config() -{ - spiffs_config cfg = {0}; - u32_t max_allowed_sector, requested_sector; - -#ifdef RBOOT_SPIFFS_0 - cfg.phys_addr = RBOOT_SPIFFS_0; -#elif RBOOT_SPIFFS_1 - cfg.phys_addr = RBOOT_SPIFFS_1; -#else -#error "Define either RBOOT_SPIFFS_0 or RBOOT_SPIFFS_1" -#endif - - cfg.phys_addr &= 0xFFFFF000; // get the start address of the sector - - max_allowed_sector = flashmem_get_sector_of_address(INTERNAL_FLASH_SIZE - 1); - requested_sector = flashmem_get_sector_of_address((cfg.phys_addr + SPIFF_SIZE) - 1); - if(requested_sector > max_allowed_sector) { - debug_w("The requested SPIFFS size is too big."); - requested_sector = max_allowed_sector; - } - // get the max size until the sector end - cfg.phys_size = ((requested_sector + 1) * INTERNAL_FLASH_SECTOR_SIZE) - cfg.phys_addr; - return cfg; -} diff --git a/Sming/Components/rboot/component.mk b/Sming/Components/rboot/component.mk index 7cd47cbdeb..b5ab0f3771 100644 --- a/Sming/Components/rboot/component.mk +++ b/Sming/Components/rboot/component.mk @@ -7,8 +7,8 @@ RBOOT_EMULATION := 1 endif COMPONENT_SUBMODULES := rboot -COMPONENT_INCDIRS := rboot appcode rboot/appcode include -COMPONENT_SRCDIRS := src +COMPONENT_INCDIRS := rboot rboot/appcode include +COMPONENT_SRCDIRS := src src/Arch/$(SMING_ARCH) RBOOT_DIR := $(COMPONENT_PATH) @@ -35,23 +35,22 @@ $(error Cannot enable RBOOT_GPIO_ENABLED and RBOOT_GPIO_SKIP_ENABLED at the same endif ### ROM Addresses ### -# Make sure that your ROM slots and SPIFFS slot(s) do not overlap! -CONFIG_VARS += RBOOT_ROM0_ADDR RBOOT_ROM1_ADDR RBOOT_ROM2_ADDR +DEBUG_VARS += RBOOT_ROM0_ADDR RBOOT_ROM1_ADDR RBOOT_ROM2_ADDR -# Loation of first ROM (default is sector 2 after rboot and rboot config sector) -RBOOT_ROM0_ADDR ?= 0x002000 +# Location of first ROM +RBOOT_ROM0_ADDR := $(PARTITION_rom0_ADDRESS) # The parameter below specifies the location of the second ROM. # You need a second slot for any kind of firmware update mechanism. # Leave empty if you don't need a second ROM slot. -RBOOT_ROM1_ADDR ?= +RBOOT_ROM1_ADDR := $(PARTITION_rom1_ADDRESS) # The parameter below specifies the location of the GPIO ROM. # It is only used when RBOOT_GPIO_ENABLED = 1 # Note that setting this parameter will only configure rboot. # The Sming build system does not create a ROM for this slot. -RBOOT_ROM2_ADDR ?= +RBOOT_ROM2_ADDR := $(PARTITION_rom2_ADDRESS) ifeq ($(RBOOT_GPIO_ENABLED),0) ifneq ($(RBOOT_ROM2_ADDR),) @@ -86,24 +85,16 @@ RBOOT_LD_TEMPLATE ?= $(RBOOT_DIR)/rboot.rom0.ld RBOOT_LD_0 := $(BUILD_BASE)/$(RBOOT_ROM_0).ld RBOOT_LD_1 := $(BUILD_BASE)/$(RBOOT_ROM_1).ld -# -CONFIG_VARS += RBOOT_SPIFFS_0 RBOOT_SPIFFS_1 -RBOOT_SPIFFS_0 ?= 0x100000 -RBOOT_SPIFFS_1 ?= 0x300000 -APP_CFLAGS += -DRBOOT_SPIFFS_0=$(RBOOT_SPIFFS_0) -APP_CFLAGS += -DRBOOT_SPIFFS_1=$(RBOOT_SPIFFS_1) - -SPIFF_START_ADDR ?= $(RBOOT_SPIFFS_0) - # filenames and options for generating rBoot rom images with esptool2 RBOOT_E2_SECTS ?= .text .text1 .data .rodata RBOOT_E2_USER_ARGS ?= -quiet -bin -boot2 +DEBUG_VARS += RBOOT_ROM_0_BIN RBOOT_ROM_1_BIN RBOOT_ROM_0_BIN := $(FW_BASE)/$(RBOOT_ROM_0).bin RBOOT_ROM_1_BIN := $(FW_BASE)/$(RBOOT_ROM_1).bin -COMPONENT_APPCODE := appcode rboot/appcode $(if $(RBOOT_EMULATION),host) +COMPONENT_APPCODE := rboot/appcode APP_CFLAGS += -DRBOOT_INTEGRATION # these are exported for use by the rBoot Makefile @@ -133,11 +124,24 @@ ifeq ($(RBOOT_GPIO_SKIP_ENABLED),1) APP_CFLAGS += -DBOOT_GPIO_SKIP_ENABLED endif -ifndef RBOOT_EMULATION +COMPONENT_CXXFLAGS += \ + -DRBOOT_ROM0_ADDR=$(RBOOT_ROM0_ADDR) \ + -DRBOOT_ROM1_ADDR=$(RBOOT_ROM1_ADDR) \ + -DPARTITION_TABLE_OFFSET=$(PARTITION_TABLE_OFFSET) + +COMPONENT_CFLAGS += \ + -DPARTITION_TABLE_OFFSET=$(PARTITION_TABLE_OFFSET) + +ifdef RBOOT_EMULATION +FLASH_BOOT_CHUNKS = 0x00000=$(BLANK_BIN) +FLASH_RBOOT_ERASE_CONFIG_CHUNKS := 0x01000=$(BLANK_BIN) +else +export RBOOT_ROM0_ADDR +export RBOOT_ROM1_ADDR RBOOT_BIN := $(FW_BASE)/rboot.bin CUSTOM_TARGETS += $(RBOOT_BIN) $(RBOOT_BIN): - $(Q) $(MAKE) -C $(RBOOT_DIR)/rboot + $(Q) $(MAKE) -C $(RBOOT_DIR)/rboot PARTITION_TABLE_OFFSET=$(PARTITION_TABLE_OFFSET) $(RBOOT_CFLAGS) EXTRA_LDFLAGS := -u Cache_Read_Enable_New @@ -151,14 +155,9 @@ endef LIBMAIN_COMMANDS += $(RBOOT_LIBMAIN_COMMANDS) endif -endif # RBOOT_EMULATION - # Define our flash chunks -FLASH_RBOOT_BOOT_CHUNKS := 0x00000=$(RBOOT_BIN) -FLASH_RBOOT_APP_CHUNKS := $(RBOOT_ROM0_ADDR)=$(RBOOT_ROM_0_BIN) -FLASH_RBOOT_ERASE_CONFIG_CHUNKS := 0x01000=$(SDK_BASE)/bin/blank.bin - -ifndef RBOOT_EMULATION +FLASH_BOOT_CHUNKS := 0x00000=$(RBOOT_BIN) +FLASH_RBOOT_ERASE_CONFIG_CHUNKS := 0x01000=$(BLANK_BIN) # => Automatic linker script generation from template # $1 -> application target diff --git a/Sming/Components/rboot/include/Data/Stream/RbootOutputStream.h b/Sming/Components/rboot/include/Data/Stream/RbootOutputStream.h index 30c8b85db8..1873d01885 100644 --- a/Sming/Components/rboot/include/Data/Stream/RbootOutputStream.h +++ b/Sming/Components/rboot/include/Data/Stream/RbootOutputStream.h @@ -17,6 +17,7 @@ #include #include +#include /** * @brief Write-only stream type used during rBoot firmware updates @@ -25,13 +26,23 @@ class RbootOutputStream : public ReadWriteStream { public: /** + * @brief Construct a stream using raw flash address/size * @param startAddress the start address on the storage media * @param maxLength the maximum allowed length of the rom. Use 0 if unlimited. + * @note This should be avoided, use partition where possible */ RbootOutputStream(uint32_t startAddress, size_t maxLength = 0) : startAddress(startAddress), maxLength(maxLength) { } + /** + * @brief Construct a stream for the given partition + * @param partition + */ + RbootOutputStream(Storage::Partition partition) : startAddress(partition.address()), maxLength(partition.size()) + { + } + virtual ~RbootOutputStream() { close(); @@ -77,11 +88,11 @@ class RbootOutputStream : public ReadWriteStream } protected: - bool initialized = false; - rboot_write_status rBootWriteStatus = {}; - size_t written = 0; // << the number of written bytes - uint32_t startAddress = 0; // << the start address on the storage - size_t maxLength = 0; // << maximum allowed length + bool initialized{false}; + rboot_write_status rBootWriteStatus{}; + size_t written{0}; // << the number of written bytes + uint32_t startAddress{0}; // << the start address on the storage + size_t maxLength{0}; // << maximum allowed length protected: virtual bool init(); diff --git a/Sming/Components/rboot/include/Network/RbootHttpUpdater.h b/Sming/Components/rboot/include/Network/RbootHttpUpdater.h index 3a6ac65c46..2100626770 100644 --- a/Sming/Components/rboot/include/Network/RbootHttpUpdater.h +++ b/Sming/Components/rboot/include/Network/RbootHttpUpdater.h @@ -18,32 +18,101 @@ #include #include "../Data/Stream/RbootOutputStream.h" -#define NO_ROM_SWITCH 0xff +/** + * @brief Magic value for ROM slot indicating slot won't change after successful OTA + */ +constexpr uint8_t NO_ROM_SWITCH{0xff}; class RbootHttpUpdater; -typedef Delegate OtaUpdateDelegate; - -struct RbootHttpUpdaterItem { - String url; - uint32_t targetOffset; - size_t size; // << max allowed size - RbootOutputStream* stream = nullptr; // (optional) output stream to use. -}; +using OtaUpdateDelegate = Delegate; class RbootHttpUpdater : protected HttpClient { public: - virtual ~RbootHttpUpdater() + struct Item { + String url; + uint32_t targetOffset; + size_t size; // << max allowed size + RbootOutputStream* stream{nullptr}; // (optional) output stream to use. + + Item(String url, uint32_t targetOffset, size_t size, RbootOutputStream* stream) + : url(url), targetOffset(targetOffset), size(size), stream(stream) + { + } + + ~Item() + { + delete stream; + } + + RbootOutputStream* getStream() + { + if(stream == nullptr) { + stream = new RbootOutputStream(targetOffset, size); + } + return stream; + } + }; + + class ItemList : public Vector { - cleanup(); + public: + bool addNew(Item* it) + { + if(addElement(it)) { + return true; + } + delete it; + return false; + } + }; + + /** + * @brief Add an item to update + * @param offset + * @param firmwareFileUrl + * @param maxSize + * @retval bool + * @note Use the `Partition` overload where possible + */ + bool addItem(uint32_t offset, const String& firmwareFileUrl, size_t maxSize = 0) + { + return items.addNew(new Item{firmwareFileUrl, offset, maxSize, nullptr}); + } + + /** + * @brief Add an item to update + * @param firmwareFileUrl + * @param partition Target partition to write + * @retval bool + */ + bool addItem(const String& firmwareFileUrl, Storage::Partition partition) + { + return addItem(partition.address(), firmwareFileUrl, partition.size()); } - bool addItem(int offset, const String& firmwareFileUrl, size_t maxSize = 0); - bool addItem(const String& firmwareFileUrl, RbootOutputStream* stream = nullptr); + /** + * @brief Add an item to update use a pre-constructed stream + * @param firmwareFileUrl + * @param stream + * @retval bool + */ + bool addItem(const String& firmwareFileUrl, RbootOutputStream* stream) + { + if(stream == nullptr) { + return false; + } + + return items.addNew(new Item{firmwareFileUrl, stream->getStartAddress(), stream->getMaxLength(), stream}); + } void start(); + /** + * @brief On completion, switch to the given ROM slot + * @param romSlot specify NO_ROM_SWITCH (the default) to cancel any previously set switch + */ void switchToRom(uint8_t romSlot) { this->romSlot = romSlot; @@ -59,11 +128,13 @@ class RbootHttpUpdater : protected HttpClient this->updateDelegate = reqUpdateDelegate; } - /* Sets the base request that can be used to pass - * - default request parameters, like request headers... - * - default SSL options - * - default SSL fingeprints - * - default SSL client certificates + /** + * @brief Sets the base request that can be used to pass + * + * - default request parameters, like request headers... + * - default SSL options + * - default SSL fingeprints + * - default SSL client certificates * * @param request */ @@ -72,10 +143,21 @@ class RbootHttpUpdater : protected HttpClient baseRequest = request; } - // Allow reading items - RbootHttpUpdaterItem getItem(unsigned int index) + /** + * @brief Allow reading items + * @deprecated Access list directly using `getItems()` + */ + const Item& getItem(unsigned int index) const SMING_DEPRECATED + { + return items[index]; + } + + /** + * @brief Allow read access to item list + */ + const ItemList& getItems() const { - return items.elementAt(index); + return items; } protected: @@ -86,24 +168,16 @@ class RbootHttpUpdater : protected HttpClient virtual int updateComplete(HttpConnection& client, bool success); protected: - Vector items; - int currentItem = 0; - rboot_write_status rbootWriteStatus; - uint8_t romSlot = NO_ROM_SWITCH; - OtaUpdateDelegate updateDelegate = nullptr; - - HttpRequest* baseRequest = nullptr; - -private: - void cleanup() - { - for(unsigned i = 0; i < items.count(); i++) { - delete items[i].stream; - items[i].stream = nullptr; - } - items.clear(); - } + ItemList items; + OtaUpdateDelegate updateDelegate; + HttpRequest* baseRequest{nullptr}; + uint8_t romSlot{NO_ROM_SWITCH}; + uint8_t currentItem{0}; + rboot_write_status rbootWriteStatus{}; }; /** @deprecated Use `RbootHttpUpdater` */ typedef RbootHttpUpdater rBootHttpUpdate SMING_DEPRECATED; + +/** @deprecated Use 'auto' in expressions or `RbootHttpUpdater::Item` */ +typedef RbootHttpUpdater::Item RbootHttpUpdaterItem SMING_DEPRECATED; diff --git a/Sming/Components/rboot/appcode/rboot-integration.h b/Sming/Components/rboot/include/rboot-integration.h similarity index 100% rename from Sming/Components/rboot/appcode/rboot-integration.h rename to Sming/Components/rboot/include/rboot-integration.h diff --git a/Sming/Components/rboot/rboot b/Sming/Components/rboot/rboot index 24f6021da8..4ad3ba2a8f 160000 --- a/Sming/Components/rboot/rboot +++ b/Sming/Components/rboot/rboot @@ -1 +1 @@ -Subproject commit 24f6021da82b303262a939c955f6a984bce6208f +Subproject commit 4ad3ba2a8f6d48d8ab90e0e7c67d42cac49f4da3 diff --git a/Sming/Components/rboot/host/rboot.cpp b/Sming/Components/rboot/src/Arch/Host/rboot.cpp similarity index 56% rename from Sming/Components/rboot/host/rboot.cpp rename to Sming/Components/rboot/src/Arch/Host/rboot.cpp index 755d942c8d..a75031fb6f 100644 --- a/Sming/Components/rboot/host/rboot.cpp +++ b/Sming/Components/rboot/src/Arch/Host/rboot.cpp @@ -13,6 +13,16 @@ #include #include +static uint32_t SPIRead(uint32_t addr, void* outptr, uint32_t len) +{ + return (flashmem_read(outptr, addr, len) == len) ? 0 : 1; +} + +#define SPIEraseSector(sector) flashmem_erase_sector(sector) +#define echof(fmt, ...) host_debug_i(fmt, ##__VA_ARGS__) + +#include + /* * Called from emulator startup code after flash has been initialised. */ @@ -20,18 +30,14 @@ void host_init_bootloader() { rboot_config romconf = rboot_get_config(); - bool init; - // fresh install or old version? + bool init = false; if(romconf.magic != BOOT_CONFIG_MAGIC) { - hostmsg("MAGIC mismatch"); + host_debug_w("MAGIC mismatch"); init = true; } else if(romconf.version != BOOT_CONFIG_VERSION) { - hostmsg("VERSION mismatch: %u found, current %u", romconf.version, BOOT_CONFIG_VERSION); + host_debug_w("VERSION mismatch: %u found, current %u", romconf.version, BOOT_CONFIG_VERSION); init = true; - } else { - // OK - init = false; } if(init) { @@ -39,16 +45,22 @@ void host_init_bootloader() memset(&romconf, 0, sizeof(romconf)); romconf.magic = BOOT_CONFIG_MAGIC; romconf.version = BOOT_CONFIG_VERSION; - romconf.count = 2; - romconf.roms[0] = SECTOR_SIZE * (BOOT_CONFIG_SECTOR + 1); -#ifdef BOOT_ROM1_ADDR - romconf.roms[1] = BOOT_ROM1_ADDR; -#else - size_t flashsize = flashmem_get_size_bytes(); - romconf.roms[1] = (flashsize / 2) + (SECTOR_SIZE * (BOOT_CONFIG_SECTOR + 1)); -#endif + } + + // Read ROM locations from partition table + bool config_changed = false; + if(scan_partitions(&romconf)) { + config_changed = true; + } + + if(romconf.count == 0) { + host_debug_w("Note: No App partitions found\r\n"); + return; + } + + if(init || config_changed) { bool ok = rboot_set_config(&romconf); - hostmsg("Write default config: %s", ok ? "OK" : "FAIL"); + host_debug_i("Update rBoot config: %s", ok ? "OK" : "FAIL"); } String addr; @@ -58,5 +70,5 @@ void host_init_bootloader() } addr += '#' + String(i) + ": 0x" + String(romconf.roms[i], 16); } - hostmsg("ROM count = %u, current = %u, %s", romconf.count, romconf.current_rom, addr.c_str()); + host_debug_i("ROM count = %u, current = %u, %s", romconf.count, romconf.current_rom, addr.c_str()); } diff --git a/Sming/Components/rboot/src/RbootHttpUpdater.cpp b/Sming/Components/rboot/src/RbootHttpUpdater.cpp index 077d9697da..78ed79e27a 100644 --- a/Sming/Components/rboot/src/RbootHttpUpdater.cpp +++ b/Sming/Components/rboot/src/RbootHttpUpdater.cpp @@ -15,36 +15,13 @@ #include -bool RbootHttpUpdater::addItem(int offset, const String& firmwareFileUrl, size_t maxSize) -{ - RbootHttpUpdaterItem add; - add.targetOffset = offset; - add.url = firmwareFileUrl; - add.size = maxSize; - add.stream = nullptr; - return items.add(add); -} - -bool RbootHttpUpdater::addItem(const String& firmwareFileUrl, RbootOutputStream* stream) -{ - if(stream == nullptr) { - return false; - } - - RbootHttpUpdaterItem add; - add.targetOffset = stream->getStartAddress(); - add.url = firmwareFileUrl; - add.size = stream->getMaxLength(); - add.stream = stream; - - return items.add(add); -} - void RbootHttpUpdater::start() { for(unsigned i = 0; i < items.count(); i++) { - RbootHttpUpdaterItem& it = items[i]; - debug_d("Download file:\r\n (%d) %s -> %X", currentItem, it.url.c_str(), it.targetOffset); + auto& it = items[i]; + debug_d("Download file:\r\n" + " (%u) %s -> %X", + currentItem, it.url.c_str(), it.targetOffset); HttpRequest* request; if(baseRequest != nullptr) { @@ -55,12 +32,7 @@ void RbootHttpUpdater::start() } request->setMethod(HTTP_GET); - - if(it.stream == nullptr) { - it.stream = new RbootOutputStream(it.targetOffset, it.size); - } - - request->setResponseStream(it.stream); + request->setResponseStream(it.getStream()); if(i == items.count() - 1) { request->onRequestComplete(RequestCompletedDelegate(&RbootHttpUpdater::updateComplete, this)); @@ -82,8 +54,8 @@ int RbootHttpUpdater::itemComplete(HttpConnection& client, bool success) return -1; } - RbootHttpUpdaterItem& it = items[currentItem]; - debug_d("Finished: URL: %s, Offset: %d, Length: %d", it.url.c_str(), it.stream->getStartAddress(), + auto& it = items[currentItem]; + debug_d("Finished: URL: %s, Offset: 0x%X, Length: %u", it.url.c_str(), it.stream->getStartAddress(), it.stream->available()); it.stream = nullptr; // the actual deletion will happen outside of this class @@ -94,14 +66,14 @@ int RbootHttpUpdater::itemComplete(HttpConnection& client, bool success) int RbootHttpUpdater::updateComplete(HttpConnection& client, bool success) { - auto hasError = itemComplete(client, success); - if(hasError) { + int hasError = itemComplete(client, success); + if(hasError != 0) { return hasError; } debug_d("\r\nFirmware download finished!"); for(unsigned i = 0; i < items.count(); i++) { - debug_d(" - item: %d, addr: %X, url: %s", i, items[i].targetOffset, items[i].url.c_str()); + debug_d(" - item: %u, addr: 0x%X, url: %s", i, items[i].targetOffset, items[i].url.c_str()); } if(!success) { @@ -124,19 +96,19 @@ void RbootHttpUpdater::updateFailed() if(updateDelegate) { updateDelegate(*this, false); } - cleanup(); + items.clear(); } void RbootHttpUpdater::applyUpdate() { - cleanup(); + items.clear(); if(romSlot == NO_ROM_SWITCH) { debug_d("Firmware updated."); return; } // set to boot new rom and then reboot - debug_d("Firmware updated, rebooting to rom %d...\r\n", romSlot); + debug_d("Firmware updated, rebooting to rom %u...\r\n", romSlot); rboot_set_current_rom(romSlot); System.restart(); } diff --git a/Sming/Components/rboot/src/RbootOutputStream.cpp b/Sming/Components/rboot/src/RbootOutputStream.cpp index 5fea78dff2..e7b71e4eb5 100644 --- a/Sming/Components/rboot/src/RbootOutputStream.cpp +++ b/Sming/Components/rboot/src/RbootOutputStream.cpp @@ -33,12 +33,12 @@ size_t RbootOutputStream::write(const uint8_t* data, size_t size) initialized = true; } - if(maxLength && (written + size > maxLength)) { + if(maxLength != 0 && (written + size > maxLength)) { debug_e("The ROM size is bigger than the maximum allowed"); return 0; } - if(!rboot_write_flash(&rBootWriteStatus, (uint8_t*)data, size)) { + if(!rboot_write_flash(&rBootWriteStatus, data, size)) { debug_e("rboot_write_flash: Failed. Size: %d", size); return 0; } diff --git a/Sming/Components/spiffs/README.rst b/Sming/Components/spiffs/README.rst index 7f2792eb57..3ac089846f 100644 --- a/Sming/Components/spiffs/README.rst +++ b/Sming/Components/spiffs/README.rst @@ -1,36 +1,47 @@ SPIFFS for Sming ================ -This Component provides additional functionality to support SPIFFS running on both Esp8266 and Host architectures. +This Component provides SPIFFS filesystem support for all architectures. -.. envvar:: DISABLE_SPIFFS +A single SPIFFS partition is defined using :envvar:`HWCONFIG` ``=spiffs``, which supports these build variables: - 0 (default) - Enable filesystem generation - - 1 - Disable filesystem generation + .. envvar:: DISABLE_SPIFFS - The value is also #defined for application use. + [deprecated and removed] - Note this doesn't actually disable SPIFFS support in the application! + This value is no longer supported. Please remove it from your project's component.mk file. -.. envvar:: SPIFF_SIZE + .. envvar:: SPIFF_SIZE - Size (in bytes) of the SPIFFS area in Flash memory. + [deprecated and removed] + Size (in bytes) of the SPIFFS area in Flash memory. To change this, edit the :ref:`hardware_config`. -.. envvar:: SPIFF_FILES + .. envvar:: SPIFF_FILES - default: ``files`` + .. envvar:: SPIFF_FILES - The SPIFFS image is built using files from this directory. + default: ``files`` + The SPIFFS image is built using files from this directory, which must exist or the build will fail. -.. envvar:: SPIFF_BIN + If you set this to an empty value, then an empty filesystem will be created. + + + .. envvar:: SPIFF_BIN + + Filename to use for the generated SPIFFS filesystem image. The default is ``spiff_rom``. + + + .. envvar:: SPIFF_BIN_OUT + + [read-only] Shows the full path to the generated image file. + + +For more control over the SPIFFS partition you can create your own partition definition in a +custom :ref:`hardware_config`. - Path to the generated SPIFFS filesystem image. .. envvar:: SPIFF_FILEDESC_COUNT @@ -38,3 +49,13 @@ This Component provides additional functionality to support SPIFFS running on bo Default: 7 Number of file descriptors allocated. This sets the maximum number of files which may be opened at once. + + +.. envvar:: SPIFFS_OBJ_META_LEN + + Default: 16 + + Maximum size of metadata which SPIFFS stores in each file index header (after the filename). + If this value is changed, existing SPIFFS images will not be readable. + + The default value given here is provided to support :component:`IFS` extended file attribute information. diff --git a/Sming/Components/spiffs/blankfs.bin b/Sming/Components/spiffs/blankfs.bin deleted file mode 100644 index 171fa2bbce..0000000000 --- a/Sming/Components/spiffs/blankfs.bin +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/Sming/Components/spiffs/component.mk b/Sming/Components/spiffs/component.mk index e9019a6382..68954b98b2 100644 --- a/Sming/Components/spiffs/component.mk +++ b/Sming/Components/spiffs/component.mk @@ -4,41 +4,18 @@ COMPONENT_SRCDIRS := . spiffs/src COMPONENT_INCDIRS := . spiffs/src COMPONENT_DOXYGEN_INPUT := spiffs/src -## Spiffy tool - -SPIFFY := $(TOOLS_BASE)/spiffy$(TOOL_EXT) -COMPONENT_TARGETS += $(SPIFFY) -$(COMPONENT_RULE)$(SPIFFY): - $(call MakeTarget,spiffy/Makefile) - - ## Application -# This controls filesystem generation, it doesn't actually disable SPIFFS support in the application -CONFIG_VARS += DISABLE_SPIFFS -DISABLE_SPIFFS ?= 0 -APP_CFLAGS += -DDISABLE_SPIFFS=$(DISABLE_SPIFFS) +ifdef DISABLE_SPIFFS +$(error DISABLE_SPIFFS is no longer supported; please remove this option from your component.mk file) +endif CACHE_VARS += SPIFF_FILES SPIFF_BIN SPIFF_FILES ?= files SPIFF_BIN ?= spiff_rom -SPIFF_BIN_OUT := $(FW_BASE)/$(SPIFF_BIN).bin -CUSTOM_TARGETS += $(SPIFF_BIN_OUT) - -CONFIG_VARS += SPIFF_SIZE -ifeq ($(SPI_SIZE), 256K) - SPIFF_SIZE ?= 131072 #128K -else ifeq ($(SPI_SIZE), 1M) - SPIFF_SIZE ?= 524288 #512K -else ifeq ($(SPI_SIZE), 2M) - SPIFF_SIZE ?= 524288 #512K -else ifeq ($(SPI_SIZE), 4M) - SPIFF_SIZE ?= 524288 #512K -else - SPIFF_SIZE ?= 196608 #192K -endif -APP_CFLAGS += -DSPIFF_SIZE=$(SPIFF_SIZE) +DEBUG_VARS += SPIFF_BIN_OUT +SPIFF_BIN_OUT := $(FW_BASE)/$(SPIFF_BIN).bin COMPONENT_RELINK_VARS += SPIFF_FILEDESC_COUNT SPIFF_FILEDESC_COUNT ?= 7 @@ -46,57 +23,25 @@ COMPONENT_CFLAGS += -DSPIFF_FILEDESC_COUNT=$(SPIFF_FILEDESC_COUNT) COMPONENT_CFLAGS += -Wno-tautological-compare - -ifeq ($(SMING_ARCH),Esp32) -SPIFF_START_ADDR ?= 0x200000 -APP_CFLAGS += -DSPIFF_SIZE=$(SPIFF_SIZE) -DSPIFF_START_ADDR=$(SPIFF_START_ADDR) -endif +COMPONENT_RELINK_VARS += SPIFFS_OBJ_META_LEN +SPIFFS_OBJ_META_LEN ?= 16 +COMPONENT_CFLAGS += -DSPIFFS_OBJ_META_LEN=$(SPIFFS_OBJ_META_LEN) ##@Building -.PHONY: spiffs-image-update -spiffs-image-update: spiffs-image-clean $(SPIFF_BIN_OUT) ##Rebuild the SPIFFS filesystem image - -##@Cleaning - -.PHONY: spiffs-image-clean -spiffs-image-clean: ##Remove SPIFFS image file - $(info Cleaning $(SPIFF_BIN_OUT)) - $(Q) rm -f $(SPIFF_BIN_OUT) - -# Generating spiffs_bin -$(SPIFF_BIN_OUT): $(SPIFFY) -ifeq ($(DISABLE_SPIFFS), 1) - $(info (!) Spiffs support disabled. Remove 'DISABLE_SPIFFS' make argument to enable spiffs.) -else - $(Q) mkdir -p $(dir $(SPIFF_BIN_OUT)) - $(info Checking for spiffs files) - $(Q) if [ -d "$(SPIFF_FILES)" ]; then \ - echo "$(SPIFF_FILES) directory exists. Creating $(SPIFF_BIN_OUT)"; \ - $(SPIFFY) $(SPIFF_SIZE) $(SPIFF_FILES) $(SPIFF_BIN_OUT); \ - else \ - echo "No files found in ./$(SPIFF_FILES)."; \ - echo "Creating empty $(SPIFF_BIN_OUT)"; \ - $(SPIFFY) $(SPIFF_SIZE) dummy.dir $(SPIFF_BIN_OUT); \ - fi +# Spiffs image generation tool +SPIFFSGEN := $(PYTHON) $(COMPONENT_PATH)/spiffsgen.py +SPIFFSGEN_SMING = $(SPIFFSGEN) --meta-len=$(SPIFFS_OBJ_META_LEN) --block-size=8192 + +# Target invoked via partition table +ifneq (,$(filter spiffsgen,$(MAKECMDGOALS))) +PART_TARGET := $(PARTITION_$(PART)_FILENAME) +$(eval PART_FILES := $(call HwExpr,part.build['files'])) +.PHONY: spiffsgen +spiffsgen: +ifneq (,$(PART_TARGET)) + @echo "Creating SPIFFS image '$(PART_TARGET)'" + $(Q) mkdir -p $(dir $(PART_TARGET)) + $(Q) $(SPIFFSGEN_SMING) $(PARTITION_$(PART)_SIZE_BYTES) "$(or $(PART_FILES),)" $(PART_TARGET) endif - -##@Flashing - -BLANKFS_BIN := $(COMPONENT_PATH)/blankfs.bin - -# If enabled, add the SPIFFS image to the chunks to write -ifeq ($(DISABLE_SPIFFS), 1) -FLASH_SPIFFS_CHUNKS := -else -FLASH_SPIFFS_CHUNKS := $(SPIFF_START_ADDR)=$(SPIFF_BIN_OUT) -FLASH_INIT_CHUNKS += $(SPIFF_START_ADDR)=$(BLANKFS_BIN) -endif - -.PHONY: flashfs -flashfs: $(SPIFF_BIN_OUT) ##Write just the SPIFFS filesystem image -ifeq ($(DISABLE_SPIFFS), 1) - $(info SPIFFS image creation disabled!) -else - $(call WriteFlash,$(FLASH_SPIFFS_CHUNKS)) endif diff --git a/Sming/Components/spiffs/spiffs_config.h b/Sming/Components/spiffs/spiffs_config.h index fe7c0b6dd7..6699389ed5 100644 --- a/Sming/Components/spiffs/spiffs_config.h +++ b/Sming/Components/spiffs/spiffs_config.h @@ -9,11 +9,7 @@ #define SPIFFS_CONFIG_H_ // ----------- 8< ------------ -#ifdef __ets__ - #include -#else - #include "spiffy_host.h" -#endif /* __ets__ */ +#include // ----------- >8 ------------ // compile time switches @@ -117,7 +113,7 @@ // logical_page_size - (SPIFFS_OBJ_NAME_LEN + sizeof(spiffs_page_header) + // spiffs_object_ix_header fields + at least some LUT entries) #ifndef SPIFFS_OBJ_META_LEN -#define SPIFFS_OBJ_META_LEN (0) +#define SPIFFS_OBJ_META_LEN (16) #endif // Size of buffer allocated on stack used when copying data. @@ -181,9 +177,7 @@ #endif // Enable this if you want the HAL callbacks to be called with the spiffs struct -#ifndef SPIFFS_HAL_CALLBACK_EXTRA -#define SPIFFS_HAL_CALLBACK_EXTRA 0 -#endif +#define SPIFFS_HAL_CALLBACK_EXTRA 1 // Enable this if you want to add an integer offset to all file handles // (spiffs_file). This is useful if running multiple instances of spiffs on diff --git a/Sming/Components/spiffs/spiffs_sming.cpp b/Sming/Components/spiffs/spiffs_sming.cpp index 0168d93b2b..def31d36b6 100644 --- a/Sming/Components/spiffs/spiffs_sming.cpp +++ b/Sming/Components/spiffs/spiffs_sming.cpp @@ -9,154 +9,38 @@ ****/ #include "spiffs_sming.h" -#include -extern "C" { -#include "spiffs/src/spiffs_nucleus.h" -} - -spiffs _filesystemStorageHandle; - -#ifndef SPIFF_FILEDESC_COUNT -#define SPIFF_FILEDESC_COUNT 7 -#endif - -namespace -{ -uint16_t spiffs_work_buf[LOG_PAGE_SIZE]; -spiffs_fd spiffs_fds[SPIFF_FILEDESC_COUNT]; -uint32_t spiffs_cache_buf[LOG_PAGE_SIZE + 32]; - -s32_t api_spiffs_read(u32_t addr, u32_t size, u8_t* dst) -{ - return (flashmem_read(dst, addr, size) == size) ? SPIFFS_OK : SPIFFS_ERR_INTERNAL; -} - -s32_t api_spiffs_write(u32_t addr, u32_t size, u8_t* src) -{ - //debugf("api_spiffs_write"); - return (flashmem_write(src, addr, size) == size) ? SPIFFS_OK : SPIFFS_ERR_INTERNAL; -} +#include +#include +#include -s32_t api_spiffs_erase(u32_t addr, u32_t size) -{ - debugf("api_spiffs_erase"); - uint32_t sect_first = flashmem_get_sector_of_address(addr); - uint32_t sect_last = sect_first; - while(sect_first <= sect_last) { - if(!flashmem_erase_sector(sect_first++)) { - return SPIFFS_ERR_INTERNAL; - } - } - return SPIFFS_OK; -} - -bool tryMount(const spiffs_config& cfg) -{ - int res = SPIFFS_mount(&_filesystemStorageHandle, const_cast(&cfg), - reinterpret_cast(spiffs_work_buf), reinterpret_cast(spiffs_fds), - sizeof(spiffs_fds), spiffs_cache_buf, sizeof(spiffs_cache_buf), nullptr); - debugf("mount res: %d", res); - - return res >= 0; -} - -bool spiffs_format_internal(const spiffs_config& cfg) +void spiffs_unmount() { - spiffs_unmount(); - if(tryMount(cfg)) { - spiffs_unmount(); + if(fileSystemType() == IFS::IFileSystem::Type::SPIFFS) { + fileFreeFileSystem(); } - - int res = SPIFFS_format(&_filesystemStorageHandle); - return res >= 0; } -bool spiffs_mount_internal(const spiffs_config& cfg) +bool spiffs_format() { - debugf("fs.start: size:%u Kb, offset:0x%X\n", cfg.phys_size / 1024U, cfg.phys_addr); - - // Simple check of the erase count to see if flash looks like it's already been formatted - spiffs_obj_id dat; - flashmem_read(&dat, cfg.phys_addr + cfg.log_page_size - sizeof(spiffs_obj_id), sizeof(spiffs_obj_id)); - //debugf("%X", dat); - bool isFormatted = (dat != spiffs_obj_id(UINT32_MAX)); - - if(!isFormatted) { - debugf("First init file system"); - spiffs_format_internal(cfg); - } - - if(!tryMount(cfg)) { + if(fileSystemType() != IFS::IFileSystem::Type::SPIFFS) { return false; } - - if(!isFormatted) { - spiffs_file fd = SPIFFS_open(&_filesystemStorageHandle, _F("initialize_fs_header.dat"), - SPIFFS_CREAT | SPIFFS_TRUNC | SPIFFS_RDWR, 0); - uint8_t c{1}; - SPIFFS_write(&_filesystemStorageHandle, fd, &c, 1); - SPIFFS_fremove(&_filesystemStorageHandle, fd); - SPIFFS_close(&_filesystemStorageHandle, fd); - } - - return true; + return fileSystemFormat() == FS_OK; } -bool initConfig(spiffs_config& cfg, uint32_t phys_addr, uint32_t phys_size) +bool spiffs_format(Storage::Partition& partition) { - if(phys_addr == 0) { - SYSTEM_ERROR("SPIFFS: Start address invalid"); - return false; + if(fileSystemType() == IFS::IFileSystem::Type::SPIFFS) { + fileFreeFileSystem(); } - - cfg = spiffs_config{ - .hal_read_f = api_spiffs_read, - .hal_write_f = api_spiffs_write, - .hal_erase_f = api_spiffs_erase, - .phys_size = phys_size, - .phys_addr = phys_addr, - .phys_erase_block = INTERNAL_FLASH_SECTOR_SIZE, - .log_block_size = INTERNAL_FLASH_SECTOR_SIZE * 2, - .log_page_size = LOG_PAGE_SIZE, - }; - - return true; -} - -} // namespace - -bool spiffs_mount() -{ - spiffs_config cfg = spiffs_get_storage_config(); - return spiffs_mount_manual(cfg.phys_addr, cfg.phys_size); -} - -bool spiffs_mount_manual(uint32_t phys_addr, uint32_t phys_size) -{ - spiffs_config cfg; - if(!initConfig(cfg, phys_addr, phys_size)) { + auto fs = new IFS::SPIFFS::FileSystem(partition); + int err = fs->format(); + if(err < 0) { + debug_e("SPIFFS format failed: %s", fs->getErrorString(err).c_str()); + delete fs; return false; } - return spiffs_mount_internal(cfg); -} -void spiffs_unmount() -{ - SPIFFS_unmount(&_filesystemStorageHandle); -} - -bool spiffs_format() -{ - auto cfg = spiffs_get_storage_config(); - return spiffs_format_manual(cfg.phys_addr, cfg.phys_size); -} - -bool spiffs_format_manual(uint32_t phys_addr, uint32_t phys_size) -{ - spiffs_config cfg; - if(!initConfig(cfg, phys_addr, phys_size)) { - return false; - } - spiffs_format_internal(cfg); - return spiffs_mount_internal(cfg); + fileSetFileSystem(fs); + return true; } diff --git a/Sming/Components/spiffs/spiffs_sming.h b/Sming/Components/spiffs/spiffs_sming.h index c8619f9d81..9480c00db1 100644 --- a/Sming/Components/spiffs/spiffs_sming.h +++ b/Sming/Components/spiffs/spiffs_sming.h @@ -9,64 +9,25 @@ ****/ #pragma once -#if defined(__cplusplus) -extern "C" { -#endif - -#include "spiffs.h" -#include - -#define LOG_PAGE_SIZE 256 - -/** - * @brief Mount the SPIFFS volume using default configuration - * @retval bool true on success - * - * Configuration is obtained `spiffs_get_storage_config()`. - */ -bool spiffs_mount(); +#include +#include /** - * @brief Mount a SPIFFS volume using custom location and size - * @param phys_addr The flash memory address (offset) for the volume - * @param phys_size The volume size, in bytes - * @retval bool true on success, false on failure - * @note If the given flash memory range appears to be empty then it is - * formatted, erasing any existing content. + * @brief unmount SPIFFS filesystem + * @deprecated use fileFreeFileSystem() instead + * @note this will do nothing if the active filesystem is not SPIFFS */ -bool spiffs_mount_manual(uint32_t phys_addr, uint32_t phys_size); +void spiffs_unmount() SMING_DEPRECATED; -/** - * @brief Unmount a previously mounted volume +/** @brief Format and mount a SPIFFS filesystem + * @deprecated use fileSystemFormat() instead + * @note this will fail if the active filesystem is not SPIFFS */ -void spiffs_unmount(); +bool spiffs_format() SMING_DEPRECATED; /** - * @brief Format and mount a SPIFFS volume using default configuration + * @brief Format and mount a SPIFFS volume using given partition + * @param partition * @retval bool true on success */ -bool spiffs_format(); - -/** - * @brief Format and mount a SPIFFS volume using custom location and size - * @param phys_addr The flash memory address (offset) for the volume - * @param phys_size The volume size, in bytes - * @retval bool true on success - */ -bool spiffs_format_manual(uint32_t phys_addr, uint32_t phys_size); - -/** - * @brief Obtain the default SPIFFS configuration information - * @retval spiffs_config - * @note Only `phys_addr` and `phys_size` are used, all other parameters are overridden. - */ -spiffs_config spiffs_get_storage_config(); - -/** - * @brief Global SPIFFS instance used by FileSystem API - */ -extern spiffs _filesystemStorageHandle; - -#if defined(__cplusplus) -} -#endif +bool spiffs_format(Storage::Partition& partition); diff --git a/Sming/Components/spiffs/spiffsgen.py b/Sming/Components/spiffs/spiffsgen.py new file mode 100644 index 0000000000..0ca153dcca --- /dev/null +++ b/Sming/Components/spiffs/spiffsgen.py @@ -0,0 +1,566 @@ +#!/usr/bin/env python +# +# spiffsgen is a tool used to generate a spiffs image from a directory +# +# Copyright 2019 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import division, print_function + +import argparse +import ctypes +import io +import math +import os +import struct +import sys + +SPIFFS_PH_FLAG_USED_FINAL_INDEX = 0xF8 +SPIFFS_PH_FLAG_USED_FINAL = 0xFC + +SPIFFS_PH_FLAG_LEN = 1 +SPIFFS_PH_IX_SIZE_LEN = 4 +SPIFFS_PH_IX_OBJ_TYPE_LEN = 1 +SPIFFS_TYPE_FILE = 1 + +# Based on typedefs under spiffs_config.h +SPIFFS_OBJ_ID_LEN = 2 # spiffs_obj_id +SPIFFS_SPAN_IX_LEN = 2 # spiffs_span_ix +SPIFFS_PAGE_IX_LEN = 2 # spiffs_page_ix +SPIFFS_BLOCK_IX_LEN = 2 # spiffs_block_ix + +# Fixed config (see spiffs_config.h) +SPIFFS_ALIGNED_OBJECT_INDEX_TABLES = True + +class SpiffsBuildConfig(): + def __init__(self, page_size, page_ix_len, block_size, + block_ix_len, meta_len, obj_name_len, obj_id_len, + span_ix_len, packed, aligned, endianness, use_magic, use_magic_len): + if block_size % page_size != 0: + raise RuntimeError('block size should be a multiple of page size') + + self.page_size = page_size + self.block_size = block_size + self.obj_id_len = obj_id_len + self.span_ix_len = span_ix_len + self.packed = packed + self.aligned = aligned + self.obj_name_len = obj_name_len + self.meta_len = meta_len + self.page_ix_len = page_ix_len + self.block_ix_len = block_ix_len + self.endianness = endianness + self.use_magic = use_magic + self.use_magic_len = use_magic_len + + self.PAGES_PER_BLOCK = self.block_size // self.page_size + self.OBJ_LU_PAGES_PER_BLOCK = int(math.ceil(self.block_size / self.page_size * self.obj_id_len / self.page_size)) + self.OBJ_USABLE_PAGES_PER_BLOCK = self.PAGES_PER_BLOCK - self.OBJ_LU_PAGES_PER_BLOCK + + self.OBJ_LU_PAGES_OBJ_IDS_LIM = self.page_size // self.obj_id_len + + self.OBJ_DATA_PAGE_HEADER_LEN = self.obj_id_len + self.span_ix_len + SPIFFS_PH_FLAG_LEN + + pad = 4 - (4 if self.OBJ_DATA_PAGE_HEADER_LEN % 4 == 0 else self.OBJ_DATA_PAGE_HEADER_LEN % 4) + + self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED = self.OBJ_DATA_PAGE_HEADER_LEN + pad + self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED_PAD = pad + self.OBJ_DATA_PAGE_CONTENT_LEN = self.page_size - self.OBJ_DATA_PAGE_HEADER_LEN + + self.OBJ_INDEX_PAGES_HEADER_LEN = (self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED + SPIFFS_PH_IX_SIZE_LEN + + SPIFFS_PH_IX_OBJ_TYPE_LEN + self.obj_name_len + self.meta_len) + self.OBJ_INDEX_PAGES_OBJ_IDS_HEAD_LIM = (self.page_size - self.OBJ_INDEX_PAGES_HEADER_LEN) // self.block_ix_len + self.OBJ_INDEX_PAGES_OBJ_IDS_LIM = (self.page_size - self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED) / self.block_ix_len + + +class SpiffsFullError(RuntimeError): + def __init__(self, message=None): + super(SpiffsFullError, self).__init__(message) + + +class SpiffsPage(): + _endianness_dict = { + 'little': '<', + 'big': '>' + } + + _len_dict = { + 1: 'B', + 2: 'H', + 4: 'I', + 8: 'Q' + } + + _type_dict = { + 1: ctypes.c_ubyte, + 2: ctypes.c_ushort, + 4: ctypes.c_uint, + 8: ctypes.c_ulonglong + } + + def __init__(self, bix, build_config): + self.build_config = build_config + self.bix = bix + + +class SpiffsObjLuPage(SpiffsPage): + def __init__(self, bix, build_config): + SpiffsPage.__init__(self, bix, build_config) + + self.obj_ids_limit = self.build_config.OBJ_LU_PAGES_OBJ_IDS_LIM + self.obj_ids = list() + + def _calc_magic(self, blocks_lim): + # Calculate the magic value mirrorring computation done by the macro SPIFFS_MAGIC defined in + # spiffs_nucleus.h + magic = 0x20140529 ^ self.build_config.page_size + if self.build_config.use_magic_len: + magic = magic ^ (blocks_lim - self.bix) + magic = SpiffsPage._type_dict[self.build_config.obj_id_len](magic) + return magic.value + + def register_page(self, page): + if not self.obj_ids_limit > 0: + raise SpiffsFullError() + + obj_id = (page.obj_id, page.__class__) + self.obj_ids.append(obj_id) + self.obj_ids_limit -= 1 + + def to_binary(self): + global test + img = b'' + + for (obj_id, page_type) in self.obj_ids: + if page_type == SpiffsObjIndexPage: + obj_id ^= (1 << ((self.build_config.obj_id_len * 8) - 1)) + img += struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + + SpiffsPage._len_dict[self.build_config.obj_id_len], obj_id) + + assert(len(img) <= self.build_config.page_size) + + img += b'\xFF' * (self.build_config.page_size - len(img)) + + return img + + def magicfy(self, blocks_lim): + # Only use magic value if no valid obj id has been written to the spot, which is the + # spot taken up by the last obj id on last lookup page. The parent is responsible + # for determining which is the last lookup page and calling this function. + remaining = self.obj_ids_limit + empty_obj_id_dict = { + 1: 0xFF, + 2: 0xFFFF, + 4: 0xFFFFFFFF, + 8: 0xFFFFFFFFFFFFFFFF + } + if (remaining >= 2): + for i in range(remaining): + if i == remaining - 2: + self.obj_ids.append((self._calc_magic(blocks_lim), SpiffsObjDataPage)) + break + else: + self.obj_ids.append((empty_obj_id_dict[self.build_config.obj_id_len], SpiffsObjDataPage)) + self.obj_ids_limit -= 1 + + +class SpiffsObjIndexPage(SpiffsPage): + def __init__(self, obj_id, span_ix, size, name, build_config): + SpiffsPage.__init__(self, 0, build_config) + self.obj_id = obj_id + self.span_ix = span_ix + self.name = name + self.size = size + + if self.span_ix == 0: + self.pages_lim = self.build_config.OBJ_INDEX_PAGES_OBJ_IDS_HEAD_LIM + else: + self.pages_lim = self.build_config.OBJ_INDEX_PAGES_OBJ_IDS_LIM + + self.pages = list() + + def register_page(self, page): + if not self.pages_lim > 0: + raise SpiffsFullError + + self.pages.append(page.offset) + self.pages_lim -= 1 + + def to_binary(self): + obj_id = self.obj_id ^ (1 << ((self.build_config.obj_id_len * 8) - 1)) + img = struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + + SpiffsPage._len_dict[self.build_config.obj_id_len] + + SpiffsPage._len_dict[self.build_config.span_ix_len] + + SpiffsPage._len_dict[SPIFFS_PH_FLAG_LEN], + obj_id, + self.span_ix, + SPIFFS_PH_FLAG_USED_FINAL_INDEX) + + # Add padding before the object index page specific information + img += b'\xFF' * self.build_config.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED_PAD + + # If this is the first object index page for the object, add filname, type + # and size information + if self.span_ix == 0: + img += struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + + SpiffsPage._len_dict[SPIFFS_PH_IX_SIZE_LEN] + + SpiffsPage._len_dict[SPIFFS_PH_FLAG_LEN], + self.size, + SPIFFS_TYPE_FILE) + + # Append name + name = self.name.encode() + img += name + (b'\x00' * (self.build_config.obj_name_len - len(name))) + + if self.build_config.meta_len >= 16: + # Build default IFS metadata + magic = 0xE3457A77 + mtime = self.mtime + attr = 0 + userRole_admin = 0x04 + compression_none = 0 + original_size = 0 + meta = struct.pack(b'> int(math.log(self.build_config.page_size, 2)) + img += struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + + SpiffsPage._len_dict[self.build_config.page_ix_len], page) + + assert(len(img) <= self.build_config.page_size) + + img += b'\xFF' * (self.build_config.page_size - len(img)) + + return img + + +class SpiffsObjDataPage(SpiffsPage): + def __init__(self, offset, obj_id, span_ix, contents, build_config): + SpiffsPage.__init__(self, 0, build_config) + self.obj_id = obj_id + self.span_ix = span_ix + self.contents = contents + self.offset = offset + + def to_binary(self): + img = struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + + SpiffsPage._len_dict[self.build_config.obj_id_len] + + SpiffsPage._len_dict[self.build_config.span_ix_len] + + SpiffsPage._len_dict[SPIFFS_PH_FLAG_LEN], + self.obj_id, + self.span_ix, + SPIFFS_PH_FLAG_USED_FINAL) + + img += self.contents + + assert(len(img) <= self.build_config.page_size) + + img += b'\xFF' * (self.build_config.page_size - len(img)) + + return img + + +class SpiffsBlock(): + def _reset(self): + self.cur_obj_index_span_ix = 0 + self.cur_obj_data_span_ix = 0 + self.cur_obj_id = 0 + self.cur_obj_idx_page = None + + def __init__(self, bix, blocks_lim, build_config): + self.build_config = build_config + self.offset = bix * self.build_config.block_size + self.remaining_pages = self.build_config.OBJ_USABLE_PAGES_PER_BLOCK + self.pages = list() + self.bix = bix + + lu_pages = list() + for i in range(self.build_config.OBJ_LU_PAGES_PER_BLOCK): + page = SpiffsObjLuPage(self.bix, self.build_config) + lu_pages.append(page) + + self.pages.extend(lu_pages) + + self.lu_page_iter = iter(lu_pages) + self.lu_page = next(self.lu_page_iter) + + self._reset() + + def _register_page(self, page): + if isinstance(page, SpiffsObjDataPage): + self.cur_obj_idx_page.register_page(page) # can raise SpiffsFullError + + try: + self.lu_page.register_page(page) + except SpiffsFullError: + self.lu_page = next(self.lu_page_iter) + try: + self.lu_page.register_page(page) + except AttributeError: # no next lookup page + # Since the amount of lookup pages is pre-computed at every block instance, + # this should never occur + raise RuntimeError('invalid attempt to add page to a block when there is no more space in lookup') + + self.pages.append(page) + + def begin_obj(self, obj_id, size, name, mtime, obj_index_span_ix=0, obj_data_span_ix=0): + if not self.remaining_pages > 0: + raise SpiffsFullError() + self._reset() + + self.cur_obj_id = obj_id + self.cur_obj_index_span_ix = obj_index_span_ix + self.cur_obj_data_span_ix = obj_data_span_ix + + page = SpiffsObjIndexPage(obj_id, self.cur_obj_index_span_ix, size, name, self.build_config) + page.mtime = mtime + self._register_page(page) + + self.cur_obj_idx_page = page + + self.remaining_pages -= 1 + self.cur_obj_index_span_ix += 1 + + def update_obj(self, contents): + if not self.remaining_pages > 0: + raise SpiffsFullError() + page = SpiffsObjDataPage(self.offset + (len(self.pages) * self.build_config.page_size), + self.cur_obj_id, self.cur_obj_data_span_ix, contents, self.build_config) + + self._register_page(page) + + self.cur_obj_data_span_ix += 1 + self.remaining_pages -= 1 + + def end_obj(self): + self._reset() + + def is_full(self): + return self.remaining_pages <= 0 + + def to_binary(self, blocks_lim): + img = b'' + + if self.build_config.use_magic: + for (idx, page) in enumerate(self.pages): + if idx == self.build_config.OBJ_LU_PAGES_PER_BLOCK - 1: + page.magicfy(blocks_lim) + img += page.to_binary() + else: + for page in self.pages: + img += page.to_binary() + + assert(len(img) <= self.build_config.block_size) + + img += b'\xFF' * (self.build_config.block_size - len(img)) + return img + + +class SpiffsFS(): + def __init__(self, img_size, build_config): + if img_size % build_config.block_size != 0: + raise RuntimeError('image size should be a multiple of block size (0x%08x, 0x%04x)' % (img_size, build_config.block_size)) + + self.img_size = img_size + self.build_config = build_config + + self.blocks = list() + self.blocks_lim = self.img_size // self.build_config.block_size + self.remaining_blocks = self.blocks_lim + self.cur_obj_id = 1 # starting object id + + def _create_block(self): + if self.is_full(): + raise SpiffsFullError('the image size has been exceeded') + + block = SpiffsBlock(len(self.blocks), self.blocks_lim, self.build_config) + self.blocks.append(block) + self.remaining_blocks -= 1 + return block + + def is_full(self): + return self.remaining_blocks <= 0 + + def create_file(self, img_path, file_path): + contents = None + + if len(img_path) > self.build_config.obj_name_len: + raise RuntimeError("object name '%s' too long" % img_path) + + name = img_path + + with open(file_path, 'rb') as obj: + contents = obj.read() + + stream = io.BytesIO(contents) + + mtime = int(os.path.getmtime(file_path)) + try: + block = self.blocks[-1] + block.begin_obj(self.cur_obj_id, len(contents), name, mtime) + except (IndexError, SpiffsFullError): + block = self._create_block() + block.begin_obj(self.cur_obj_id, len(contents), name, mtime) + + contents_chunk = stream.read(self.build_config.OBJ_DATA_PAGE_CONTENT_LEN) + + while contents_chunk: + try: + block = self.blocks[-1] + try: + # This can fail because either (1) all the pages in block have been + # used or (2) object index has been exhausted. + block.update_obj(contents_chunk) + except SpiffsFullError: + # If its (1), use the outer exception handler + if block.is_full(): + raise SpiffsFullError + # If its (2), write another object index page + block.begin_obj(self.cur_obj_id, len(contents), name, mtime, + obj_index_span_ix=block.cur_obj_index_span_ix, + obj_data_span_ix=block.cur_obj_data_span_ix) + continue + except (IndexError, SpiffsFullError): + # All pages in the block have been exhausted. Create a new block, copying + # the previous state of the block to a new one for the continuation of the + # current object + prev_block = block + block = self._create_block() + block.cur_obj_id = prev_block.cur_obj_id + block.cur_obj_idx_page = prev_block.cur_obj_idx_page + block.cur_obj_data_span_ix = prev_block.cur_obj_data_span_ix + block.cur_obj_index_span_ix = prev_block.cur_obj_index_span_ix + continue + + contents_chunk = stream.read(self.build_config.OBJ_DATA_PAGE_CONTENT_LEN) + + block.end_obj() + + self.cur_obj_id += 1 + + def to_binary(self): + img = b'' + for block in self.blocks: + img += block.to_binary(self.blocks_lim) + bix = len(self.blocks) + if self.build_config.use_magic: + # Create empty blocks with magic numbers + while self.remaining_blocks > 0: + block = SpiffsBlock(bix, self.blocks_lim, self.build_config) + img += block.to_binary(self.blocks_lim) + self.remaining_blocks -= 1 + bix += 1 + else: + # Just fill remaining spaces FF's + img += b'\xFF' * (self.img_size - len(img)) + return img + + +def main(): + if sys.version_info[0] < 3: + print('WARNING: Support for Python 2 is deprecated and will be removed in future versions.', file=sys.stderr) + elif sys.version_info[0] == 3 and sys.version_info[1] < 6: + print('WARNING: Python 3 versions older than 3.6 are not supported.', file=sys.stderr) + parser = argparse.ArgumentParser(description='SPIFFS Image Generator', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + parser.add_argument('image_size', + help='Size of the created image') + + parser.add_argument('base_dir', + help='Path to directory from which the image will be created') + + parser.add_argument('output_file', + help='Created image output file path') + + parser.add_argument('--page-size', + help='Logical page size. Set to value same as CONFIG_SPIFFS_PAGE_SIZE.', + type=int, + default=256) + + parser.add_argument('--block-size', + help="Logical block size. Set to the same value as the flash chip's sector size (g_rom_flashchip.sector_size).", + type=int, + default=4096) + + parser.add_argument('--obj-name-len', + help='File full path maximum length. Set to value same as CONFIG_SPIFFS_OBJ_NAME_LEN.', + type=int, + default=32) + + parser.add_argument('--meta-len', + help='File metadata length. Set to value same as CONFIG_SPIFFS_META_LENGTH.', + type=int, + default=4) + + parser.add_argument('--use-magic', + help='Use magic number to create an identifiable SPIFFS image. Specify if CONFIG_SPIFFS_USE_MAGIC.', + action='store_true', + default=False) + + parser.add_argument('--follow-symlinks', + help='Take into account symbolic links during partition image creation.', + action='store_true', + default=False) + + parser.add_argument('--use-magic-len', + help='Use position in memory to create different magic numbers for each block. Specify if CONFIG_SPIFFS_USE_MAGIC_LENGTH.', + action='store_true', + default=False) + + parser.add_argument('--big-endian', + help='Specify if the target architecture is big-endian. If not specified, little-endian is assumed.', + action='store_true', + default=False) + + args = parser.parse_args() + + if args.base_dir != '' and not os.path.exists(args.base_dir): + raise RuntimeError('given base directory %s does not exist' % args.base_dir) + + image_size = int(args.image_size, 0) + spiffs_build_default = SpiffsBuildConfig(args.page_size, SPIFFS_PAGE_IX_LEN, + args.block_size, SPIFFS_BLOCK_IX_LEN, args.meta_len, + args.obj_name_len, SPIFFS_OBJ_ID_LEN, SPIFFS_SPAN_IX_LEN, + True, True, 'big' if args.big_endian else 'little', + args.use_magic, args.use_magic_len) + + spiffs = SpiffsFS(image_size, spiffs_build_default) + + if args.base_dir != '': + for root, dirs, files in os.walk(args.base_dir, followlinks=args.follow_symlinks): + for f in files: + full_path = os.path.join(root, f) + rel_path = os.path.relpath(full_path, args.base_dir) + print("'%s', '%s', '%s'" % (rel_path, full_path, args.base_dir)) + spiffs.create_file(rel_path.replace('\\', '/'), full_path) + + image = spiffs.to_binary() + open(args.output_file, 'wb').write(image) + + +if __name__ == '__main__': + main() diff --git a/Sming/Components/spiffs/spiffy/Makefile b/Sming/Components/spiffs/spiffy/Makefile deleted file mode 100644 index b830ed7ce4..0000000000 --- a/Sming/Components/spiffs/spiffy/Makefile +++ /dev/null @@ -1,41 +0,0 @@ -# -# Makefile for spiffy -# -# Pass in TARGET, BUILD_DIR , SPIFFS_SMING, SPIFFS_BASE -# - -HOST_CC ?= gcc -HOST_LD ?= gcc - -INCDIR := -I.. -I../spiffs/src -CFLAGS := -O2 -Wall -Wno-unused-value - -ifeq ("$(V)","1") -Q := -vecho := @true -else -Q := @ -vecho := @echo -endif - -BUILD_DIR ?= . -TARGET ?= spiffy - -OBJS := $(addprefix $(BUILD_DIR)/,spiffy.o spiffs_cache.o spiffs_nucleus.o spiffs_hydrogen.o spiffs_gc.o spiffs_check.o) - -all: $(TARGET) - -$(BUILD_DIR)/%.o: ../spiffs/src/%.c - $(vecho) "CC $<" - $(Q) $(HOST_CC) $(CFLAGS) $(INCDIR) -c $< -o $@ - -$(BUILD_DIR)/spiffy.o: spiffy.c - $(vecho) "CC $<" - $(Q) $(HOST_CC) $(CFLAGS) $(INCDIR) -c $< -o $@ - -$(TARGET): $(OBJS) - $(vecho) "LD $@" - $(Q) $(HOST_LD) -o $@ $^ - -clean: - rm -f $(OBJS) $(TARGET) diff --git a/Sming/Components/spiffs/spiffy/spiffy.c b/Sming/Components/spiffs/spiffy/spiffy.c deleted file mode 100644 index 485e4a87c5..0000000000 --- a/Sming/Components/spiffs/spiffy/spiffy.c +++ /dev/null @@ -1,309 +0,0 @@ -#include -#include -#include -#include -#include - -#define LOG_PAGE_SIZE 256 -#define SPI_FLASH_SEC_SIZE 4096 - -#define ROM_ERASE 0xFF - -#define DEFAULT_FOLDER "files" -#define DEFAULT_ROM_NAME "spiff_rom.bin" -#define DEFAULT_ROM_SIZE 0x30000 - -static spiffs fs; -static u8_t spiffs_work_buf[LOG_PAGE_SIZE*2]; -static u8_t spiffs_fds[32*4]; -static u8_t spiffs_cache_buf[(LOG_PAGE_SIZE+32)*4]; - -#define S_DBG(fmt, ...) -//#define S_DBG printf - -static FILE *rom = 0; - -void hexdump_mem(u8_t *b, u32_t len) { - int i; - for (i = 0; i < len; i++) { - S_DBG("%02x", *b++); - if ((i % 16) == 15) S_DBG("\n"); - else if ((i % 16) == 7) S_DBG(" "); - } - if ((i % 16) != 0) S_DBG("\n"); -} - -static s32_t my_spiffs_read(u32_t addr, u32_t size, u8_t *dst) { - - int res; - - if (fseek(rom, addr, SEEK_SET)) { - printf("Unable to seek to %d.\n", addr); - return SPIFFS_ERR_END_OF_OBJECT; - } - - res = fread(dst, 1, size, rom); - if (res != size) { - printf("Unable to read - tried to get %d bytes only got %d.\n", size, res); - return SPIFFS_ERR_NOT_READABLE; - } - - S_DBG("Read %d bytes from offset %d.\n", size, addr); - //hexdump_mem(dst, size); - return SPIFFS_OK; -} - -static s32_t my_spiffs_write(u32_t addr, u32_t size, u8_t *src) { - - int ret = SPIFFS_OK; - u8_t *buff = 0; - - buff = malloc(size); - if (!buff) { - printf("Unable to malloc %d bytes.\n", size); - ret = SPIFFS_ERR_INTERNAL; - } else { - ret = my_spiffs_read(addr, size, buff); - if (ret == SPIFFS_OK) { - int i; - for(i = 0; i < size; i++) buff[i] &= src[i]; - //hexdump_mem(buff, size); - if (fseek(rom, addr, SEEK_SET)) { - printf("Unable to seek to %d.\n", addr); - ret = SPIFFS_ERR_END_OF_OBJECT; - } else { - if (fwrite(src, 1, size, rom) != size) { - printf("Unable to write.\n"); - ret = SPIFFS_ERR_NOT_WRITABLE; - } else { - fflush(rom); - S_DBG("Wrote %d bytes to offset %d.\n", size, addr); - } - } - } - } - - if (buff) free (buff); - return ret; -} - -static s32_t my_spiffs_erase(u32_t addr, u32_t size) { - - int i; - - if (fseek(rom, addr, SEEK_SET)) { - printf("Unable to seek to %d.\n", addr); - return SPIFFS_ERR_END_OF_OBJECT; - } - - for (i = 0; i < size; i++) { - if (fputc(ROM_ERASE, rom) == EOF) { - printf("Unable to write.\n"); - return SPIFFS_ERR_NOT_WRITABLE; - } - } - - fflush(rom); - S_DBG("Erased %d bytes at offset %d.\n", size, addr); - - return SPIFFS_OK; -} - -int my_spiffs_mount(u32_t msize) { - - spiffs_config cfg; - - cfg.phys_size = msize; - cfg.phys_addr = 0; - - cfg.phys_erase_block = SPI_FLASH_SEC_SIZE; - cfg.log_block_size = SPI_FLASH_SEC_SIZE * 2; - cfg.log_page_size = LOG_PAGE_SIZE; - - cfg.hal_read_f = my_spiffs_read; - cfg.hal_write_f = my_spiffs_write; - cfg.hal_erase_f = my_spiffs_erase; - - int res = SPIFFS_mount(&fs, - &cfg, - spiffs_work_buf, - spiffs_fds, - sizeof(spiffs_fds), - spiffs_cache_buf, - sizeof(spiffs_cache_buf), - 0); - S_DBG("Mount result: %d.\n", res); - - return res; -} - -void my_spiffs_unmount() { - SPIFFS_unmount(&fs); -} - -int my_spiffs_format() { - int res = SPIFFS_format(&fs); - S_DBG("Format result: %d.\n", res); - return res; -} - -int write_to_spiffs(char *fname, u8_t *data, int size) { - - int ret = 0; - - spiffs_file fd = SPIFFS_open(&fs, fname, SPIFFS_CREAT | SPIFFS_TRUNC | SPIFFS_RDWR, 0); - if (fd < 0) { - printf("Unable to open spiffs file '%s', error %d.\n", fname, fd); - } else { - S_DBG("Opened spiffs file '%s'.\n", fname); - if (SPIFFS_write(&fs, fd, (u8_t *)data, size) < SPIFFS_OK) { - printf("Unable to write to spiffs file '%s', errno %d.\n", fname, SPIFFS_errno(&fs)); - } else { - ret = 1; - } - } - - if (fd >= 0) { - SPIFFS_close(&fs, fd); - S_DBG("Closed spiffs file '%s'.\n", fname); - } - return ret; -} - -int add_file(const char* fdir, char* fname) { - - int ret = 0; - u8_t *buff = 0; - FILE *fp = 0; - char *path = 0; - - path = malloc(1024); - if (!path) { - printf("Unable to malloc %d bytes.\n", 1024); - } else { - struct stat st; - sprintf(path, "%s/%s", fdir, fname); - if (stat(path, &st) || !S_ISREG(st.st_mode)) { - S_DBG("Skipping non-file '%s'.\n", fname); - } else { - fp = fopen(path, "rb"); - if (!fp) { - S_DBG("Unable to open '%s'.\n", fname); - } else { - int size = (int)st.st_size; - buff = malloc(size); - if (!buff) { - printf("Unable to malloc %d bytes.\n", size); - } else { - if (fread(buff, 1, size, fp) != size) { - printf("Unable to read file '%s'.\n", fname); - } else { - S_DBG("%d bytes read from '%s'.\n", size, fname); - if (write_to_spiffs(fname, buff, size)) { - printf("Added '%s' to spiffs (%d bytes).\n", fname, size); - ret = 1; - } - } - } - } - } - } - - if (buff) free(buff); - if (path) free(path); - if (fp) fclose(fp); - - return ret; -} - -int get_rom_size (const char *str) { - - long val; - - // accept decimal or hex, but not octal - if ((strlen(str) > 2) && (str[0] == '0') && - (((str[1] == 'x')) || ((str[1] == 'X')))) { - val = strtol(str, NULL, 16); - } else { - val = strtol(str, NULL, 10); - } - - return (int)val; -} - -int main(int argc, char **argv) { - - const char *folder; - const char *romfile; - int romsize; - int ret = EXIT_SUCCESS; - - if (argc == 1) { - romsize = DEFAULT_ROM_SIZE; - folder = DEFAULT_FOLDER; - romfile = DEFAULT_ROM_NAME; - printf("Usage: %s maxFsSizeinByte spiffsBaseDir [outfile.bin]\n" - "There is no specific size or files directory.\n" - "Starting in compatibility mode.\n" - "Default fs size is 0x%x (%d) bytes and directory is '%s'.\n", - argv[0], romsize, romsize, DEFAULT_FOLDER); - } else if (argc == 3) { - romsize = get_rom_size(argv[1]); - folder = argv[2]; - romfile = DEFAULT_ROM_NAME; - } else if (argc == 4) { - romsize = get_rom_size(argv[1]); - folder = argv[2]; - romfile = argv[3]; - } else { - printf ("Usage: %s [OutFile.bin]\n", argv[0]); - printf (" FsSizeInBytes can be in hex (starting with 0x) or decimal.\n"); - printf (" To create an empty filesystem pass 'dummy.dir' as FilesDir.\n"); - exit(EXIT_FAILURE); - } - - printf("Creating rom '%s' of size 0x%x (%d) bytes.\n", romfile, romsize, romsize); - rom = fopen(romfile, "wb+"); - - if (!rom) { - printf("Unable to open file '%s' for writing.\n", romfile); - } else { - - fseek(rom, romsize - 1, SEEK_SET); - fputc(ROM_ERASE, rom); - - // we have to do this before calling format - if (!my_spiffs_mount(romsize)) { - my_spiffs_unmount(); - } - - int res; - if ((res = my_spiffs_format()) != SPIFFS_OK) { - printf("Failed to format spiffs, error %d.\n", res); - ret = EXIT_FAILURE; - } else if ((res = my_spiffs_mount(romsize)) != SPIFFS_OK) { - printf("Failed to mount spiffs, error %d.\n", res); - ret = EXIT_FAILURE; - } else { - DIR *dir; - struct dirent *ent; - if (!strcmp(folder, "dummy.dir")) { - printf("Creating empty filesystem.\n"); - } else if ((dir = opendir(folder)) != NULL) { - printf("Adding files in directory '%s'.\n", folder); - while ((ent = readdir(dir)) != NULL) { - add_file(folder, ent->d_name); - } - closedir(dir); - } else { - printf("Unable to open directory '%s'.\n", folder); - ret = EXIT_FAILURE; - } - my_spiffs_unmount(); - } - } - - if (rom) fclose(rom); - if (ret == EXIT_FAILURE) unlink(romfile); - exit(ret); -} diff --git a/Sming/Components/spiffs/spiffy_host.h b/Sming/Components/spiffs/spiffy_host.h deleted file mode 100644 index f621ef6557..0000000000 --- a/Sming/Components/spiffs/spiffy_host.h +++ /dev/null @@ -1,16 +0,0 @@ - -#pragma once - -#include -#include -#include -#include -#include -#include - -typedef signed int s32_t; -typedef unsigned int u32_t; -typedef signed short s16_t; -typedef unsigned short u16_t; -typedef signed char s8_t; -typedef unsigned char u8_t; diff --git a/Sming/Components/ssl/Tools/make_certs.sh b/Sming/Components/ssl/Tools/make_certs.sh old mode 100644 new mode 100755 index b79373e175..be1adde987 --- a/Sming/Components/ssl/Tools/make_certs.sh +++ b/Sming/Components/ssl/Tools/make_certs.sh @@ -149,7 +149,7 @@ openssl x509 -req -in x509_512.req -out x509_bad_after.pem \ # some cleanup rm *.req -rm srl +rm *.srl rm *.conf # need this for the client tests @@ -177,6 +177,6 @@ cat ca_x509.pem >> x509_device.pem # set default key/cert for use in the server xxd -i x509_1024.cer | sed -e \ - "s/axTLS_x509_1024_cer/default_certificate/" > $SSL_INCLUDE_DIR/cert.h + "s/x509_1024_cer/default_certificate/" > $SSL_INCLUDE_DIR/cert.h xxd -i key_1024 | sed -e \ - "s/axTLS_key_1024/default_private_key/" > $SSL_INCLUDE_DIR/private_key.h + "s/key_1024/default_private_key/" > $SSL_INCLUDE_DIR/private_key.h diff --git a/Sming/Components/ssl/component.mk b/Sming/Components/ssl/component.mk index bd60b5182c..41ba7ae644 100644 --- a/Sming/Components/ssl/component.mk +++ b/Sming/Components/ssl/component.mk @@ -46,13 +46,14 @@ CUSTOM_TARGETS += include/ssl/private_key.h SSL_TOOLS_PATH := $(COMPONENT_PATH)/Tools SSL_INCLUDE_DIR := $(PROJECT_DIR)/include/ssl -SSL_CERT_DIR := $(PROJECT_DIR)/cert +OUT_SSL := out/ssl/ +SSL_CERT_DIR := $(OUT_SSL)/cert include/ssl/private_key.h: $(info Generating unique certificate and key. This may take some time...) $(Q) mkdir -p $(SSL_INCLUDE_DIR) $(SSL_CERT_DIR) $(Q) chmod a+x $(SSL_TOOLS_PATH)/make_certs.sh - cd $(SSL_CERT_DIR); SSL_INCLUDE_DIR=$(SSL_INCLUDE_DIR) $(SSL_TOOLS_PATH)/make_certs.sh + $(Q) cd $(SSL_CERT_DIR); SSL_INCLUDE_DIR=$(SSL_INCLUDE_DIR) $(SSL_TOOLS_PATH)/make_certs.sh endif diff --git a/Sming/Components/ssl/src/Session.cpp b/Sming/Components/ssl/src/Session.cpp index e3034e0012..208fee87c0 100644 --- a/Sming/Components/ssl/src/Session.cpp +++ b/Sming/Components/ssl/src/Session.cpp @@ -145,7 +145,10 @@ void Session::close() int Session::read(InputBuffer& input, uint8_t*& output) { - assert(connection != nullptr); + if(connection == nullptr) { + debug_w("SSL: no connection"); + return -1; + } int len = connection->read(input, output); if(len < 0) { debug_w("SSL: Got error: %d (%s)", len, connection->getErrorString(len).c_str()); diff --git a/Sming/Core/Data/BitSet.h b/Sming/Core/Data/BitSet.h index 10dd5164ad..27aaef3d76 100644 --- a/Sming/Core/Data/BitSet.h +++ b/Sming/Core/Data/BitSet.h @@ -377,7 +377,7 @@ template class BitSet template inline constexpr BitSet operator&(const BitSet& x, const BitSet& y) { - return BitSet(S(x) & ~S(y)); + return BitSet(S(x) & S(y)); } template @@ -416,6 +416,18 @@ inline constexpr BitSet operator-(const BitSet& x, E b return BitSet(S(x) & ~BitSet::bitVal(b)); } +template +inline constexpr BitSet operator^(BitSet x, BitSet y) +{ + return BitSet(S(x) ^ S(y)); +} + +template +inline constexpr BitSet operator^(BitSet x, E b) +{ + return x ^ BitSet(b); +} + /* * These allow construction of a maximally-sized BitSet in an expression, * which is then copy-constructed to the actual value. For example: @@ -441,7 +453,10 @@ constexpr return a | b; } -String toString(uint8_t value); +template typename std::enable_if::value, String>::type toString(T value) +{ + return String(value); +} /** * @brief Class template to print the contents of a BitSet to a String diff --git a/Sming/Core/Data/CString.h b/Sming/Core/Data/CString.h index 9dea4b4d9b..4335b95bc7 100644 --- a/Sming/Core/Data/CString.h +++ b/Sming/Core/Data/CString.h @@ -13,6 +13,7 @@ #pragma once #include +#include #include /** @@ -27,13 +28,21 @@ class CString : public std::unique_ptr public: CString() = default; - CString(const CString& src) = default; + CString(const CString& src) + { + assign(src.get()); + } CString(const String& src) { assign(src); } + CString(const char* src) + { + assign(src); + } + void assign(const String& src) { if(src) { @@ -59,6 +68,12 @@ class CString : public std::unique_ptr } } + CString& operator=(const CString& src) + { + assign(src.get()); + return *this; + } + CString& operator=(const String& src) { assign(src); @@ -81,14 +96,55 @@ class CString : public std::unique_ptr return get() ?: ""; } - bool operator==(const CString& other) const + bool equals(const CString& other) const { return strcmp(c_str(), other.c_str()) == 0; } + bool equals(const String& other) const + { + return other.equals(c_str()); + } + + bool equals(const char* other) const + { + if(other == nullptr) { + return length() == 0; + } + return strcmp(c_str(), other) == 0; + } + + bool equalsIgnoreCase(const CString& other) const + { + return strcasecmp(c_str(), other.c_str()) == 0; + } + + bool equalsIgnoreCase(const String& other) const + { + return other.equalsIgnoreCase(c_str()); + } + + bool equalsIgnoreCase(const char* other) const + { + if(other == nullptr) { + return length() == 0; + } + return strcasecmp(c_str(), other) == 0; + } + + bool operator==(const CString& other) const + { + return equals(other); + } + bool operator==(const String& other) const { - return strcmp(c_str(), other.c_str()) == 0; + return equals(other); + } + + bool operator==(const char* other) const + { + return equals(other); } size_t length() const diff --git a/Sming/Core/Data/LinkedObject.h b/Sming/Core/Data/LinkedObject.h new file mode 100644 index 0000000000..7b4cecbd86 --- /dev/null +++ b/Sming/Core/Data/LinkedObject.h @@ -0,0 +1,144 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * LinkedObject.h + * + ****/ +#pragma once + +#include +#include + +/** + * @brief Base virtual class to allow objects to be linked together + * + * This can be more efficient than defining a separate list, as each object + * requires only an additional pointer field for 'next'. + */ +class LinkedObject +{ +public: + virtual ~LinkedObject() + { + } + + virtual LinkedObject* next() const + { + return mNext; + } + + LinkedObject* getNext() const + { + return mNext; + } + + bool operator==(const LinkedObject& other) const + { + return this == &other; + } + + bool operator!=(const LinkedObject& other) const + { + return this != &other; + } + +private: + friend class LinkedObjectList; + LinkedObject* mNext{nullptr}; +}; + +/** + * @brief Base class template for linked items with type casting + */ +template class LinkedObjectTemplate : public LinkedObject +{ +public: + template + class IteratorTemplate : public std::iterator + { + public: + IteratorTemplate(TPtr x) : mObject(x) + { + } + + IteratorTemplate(TRef& x) : mObject(&x) + { + } + + IteratorTemplate(const IteratorTemplate& other) : mObject(other.mObject) + { + } + + IteratorTemplate& operator++() + { + mObject = mObject->getNext(); + return *this; + } + + IteratorTemplate operator++(int) + { + Iterator tmp(*this); + operator++(); + return tmp; + } + + bool operator==(const IteratorTemplate& rhs) const + { + return mObject == rhs.mObject; + } + + bool operator!=(const IteratorTemplate& rhs) const + { + return mObject != rhs.mObject; + } + + TRef operator*() + { + return *mObject; + } + + TPtr operator->() + { + return mObject; + } + + operator TPtr() + { + return mObject; + } + + private: + TPtr mObject; + }; + + using Iterator = IteratorTemplate; + using ConstIterator = IteratorTemplate; + + ObjectType* getNext() const + { + return reinterpret_cast(this->next()); + } + + Iterator begin() const + { + return Iterator(this); + } + + Iterator end() const + { + return Iterator(nullptr); + } + + Iterator cbegin() const + { + return ConstIterator(this); + } + + Iterator cend() const + { + return ConstIterator(nullptr); + } +}; diff --git a/Sming/Core/Data/LinkedObjectList.cpp b/Sming/Core/Data/LinkedObjectList.cpp new file mode 100644 index 0000000000..a1634d7e2f --- /dev/null +++ b/Sming/Core/Data/LinkedObjectList.cpp @@ -0,0 +1,61 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * LinkedObjectList.cpp + * + ****/ + +#include "LinkedObjectList.h" + +bool LinkedObjectList::add(LinkedObject* object) +{ + if(object == nullptr) { + return false; + } + + LinkedObject* prev = nullptr; + auto it = mHead; + while(it != nullptr) { + if(it == object) { + // Already in list + return true; + } + prev = it; + it = it->mNext; + } + + if(prev == nullptr) { + mHead = object; + } else { + prev->mNext = object; + } + object->mNext = it; + return true; +} + +bool LinkedObjectList::remove(LinkedObject* object) +{ + if(object == nullptr || mHead == nullptr) { + return false; + } + + if(mHead == object) { + mHead = object->mNext; + return true; + } + + auto it = mHead; + while(it->mNext != nullptr) { + if(it->mNext == object) { + it->mNext = object->mNext; + object->mNext = nullptr; + return true; + } + it = it->mNext; + } + + return false; +} diff --git a/Sming/Core/Data/LinkedObjectList.h b/Sming/Core/Data/LinkedObjectList.h new file mode 100644 index 0000000000..1d5908f10f --- /dev/null +++ b/Sming/Core/Data/LinkedObjectList.h @@ -0,0 +1,140 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * LinkedObjectList.h + * + ****/ +#pragma once + +#include "LinkedObject.h" + +/** + * @brief Singly-linked list of objects + * @note We don't own the items, just keep references to them + */ +class LinkedObjectList +{ +public: + LinkedObjectList() + { + } + + LinkedObjectList(LinkedObject* object) : mHead(object) + { + } + + bool add(LinkedObject* object); + + bool add(const LinkedObject* object) + { + return add(const_cast(object)); + } + + bool remove(LinkedObject* object); + + void clear() + { + mHead = nullptr; + } + + LinkedObject* head() + { + return mHead; + } + + const LinkedObject* head() const + { + return mHead; + } + + bool isEmpty() const + { + return mHead == nullptr; + } + +protected: + LinkedObject* mHead{nullptr}; +}; + +template class LinkedObjectListTemplate : public LinkedObjectList +{ +public: + LinkedObjectListTemplate() = default; + + LinkedObjectListTemplate(ObjectType* object) : LinkedObjectList(object) + { + } + + ObjectType* head() + { + return reinterpret_cast(mHead); + } + + const ObjectType* head() const + { + return reinterpret_cast(mHead); + } + + typename ObjectType::Iterator begin() + { + return head(); + } + + typename ObjectType::Iterator end() + { + return nullptr; + } + + typename ObjectType::ConstIterator begin() const + { + return head(); + } + + typename ObjectType::ConstIterator end() const + { + return nullptr; + } + + size_t count() const + { + size_t n{0}; + for(auto p = mHead; p != nullptr; ++n, p = p->next()) { + } + return n; + } + + bool contains(const ObjectType& object) const + { + return std::find(begin(), end(), object); + } +}; + +/** + * @brief Class template for singly-linked list of objects + * @note We own the objects so are responsible for destroying them when removed + */ +template class OwnedLinkedObjectListTemplate : public LinkedObjectListTemplate +{ +public: + ~OwnedLinkedObjectListTemplate() + { + clear(); + } + + bool remove(ObjectType* object) + { + bool res = LinkedObjectList::remove(object); + delete object; + return res; + } + + void clear() + { + while(remove(this->head())) { + // + } + } +}; diff --git a/Sming/Core/Data/Packet.h b/Sming/Core/Data/Packet.h new file mode 100644 index 0000000000..b37a8ada06 --- /dev/null +++ b/Sming/Core/Data/Packet.h @@ -0,0 +1,146 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * Packet.h + * + ****/ + +#pragma once + +#include + +/** + * @brief Helper class for reading/writing packet content + */ +struct Packet { + uint8_t* data; + mutable uint16_t pos; + + Packet(void* data, uint16_t pos = 0) : data(static_cast(data)), pos(pos) + { + } + + const uint8_t* ptr() const + { + return data + pos; + } + + uint8_t* ptr() + { + return data + pos; + } + + void skip(uint16_t len) const + { + pos += len; + } + + uint8_t peek8() const + { + return data[pos]; + } + + uint8_t read8() const + { + return data[pos++]; + } + + void read(void* buffer, uint16_t len) const + { + memcpy(buffer, ptr(), len); + pos += len; + } + + String readString(uint16_t length) const + { + String s; + if(s.setLength(length)) { + read(s.begin(), length); + } else { + pos += length; + } + return s; + } + + void write8(uint8_t value) + { + data[pos++] = value; + } + + void write(const void* s, uint16_t len) + { + memcpy(ptr(), s, len); + pos += len; + } +}; + +/** + * @brief Helper class for reading/writing packet content in network byte-order (MSB first) + */ +struct NetworkPacket : public Packet { + using Packet::Packet; + + uint16_t peek16() const + { + return (data[pos] << 8) | data[pos + 1]; + } + + uint16_t read16() const + { + return (read8() << 8) | read8(); + } + + uint32_t read32() const + { + return (read16() << 16) | read16(); + } + + void write16(uint16_t value) + { + write8(value >> 8); + write8(value & 0xff); + } + + void write32(uint32_t value) + { + write16(value >> 16); + write16(value & 0xffff); + } +}; + +/** + * @brief Helper class for reading/writing packet content in host byte-order (LSB first) + */ +struct HostPacket : public Packet { + using Packet::Packet; + + uint16_t peek16() const + { + return data[pos] | (data[pos + 1] << 8); + } + + uint16_t read16() const + { + return read8() | (read8() << 8); + } + + uint32_t read32() const + { + return read16() | (read16() << 16); + } + + void write16(uint16_t value) + { + write8(value & 0xff); + write8(value >> 8); + } + + void write32(uint32_t value) + { + write16(value & 0xffff); + write16(value >> 16); + } +}; diff --git a/Sming/Core/Data/Stream/Base64OutputStream.cpp b/Sming/Core/Data/Stream/Base64OutputStream.cpp index dbe3df1c38..5bc2df6e0c 100644 --- a/Sming/Core/Data/Stream/Base64OutputStream.cpp +++ b/Sming/Core/Data/Stream/Base64OutputStream.cpp @@ -23,7 +23,7 @@ Base64OutputStream::Base64OutputStream(IDataSourceStream* stream, size_t resultS size_t Base64OutputStream::transform(const uint8_t* source, size_t sourceLength, uint8_t* target, size_t targetLength) { - size_t count = 0; + size_t count; if(sourceLength == 0) { count = base64_encode_blockend((char*)target, &state); } else { @@ -32,13 +32,3 @@ size_t Base64OutputStream::transform(const uint8_t* source, size_t sourceLength, return count; } - -void Base64OutputStream::saveState() -{ - memcpy(&lastState, &state, sizeof(base64_encodestate)); -} - -void Base64OutputStream::restoreState() -{ - memcpy(&state, &lastState, sizeof(base64_encodestate)); -} diff --git a/Sming/Core/Data/Stream/Base64OutputStream.h b/Sming/Core/Data/Stream/Base64OutputStream.h index 206224c2fe..a81af415a7 100644 --- a/Sming/Core/Data/Stream/Base64OutputStream.h +++ b/Sming/Core/Data/Stream/Base64OutputStream.h @@ -31,17 +31,17 @@ class Base64OutputStream : public StreamTransformer size_t transform(const uint8_t* source, size_t sourceLength, uint8_t* target, size_t targetLength) override; - /** - * @brief A method that backs up the current state - */ - void saveState() override; + void saveState() override + { + savedState = state; + } - /** - * @brief A method that restores the last backed up state - */ - void restoreState() override; + void restoreState() override + { + state = savedState; + } private: - base64_encodestate state = {}; - base64_encodestate lastState = {}; + base64_encodestate state{}; + base64_encodestate savedState{}; }; diff --git a/Sming/Core/Data/Stream/FileStream.cpp b/Sming/Core/Data/Stream/FileStream.cpp deleted file mode 100644 index 8837e4bb6a..0000000000 --- a/Sming/Core/Data/Stream/FileStream.cpp +++ /dev/null @@ -1,155 +0,0 @@ -/**** - * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. - * Created 2015 by Skurydin Alexey - * http://github.com/SmingHub/Sming - * All files of the Sming Core are provided under the LGPL v3 license. - * - * FileStream.cpp - * - ****/ - -#include "FileStream.h" - -/* FileStream */ - -void FileStream::attach(file_t file, size_t size) -{ - close(); - if(file >= 0) { - handle = file; - this->size = size; - fileSeek(handle, 0, SeekOrigin::Start); - pos = 0; - debug_d("attached file: '%s' (%u bytes) #0x%08X", fileName().c_str(), size, this); - } -} - -bool FileStream::open(const String& fileName, FileOpenFlags openFlags) -{ - lastError = SPIFFS_OK; - - file_t file = fileOpen(fileName, openFlags); - if(!check(file)) { - debug_w("File wasn't found: %s", fileName.c_str()); - return false; - } - - // Get size - int size = fileSeek(file, 0, SeekOrigin::End); - if(check(size)) { - attach(file, size); - return true; - } - - fileClose(file); - return false; -} - -void FileStream::close() -{ - if(handle >= 0) { - fileClose(handle); - handle = -1; - } - size = 0; - pos = 0; - lastError = SPIFFS_OK; -} - -size_t FileStream::readBytes(char* buffer, size_t length) -{ - if(buffer == nullptr || length == 0 || pos >= size) { - return 0; - } - - int available = fileRead(handle, buffer, std::min(size - pos, length)); - if(!check(available)) { - return 0; - } - - pos += size_t(available); - - return available; -} - -uint16_t FileStream::readMemoryBlock(char* data, int bufSize) -{ - assert(bufSize >= 0); - size_t startPos = pos; - size_t count = readBytes(data, bufSize); - - // Move cursor back to start position - (void)fileSeek(handle, startPos, SeekOrigin::Start); - - return count; -} - -size_t FileStream::write(const uint8_t* buffer, size_t size) -{ - if(!fileExist()) { - return 0; - } - - if(pos != this->size) { - int writePos = fileSeek(handle, 0, SeekOrigin::End); - if(!check(writePos)) { - return 0; - } - - pos = size_t(writePos); - } - - int written = fileWrite(handle, buffer, size); - if(check(written)) { - pos += size_t(written); - this->size = pos; - } - - return written > 0 ? written : 0; -} - -int FileStream::seekFrom(int offset, SeekOrigin origin) -{ - // Cannot rely on return value from fileSeek - failure does not mean position hasn't changed - fileSeek(handle, offset, origin); - int newpos = fileTell(handle); - if(check(newpos)) { - pos = size_t(newpos); - if(pos > size) { - size = pos; - } - } - return newpos; -} - -String FileStream::fileName() const -{ - spiffs_stat stat; - fileStats(handle, &stat); - return String(reinterpret_cast(stat.name)); -} - -String FileStream::id() const -{ - spiffs_stat stat; - fileStats(handle, &stat); - -#define ETAG_SIZE 16 - char buf[ETAG_SIZE]; - m_snprintf(buf, ETAG_SIZE, _F("00f-%x-%x0-%x"), stat.obj_id, stat.size, - strlen(reinterpret_cast(stat.name))); - - return String(buf); -} - -bool FileStream::truncate(size_t newSize) -{ - bool res = check(fileTruncate(handle, newSize)); - if(res) { - size = newSize; - if(pos > size) { - pos = size; - } - } - return res; -} diff --git a/Sming/Core/Data/Stream/FileStream.h b/Sming/Core/Data/Stream/FileStream.h index d5b0b01037..d391a2aa42 100644 --- a/Sming/Core/Data/Stream/FileStream.h +++ b/Sming/Core/Data/Stream/FileStream.h @@ -10,17 +10,17 @@ #pragma once -#include "ReadWriteStream.h" -#include "FileSystem.h" +#include "IFS/FileStream.h" +#include /** * @brief File stream class * @ingroup stream data */ -class FileStream : public ReadWriteStream +class FileStream : public IFS::FileStream { public: - FileStream() + FileStream() : IFS::FileStream(::getFileSystem()) { } @@ -28,155 +28,21 @@ class FileStream : public ReadWriteStream * @param fileName Name of file to open * @param openFlags */ - FileStream(const String& fileName, FileOpenFlags openFlags = eFO_ReadOnly) + FileStream(const String& fileName, FileOpenFlags openFlags = File::ReadOnly) : FileStream() { open(fileName, openFlags); } - ~FileStream() + FileStream(const FileStat& stat, FileOpenFlags openFlags = File::ReadOnly) : FileStream() { - close(); + open(stat, openFlags); } - /** @brief Attach this stream object to an open file handle - * @param file - * @param size - */ - void attach(file_t file, size_t size); + using IFS::FileStream::attach; /** @deprecated Use `open()` instead */ - bool attach(const String& fileName, FileOpenFlags openFlags = eFO_ReadOnly) SMING_DEPRECATED + bool attach(const String& fileName, FileOpenFlags openFlags = File::ReadOnly) SMING_DEPRECATED { return open(fileName, openFlags); } - - /** @brief Open a file and attach this stream object to it - * @param fileName - * @param openFlags - * @retval bool true on success, false on error - * @note call getLastError() to determine cause of failure - */ - bool open(const String& fileName, FileOpenFlags openFlags = eFO_ReadOnly); - - /** @brief Close file - */ - void close(); - - StreamType getStreamType() const override - { - return eSST_File; - } - - size_t write(const uint8_t* buffer, size_t size) override; - - int read() override - { - char c; - return readBytes(&c, 1) ? static_cast(c) : -1; - } - - size_t readBytes(char* buffer, size_t length) override; - - uint16_t readMemoryBlock(char* data, int bufSize) override; - - int seekFrom(int offset, SeekOrigin origin) override; - - bool isFinished() override - { - return fileIsEOF(handle); - } - - /** @brief Filename of file stream is attached to - * @retval String invalid if stream isn't open - */ - String fileName() const; - - /** @brief Determine if file exists - * @retval bool true if stream contains valid file - */ - bool fileExist() const - { - return handle >= 0; - } - - String getName() const override - { - return fileName(); - } - - bool isValid() const override - { - return fileExist(); - } - - /** @brief Get the offset of cursor from beginning of data - * @retval size_t Cursor offset - */ - size_t getPos() const - { - return pos; - } - - /** @brief Get the total file size - * @retval size_t File size - */ - size_t getSize() const - { - return size; - } - - /** @brief Return the maximum bytes available to read, from current position - * @retval int -1 is returned when the size cannot be determined - */ - int available() override - { - return size - pos; - } - - String id() const override; - - /** @brief determine if an error occurred during operation - * @retval int filesystem error code - */ - int getLastError() - { - return lastError; - } - - /** @brief Reduce the file size - * @param newSize - * @retval bool true on success - */ - bool truncate(size_t newSize); - - /** @brief Truncate file at current position - * @retval bool true on success - */ - bool truncate() - { - return truncate(pos); - } - -private: - /** @brief Check file operation result and note error code - * @param res result of fileXXX() operation to check - * @retval bool true if operation was successful, false if error occurred - */ - bool check(int res) - { - if(res >= 0) { - return true; - } - - if(lastError >= 0) { - lastError = res; - } - return false; - } - -private: - file_t handle = -1; - size_t pos = 0; - size_t size = 0; - int lastError = SPIFFS_OK; }; diff --git a/Sming/Core/Data/Stream/GdbFileStream.cpp b/Sming/Core/Data/Stream/GdbFileStream.cpp deleted file mode 100644 index ae7d03c967..0000000000 --- a/Sming/Core/Data/Stream/GdbFileStream.cpp +++ /dev/null @@ -1,130 +0,0 @@ -/**** - * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. - * Created 2015 by Skurydin Alexey - * http://github.com/SmingHub/Sming - * All files of the Sming Core are provided under the LGPL v3 license. - * - * GdbFileStream.cpp - * - ****/ - -#include "GdbFileStream.h" -#include - -bool GdbFileStream::open(const String& fileName, FileOpenFlags openFlags) -{ - lastError = 0; - - int flags = 0; - if((openFlags & eFO_ReadWrite) == eFO_ReadWrite) { - flags = O_RDWR; - } else if(openFlags & eFO_WriteOnly) { - flags = O_WRONLY; - } else { - flags = O_RDONLY; - } - if(openFlags & eFO_CreateIfNotExist) { - flags |= O_CREAT; - } - if(openFlags & eFO_Truncate) { - flags |= O_TRUNC; - } - - int fd = gdb_syscall_open(fileName.c_str(), flags, S_IRWXU); - if(!check(fd)) { - debug_w("File wasn't found: %s", fileName.c_str()); - return false; - } - - // Get size - int size = gdb_syscall_lseek(fd, 0, SEEK_END); - if(check(size)) { - close(); - if(fd >= 0) { - handle = fd; - this->size = size; - gdb_syscall_lseek(fd, 0, SEEK_SET); - pos = 0; - debug_d("opened file: '%s' (%u bytes) #0x%08X", fileName.c_str(), size, this); - } - return true; - } - - gdb_syscall_close(fd); - return false; -} - -void GdbFileStream::close() -{ - if(handle >= 0) { - gdb_syscall_close(handle); - handle = -1; - } - size = 0; - pos = 0; - lastError = 0; -} - -uint16_t GdbFileStream::readMemoryBlock(char* data, int bufSize) -{ - if(data == nullptr || bufSize <= 0 || pos >= size) { - return 0; - } - - int available = gdb_syscall_read(handle, data, std::min(size - pos, size_t(bufSize))); - (void)check(available); - - // Don't move cursor now (waiting seek) - (void)gdb_syscall_lseek(handle, pos, SEEK_SET); - - return available > 0 ? available : 0; -} - -size_t GdbFileStream::write(const uint8_t* buffer, size_t size) -{ - if(!fileExist()) { - return 0; - } - - int writePos = gdb_syscall_lseek(handle, 0, SEEK_END); - if(!check(writePos)) { - return 0; - } - - pos = size_t(writePos); - - int written = gdb_syscall_write(handle, buffer, size); - if(check(written)) { - pos += size_t(written); - if(pos > this->size) { - this->size = pos; - } - } - - return written; -} - -int GdbFileStream::seekFrom(int offset, SeekOrigin origin) -{ - int newpos = gdb_syscall_lseek(handle, offset, int(origin)); - if(check(newpos)) { - pos = size_t(newpos); - if(pos > size) { - size = pos; - } - } - - return newpos; -} - -String GdbFileStream::id() const -{ - gdb_stat_t stat; - gdb_syscall_fstat(handle, &stat); - -#define ETAG_SIZE 16 - char buf[ETAG_SIZE]; - m_snprintf(buf, ETAG_SIZE, _F("00f-%x-%x0-%x"), stat.st_ino, stat.st_size, fileName.length()); - - return String(buf); -} diff --git a/Sming/Core/Data/Stream/GdbFileStream.h b/Sming/Core/Data/Stream/GdbFileStream.h index fbc47e2b3e..e7f564a41b 100644 --- a/Sming/Core/Data/Stream/GdbFileStream.h +++ b/Sming/Core/Data/Stream/GdbFileStream.h @@ -10,130 +10,20 @@ #pragma once -#include "ReadWriteStream.h" -#include "FileSystem.h" +#include "IFS/FileStream.h" +#include /** * @brief GDB File stream class to provide access to host files whilst running under debugger * @ingroup stream gdb_syscall */ -class GdbFileStream : public ReadWriteStream +class GdbFileStream : public IFS::FileStream { public: - GdbFileStream() + GdbFileStream() : IFS::FileStream(&fileSystem) { } - /** @brief Create a file stream - * @param fileName Name of file to open - * @param openFlags - */ - GdbFileStream(const String& fileName, FileOpenFlags openFlags = eFO_ReadOnly) - { - open(fileName, openFlags); - } - - ~GdbFileStream() - { - close(); - } - - /** @brief Open a file and attach this stream object to it - * @param fileName - * @param openFlags - * @retval bool true on success, false on error - * @note call getLastError() to determine cause of failure - */ - bool open(const String& fileName, FileOpenFlags openFlags = eFO_ReadOnly); - - /** @brief Close file - */ - void close(); - - size_t write(const uint8_t* buffer, size_t size) override; - - uint16_t readMemoryBlock(char* data, int bufSize) override; - - int seekFrom(int offset, SeekOrigin origin) override; - - bool isFinished() override - { - return pos == size; - } - - /** @brief Filename of file stream is attached to - * @retval String invalid if stream isn't open - */ - String getFileName() const - { - return fileName; - } - - /** @brief Determine if file exists - * @retval bool true if stream contains valid file - */ - bool fileExist() const - { - return handle >= 0; - } - - String getName() const override - { - return fileName; - } - - bool isValid() const override - { - return fileExist(); - } - - /** @brief Get the offset of cursor from beginning of data - * @retval size_t Cursor offset - */ - size_t getPos() const - { - return pos; - } - - /** @brief Return the total length of the stream - * @retval int -1 is returned when the size cannot be determined - */ - int available() override - { - return size - pos; - } - - String id() const override; - - /** @brief determine if an error occurred during operation - * @retval int filesystem error code - */ - int getLastError() - { - return lastError; - } - -private: - /** @brief Check file operation result and note error code - * @param res result of fileXXX() operation to check - * @retval bool true if operation was successful, false if error occurred - */ - bool check(int res) - { - if(res >= 0) { - return true; - } - - if(lastError >= 0) { - lastError = res; - } - return false; - } - private: - String fileName; - int handle = -1; - size_t pos = 0; - size_t size = 0; - int lastError = 0; + IFS::Gdb::FileSystem fileSystem; }; diff --git a/Sming/Core/Data/Stream/IFS/DirectoryTemplate.cpp b/Sming/Core/Data/Stream/IFS/DirectoryTemplate.cpp new file mode 100644 index 0000000000..ca667ef846 --- /dev/null +++ b/Sming/Core/Data/Stream/IFS/DirectoryTemplate.cpp @@ -0,0 +1,118 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/anakod/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * DirectoryStream.cpp + * + * @author mikee47 May 2019 + * + ****/ + +#include "DirectoryTemplate.h" +#include +#include + +namespace IFS +{ +namespace +{ +#define XX(name, comment) DEFINE_FSTR_LOCAL(fieldstr_##name, #name) +DIRSTREAM_FIELD_MAP(XX) +#undef XX + +#define XX(name, comment) &fieldstr_##name, +DEFINE_FSTR_VECTOR(fieldStrings, FSTR::String, DIRSTREAM_FIELD_MAP(XX)) +#undef XX + +} // namespace + +String DirectoryTemplate::getValue(const char* name) +{ + String value = SectionTemplate::getValue(name); + if(value) { + return value; + } + + int i = fieldStrings.indexOf(name); + auto field = Field(i + 1); + + auto& d = dir(); + auto& s = d.stat(); + + bool statValid = d.isValid(); + + switch(field) { + case Field::unknown: + break; + + case Field::file_id: + return statValid ? String(s.id) : nullptr; + + case Field::name: + if(statValid) { + value.setString(s.name.buffer, s.name.length); + formatter().escape(value); + return value; + } else { + return nullptr; + } + + case Field::modified: + return statValid ? DateTime(s.mtime).toISO8601() : nullptr; + + case Field::size: + return statValid ? String(s.size) : nullptr; + + case Field::original_size: + return statValid ? String(s.compression.originalSize) : nullptr; + + case Field::attr: + return statValid ? IFS::getFileAttributeString(s.attr) : nullptr; + + case Field::attr_long: + return statValid ? toString(s.attr) : nullptr; + + case Field::compression: + if(!statValid) { + return nullptr; + } else if(!s.attr[FileAttribute::Compressed]) { + return ""; + } else { + return toString(s.compression.type); + } + + case Field::access: + return statValid ? IFS::getAclString(s.acl) : nullptr; + + case Field::access_long: + return statValid ? toString(s.acl) : nullptr; + + case Field::index: + return String(d.index()); + + case Field::total_size: + return String(d.size()); + + case Field::path: + return d.getPath(); + + case Field::parent: { + value = d.getParent(); + if(value.length() != 0) { + formatter().escape(value); + } else { + value = ""; + } + return value; + } + + case Field::last_error: + return d.getLastErrorString(); + } + + return nullptr; +} + +} // namespace IFS diff --git a/Sming/Core/Data/Stream/IFS/DirectoryTemplate.h b/Sming/Core/Data/Stream/IFS/DirectoryTemplate.h new file mode 100644 index 0000000000..287c70b422 --- /dev/null +++ b/Sming/Core/Data/Stream/IFS/DirectoryTemplate.h @@ -0,0 +1,82 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/anakod/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * DirectoryTemplate.h + * + * @author mikee47 May 2019 + * + * + ****/ + +#pragma once + +#include "../SectionTemplate.h" +#include + +#define DIRSTREAM_FIELD_MAP(XX) \ + XX(file_id, "File identifier") \ + XX(name, "Filename") \ + XX(modified, "Date/time of file modification") \ + XX(size, "File size in bytes") \ + XX(original_size, "Original size of compressed file") \ + XX(attr, "File attributes (brief)") \ + XX(attr_long, "File attributes") \ + XX(compression, "Compression type") \ + XX(access, "File access information (brief)") \ + XX(access_long, "File access information") \ + XX(index, "Zero-based index of current file") \ + XX(total_size, "Total size of files processed (in bytes)") \ + XX(path, "Path to containing directory") \ + XX(parent, "Path to parent directory (if any)") \ + XX(last_error, "Last error message") + +namespace IFS +{ +/** + * @brief Directory stream class + * @ingroup stream data + */ +class DirectoryTemplate : public SectionTemplate +{ +public: + enum class Field { + unknown = 0, +#define XX(name, comment) name, + DIRSTREAM_FIELD_MAP(XX) +#undef XX + }; + + DirectoryTemplate(IDataSourceStream* source, Directory* dir) : SectionTemplate(source), directory(dir) + { + } + + ~DirectoryTemplate() + { + delete directory; + } + + Directory& dir() + { + return *directory; + } + + bool nextRecord() override + { + if(sectionIndex() == 1) { + return directory->next(); + } + + return recordIndex() < 0; + } + +protected: + String getValue(const char* name) override; + +private: + Directory* directory; +}; + +} // namespace IFS diff --git a/Sming/Core/Data/Stream/IFS/FileStream.cpp b/Sming/Core/Data/Stream/IFS/FileStream.cpp new file mode 100644 index 0000000000..e491353c86 --- /dev/null +++ b/Sming/Core/Data/Stream/IFS/FileStream.cpp @@ -0,0 +1,233 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * FileStream.cpp + * + ****/ + +#include "FileStream.h" + +#define ETAG_SIZE 16 + +namespace IFS +{ +void FileStream::attach(FileHandle file, size_t size) +{ + close(); + if(file < 0) { + return; + } + + auto fs = getFileSystem(); + if(fs == nullptr) { + return; + } + handle = file; + this->size = size; + fs->lseek(handle, 0, SeekOrigin::Start); + pos = 0; + + debug_d("attached file: '%s' (%u bytes) #0x%08X", fileName().c_str(), size, this); +} + +bool FileStream::open(const Stat& stat, OpenFlags openFlags) +{ + auto fs = getFileSystem(); + if(fs == nullptr) { + return false; + } + + lastError = FS_OK; + + FileHandle file = fs->fopen(stat, openFlags); + if(!check(file)) { + return false; + } + + attach(file, stat.size); + return true; +} + +bool FileStream::open(const String& fileName, OpenFlags openFlags) +{ + auto fs = getFileSystem(); + if(fs == nullptr) { + return false; + } + + lastError = FS_OK; + + FileHandle file = fs->open(fileName, openFlags); + if(!check(file)) { + debug_w("File '%s' open error: %s", fileName.c_str(), fs->getErrorString(file).c_str()); + return false; + } + + // Get size + int size = fs->lseek(file, 0, SeekOrigin::End); + if(check(size)) { + attach(file, size); + return true; + } + + fs->close(file); + return false; +} + +void FileStream::close() +{ + if(handle >= 0) { + auto fs = getFileSystem(); + assert(fs != nullptr); + fs->close(handle); + handle = -1; + } + size = 0; + pos = 0; + lastError = FS_OK; +} + +size_t FileStream::readBytes(char* buffer, size_t length) +{ + if(buffer == nullptr || length == 0 || pos >= size) { + return 0; + } + + auto fs = getFileSystem(); + if(fs == nullptr) { + return 0; + } + + int available = fs->read(handle, buffer, std::min(size - pos, length)); + if(!check(available)) { + return 0; + } + + pos += size_t(available); + + return available; +} + +uint16_t FileStream::readMemoryBlock(char* data, int bufSize) +{ + auto fs = getFileSystem(); + if(fs == nullptr) { + return 0; + } + + assert(bufSize >= 0); + size_t startPos = pos; + size_t count = readBytes(data, bufSize); + + // Move cursor back to start position + (void)fs->lseek(handle, startPos, SeekOrigin::Start); + pos = startPos; + + return count; +} + +size_t FileStream::write(const uint8_t* buffer, size_t size) +{ + auto fs = getFileSystem(); + if(fs == nullptr) { + return 0; + } + + if(pos != this->size) { + int writePos = fs->lseek(handle, 0, SeekOrigin::End); + if(!check(writePos)) { + return 0; + } + + pos = size_t(writePos); + } + + int written = fs->write(handle, buffer, size); + if(check(written)) { + pos += size_t(written); + this->size = pos; + } + + return written > 0 ? written : 0; +} + +int FileStream::seekFrom(int offset, SeekOrigin origin) +{ + auto fs = getFileSystem(); + if(fs == nullptr) { + return 0; + } + + // Cannot rely on return value from fileSeek - failure does not mean position hasn't changed + fs->lseek(handle, offset, origin); + int newpos = fs->tell(handle); + if(check(newpos)) { + pos = size_t(newpos); + if(pos > size) { + size = pos; + } + } + return newpos; +} + +String FileStream::fileName() const +{ + auto fs = getFileSystem(); + if(fs == nullptr) { + return nullptr; + } + + NameStat stat; + int res = fs->fstat(handle, stat); + return (res < 0 || stat.name.length == 0) ? nullptr : stat.name.buffer; +} + +MimeType FileStream::getMimeType() const +{ + String name = fileName(); + if(name.endsWith(".gz")) { + name.remove(name.length() - 3); + } + return ContentType::fromFullFileName(name, MIME_UNKNOWN); +} + +String FileStream::id() const +{ + auto fs = getFileSystem(); + if(fs == nullptr) { + return 0; + } + + Stat stat; + int res = fs->fstat(handle, stat); + if(res < 0) { + return nullptr; + } + + char buf[ETAG_SIZE]; + m_snprintf(buf, ETAG_SIZE, _F("00f-%x-%x0-%x"), stat.id, stat.size, stat.name.length); + + return String(buf); +} + +bool FileStream::truncate(size_t newSize) +{ + auto fs = getFileSystem(); + if(fs == nullptr) { + return 0; + } + + bool res = check(fs->ftruncate(handle, newSize)); + if(res) { + size = newSize; + if(pos > size) { + pos = size; + } + } + return res; +} + +} // namespace IFS diff --git a/Sming/Core/Data/Stream/IFS/FileStream.h b/Sming/Core/Data/Stream/IFS/FileStream.h new file mode 100644 index 0000000000..d9147d015f --- /dev/null +++ b/Sming/Core/Data/Stream/IFS/FileStream.h @@ -0,0 +1,148 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * FileStream.h + * + ****/ + +#pragma once + +#include "../ReadWriteStream.h" +#include + +namespace IFS +{ +/** + * @brief File stream class + * @ingroup stream data + */ +class FileStream : public FsBase, public ReadWriteStream +{ +public: + using FsBase::FsBase; + + ~FileStream() + { + close(); + } + + /** @brief Attach this stream object to an open file handle + * @param file + * @param size + */ + void attach(FileHandle file, size_t size); + + bool open(const Stat& stat, OpenFlags openFlags = OpenFlag::Read); + + /** @brief Open a file and attach this stream object to it + * @param fileName + * @param openFlags + * @retval bool true on success, false on error + * @note call getLastError() to determine cause of failure + */ + bool open(const String& fileName, IFS::OpenFlags openFlags = OpenFlag::Read); + + /** @brief Close file + */ + void close(); + + StreamType getStreamType() const override + { + return eSST_File; + } + + size_t write(const uint8_t* buffer, size_t size) override; + + int read() override + { + char c; + return readBytes(&c, 1) ? static_cast(c) : -1; + } + + size_t readBytes(char* buffer, size_t length) override; + + uint16_t readMemoryBlock(char* data, int bufSize) override; + + int seekFrom(int offset, SeekOrigin origin) override; + + bool isFinished() override + { + auto fs = getFileSystem(); + return fs == nullptr || lastError != FS_OK || fs->eof(handle) != 0; + } + + /** @brief Filename of file stream is attached to + * @retval String invalid if stream isn't open + */ + String fileName() const; + + /** @brief Determine if file exists + * @retval bool true if stream contains valid file + */ + bool fileExist() const + { + return handle >= 0; + } + + String getName() const override + { + return fileName(); + } + + MimeType getMimeType() const override; + + bool isValid() const override + { + return fileExist(); + } + + /** @brief Get the offset of cursor from beginning of data + * @retval size_t Cursor offset + */ + size_t getPos() const + { + return pos; + } + + /** @brief Get the total file size + * @retval size_t File size + */ + size_t getSize() const + { + return size; + } + + /** @brief Return the maximum bytes available to read, from current position + * @retval int -1 is returned when the size cannot be determined + */ + int available() override + { + return size - pos; + } + + String id() const override; + + /** @brief Reduce the file size + * @param newSize + * @retval bool true on success + */ + bool truncate(size_t newSize); + + /** @brief Truncate file at current position + * @retval bool true on success + */ + bool truncate() + { + return truncate(pos); + } + +private: + FileHandle handle{-1}; + size_t pos{0}; + size_t size{0}; +}; + +} // namespace IFS diff --git a/Sming/Core/Data/Stream/IFS/HtmlDirectoryTemplate.cpp b/Sming/Core/Data/Stream/IFS/HtmlDirectoryTemplate.cpp new file mode 100644 index 0000000000..256f270037 --- /dev/null +++ b/Sming/Core/Data/Stream/IFS/HtmlDirectoryTemplate.cpp @@ -0,0 +1,67 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * HtmlDirectoryTemplate.h + * + * @author mikee47 May 2019 + * + ****/ + +#include "HtmlDirectoryTemplate.h" +#include +#include + +namespace IFS +{ +String HtmlDirectoryTemplate::getValue(const char* name) +{ + String text = DirectoryTemplate::getValue(name); + if(text) { + return text; + } + + if(dir().isValid()) { + auto& stat = dir().stat(); + + if(FS("icon") == name) { + if(stat.isDir()) { + return F("📁"); + } + + auto mimeType = ContentType::fromFullFileName(stat.name, MIME_UNKNOWN); + // https://html-css-js.com/html/character-codes/icons/ + switch(mimeType) { + case MIME_TEXT: + return F("📄"); + case MIME_JS: + case MIME_CSS: + case MIME_HTML: + case MIME_XML: + case MIME_JSON: + return F("📑"); + + case MIME_JPEG: + case MIME_GIF: + case MIME_PNG: + case MIME_SVG: + case MIME_ICO: + return F("🍓"); + + case MIME_GZIP: + case MIME_ZIP: + return F("🗃"); + + case MIME_UNKNOWN: + default: + return F("•"); + } + } + } + + return nullptr; +} + +} // namespace IFS diff --git a/Sming/Core/Data/Stream/IFS/HtmlDirectoryTemplate.h b/Sming/Core/Data/Stream/IFS/HtmlDirectoryTemplate.h new file mode 100644 index 0000000000..ec60d66ba4 --- /dev/null +++ b/Sming/Core/Data/Stream/IFS/HtmlDirectoryTemplate.h @@ -0,0 +1,36 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * HtmlDirectoryTemplate.h + * + * @author mikee47 May 2019 + * + ****/ + +#pragma once + +#include "../../Format.h" +#include "DirectoryTemplate.h" + +namespace IFS +{ +/** + * @brief Read-only stream access to directory listing with HTML output + * @ingroup stream data +*/ +class HtmlDirectoryTemplate : public DirectoryTemplate +{ +public: + HtmlDirectoryTemplate(IDataSourceStream* source, Directory* dir) : DirectoryTemplate(source, dir) + { + setFormatter(Format::html); + } + +protected: + String getValue(const char* name) override; +}; + +} // namespace IFS diff --git a/Sming/Core/Data/Stream/IFS/JsonDirectoryTemplate.h b/Sming/Core/Data/Stream/IFS/JsonDirectoryTemplate.h new file mode 100644 index 0000000000..61ec5bc438 --- /dev/null +++ b/Sming/Core/Data/Stream/IFS/JsonDirectoryTemplate.h @@ -0,0 +1,33 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * JsonDirectoryTemplate.h + * + * @author mikee47 Nov 2020 + * + ****/ + +#pragma once + +#include "DirectoryTemplate.h" +#include + +namespace IFS +{ +/** + * @brief Read-only stream providing directory listing in JSON format +*/ +class JsonDirectoryTemplate : public DirectoryTemplate +{ +public: + JsonDirectoryTemplate(IDataSourceStream* source, Directory* dir) : DirectoryTemplate(source, dir) + { + setDoubleBraces(true); + setFormatter(Format::json); + } +}; + +} // namespace IFS diff --git a/Sming/Core/Data/Stream/LimitedMemoryStream.h b/Sming/Core/Data/Stream/LimitedMemoryStream.h index 4efc62726d..b414b6edef 100644 --- a/Sming/Core/Data/Stream/LimitedMemoryStream.h +++ b/Sming/Core/Data/Stream/LimitedMemoryStream.h @@ -66,6 +66,11 @@ class LimitedMemoryStream : public ReadWriteStream return writePos - readPos; } + size_t getCapacity() const + { + return capacity; + } + uint16_t readMemoryBlock(char* data, int bufSize) override; int seekFrom(int offset, SeekOrigin origin) override; diff --git a/Sming/Core/Data/Stream/MultiStream.cpp b/Sming/Core/Data/Stream/MultiStream.cpp index ac606cbc80..62cf474a4e 100644 --- a/Sming/Core/Data/Stream/MultiStream.cpp +++ b/Sming/Core/Data/Stream/MultiStream.cpp @@ -14,14 +14,13 @@ uint16_t MultiStream::readMemoryBlock(char* data, int bufSize) { - if(stream != nullptr && stream->isFinished()) { - delete stream; - stream = nullptr; + if(stream && stream->isFinished()) { + stream.reset(); } - if(stream == nullptr) { - stream = getNextStream(); - if(stream == nullptr) { + if(!stream) { + stream.reset(getNextStream()); + if(!stream) { finished = true; return 0; } diff --git a/Sming/Core/Data/Stream/MultiStream.h b/Sming/Core/Data/Stream/MultiStream.h index 78dc5a5876..425546434f 100644 --- a/Sming/Core/Data/Stream/MultiStream.h +++ b/Sming/Core/Data/Stream/MultiStream.h @@ -14,6 +14,7 @@ #pragma once #include "DataSourceStream.h" +#include /** * @brief Base class for read-only stream which generates output from multiple source streams @@ -22,21 +23,6 @@ class MultiStream : public IDataSourceStream { public: - ~MultiStream() - { - delete stream; - } - - StreamType getStreamType() const override - { - return eSST_Unknown; - } - - int available() override - { - return -1; - } - uint16_t readMemoryBlock(char* data, int bufSize) override; bool seek(int len) override; @@ -55,6 +41,6 @@ class MultiStream : public IDataSourceStream virtual IDataSourceStream* getNextStream() = 0; private: - IDataSourceStream* stream = nullptr; - bool finished = false; + std::unique_ptr stream; + bool finished{false}; }; diff --git a/Sming/Core/Data/StreamTransformer.cpp b/Sming/Core/Data/StreamTransformer.cpp index f3ac23902a..4090ec6122 100644 --- a/Sming/Core/Data/StreamTransformer.cpp +++ b/Sming/Core/Data/StreamTransformer.cpp @@ -24,57 +24,46 @@ uint16_t StreamTransformer::readMemoryBlock(char* data, int bufSize) tempStream = new CircularBuffer(NETWORK_SEND_BUFFER_SIZE + 10); } - if(tempStream->isFinished()) { - if(sourceStream->isFinished()) { - return 0; - } - - // Fill the temp stream with data... - int i = bufSize / blockSize + 1; - do { - int len = blockSize; - if(i == 1) { - len = bufSize % blockSize; - } - - len = sourceStream->readMemoryBlock(data, len); - if(!len) { - break; - } + // Use provided buffer as a temporary store for this operation + fillTempStream(data, bufSize); - saveState(); - size_t outLength = transform(reinterpret_cast(data), len, result, resultSize); - if(outLength > tempStream->room()) { - restoreState(); - break; - } - - if(tempStream->write(result, outLength) != outLength) { - debug_e("That should not happen!"); - restoreState(); - break; - } + return tempStream->readMemoryBlock(data, bufSize); +} - sourceStream->seek(len); - } while(--i); +void StreamTransformer::fillTempStream(char* buffer, size_t bufSize) +{ + size_t room; + auto maxChunkSize = std::min(bufSize, blockSize); + while((room = tempStream->room()) >= maxChunkSize) { + auto chunkSize = sourceStream->readMemoryBlock(buffer, maxChunkSize); + if(chunkSize == 0) { + return; + } - if(sourceStream->isFinished()) { - int outLength = transform(nullptr, 0, result, resultSize); - tempStream->write(result, outLength); + saveState(); + size_t outLength = transform(reinterpret_cast(buffer), chunkSize, result, resultSize); + if(outLength > room) { + restoreState(); + return; } - } /* if(tempStream->isFinished()) */ + auto written = tempStream->write(result, outLength); + assert(written == outLength); - return tempStream->readMemoryBlock(data, bufSize); + sourceStream->seek(chunkSize); + } + + if(sourceStream->isFinished()) { + auto outLength = transform(nullptr, 0, result, resultSize); + tempStream->write(result, outLength); + } } -//Use base class documentation bool StreamTransformer::seek(int len) { return tempStream->seek(len); } -//Use base class documentation bool StreamTransformer::isFinished() { return (sourceStream->isFinished() && (tempStream == nullptr || tempStream->isFinished())); diff --git a/Sming/Core/Data/StreamTransformer.h b/Sming/Core/Data/StreamTransformer.h index dee5dcfb08..fe293ec88e 100644 --- a/Sming/Core/Data/StreamTransformer.h +++ b/Sming/Core/Data/StreamTransformer.h @@ -117,9 +117,11 @@ class StreamTransformer : public IDataSourceStream StreamTransformerCallback transformCallback = nullptr; private: - IDataSourceStream* sourceStream = nullptr; - CircularBuffer* tempStream = nullptr; - uint8_t* result = nullptr; + void fillTempStream(char* buffer, size_t bufSize); + + IDataSourceStream* sourceStream{nullptr}; + CircularBuffer* tempStream{nullptr}; + uint8_t* result{nullptr}; size_t resultSize; size_t blockSize; }; diff --git a/Sming/Core/DateTime.cpp b/Sming/Core/DateTime.cpp old mode 100755 new mode 100644 diff --git a/Sming/Core/DateTime.h b/Sming/Core/DateTime.h old mode 100755 new mode 100644 diff --git a/Sming/Core/FileSystem.cpp b/Sming/Core/FileSystem.cpp index c7e1f1def1..9d239cdaab 100644 --- a/Sming/Core/FileSystem.cpp +++ b/Sming/Core/FileSystem.cpp @@ -9,216 +9,115 @@ ****/ #include "FileSystem.h" -#include +#include #include -file_t fileOpen(const String& name, FileOpenFlags flags) +namespace SmingInternal { - // Special fix to prevent known spifFS bug: manual delete file - if((flags & eFO_CreateNewAlways) == eFO_CreateNewAlways) { - if(fileExist(name)) { - fileDelete(name); - } - flags = (FileOpenFlags)((int)flags & ~eFO_Truncate); - } - - int res = SPIFFS_open(&_filesystemStorageHandle, name.c_str(), (spiffs_flags)flags, 0); - if(res < 0) { - debugf("open errno %d\n", SPIFFS_errno(&_filesystemStorageHandle)); - } - - return res; -} - -void fileClose(file_t file) -{ - SPIFFS_close(&_filesystemStorageHandle, file); +IFS::IFileSystem* activeFileSystem; } -int fileWrite(file_t file, const void* data, size_t size) +void fileSetFileSystem(IFS::IFileSystem* fileSystem) { - int res = SPIFFS_write(&_filesystemStorageHandle, file, (void*)data, size); - if(res < 0) { - debugf("write errno %d\n", SPIFFS_errno(&_filesystemStorageHandle)); + if(SmingInternal::activeFileSystem != fileSystem) { + delete SmingInternal::activeFileSystem; + SmingInternal::activeFileSystem = fileSystem; } - return res; } -int fileRead(file_t file, void* data, size_t size) +namespace { - int res = SPIFFS_read(&_filesystemStorageHandle, file, data, size); - if(res < 0) { - debugf("read errno %d\n", SPIFFS_errno(&_filesystemStorageHandle)); - } - return res; -} - -int fileSeek(file_t file, int offset, SeekOrigin origin) +template Storage::Partition findDefaultPartition(T subType) { - return SPIFFS_lseek(&_filesystemStorageHandle, file, offset, int(origin)); -} - -bool fileIsEOF(file_t file) -{ - return SPIFFS_eof(&_filesystemStorageHandle, file) != 0; -} - -int32_t fileTell(file_t file) -{ - return SPIFFS_tell(&_filesystemStorageHandle, file); -} - -int fileFlush(file_t file) -{ - return SPIFFS_fflush(&_filesystemStorageHandle, file); -} - -int fileStats(const String& name, spiffs_stat* stat) -{ - return SPIFFS_stat(&_filesystemStorageHandle, name.c_str(), stat); -} - -int fileStats(file_t file, spiffs_stat* stat) -{ - return SPIFFS_fstat(&_filesystemStorageHandle, file, stat); + auto part = *Storage::findPartition(subType); + if(part) { + debug_i("[%s] Found '%s'", part.typeString().c_str(), part.name().c_str()); + } else { + debug_e("[%s] No partition found", toString(subType).c_str()); + } + return part; } -int fileDelete(const String& name) +bool mount(IFS::IFileSystem* fs) { - return SPIFFS_remove(&_filesystemStorageHandle, name.c_str()); -} + if(fs == nullptr) { + debug_e("Failed to created filesystem object"); + return false; + } -int fileDelete(file_t file) -{ - return SPIFFS_fremove(&_filesystemStorageHandle, file); -} + int res = fs->mount(); + debug_i("mount() returned %d (%s)", res, fs->getErrorString(res).c_str()); -bool fileExist(const String& name) -{ - spiffs_stat stat = {0}; - if(fileStats(name.c_str(), &stat) < 0) { + if(res < 0) { + delete fs; return false; } - return stat.name[0] != '\0'; + + fileSetFileSystem(fs); + + debug_i("File system initialised"); + return true; } +} // namespace -int fileLastError(file_t fd) +bool spiffs_mount() { - return SPIFFS_errno(&_filesystemStorageHandle); + auto part = findDefaultPartition(Storage::Partition::SubType::Data::spiffs); + return part ? spiffs_mount(part) : false; } -void fileClearLastError(file_t fd) +bool spiffs_mount(Storage::Partition partition) { - SPIFFS_clearerr(&_filesystemStorageHandle); + auto fs = IFS::createSpiffsFilesystem(partition); + return mount(fs); } -int fileSetContent(const String& fileName, const String& content) +bool fwfs_mount() { - return fileSetContent(fileName, content.c_str(), content.length()); + auto part = findDefaultPartition(Storage::Partition::SubType::Data::fwfs); + return part ? fwfs_mount(part) : false; } -int fileSetContent(const String& fileName, const char* content, int length) +bool fwfs_mount(Storage::Partition partition) { - file_t file = fileOpen(fileName.c_str(), eFO_CreateNewAlways | eFO_WriteOnly); - if(file < 0) { - return file; - } - if(length < 0) { - length = strlen(content); - } - int res = fileWrite(file, content, length); - fileClose(file); - return res; + auto fs = IFS::createFirmwareFilesystem(partition); + return mount(fs); } -uint32_t fileGetSize(const String& fileName) +bool hyfs_mount() { - file_t file = fileOpen(fileName.c_str(), eFO_ReadOnly); - int size = fileSeek(file, 0, SeekOrigin::End); - fileClose(file); - return (size < 0) ? 0 : size; + auto fwfsPart = findDefaultPartition(Storage::Partition::SubType::Data::fwfs); + auto spiffsPart = findDefaultPartition(Storage::Partition::SubType::Data::spiffs); + return (fwfsPart && spiffsPart) ? hyfs_mount(fwfsPart, spiffsPart) : false; } -int fileRename(const String& oldName, const String& newName) +bool hyfs_mount(Storage::Partition fwfsPartition, Storage::Partition spiffsPartition) { - return SPIFFS_rename(&_filesystemStorageHandle, oldName.c_str(), newName.c_str()); + auto fs = IFS::createHybridFilesystem(fwfsPartition, spiffsPartition); + return mount(fs); } Vector fileList() { Vector result; - spiffs_DIR d; - spiffs_dirent info; - SPIFFS_opendir(&_filesystemStorageHandle, "/", &d); - while(true) { - if(!SPIFFS_readdir(&d, &info)) { - break; + DirHandle dir; + if(fileOpenDir(nullptr, dir) == FS_OK) { + FileNameStat stat; + while(fileReadDir(dir, stat) >= 0) { + result.add(stat.name.buffer); } - result.add(String((char*)info.name)); + fileCloseDir(dir); } - SPIFFS_closedir(&d); return result; } -String fileGetContent(const String& fileName) -{ - String res; - file_t file = fileOpen(fileName.c_str(), eFO_ReadOnly); - int size = fileSeek(file, 0, SeekOrigin::End); - if(size == 0) { - res = ""; // Valid String, file is empty - } else if(size > 0) { - fileSeek(file, 0, SeekOrigin::Start); - res.setLength(size); - if(fileRead(file, res.begin(), res.length()) != size) { - res = nullptr; // read failed, invalidate String - } - } - fileClose(file); - return res; -} - -size_t fileGetContent(const String& fileName, char* buffer, size_t bufSize) +IFS::IFileSystem::Type fileSystemType() { - if(buffer == nullptr || bufSize == 0) { - return 0; + if(SmingInternal::activeFileSystem == nullptr) { + return IFS::IFileSystem::Type::Unknown; } - - file_t file = fileOpen(fileName.c_str(), eFO_ReadOnly); - int size = fileSeek(file, 0, SeekOrigin::End); - if(size <= 0 || bufSize <= size_t(size)) { - size = 0; - } else { - fileSeek(file, 0, SeekOrigin::Start); - if(fileRead(file, buffer, size) != size) { - size = 0; // Error - } - } - fileClose(file); - buffer[size] = '\0'; - return size; -} - -int fileTruncate(file_t file, size_t newSize) -{ - return SPIFFS_ftruncate(&_filesystemStorageHandle, file, newSize); -} - -int fileTruncate(file_t file) -{ - int pos = fileTell(file); - return (pos < 0) ? pos : fileTruncate(file, pos); -} - -int fileTruncate(const String& fileName, size_t newSize) -{ - file_t file = fileOpen(fileName, eFO_WriteOnly); - if(file < 0) { - return file; - } - - int res = fileTruncate(file, newSize); - fileClose(file); - return res; + IFS::IFileSystem::Info info; + fileGetSystemInfo(info); + return info.type; } diff --git a/Sming/Core/FileSystem.h b/Sming/Core/FileSystem.h index aece4d25a0..750ea338e7 100644 --- a/Sming/Core/FileSystem.h +++ b/Sming/Core/FileSystem.h @@ -15,46 +15,174 @@ #pragma once +#include +#include +#include #include -#include -#include -#include "Data/Stream/SeekOrigin.h" - -using file_t = signed short; ///< File handle - -/// File open flags -enum FileOpenFlags { - eFO_ReadOnly = SPIFFS_RDONLY, ///< Read only file - eFO_WriteOnly = SPIFFS_WRONLY, ///< Write only file - eFO_ReadWrite = eFO_ReadOnly | eFO_WriteOnly, ///< Read and write file - eFO_CreateIfNotExist = SPIFFS_CREAT, ///< Create new file if file does not exist - eFO_Append = SPIFFS_APPEND, ///< Append to file - eFO_Truncate = SPIFFS_TRUNC, ///< Truncate file - eFO_CreateNewAlways = eFO_CreateIfNotExist | eFO_Truncate ///< Create new file even if file exists +#include "WVector.h" ///< @deprecated see fileList() + +using file_t = IFS::FileHandle; +typedef SeekOrigin SeekOriginFlags; ///< @deprecated Use `SeekOrigin` instead +using FileHandle = IFS::FileHandle; +using DirHandle = IFS::DirHandle; +using FileOpenFlag = IFS::OpenFlag; +using FileOpenFlags = IFS::OpenFlags; +using FileAttribute = IFS::FileAttribute; +using FileAttributes = IFS::FileAttributes; +using FileStat = IFS::Stat; +using FileNameStat = IFS::NameStat; +constexpr int FS_OK = IFS::FS_OK; + +constexpr SeekOrigin eSO_FileStart{SeekOrigin::Start}; ///< @deprecated use SeekOrigin::Start +constexpr SeekOrigin eSO_CurrentPos{SeekOrigin::Current}; ///< @deprecated use SeekOrigin::Current +constexpr SeekOrigin eSO_FileEnd{SeekOrigin::End}; ///< @deprecated use SeekOrigin::End + +namespace SmingInternal +{ +/** @brief Global file system instance + * + * Filing system implementations should use helper functions to setup and + * initialise a filing system. If successful, call fileSetFileSystem() to make + * it active. This ensures that any active filing system is dismounted destroyed + * first. + * + */ +extern IFS::IFileSystem* activeFileSystem; + +} // namespace SmingInternal + +class File : public IFS::File +{ +public: + File() : IFS::File(SmingInternal::activeFileSystem) + { + } }; -static inline FileOpenFlags operator|(FileOpenFlags lhs, FileOpenFlags rhs) +/** + * @brief Directory stream class + * @ingroup stream data + */ +class Directory : public IFS::Directory { - return (FileOpenFlags)((int)lhs | (int)rhs); +public: + Directory() : IFS::Directory(SmingInternal::activeFileSystem) + { + } +}; + +// Various file flag combinations +constexpr FileOpenFlags eFO_ReadOnly{File::ReadOnly}; ///< @deprecated use File::ReadOnly +constexpr FileOpenFlags eFO_WriteOnly{File::WriteOnly}; ///< @deprecated use File::WriteOnly +constexpr FileOpenFlags eFO_ReadWrite{File::ReadWrite}; ///< @deprecated use File::ReadWrite +constexpr FileOpenFlags eFO_CreateIfNotExist{File::Create}; ///< @deprecated use File::Create +constexpr FileOpenFlags eFO_Append{File::Append}; ///< @deprecated use File::Append +constexpr FileOpenFlags eFO_Truncate{File::Truncate}; ///< @deprecated use File::Truncate +constexpr FileOpenFlags eFO_CreateNewAlways{File::CreateNewAlways}; ///< @deprecated use File::CreateNewAlways + +/* + * Boilerplate check for file function wrappers to catch undefined filesystem. + */ +#define CHECK_FS(_method) \ + auto fileSystem = static_cast(SmingInternal::activeFileSystem); \ + if(fileSystem == nullptr) { \ + debug_e("ERROR in %s(): No active file system", __FUNCTION__); \ + return file_t(IFS::Error::NoFileSystem); \ + } + +/** + * @brief Get the currently active file system, if any + * @retval IFS::IFileSystem* + */ +inline IFS::FileSystem* getFileSystem() +{ + if(SmingInternal::activeFileSystem == nullptr) { + debug_e("ERROR: No active file system"); + } + return static_cast(SmingInternal::activeFileSystem); } -/// File seek flags -constexpr SeekOrigin eSO_FileStart{SeekOrigin::Start}; ///< @deprecated use SeekOrigin::Start -constexpr SeekOrigin eSO_CurrentPos{SeekOrigin::Current}; ///< @deprecated use SeekOrigin::Current -constexpr SeekOrigin eSO_FileEnd{SeekOrigin::End}; ///< @deprecated use SeekOrigin::End +/** @brief Sets the currently active file system + * @param fileSystem + * @note Any existing filing system is freed first. + * Typically the filing system implementation has helper functions which + * create and initialise the file system to a valid state. The last step + * is to call this function to make it active. + * Call this function with nullptr to inactivate the filing system. + */ +void fileSetFileSystem(IFS::IFileSystem* fileSystem); + +inline void fileFreeFileSystem() +{ + fileSetFileSystem(nullptr); +} + +/** + * @brief Mount the first available SPIFFS volume + * @retval bool true on success + */ +bool spiffs_mount(); + +/** + * @brief Mount SPIFFS volume from a specific partition + */ +bool spiffs_mount(Storage::Partition partition); + +/** + * @brief Mount the first available FWFS volume + * @retval bool true on success + */ +bool fwfs_mount(); + +/** + * @brief Mount SPIFFS volume from a specific partition + * @retval bool true on success + */ +bool fwfs_mount(Storage::Partition partition); + +/** + * @brief Mount the first available FWFS and SPIFFS partitions as a hybrid filesystem + * @retval bool true on success + */ +bool hyfs_mount(); + +/** + * @brief Mount the given FWFS and SPIFFS partitions as a hybrid filesystem + * @retval bool true on success + */ +bool hyfs_mount(Storage::Partition fwfsPartition, Storage::Partition spiffsPartition); /** @brief Open file * @param name File name * @param flags Mode to open file * @retval file File ID or negative error code */ -file_t fileOpen(const String& name, FileOpenFlags flags); +inline file_t fileOpen(const char* name, FileOpenFlags flags = File::ReadOnly) +{ + CHECK_FS(open) + return fileSystem->open(name, flags); +} + +inline file_t fileOpen(const String& name, FileOpenFlags flags = File::ReadOnly) +{ + return fileOpen(name.c_str(), flags); +} + +inline file_t fileOpen(const FileStat& stat, FileOpenFlags flags = File::ReadOnly) +{ + CHECK_FS(fopen) + return fileSystem->fopen(stat, flags); +} /** @brief Clode file * @param file ID of file to open * @note File ID is returned from fileOpen function */ -void fileClose(file_t file); +inline int fileClose(file_t file) +{ + CHECK_FS(close) + return fileSystem->close(file); +} /** @brief Write to file * @param file File ID @@ -62,7 +190,20 @@ void fileClose(file_t file); * @param size Quantity of data elements to write to file * @retval int Quantity of data elements actually written to file or negative error code */ -int fileWrite(file_t file, const void* data, size_t size); +inline int fileWrite(file_t file, const void* data, size_t size) +{ + CHECK_FS(write); + return fileSystem->write(file, data, size); +} + +/** @brief Update file modification time + * @param file File ID + * @retval int Error code + */ +inline int fileTouch(file_t file) +{ + return fileWrite(file, nullptr, 0); +} /** @brief Read from file * @param file File ID @@ -70,112 +211,155 @@ int fileWrite(file_t file, const void* data, size_t size); * @param size Quantity of data elements to read from file * @retval int Quantity of data elements actually read from file or negative error code */ -int fileRead(file_t file, void* data, size_t size); +inline int fileRead(file_t file, void* data, size_t size) +{ + CHECK_FS(read) + return fileSystem->read(file, data, size); +} /** @brief Position file cursor * @param file File ID * @param offset Quantity of bytes to move cursor * @param origin Position from where to move cursor - * @retval Offset within file or negative error code + * @retval int Offset within file or negative error code */ -int fileSeek(file_t file, int offset, SeekOrigin origin); +inline int fileSeek(file_t file, int offset, SeekOrigin origin) +{ + CHECK_FS(seek) + return fileSystem->lseek(file, offset, origin); +} /** @brief Check if at end of file * @param file File ID - * @retval bool True if at end of file + * @retval bool true if at end of file */ -bool fileIsEOF(file_t file); +inline bool fileIsEOF(file_t file) +{ + auto fileSystem = getFileSystem(); + return fileSystem ? (fileSystem->eof(file) != 0) : true; +} /** @brief Get position in file * @param file File ID - * @retval int32_t Read / write cursor position + * @retval int32_t Read / write cursor position or error code */ -int32_t fileTell(file_t file); +inline int fileTell(file_t file) +{ + CHECK_FS(tell) + return fileSystem->tell(file); +} /** @brief Flush pending writes * @param file File ID - * @retval int Size of last file written or negative error number - */ -int fileFlush(file_t file); - -/** @brief Get last file system error code - * @param fd Not used - * @todo Why does fileLastError have unused fd parameter? - * @retval int Error code of last file system operation + * @retval int Size of last file written or error code */ -int fileLastError(file_t fd); +inline int fileFlush(file_t file) +{ + CHECK_FS(flush) + return fileSystem->flush(file); +} -/** @brief Clear last file system error - * @param fd Not used - * @todo Why does fileClearLastError have unused fd parameter? +/** @brief get the text for a returned error code + * @param err + * @retval String */ -void fileClearLastError(file_t fd); +inline String fileGetErrorString(int err) +{ + auto fileSystem = getFileSystem(); + if(fileSystem == nullptr) { + return IFS::Error::toString(err); + } + return fileSystem->getErrorString(err); +} /** @brief Create or replace file with defined content * @param fileName Name of file to create or replace * @param content Pointer to c-string containing content to populate file with - * @param length (optional) number of characters to write - * @retval int Positive value (>= 0) represents the numbers of bytes written - * Negative value (< 0) indicates error + * @param length Number of characters to write + * @retval int Number of bytes transferred or error code * @note This function creates a new file or replaces an existing file and * populates the file with the content of a c-string buffer. - If you do not specify `length`, remember to terminate your c-string buffer with a NUL ('\0'). */ -int fileSetContent(const String& fileName, const char* content, int length = -1); +template inline int fileSetContent(const TFileName& fileName, const char* content, size_t length) +{ + CHECK_FS(setContent) + return fileSystem->setContent(fileName, content, length); +} -/** @brief Create or replace file with defined content - * @param fileName Name of file to create or replace - * @param content String containing content to populate file with - * @retval int Positive integer represents the numbers of bytes written, - * negative integer represents the error code of last file system operation. - * @note This function creates a new file or replaces an existing file and - populates the file with the content of a string. - */ -int fileSetContent(const String& fileName, const String& content); +template inline int fileSetContent(const TFileName& fileName, TContent content) +{ + CHECK_FS(setContent) + return fileSystem->setContent(fileName, content); +} /** @brief Get size of file * @param fileName Name of file - * @retval uint32_t Size of file in bytes - * @note Returns 0 if error occurs + * @retval uint32_t Size of file in bytes, 0 on error */ -uint32_t fileGetSize(const String& fileName); +template inline uint32_t fileGetSize(const TFileName& fileName) +{ + auto fileSystem = getFileSystem(); + return fileSystem ? fileSystem->getSize(fileName) : 0; +} /** @brief Truncate (reduce) the size of an open file * @param file Open file handle, must have Write access * @param newSize - * @retval int error code + * @retval int Error code * @note In POSIX `ftruncate()` can also make the file bigger, however SPIFFS can only * reduce the file size and will return an error if newSize > fileSize */ -int fileTruncate(file_t file, size_t newSize); +inline int fileTruncate(file_t file, size_t newSize) +{ + CHECK_FS(truncate); + return fileSystem->ftruncate(file, newSize); +} /** @brief Truncate an open file at the current cursor position * @param file Open file handle, must have Write access - * @retval int error code + * @retval int Error code */ -int fileTruncate(file_t file); +inline int fileTruncate(file_t file) +{ + CHECK_FS(truncate); + return fileSystem->ftruncate(file); +} /** @brief Truncate (reduce) the size of a file * @param fileName * @param newSize - * @retval int error code + * @retval int Error code * @note In POSIX `truncate()` can also make the file bigger, however SPIFFS can only * reduce the file size and will return an error if newSize > fileSize */ -int fileTruncate(const String& fileName, size_t newSize); +template int fileTruncate(const TFileName& fileName, size_t newSize) +{ + CHECK_FS(truncate); + return fileSystem->truncate(fileName, newSize); +} /** @brief Rename file * @param oldName Original name of file to rename * @param newName New name for file - * @retval int error code + * @retval int Error code */ -int fileRename(const String& oldName, const String& newName); +inline int fileRename(const char* oldName, const char* newName) +{ + CHECK_FS(rename) + return fileSystem->rename(oldName, newName); +} + +inline int fileRename(const String& oldName, const String& newName) +{ + return fileRename(oldName.c_str(), newName.c_str()); +} /** @brief Get list of files on file system * @retval Vector Vector of strings. Each string element contains the name of a file on the file system + @deprecated use `Directory` object (or fileOpenDir / fileReadDir / fileCloseDir) */ -Vector fileList(); +Vector fileList() SMING_DEPRECATED; /** @brief Read content of a file * @param fileName Name of file to read from @@ -184,7 +368,11 @@ Vector fileList(); * The result will be an invalid String (equates to `false`) if the file could not be read. * If the file exists, but is empty, the result will be an empty string "". */ -String fileGetContent(const String& fileName); +template String fileGetContent(const TFileName& fileName) +{ + auto fileSystem = getFileSystem(); + return fileSystem ? fileSystem->getContent(fileName) : nullptr; +} /** @brief Read content of a file * @param fileName Name of file to read from @@ -192,46 +380,224 @@ String fileGetContent(const String& fileName); * @param bufSize Quantity of bytes to read from file * @retval size_t Quantity of bytes read from file or zero on failure * @note After calling this function the content of the file is placed in to a c-string - Ensure there is sufficient space in the buffer for file content - plus extra trailing null, i.e. at least bufSize + 1 - @note Returns 0 if the file could not be read + * Ensure there is sufficient space in the buffer for file content + * plus extra trailing null, i.e. at least bufSize + 1 + * Always check the return value! + * @note Returns 0 if the file could not be read */ -size_t fileGetContent(const String& fileName, char* buffer, size_t bufSize); +template inline size_t fileGetContent(const TFileName& fileName, char* buffer, size_t bufSize) +{ + auto fileSystem = getFileSystem(); + return fileSystem ? fileSystem->getContent(fileName, buffer, bufSize) : 0; +} -/** brief Get file statistics +template inline size_t fileGetContent(const TFileName& fileName, char* buffer) +{ + auto fileSystem = getFileSystem(); + return fileSystem ? fileSystem->getContent(fileName, buffer) : 0; +} + +/** @brief Get file statistics * @param name File name * @param stat Pointer to SPIFFS statistic structure to populate - * @retval int -1 on error - * @note Pass a pointer to an instantiated fileStats structure - * @todo Document the return value of fileStats + * @retval int Error code */ -int fileStats(const String& name, spiffs_stat* stat); +inline int fileStats(const char* fileName, FileStat& stat) +{ + CHECK_FS(stat) + return fileSystem->stat(fileName, &stat); +} + +inline int fileStats(const String& fileName, FileStat& stat) +{ + return fileStats(fileName.c_str(), stat); +} /** brief Get file statistics * @param file File ID * @param stat Pointer to SPIFFS statistic structure to populate - * @retval int -1 on error - * @note Pass a pointer to an instantiated fileStats structure - * @todo Document the return value of fileStats + * @retval int Error code */ -int fileStats(file_t file, spiffs_stat* stat); +inline int fileStats(file_t file, FileStat& stat) +{ + CHECK_FS(fstat) + return fileSystem->fstat(file, &stat); +} /** @brief Delete file * @param name Name of file to delete - * @retval int error code, 0 on success + * @retval int Error code */ -int fileDelete(const String& name); +inline int fileDelete(const char* fileName) +{ + CHECK_FS(remove) + return fileSystem->remove(fileName); +} + +inline int fileDelete(const String& fileName) +{ + return fileDelete(fileName.c_str()); +} /** @brief Delete file - * @param file ID of file to delete - * @retval int error code, 0 on success + * @param file handle of file to delete + * @retval int Error code */ -int fileDelete(file_t file); +inline int fileDelete(file_t file) +{ + CHECK_FS(fremove) + return fileSystem->fremove(file); +} /** @brief Check if a file exists on file system * @param name Name of file to check for - * @retval bool True if file exists + * @retval bool true if file exists + */ +inline bool fileExist(const char* fileName) +{ + CHECK_FS(stat) + return fileSystem->stat(fileName, nullptr) >= 0; +} + +inline bool fileExist(const String& fileName) +{ + return fileExist(fileName.c_str()); +} + +/** @brief Open a named directory for reading + * @param name Name of directory to open, empty or "/" for root + * @param dir Directory object + * @retval int Error code + * + */ +inline int fileOpenDir(const char* dirName, DirHandle& dir) +{ + CHECK_FS(opendir) + return fileSystem->opendir(dirName, dir); +} + +inline int fileOpenDir(const String& dirName, DirHandle& dir) +{ + return fileOpenDir(dirName.c_str(), dir); +} + +inline int fileOpenRootDir(DirHandle& dir) +{ + return fileOpenDir(nullptr, dir); +} + +/** @brief Open a sub-directory for reading + * @param stat Details of directory to open, nullptr for root directory + * @param dir Directory object + * @retrval 0 on success or negative error */ -bool fileExist(const String& name); +inline int fileOpenDir(const FileStat& stat, DirHandle& dir) +{ + CHECK_FS(opendir) + return fileSystem->fopendir(&stat, dir); +} + +/** @brief close a directory object + * @param dir directory to close + * @retval int Error code + */ +inline int fileCloseDir(DirHandle dir) +{ + CHECK_FS(closedir) + return fileSystem->closedir(dir); +} + +/** @brief Read a directory entry + * @param dir The directory object returned by fileOpenDir() + * @param stat The returned information, owned by DirHandle object + * @retval int Error code + */ +inline int fileReadDir(DirHandle dir, FileStat& stat) +{ + CHECK_FS(readdir) + return fileSystem->readdir(dir, stat); +} + +/** @brief Rewind to start of directory entries + * @param dir The directory object returned by fileOpenDir() + * @retval int Error code + */ +inline int fileRewindDir(DirHandle dir) +{ + CHECK_FS(rewinddir) + return fileSystem->rewinddir(dir); +} + +/** @brief Get basic file system information + * @retval int Error code + */ +inline int fileGetSystemInfo(IFS::IFileSystem::Info& info) +{ + CHECK_FS(getinfo) + return fileSystem->getinfo(info); +} + +/** @brief Get the type of file system currently mounted (if any) + * @retval FileSystemType the file system type + */ +IFS::IFileSystem::Type fileSystemType(); + +/** @brief Format the active file system + * @retval int Error code + */ +inline int fileSystemFormat() +{ + CHECK_FS(format) + return fileSystem->format(); +} + +/** @brief Perform a consistency check/repair on the active file system + * @retval int 0 if OK, < 0 unrecoverable errors, > 0 repairs required + */ +inline int fileSystemCheck() +{ + CHECK_FS(check) + return fileSystem->check(); +} + +/** @brief Set access control information + * @param file File handle + * @param acl + * @retval int Error code + */ +inline int fileSetACL(file_t file, const IFS::ACL& acl) +{ + CHECK_FS(setacl) + return fileSystem->setacl(file, acl); +} + +/** @name Set file attributes + * @param filename + * @param attr + * @retval int Error code + * @{ + */ +inline int fileSetAttr(const char* filename, FileAttributes attr) +{ + CHECK_FS(setattr) + return fileSystem->setattr(filename, attr); +} + +inline int fileSetAttr(const String& filename, FileAttributes attr) +{ + return fileSetAttr(filename.c_str(), attr); +} +/** @} */ + +/** @brief Set access control information for file + * @param file handle to open file, must have write access + * @retval int Error code + * @note any writes to file will reset this to current time + */ +inline int fileSetTime(file_t file, time_t mtime) +{ + CHECK_FS(settime) + return fileSystem->settime(file, mtime); +} /** @} */ diff --git a/Sming/Core/Network/Ftp/FtpDataFileList.h b/Sming/Core/Network/Ftp/FtpDataFileList.h index 6e9808fe7b..637804ddf7 100644 --- a/Sming/Core/Network/Ftp/FtpDataFileList.h +++ b/Sming/Core/Network/Ftp/FtpDataFileList.h @@ -11,13 +11,36 @@ #pragma once #include "FtpDataStream.h" -#include "FileSystem.h" +#include +#include class FtpDataFileList : public FtpDataStream { public: - explicit FtpDataFileList(FtpServerConnection& connection) : FtpDataStream(connection) + explicit FtpDataFileList(FtpServerConnection& connection, const String& path, bool namesOnly = false) + : FtpDataStream(connection), dir(connection.getFileSystem()), namesOnly(namesOnly) { + auto fs = dir.getFileSystem(); + assert(fs != nullptr); + auto& stat = const_cast(dir.stat()); + auto err = fs->stat(path, stat); + if(err != FS_OK) { + completed = true; + return; + } + + if(stat.isDir()) { + dir.open(path); + statValid = dir.next(); + } else { + statValid = true; + } + + if(SystemClock.isSet()) { + year = DateTime(SystemClock.now()).Year; + } else { + debug_w("[FTP] Warning: system clock hasn't been set!"); + } } void transferData(TcpConnectionEvent sourceEvent) override @@ -25,17 +48,50 @@ class FtpDataFileList : public FtpDataStream if(completed) { return; } - Vector list = fileList(); - debug_d("send file list: %d", list.count()); - for(unsigned i = 0; i < list.count(); i++) { - String s = F("01-01-15 01:00AM "); - s += fileGetSize(list[i]); - s += ' '; - s += list[i]; - s += "\r\n"; - writeString(s); + + String line; + while(statValid) { + getStatLine(dir.stat(), line); + line += "\r\n"; + if(line.length() > getAvailableWriteSize()) { + return; + } + int written = writeString(line); + if(written < 0) { + return; + } + statValid = dir.next(); } + + debug_d("sent file list: %u", dir.count()); completed = true; finishTransfer(); } + + void getStatLine(const FileStat& stat, String& line) + { + line.setLength(0); + if(!namesOnly) { + auto& user = control.getUser(); + DateTime dt{stat.mtime}; + String timestr = dt.format((dt.Year == year) ? _F("%b %e %H:%M") : _F("%b %e %Y")); + char buf[128]; + PSTR_ARRAY(fmt, "--- %-6u %s "); + m_snprintf(buf, sizeof(buf), fmt, stat.size, timestr.c_str()); + buf[0] = stat.isDir() ? 'd' : '-'; + buf[1] = (user.role >= stat.acl.readAccess) ? 'r' : '-'; + buf[2] = stat.attr[FileAttribute::ReadOnly] && (user.role >= stat.acl.writeAccess) ? 'w' : '-'; + line = buf; + } + line.concat(stat.name, stat.name.length); + if(stat.isDir()) { + line += '/'; + } + } + +private: + IFS::Directory dir; + uint16_t year; + bool namesOnly; + bool statValid; }; diff --git a/Sming/Core/Network/Ftp/FtpDataRetrieve.h b/Sming/Core/Network/Ftp/FtpDataRetrieve.h index 0fecb0821f..70cc360bc3 100644 --- a/Sming/Core/Network/Ftp/FtpDataRetrieve.h +++ b/Sming/Core/Network/Ftp/FtpDataRetrieve.h @@ -17,7 +17,7 @@ class FtpDataRetrieve : public FtpDataStream { public: FtpDataRetrieve(FtpServerConnection& connection, const String& fileName) - : FtpDataStream(connection), file(fileOpen(fileName, eFO_ReadOnly)) + : FtpDataStream(connection), file(fileOpen(fileName, File::ReadOnly)) { } diff --git a/Sming/Core/Network/Ftp/FtpDataStore.h b/Sming/Core/Network/Ftp/FtpDataStore.h index aa33032190..a713b97af0 100644 --- a/Sming/Core/Network/Ftp/FtpDataStore.h +++ b/Sming/Core/Network/Ftp/FtpDataStore.h @@ -17,7 +17,7 @@ class FtpDataStore : public FtpDataStream { public: FtpDataStore(FtpServerConnection& connection, const String& fileName) - : FtpDataStream(connection), file(fileOpen(fileName, eFO_WriteOnly | eFO_CreateNewAlways)) + : FtpDataStream(connection), file(fileOpen(fileName, File::WriteOnly | File::CreateNewAlways)) { } diff --git a/Sming/Core/Network/Ftp/FtpDataStream.h b/Sming/Core/Network/Ftp/FtpDataStream.h index 499a69083a..13f2018c59 100644 --- a/Sming/Core/Network/Ftp/FtpDataStream.h +++ b/Sming/Core/Network/Ftp/FtpDataStream.h @@ -13,56 +13,66 @@ #include "FtpServerConnection.h" #include "Network/TcpConnection.h" +/* + RFC959: + + User-DTP listens on data port when required. + + Server is responsible for creating the data connection. + + The server MUST close the data connection under the following conditions: + + 1. The server has completed sending data in a transfer mode + that requires a close to indicate EOF. + + 2. The server receives an ABORT command from the user. + + 3. The port specification is changed by a command from the user. + + 4. The control connection is closed legally or otherwise. + + 5. An irrecoverable error condition occurs. + + Otherwise the close is a server option, the exercise of which the + server must indicate to the user-process by either a 250 or 226 + reply only. +*/ class FtpDataStream : public TcpConnection { public: - explicit FtpDataStream(FtpServerConnection& connection) : TcpConnection(true), parent(connection) + explicit FtpDataStream(FtpServerConnection& control) : TcpConnection(true), control(control) { } - err_t onConnected(err_t err) override + ~FtpDataStream() { - //response(125, "Connected"); - setTimeOut(300); // Update timeout - return TcpConnection::onConnected(err); + control.dataStreamDestroyed(this); } - err_t onSent(uint16_t len) override + err_t onConnected(err_t err) override { - sent += len; - if(written < sent || !completed) { - return TcpConnection::onSent(len); - } - finishTransfer(); - return TcpConnection::onSent(len); + setTimeOut(300); + return TcpConnection::onConnected(err); } void finishTransfer() { close(); - parent.dataTransferFinished(this); + control.dataTransferFinished(this); } void response(int code, String text = nullptr) { - parent.response(code, text); - } - - int write(const char* data, int len, uint8_t apiflags = 0) override - { - written += len; - return TcpConnection::write(data, len, apiflags); + control.response(code, text); } void onReadyToSendData(TcpConnectionEvent sourceEvent) override { - if(!parent.isCanTransfer()) { - return; - } - if(completed && written == 0) { + if(completed) { finishTransfer(); + } else { + transferData(sourceEvent); } - transferData(sourceEvent); } virtual void transferData(TcpConnectionEvent sourceEvent) @@ -70,8 +80,6 @@ class FtpDataStream : public TcpConnection } protected: - FtpServerConnection& parent; + FtpServerConnection& control; bool completed{false}; - unsigned written{0}; - unsigned sent{0}; }; diff --git a/Sming/Core/Network/Ftp/FtpServerConnection.cpp b/Sming/Core/Network/Ftp/FtpServerConnection.cpp index 2fe610a7a3..ce37fae343 100644 --- a/Sming/Core/Network/Ftp/FtpServerConnection.cpp +++ b/Sming/Core/Network/Ftp/FtpServerConnection.cpp @@ -14,9 +14,53 @@ #include "FtpDataFileList.h" #include "../FtpServer.h" #include "Network/NetUtils.h" +#include + +// Name, Comment +#define FTP_COMMAND_MAP(XX) \ + XX(ACCT, "Identifies user's account") \ + XX(CWD, "Change working directory") \ + XX(CDUP, "Change to parent working directory") \ + XX(DELE, "Delete file") \ + XX(HELP, "List recognised commands") \ + XX(LIST, "List file or directory information") \ + XX(NLST, "List file or directory names") \ + XX(NOOP, "") \ + XX(PWD, "Get current working directory") \ + XX(PASS, "Must follow user") \ + XX(PASV, "") \ + XX(PORT, "") \ + XX(QUIT, "") \ + XX(RNFR, "Rename file: FROM") \ + XX(RNTO, "Rename file: TO") \ + XX(RETR, "Retrieve file content") \ + XX(SIZE, "Get file size") \ + XX(STOR, "Store file data") \ + XX(SYST, "Get system information") \ + XX(TYPE, "") \ + XX(USER, "Start login sequence, flushing any current account information") \ + XX(XCWD, "Deprecated alias for CWD") \ + XX(XPWD, "Deprecated alias for PWD") namespace { +#define XX(name, comment) #name "\0" +DEFINE_FSTR(fstrCommandList, FTP_COMMAND_MAP(XX)) +#undef XX + +enum class Command { + UNKNOWN, +#define XX(name, comment) name, + FTP_COMMAND_MAP(XX) +#undef XX +}; + +Command parseCommand(const String& data) +{ + int i = CStringArray(fstrCommandList).indexOf(data); + return Command(i + 1); +} + int getSplitterPos(const String& data, char splitter, uint8_t number) { uint8_t k = 0; @@ -33,27 +77,26 @@ int getSplitterPos(const String& data, char splitter, uint8_t number) return -1; } -String makeFileName(String name, bool shortIt) +String dirname(const char* path) { - if(name[0] == '/') { - name.remove(0, 1); + if(path == nullptr) { + return nullptr; } - - if(shortIt && name.length() > 20) { - String ext; - int dotPos = name.lastIndexOf('.'); - if(dotPos >= 0) { - ext = name.substring(dotPos); - } - - return name.substring(0, 16) + ext; + auto c = strrchr(path, '/'); + if(c == nullptr) { + return nullptr; } - - return name; + return String(path, c - path); } } // namespace +FtpServerConnection::FtpServerConnection(CustomFtpServer& parentServer, tcp_pcb* clientTcp) + : TcpConnection(clientTcp, true), server(parentServer) +{ + writeString(_F("220 Welcome to Sming FTP\r\n")); +} + err_t FtpServerConnection::onReceive(pbuf* buf) { if(buf == nullptr) { @@ -62,6 +105,7 @@ err_t FtpServerConnection::onReceive(pbuf* buf) int prev = 0; while(true) { + debug_hex(DBG, "CTRL < ", buf->payload, buf->len); int p = NetUtils::pbufFindStr(buf, "\r\n", prev); if(p < 0 || size_t(p - prev) > MAX_FTP_CMD) { break; @@ -103,234 +147,316 @@ void FtpServerConnection::cmdPort(const String& data) response(200); } -void FtpServerConnection::onCommand(String cmd, String data) +IFS::FileSystem* FtpServerConnection::getFileSystem() +{ + auto fs = server.getFileSystem(); + if(fs == nullptr) { + response(550, F("No active file system")); + } + return fs; +} + +String FtpServerConnection::resolvePath(const char* name) +{ + if(name != nullptr && name[0] == '/') { + return &name[1]; + } + + String path; + path += cwd.c_str(); + if(name != nullptr && *name != '\0') { + if(path.length() != 0) { + path += '/'; + } + path += name; + // Remove any trailing path separator + auto len = path.length(); + if(path[len - 1] == '/') { + path.setLength(len - 1); + } + } + return path; +} + +bool FtpServerConnection::checkFileAccess(const char* filename, IFS::OpenFlags flags) { - cmd.toUpperCase(); + auto fs = getFileSystem(); + if(fs == nullptr) { + return false; + } - /* Use macros to make code easier to read */ -#define SWITCH while(true) -#define CASE(s) if(cmd == _F(s)) -#define DEFAULT + FileStat stat; + int err = fs->stat(filename, stat); + if(err == IFS::Error::NotFound && flags[IFS::OpenFlag::Write]) { + // File doesn't exist - check permissions on parent directory + String path = dirname(filename); + err = fs->stat(path, stat); + } + if(err != FS_OK) { + response(550, fileGetErrorString(err)); // Not found / no access + return false; + } + if(flags[IFS::OpenFlag::Write]) { + if(stat.attr[FileAttribute::ReadOnly]) { + response(550, fileGetErrorString(IFS::Error::ReadOnly)); + } + if(user.role < stat.acl.writeAccess) { + response(550, _F("Write access denied")); // Not found / no access + return false; + } + } + if(user.role < stat.acl.readAccess) { + response(550, _F("Read access denied")); // Not found / no access + return false; + } + return true; +} +void FtpServerConnection::onCommand(String cmd, String data) +{ + auto command = parseCommand(cmd); + switch(command) { // We ready to quit always :) - CASE("QUIT") - { - response(221); + case Command::QUIT: + response(221); // Service closing control connection close(); - return; - } + break; // Strong security check :) - switch(state) { - case State::Authorization: - SWITCH - { - CASE("USER") - { - userName = data; - response(331); - break; - } - - CASE("PASS") - { - if(server.checkUser(userName, data)) { - userName = ""; - state = State::Active; - response(230); - } else { - response(430); - } - break; - } + case Command::USER: + // Authenticate or re-authenticate, wiping existing credentials + user = User{}; + user.name = data; + response(331); // User name OK, need password + break; - DEFAULT - { - response(530); - break; - } - } // SWITCH + case Command::PASS: { + if(!user.name) { + response(332); // Need account for login + } else { + user.role = server.validateUser(user.name.c_str(), data.c_str()); + response(user.isValid() ? 230 : 430); + } + user.name = nullptr; break; + } - case State::Active: - SWITCH - { - CASE("SYST") - { - response(215, F("Windows_NT: Sming Framework")); // Why not? It's look like Windows :) - break; - } + case Command::SYST: + response(215, F("Windows_NT: Sming Framework")); // Why not? It's look like Windows :) + break; - CASE("PWD") - { - response(257, F("\"/\"")); - break; - } + case Command::PWD: + case Command::XPWD: { + String s; + s += "\"/"; + s += cwd.c_str(); + s += '"'; + response(257, s); + break; + } - CASE("PORT") - { - cmdPort(data); - break; - } + case Command::PORT: + cmdPort(data); + break; - CASE("CWD") - { - if(data == "/") { - response(250); - } else { - response(550); - } - break; - } + case Command::CWD: + case Command::XCWD: { + String path = resolvePath(data.c_str()); + if(!checkFileAccess(path.c_str(), IFS::OpenFlag::Read)) { + break; + } + debug_i("CWD: '%s'", path.c_str()); + FileStat stat; + if(getFileSystem()->stat(path, stat) == FS_OK && stat.isDir()) { + cwd = path; + response(250, F("directory changed to /") + cwd.c_str()); // OK + } else { + response(550); // Not found / no access + } + break; + } - CASE("TYPE") - { - response(250); - break; - } + case Command::CDUP: { + cwd = dirname(cwd.c_str()); + response(250); // OK + break; + } - CASE("SIZE") - { - auto fileName = makeFileName(data, false); - if(fileExist(fileName)) { - auto fileSize = fileGetSize(fileName); - response(213, String(fileSize)); - } else { - response(550); - } - break; - } + case Command::TYPE: + response(250); // OK + break; - CASE("DELE") - { - String name = makeFileName(data, false); - if(fileExist(name)) { - fileDelete(name); - response(250); - } else { - response(550); - } - break; - } + case Command::SIZE: { + auto path = resolvePath(data.c_str()); + if(!checkFileAccess(cwd.c_str(), IFS::OpenFlag::Read)) { + break; + } + auto fs = getFileSystem(); + FileStat stat; + int err = fs->stat(path, stat); + if(err == FS_OK) { + response(213, String(stat.size)); // File status + } else { + response(550, fs->getErrorString(err)); // Not found / no access + } + break; + } - CASE("RNFR") - { - renameFrom = data; - response(350); - break; - } + case Command::DELE: { + String path = resolvePath(data.c_str()); + if(!checkFileAccess(cwd.c_str(), IFS::OpenFlag::Write | IFS::OpenFlag::Truncate)) { + break; + } + auto fs = getFileSystem(); + int err = fs->remove(path); + if(err == FS_OK) { + response(250); + } else { + response(550, getFileSystem()->getErrorString(err)); + } + break; + } - CASE("RNTO") - { - if(fileExist(renameFrom)) { - fileRename(renameFrom, data); - response(250); - } else { - response(550); - } - break; - } + case Command::RNFR: { + renameFrom = data; + response(350); // Pending further information + break; + } - CASE("RETR") - { - createDataConnection(new FtpDataRetrieve(*this, makeFileName(data, false))); - break; - } + case Command::RNTO: { + if(!renameFrom) { + response(550, F("Rename FROM unspecified")); + break; + } + String pathFrom = resolvePath(renameFrom.c_str()); + if(!checkFileAccess(pathFrom.c_str(), IFS::OpenFlag::Write)) { + break; + } + String pathTo = resolvePath(data.c_str()); + if(!checkFileAccess(pathTo.c_str(), IFS::OpenFlag::Write)) { + break; + } + auto fs = getFileSystem(); + int err = fs->rename(pathFrom, pathTo); + if(err == FS_OK) { + response(250); + } else { + response(550, fs->getErrorString(err)); + } + break; + } - CASE("STOR") - { - createDataConnection(new FtpDataStore(*this, makeFileName(data, true))); - break; - } + case Command::RETR: { + String path = resolvePath(data.c_str()); + if(checkFileAccess(path.c_str(), IFS::OpenFlag::Read)) { + setDataConnection(new FtpDataRetrieve(*this, path)); + } + break; + } - CASE("LIST") - { - createDataConnection(new FtpDataFileList(*this)); - break; - } + case Command::STOR: { + String path = resolvePath(data.c_str()); + if(checkFileAccess(path.c_str(), IFS::OpenFlag::Write)) { + setDataConnection(new FtpDataStore(*this, path)); + } + break; + } - CASE("PASV") - { - response(500, F("Passive mode not supported")); - break; - } + case Command::LIST: + case Command::NLST: { + String path = resolvePath(data.c_str()); + if(checkFileAccess(path.c_str(), IFS::OpenFlag::Read)) { + setDataConnection(new FtpDataFileList(*this, path, command == Command::NLST)); + } + break; + } - CASE("NOOP") - { - response(200); - break; - } + case Command::PASV: { + response(500, F("Passive mode not supported")); + break; + } - DEFAULT - { - if(!server.onCommand(cmd, data, *this)) { - response(502, F("Not supported")); - } - break; - } - } // SWITCH + case Command::NOOP: { + response(200); + break; + } + case Command::HELP: { + response(214, _F("The following commands are recognized."), '-'); + String s(fstrCommandList); + s.replace('\0', ' '); + writeString(" "); + writeString(s); + writeString("\r\n"); + response(214, _F("Help OK.")); break; + } + case Command::UNKNOWN: default: - debug_e("Invalid state"); - assert(false); + cmd.toUpperCase(); + if(!server.onCommand(cmd, data, *this)) { + response(502, F("Not supported")); + } } } err_t FtpServerConnection::onSent(uint16_t len) { - canTransfer = true; + if(dataConnection != nullptr) { + dataConnection->onReadyToSendData(eTCE_Poll); + } return ERR_OK; } -void FtpServerConnection::createDataConnection(TcpConnection* connection) +void FtpServerConnection::setDataConnection(FtpDataStream* connection) { + if(dataConnection != nullptr) { + SYSTEM_ERROR("[FTP] Data connection already exists!"); + } dataConnection = connection; dataConnection->connect(ip, port); response(150, F("Connecting")); } -void FtpServerConnection::dataTransferFinished(TcpConnection* connection) +void FtpServerConnection::dataStreamDestroyed(TcpConnection* connection) { if(connection != dataConnection) { SYSTEM_ERROR("FTP Wrong state: connection != dataConnection"); } dataConnection = nullptr; +} + +void FtpServerConnection::dataTransferFinished(TcpConnection* connection) +{ + if(dataConnection != nullptr) { + if(connection != dataConnection) { + SYSTEM_ERROR("FTP Wrong state: connection != dataConnection"); + } + dataConnection = nullptr; + } + response(226, F("Transfer Complete.")); } -void FtpServerConnection::response(int code, String text) +void FtpServerConnection::response(int code, String text, char sep) { String response = String(code, DEC); + response += sep; if(text.length() == 0) { if(code >= 200 && code <= 399) { // Just for simplify - response += _F(" OK"); + response += _F("OK"); } else { - response += _F(" FAIL"); + response += _F("FAIL"); } } else { - response += ' '; response += text; } response += "\r\n"; debug_d("> %s", response.c_str()); - writeString(response, TCP_WRITE_FLAG_COPY); // Dynamic memory, should copy - canTransfer = false; + writeString(response); flush(); } - -void FtpServerConnection::onReadyToSendData(TcpConnectionEvent sourceEvent) -{ - switch(state) { - case State::Ready: - writeString(_F("220 Welcome to Sming FTP\r\n"), 0); - state = State::Authorization; - break; - - default:; // Do nothing - } -} diff --git a/Sming/Core/Network/Ftp/FtpServerConnection.h b/Sming/Core/Network/Ftp/FtpServerConnection.h index 901834db6c..b0f9a7c69e 100644 --- a/Sming/Core/Network/Ftp/FtpServerConnection.h +++ b/Sming/Core/Network/Ftp/FtpServerConnection.h @@ -13,6 +13,7 @@ #include "Network/TcpConnection.h" #include "IpAddress.h" #include "WString.h" +#include /** * @defgroup ftp FTP @@ -20,55 +21,63 @@ * @{ */ -class FtpServer; +class CustomFtpServer; +class FtpDataStream; class FtpServerConnection : public TcpConnection { friend class FtpDataStream; - friend class FtpServer; + friend class CustomFtpServer; public: + struct User { + CString name; + IFS::UserRole role{}; + + bool isValid() const + { + return role != IFS::UserRole::None; + } + }; + static constexpr size_t MAX_FTP_CMD{255}; - FtpServerConnection(FtpServer& parentServer, tcp_pcb* clientTcp) - : TcpConnection(clientTcp, true), server(parentServer) - { - } + FtpServerConnection(CustomFtpServer& parentServer, tcp_pcb* clientTcp); err_t onReceive(pbuf* buf) override; err_t onSent(uint16_t len) override; - void onReadyToSendData(TcpConnectionEvent sourceEvent) override; void dataTransferFinished(TcpConnection* connection); + void dataStreamDestroyed(TcpConnection* connection); + + const User& getUser() const + { + return user; + } + + // Get the active filesystem, send an error response if undefined + IFS::FileSystem* getFileSystem(); + + virtual void response(int code, String text = nullptr, char sep = ' '); protected: virtual void onCommand(String cmd, String data); - virtual void response(int code, String text = ""); void cmdPort(const String& data); - void createDataConnection(TcpConnection* connection); - - bool isCanTransfer() - { - return canTransfer; - } + void setDataConnection(FtpDataStream* connection); + String resolvePath(const char* name); + bool checkFileAccess(const char* filename, IFS::OpenFlags flags); private: - enum class State { - Ready, - Authorization, - Active, - }; - - FtpServer& server; - State state{State::Ready}; - String userName; - String renameFrom; + CustomFtpServer& server; + User user{}; + CString renameFrom; IpAddress ip; - int port{0}; - TcpConnection* dataConnection{nullptr}; - bool canTransfer{true}; + uint16_t port{20}; + bool readyForData{false}; + CString cwd; + FtpDataStream* dataConnection{nullptr}; }; typedef FtpServerConnection FTPServerConnection SMING_DEPRECATED; // @deprecated Use `FtpServerConnection` instead diff --git a/Sming/Core/Network/FtpServer.cpp b/Sming/Core/Network/FtpServer.cpp index b2515665c5..985fa77aeb 100644 --- a/Sming/Core/Network/FtpServer.cpp +++ b/Sming/Core/Network/FtpServer.cpp @@ -10,34 +10,33 @@ #include "FtpServer.h" #include "Ftp/FtpServerConnection.h" -#include "FileSystem.h" -TcpConnection* FtpServer::createClient(tcp_pcb* clientTcp) +TcpConnection* CustomFtpServer::createClient(tcp_pcb* clientTcp) { return new FtpServerConnection(*this, clientTcp); } -void FtpServer::addUser(const String& login, const String& pass) +void FtpServer::addUser(const String& login, const String& pass, IFS::UserRole role) { - debug_d("addUser: %s %s", login.c_str(), pass.c_str()); - users[login] = pass; + debug_d("addUser: %s %s (%s)", login.c_str(), pass.c_str(), toString(role).c_str()); + users[login] = User{pass, role}; } -bool FtpServer::checkUser(const String& login, const String& pass) +IFS::UserRole FtpServer::validateUser(const char* login, const char* pass) { - debug_d("checkUser: %s %s", login.c_str(), pass.c_str()); - if(!users.contains(login)) { - return false; - } - - return users[login] == pass; + debug_d("validateUser: %s %s", login, pass); + auto& user = static_cast(users)[login]; + return (user.password == pass) ? user.role : IFS::UserRole::None; } bool FtpServer::onCommand(String cmd, String data, FtpServerConnection& connection) { if(cmd == _F("FSFORMAT")) { - spiffs_format(); - connection.response(200, F("File system successfully formatted")); + auto fs = connection.getFileSystem(); + if(fs != nullptr) { + int err = fs->format(); + connection.response(200, F("File system format: ") + fileGetErrorString(err)); + } return true; } return false; diff --git a/Sming/Core/Network/FtpServer.h b/Sming/Core/Network/FtpServer.h index 5bf1df7da1..15be35353a 100644 --- a/Sming/Core/Network/FtpServer.h +++ b/Sming/Core/Network/FtpServer.h @@ -12,34 +12,85 @@ #include "TcpServer.h" #include "WHashMap.h" -#include "WString.h" +#include class FtpServerConnection; /** @defgroup ftpserver FTP server * @ingroup tcpserver - * @brief Provides FTP server + * @brief Base implementation for FTP server */ -class FtpServer : public TcpServer +class CustomFtpServer : public TcpServer { friend class FtpServerConnection; public: - FtpServer() + CustomFtpServer(IFS::IFileSystem* fileSystem = nullptr) : fileSystem(fileSystem) { - setTimeOut(900); // Update timeout + setTimeOut(900); } - void addUser(const String& login, const String& pass); - bool checkUser(const String& login, const String& pass); + /** + * @brief Validate user + * @param login User name + * @param pass User password + * @retval IFS::UserRole Returns assigned user role, None if user not validated + */ + virtual IFS::UserRole validateUser(const char* login, const char* pass) = 0; protected: TcpConnection* createClient(tcp_pcb* clientTcp) override; - virtual bool onCommand(String cmd, String data, FtpServerConnection& connection); + /** + * @brief Handle an incomding command + * @param cmd The command identifier, e.g. LIST + * @param data Any command arguments + * @param connection The associated TCP connection to receive any response + * @retval bool true if command handled and response sent + */ + virtual bool onCommand(String cmd, String data, FtpServerConnection& connection) + { + return false; + } + + IFS::FileSystem* getFileSystem() const + { + return fileSystem ? IFS::FileSystem::cast(fileSystem) : ::getFileSystem(); + } + +private: + IFS::IFileSystem* fileSystem; +}; + +/** @defgroup ftpserver FTP server + * @ingroup tcpserver + * @brief Provides FTP server + */ +class FtpServer : public CustomFtpServer +{ +public: + void addUser(const String& login, const String& pass, IFS::UserRole userRole = IFS::UserRole::Admin); + IFS::UserRole validateUser(const char* login, const char* pass) override; + + /** + * @brief Legacy user validation + * @deprecated Use `validateUser()` instead + */ + bool checkUser(const String& login, const String& pass) SMING_DEPRECATED + { + return validateUser(login.c_str(), pass.c_str()) != IFS::UserRole::None; + } + +protected: + bool onCommand(String cmd, String data, FtpServerConnection& connection) override; private: - HashMap users; + struct User { + String password; + IFS::UserRole role; + }; + using UserList = HashMap; + UserList users; }; /** diff --git a/Sming/Core/Network/Http/HttpHeaderFields.cpp b/Sming/Core/Network/Http/HttpHeaderFields.cpp index 66d5c2f39f..5096b576ed 100644 --- a/Sming/Core/Network/Http/HttpHeaderFields.cpp +++ b/Sming/Core/Network/Http/HttpHeaderFields.cpp @@ -14,14 +14,28 @@ #include // Define field name strings and a lookup table -#define XX(_tag, _str, _comment) DEFINE_FSTR_LOCAL(hhfnStr_##_tag, _str); +#define XX(tag, str, flags, comment) DEFINE_FSTR_LOCAL(hhfnStr_##tag, str); HTTP_HEADER_FIELDNAME_MAP(XX) #undef XX -#define XX(_tag, _str, _comment) &hhfnStr_##_tag, +#define XX(tag, str, flags, comment) &hhfnStr_##tag, DEFINE_FSTR_VECTOR_LOCAL(fieldNameStrings, FlashString, HTTP_HEADER_FIELDNAME_MAP(XX)); #undef XX +HttpHeaderFields::Flags HttpHeaderFields::getFlags(HttpHeaderFieldName name) const +{ + switch(name) { +#define XX(tag, str, flags, comment) \ + case HttpHeaderFieldName::tag: \ + return flags; + HTTP_HEADER_FIELDNAME_MAP(XX) +#undef XX + default: + // Custom fields + return 0; + } +} + String HttpHeaderFields::toString(HttpHeaderFieldName name) const { if(name == HTTP_HEADER_UNKNOWN) { @@ -35,6 +49,26 @@ String HttpHeaderFields::toString(HttpHeaderFieldName name) const return customFieldNames[unsigned(name) - unsigned(HTTP_HEADER_CUSTOM)]; } +String HttpHeaderFields::toString(HttpHeaderFieldName name, const String& value) const +{ + String tag = toString(name); + if(getFlags(name)[Flag::Multi]) { + tag += ": "; + CStringArray values(value); + String s; + s.reserve((tag.length() + 2) * values.count() + value.length()); + for(auto p : values) { + s += tag; + s += p; + s += "\r\n"; + } + + return s; + } + + return toString(tag, value); +} + String HttpHeaderFields::toString(const String& name, const String& value) { String s; diff --git a/Sming/Core/Network/Http/HttpHeaderFields.h b/Sming/Core/Network/Http/HttpHeaderFields.h index de05acd30b..2ec1c14431 100644 --- a/Sming/Core/Network/Http/HttpHeaderFields.h +++ b/Sming/Core/Network/Http/HttpHeaderFields.h @@ -14,6 +14,7 @@ #include "Data/CStringArray.h" #include "WString.h" +#include /* * Common HTTP header field names. Enumerating these simplifies matching @@ -25,68 +26,91 @@ * A brief description of each header field is given for information purposes. * For details see https://www.iana.org/assignments/message-headers/message-headers.xhtml * + * Entries are formatted thus: XX(tag, str, flags, comment) + * tag: Identifier used in code (without HTTP_HEADER_ prefix) + * str: String used in HTTP protocol + * flags: Additional flags (see HttpHeaderFields::Flag) + * comment: Further details */ #define HTTP_HEADER_FIELDNAME_MAP(XX) \ - XX(ACCEPT, "Accept", "Limit acceptable response types") \ - XX(ACCESS_CONTROL_ALLOW_ORIGIN, "Access-Control-Allow-Origin", "") \ - XX(AUTHORIZATION, "Authorization", "Basic user agent authentication") \ - XX(CC, "Cc", "email field") \ - XX(CONNECTION, "Connection", "Indicates sender's desired control options") \ - XX(CONTENT_DISPOSITION, "Content-Disposition", "Additional information about how to process response payload") \ - XX(CONTENT_ENCODING, "Content-Encoding", "Applied encodings in addition to content type") \ - XX(CONTENT_LENGTH, "Content-Length", "Anticipated size for payload when not using transfer encoding") \ - XX(CONTENT_TYPE, "Content-Type", \ + XX(ACCEPT, "Accept", 0, "Limit acceptable response types") \ + XX(ACCESS_CONTROL_ALLOW_ORIGIN, "Access-Control-Allow-Origin", 0, "") \ + XX(AUTHORIZATION, "Authorization", 0, "Basic user agent authentication") \ + XX(CC, "Cc", 0, "email field") \ + XX(CONNECTION, "Connection", 0, "Indicates sender's desired control options") \ + XX(CONTENT_DISPOSITION, "Content-Disposition", 0, "Additional information about how to process response payload") \ + XX(CONTENT_ENCODING, "Content-Encoding", 0, "Applied encodings in addition to content type") \ + XX(CONTENT_LENGTH, "Content-Length", 0, "Anticipated size for payload when not using transfer encoding") \ + XX(CONTENT_TYPE, "Content-Type", 0, \ "Payload media type indicating both data format and intended manner of processing by recipient") \ - XX(CONTENT_TRANSFER_ENCODING, "Content-Transfer-Encoding", "Coding method used in a MIME message body part") \ - XX(CACHE_CONTROL, "Cache-Control", "Directives for caches along the request/response chain") \ - XX(DATE, "Date", "Message originating date/time") \ - XX(EXPECT, "Expect", "Behaviours to be supported by the server in order to properly handle this request.") \ - XX(ETAG, "ETag", \ + XX(CONTENT_TRANSFER_ENCODING, "Content-Transfer-Encoding", 0, "Coding method used in a MIME message body part") \ + XX(CACHE_CONTROL, "Cache-Control", 0, "Directives for caches along the request/response chain") \ + XX(DATE, "Date", 0, "Message originating date/time") \ + XX(EXPECT, "Expect", 0, "Behaviours to be supported by the server in order to properly handle this request.") \ + XX(ETAG, "ETag", 0, \ "Validates resource, such as a file, so recipient can confirm whether it has changed - generally more " \ "reliable than Date") \ - XX(FROM, "From", "email address of human user who controls the requesting user agent") \ - XX(HOST, "Host", \ + XX(FROM, "From", 0, "email address of human user who controls the requesting user agent") \ + XX(HOST, "Host", 0, \ "Request host and port information for target URI; allows server to service requests for multiple hosts on a " \ "single IP address") \ - XX(IF_MATCH, "If-Match", \ + XX(IF_MATCH, "If-Match", 0, \ "Precondition check using ETag to avoid accidental overwrites when servicing multiple user requests. Ensures " \ "resource entity tag matches before proceeding.") \ - XX(IF_MODIFIED_SINCE, "If-Modified-Since", "Precondition check using Date") \ - XX(LAST_MODIFIED, "Last-Modified", "Server timestamp indicating date and time resource was last modified") \ - XX(LOCATION, "Location", "Used in redirect responses, amongst other places") \ - XX(SEC_WEBSOCKET_ACCEPT, "Sec-WebSocket-Accept", "Server response to opening Websocket handshake") \ - XX(SEC_WEBSOCKET_VERSION, "Sec-WebSocket-Version", \ + XX(IF_MODIFIED_SINCE, "If-Modified-Since", 0, "Precondition check using Date") \ + XX(LAST_MODIFIED, "Last-Modified", 0, "Server timestamp indicating date and time resource was last modified") \ + XX(LOCATION, "Location", 0, "Used in redirect responses, amongst other places") \ + XX(SEC_WEBSOCKET_ACCEPT, "Sec-WebSocket-Accept", 0, "Server response to opening Websocket handshake") \ + XX(SEC_WEBSOCKET_VERSION, "Sec-WebSocket-Version", 0, \ "Websocket opening request indicates acceptable protocol version. Can appear more than once.") \ - XX(SEC_WEBSOCKET_KEY, "Sec-WebSocket-Key", "Websocket opening request validation key") \ - XX(SEC_WEBSOCKET_PROTOCOL, "Sec-WebSocket-Protocol", \ + XX(SEC_WEBSOCKET_KEY, "Sec-WebSocket-Key", 0, "Websocket opening request validation key") \ + XX(SEC_WEBSOCKET_PROTOCOL, "Sec-WebSocket-Protocol", 0, \ "Websocket opening request indicates supported protocol(s), response contains negotiated protocol(s)") \ - XX(SERVER, "Server", "Identifies software handling requests") \ - XX(SET_COOKIE, "Set-Cookie", "Server may pass name/value pairs and associated metadata to user agent (client)") \ - XX(SUBJECT, "Subject", "email subject line") \ - XX(TO, "To", "email intended recipient address") \ - XX(TRANSFER_ENCODING, "Transfer-Encoding", "e.g. Chunked, compress, deflate, gzip") \ - XX(UPGRADE, "Upgrade", \ + XX(SERVER, "Server", 0, "Identifies software handling requests") \ + XX(SET_COOKIE, "Set-Cookie", Flag::Multi, \ + "Server may pass name/value pairs and associated metadata to user agent (client)") \ + XX(SUBJECT, "Subject", 0, "email subject line") \ + XX(TO, "To", 0, "email intended recipient address") \ + XX(TRANSFER_ENCODING, "Transfer-Encoding", 0, "e.g. Chunked, compress, deflate, gzip") \ + XX(UPGRADE, "Upgrade", 0, \ "Used to transition from HTTP to some other protocol on the same connection. e.g. Websocket") \ - XX(USER_AGENT, "User-Agent", "Information about the user agent originating the request") \ - XX(WWW_AUTHENTICATE, "WWW-Authenticate", "Indicates HTTP authentication scheme(s) and applicable parameters") + XX(USER_AGENT, "User-Agent", 0, "Information about the user agent originating the request") \ + XX(WWW_AUTHENTICATE, "WWW-Authenticate", Flag::Multi, \ + "Indicates HTTP authentication scheme(s) and applicable parameters") \ + XX(PROXY_AUTHENTICATE, "Proxy-Authenticate", Flag::Multi, \ + "Indicates proxy authentication scheme(s) and applicable parameters") enum class HttpHeaderFieldName { UNKNOWN = 0, -#define XX(tag, str, comment) tag, +#define XX(tag, str, flags, comment) tag, HTTP_HEADER_FIELDNAME_MAP(XX) #undef XX CUSTOM // First custom header tag value }; -#define XX(tag, str, comment) constexpr HttpHeaderFieldName HTTP_HEADER_##tag = HttpHeaderFieldName::tag; -XX(UNKNOWN, "", "") +#define XX(tag, str, flags, comment) constexpr HttpHeaderFieldName HTTP_HEADER_##tag = HttpHeaderFieldName::tag; +XX(UNKNOWN, "", 0, "") HTTP_HEADER_FIELDNAME_MAP(XX) -XX(CUSTOM, "", "") +XX(CUSTOM, "", 0, "") #undef XX class HttpHeaderFields { public: + /** + * @brief Flag values providing additional information about header fields + */ + enum class Flag { + Multi, ///< Field may have multiple values + }; + using Flags = BitSet; + + /** + * @brief Get flags (if any) for given header field + * @retval Flags + */ + Flags getFlags(HttpHeaderFieldName name) const; + String toString(HttpHeaderFieldName name) const; /** @brief Produce a string for output in the HTTP header, with line ending @@ -96,10 +120,7 @@ class HttpHeaderFields */ static String toString(const String& name, const String& value); - String toString(HttpHeaderFieldName name, const String& value) const - { - return toString(toString(name), value); - } + String toString(HttpHeaderFieldName name, const String& value) const; /** @brief Find the enumerated value for the given field name string * @param name diff --git a/Sming/Core/Network/Http/HttpHeaders.cpp b/Sming/Core/Network/Http/HttpHeaders.cpp new file mode 100644 index 0000000000..9f98985693 --- /dev/null +++ b/Sming/Core/Network/Http/HttpHeaders.cpp @@ -0,0 +1,48 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * HttpHeaders.cpp + * + ****/ + +#include "HttpHeaders.h" +#include + +const String& HttpHeaders::operator[](const String& name) const +{ + auto field = fromString(name); + if(field == HTTP_HEADER_UNKNOWN) { + return nil; + } + return operator[](field); +} + +bool HttpHeaders::append(const HttpHeaderFieldName& name, const String& value) +{ + int i = indexOf(name); + if(i < 0) { + operator[](name) = value; + return true; + } + + if(!getFlags(name)[Flag::Multi]) { + debug_w("[HTTP] Append not supported for header field '%s'", toString(name).c_str()); + return false; + } + + valueAt(i) += '\0' + value; + + return true; +} + +void HttpHeaders::setMultiple(const HttpHeaders& headers) +{ + for(unsigned i = 0; i < headers.count(); i++) { + HttpHeaderFieldName fieldName = headers.keyAt(i); + auto fieldNameString = headers.toString(fieldName); + operator[](fieldNameString) = headers.valueAt(i); + } +} diff --git a/Sming/Core/Network/Http/HttpHeaders.h b/Sming/Core/Network/Http/HttpHeaders.h index 9c51d85857..0c3fd579e4 100644 --- a/Sming/Core/Network/Http/HttpHeaders.h +++ b/Sming/Core/Network/Http/HttpHeaders.h @@ -48,14 +48,7 @@ class HttpHeaders : public HttpHeaderFields, private HashMap= 0) { + debug_d("found %s", stat.name); + stat.compression.type = IFS::Compression::Type::GZip; + stat.name = IFS::NameBuffer(const_cast(fileName)); + return sendFile(stat); + } } - headers[HTTP_HEADER_CONTENT_TYPE] = ContentType::fromFullFileName(fileName); + if(fileStats(fileName, stat) >= 0) { + debug_d("found %s", fileName.c_str()); + stat.name = IFS::NameBuffer(const_cast(fileName)); + return sendFile(stat); + } - return sendNamedStream(stream); + code = HTTP_STATUS_NOT_FOUND; + return false; } bool HttpResponse::sendNamedStream(IDataSourceStream* newDataStream) diff --git a/Sming/Core/Network/Http/HttpResponse.h b/Sming/Core/Network/Http/HttpResponse.h index 1debc961d0..d12d7f4725 100644 --- a/Sming/Core/Network/Http/HttpResponse.h +++ b/Sming/Core/Network/Http/HttpResponse.h @@ -77,7 +77,7 @@ class HttpResponse return setContentType(::toString(type)); } - HttpResponse* setCookie(const String& name, const String& value); + HttpResponse* setCookie(const String& name, const String& value, bool append = false); HttpResponse* setHeader(const String& name, const String& value) { @@ -94,6 +94,12 @@ class HttpResponse return this; } + /* + * Send file by stat, indicates whether file is compressed + * A name is required in stat to get the appropriate content type + */ + bool sendFile(const FileStat& stat); + /** * @brief Send file by name * @param fileName diff --git a/Sming/Core/Network/HttpClient.cpp b/Sming/Core/Network/HttpClient.cpp index faff09432a..723788e3fe 100644 --- a/Sming/Core/Network/HttpClient.cpp +++ b/Sming/Core/Network/HttpClient.cpp @@ -54,7 +54,7 @@ bool HttpClient::downloadFile(const Url& url, const String& saveFileName, Reques } auto fileStream = new FileStream(); - if(!fileStream->open(file, eFO_CreateNewAlways | eFO_WriteOnly)) { + if(!fileStream->open(file, File::CreateNewAlways | File::WriteOnly)) { debug_e("HttpClient failed to open \"%s\"", file.c_str()); delete fileStream; return false; diff --git a/Sming/Core/Network/MqttClient.cpp b/Sming/Core/Network/MqttClient.cpp index c9b2377c4c..1647272fce 100644 --- a/Sming/Core/Network/MqttClient.cpp +++ b/Sming/Core/Network/MqttClient.cpp @@ -168,13 +168,11 @@ int MqttClient::onMessageEnd(mqtt_message_t* message) // failure clearBits(flags, MQTT_CLIENT_CONNECTED); setTimeOut(1); // schedule the connection for closing - - return message->connack.return_code; + } else { + // success + setTimeOut(USHRT_MAX); + setBits(flags, MQTT_CLIENT_CONNECTED); } - - // success - setTimeOut(USHRT_MAX); - setBits(flags, MQTT_CLIENT_CONNECTED); } auto& handler = static_cast(eventHandlers)[message->common.type]; @@ -388,13 +386,20 @@ void MqttClient::onReadyToSendData(TcpConnectionEvent sourceEvent) break; } + if(outgoingMessage->common.type == MQTT_TYPE_PUBLISH && payloadStream != nullptr) { + // The packetLength should be big enought for the header ONLY. + // Payload will be attached as a second stream + packetLength -= outgoingMessage->publish.content.length; + outgoingMessage->publish.content.data = nullptr; + } + uint8_t packet[packetLength]; mqtt_serialiser_write(&serialiser, outgoingMessage, packet, packetLength); delete stream; auto headerStream = new MemoryDataStream(); headerStream->write(packet, packetLength); - if(outgoingMessage->common.type == MQTT_TYPE_PUBLISH && payloadStream != nullptr) { + if(payloadStream != nullptr) { auto streamChain = new StreamChain(); streamChain->attachStream(headerStream); streamChain->attachStream(payloadStream); diff --git a/Sming/Core/Network/NetUtils.cpp b/Sming/Core/Network/NetUtils.cpp index b3c01fe854..f8aed2765a 100644 --- a/Sming/Core/Network/NetUtils.cpp +++ b/Sming/Core/Network/NetUtils.cpp @@ -13,13 +13,17 @@ #include #include +namespace +{ #ifdef FIX_NETWORK_ROUTING -bool NetUtils::ipClientRoutingFixed = false; +bool ipClientRoutingFixed{false}; #endif -/// HELPERS /// +} // namespace -int NetUtils::pbufFindChar(const pbuf* buf, char wtf, unsigned startPos) +namespace NetUtils +{ +int pbufFindChar(const pbuf* buf, char wtf, unsigned startPos) { unsigned ofs = 0; @@ -48,7 +52,7 @@ int NetUtils::pbufFindChar(const pbuf* buf, char wtf, unsigned startPos) return -1; } -bool NetUtils::pbufIsStrEqual(const pbuf* buf, const char* compared, unsigned startPos) +bool pbufIsStrEqual(const pbuf* buf, const char* compared, unsigned startPos) { unsigned cur = startPos; @@ -78,7 +82,7 @@ bool NetUtils::pbufIsStrEqual(const pbuf* buf, const char* compared, unsigned st return true; } -int NetUtils::pbufFindStr(const pbuf* buf, const char* wtf, unsigned startPos) +int pbufFindStr(const pbuf* buf, const char* wtf, unsigned startPos) { if(wtf == nullptr) { return -1; @@ -100,7 +104,7 @@ int NetUtils::pbufFindStr(const pbuf* buf, const char* wtf, unsigned startPos) return -1; } -char* NetUtils::pbufAllocateStrCopy(const pbuf* buf, unsigned startPos, unsigned length) +char* pbufAllocateStrCopy(const pbuf* buf, unsigned startPos, unsigned length) { char* stringPtr = new char[length + 1]; if(stringPtr == nullptr) { @@ -111,7 +115,7 @@ char* NetUtils::pbufAllocateStrCopy(const pbuf* buf, unsigned startPos, unsigned return stringPtr; } -String NetUtils::pbufStrCopy(const pbuf* buf, unsigned startPos, unsigned length) +String pbufStrCopy(const pbuf* buf, unsigned startPos, unsigned length) { char* stringPtr = new char[length + 1]; if(stringPtr == nullptr) { @@ -125,7 +129,7 @@ String NetUtils::pbufStrCopy(const pbuf* buf, unsigned startPos, unsigned length } #ifdef FIX_NETWORK_ROUTING -bool NetUtils::FixNetworkRouting() +bool FixNetworkRouting() { if(ipClientRoutingFixed) { return true; @@ -187,9 +191,11 @@ static void debugPrintTcp(const char* type, const struct tcp_pcb* pcb) } } -void NetUtils::debugPrintTcpList() +void debugPrintTcpList() { debugPrintTcp(_F("Active"), tcp_active_pcbs); debugPrintTcp(_F("Listen"), tcp_listen_pcbs.pcbs); debugPrintTcp(_F("TIME-WAIT"), tcp_tw_pcbs); } + +} // namespace NetUtils diff --git a/Sming/Core/Network/NetUtils.h b/Sming/Core/Network/NetUtils.h index a5e6e665f4..46e5192dc0 100644 --- a/Sming/Core/Network/NetUtils.h +++ b/Sming/Core/Network/NetUtils.h @@ -8,10 +8,6 @@ * ****/ -/** @defgroup networking Networking - * @{ - */ - #pragma once #ifdef ARCH_ESP8266 @@ -23,32 +19,31 @@ struct pbuf; class String; -class NetUtils +/** @defgroup networking Networking + * @{ + */ + +namespace NetUtils { -public: - // Helpers - static bool pbufIsStrEqual(const pbuf* buf, const char* compared, unsigned startPos); - static int pbufFindChar(const pbuf* buf, char wtf, unsigned startPos = 0); - static int pbufFindStr(const pbuf* buf, const char* wtf, unsigned startPos = 0); - static char* pbufAllocateStrCopy(const pbuf* buf, unsigned startPos, unsigned length); - static String pbufStrCopy(const pbuf* buf, unsigned startPos, unsigned length); +// Helpers +bool pbufIsStrEqual(const pbuf* buf, const char* compared, unsigned startPos); +int pbufFindChar(const pbuf* buf, char wtf, unsigned startPos = 0); +int pbufFindStr(const pbuf* buf, const char* wtf, unsigned startPos = 0); +char* pbufAllocateStrCopy(const pbuf* buf, unsigned startPos, unsigned length); +String pbufStrCopy(const pbuf* buf, unsigned startPos, unsigned length); #ifdef FIX_NETWORK_ROUTING - static bool FixNetworkRouting(); +bool FixNetworkRouting(); #else - static bool FixNetworkRouting() - { - return true; // Should work on standard lwip - } +inline bool FixNetworkRouting() +{ + return true; // Should work on standard lwip +} #endif - // Debug - static void debugPrintTcpList(); +// Debug +void debugPrintTcpList(); -private: -#ifdef FIX_NETWORK_ROUTING - static bool ipClientRoutingFixed; -#endif -}; +}; // namespace NetUtils /** @} */ diff --git a/Sming/Core/Network/NtpClient.cpp b/Sming/Core/Network/NtpClient.cpp index c79698164b..ed49a6d819 100644 --- a/Sming/Core/Network/NtpClient.cpp +++ b/Sming/Core/Network/NtpClient.cpp @@ -46,13 +46,14 @@ void NtpClient::requestTime() } ip_addr_t resolvedIp; - int result = dns_gethostbyname(server.c_str(), &resolvedIp, - [](const char* name, LWIP_IP_ADDR_T* ip, void* arg) { - // We do a new request since the last one was never done. - if(ip) - reinterpret_cast(arg)->internalRequestTime(*ip); - }, - this); + int result = dns_gethostbyname( + server.c_str(), &resolvedIp, + [](const char* name, LWIP_IP_ADDR_T* ip, void* arg) { + // We do a new request since the last one was never done. + if(ip) + reinterpret_cast(arg)->internalRequestTime(*ip); + }, + this); debug_d("dns_gethostbyname() returned %d", result); diff --git a/Sming/Core/Network/SmtpClient.cpp b/Sming/Core/Network/SmtpClient.cpp index 321898003a..c78a04cee0 100644 --- a/Sming/Core/Network/SmtpClient.cpp +++ b/Sming/Core/Network/SmtpClient.cpp @@ -35,7 +35,7 @@ } #define ADVANCE_UNTIL_EOL \ do { \ - if(*(buffer - 1) == '\r' && *buffer == '\n') { \ + if(buffer[-1] == '\r' && buffer[0] == '\n') { \ ADVANCE_AND_BREAK; \ } \ ADVANCE; \ @@ -44,7 +44,7 @@ #define ADVANCE_UNTIL_EOL_OR_BREAK \ { \ ADVANCE_UNTIL_EOL; \ - if(*(buffer - 1) != '\n') { \ + if(buffer[-1] != '\n') { \ break; \ } \ } @@ -61,10 +61,6 @@ break; \ } -SmtpClient::SmtpClient(bool autoDestroy) : TcpClient(autoDestroy) -{ -} - SmtpClient::~SmtpClient() { delete outgoingMail; @@ -143,45 +139,39 @@ void SmtpClient::onReadyToSendData(TcpConnectionEvent sourceEvent) } case eSMTP_SendAuth: { - if(authMethods.count()) { - auto methodPlain = F("PLAIN"); - auto methodCramMd5 = F("CRAM-MD5"); - // TODO: Simplify the code in that block... - Vector preferredOrder; - if(useSsl) { - preferredOrder.addElement(methodPlain); - preferredOrder.addElement(methodCramMd5); - } else { - preferredOrder.addElement(methodCramMd5); - preferredOrder.addElement(methodPlain); - } - - for(unsigned i = 0; i < preferredOrder.count(); i++) { - if(authMethods.contains(preferredOrder[i])) { - if(preferredOrder[i] == methodPlain) { - // base64('\0' + username + '\0' + password) - String token = '\0' + url.User + '\0' + url.Password; - String hash = base64_encode(token); - sendString(F("AUTH PLAIN ") + hash + "\r\n"); - state = eSMTP_SendingAuth; - break; - } else if(preferredOrder[i] == methodCramMd5) { - // otherwise we can try the slow cram-md5 authentication... - sendString(F("AUTH CRAM-MD5\r\n")); - state = eSMTP_RequestingAuthChallenge; - break; - } - } + auto authPlain = [this]() { + // base64('\0' + username + '\0' + password) + String token = '\0' + url.User + '\0' + url.Password; + String hash = base64_encode(token); + sendString(F("AUTH PLAIN ") + hash + "\r\n"); + state = eSMTP_SendingAuth; + }; + + auto authCramMd5 = [this]() { + // Slower cram-md5 authentication + sendString(F("AUTH CRAM-MD5\r\n")); + state = eSMTP_RequestingAuthChallenge; + }; + + DEFINE_FSTR_LOCAL(methodPlain, "PLAIN") + DEFINE_FSTR_LOCAL(methodCramMd5, "CRAM-MD5") + if(useSsl) { + if(authMethods.contains(methodPlain)) { + authPlain(); + } else if(authMethods.contains(methodCramMd5)) { + authCramMd5(); } - } /* authMethods.count */ + } else if(authMethods.contains(methodCramMd5)) { + authCramMd5(); + } else if(authMethods.contains(methodPlain)) { + authPlain(); + } if(state == eSMTP_SendAuth) { state = eSMTP_Ready; } break; - - default:; // Do nothing } case eSMTP_SendAuthResponse: { @@ -264,10 +254,12 @@ void SmtpClient::onReadyToSendData(TcpConnectionEvent sourceEvent) } case eSMTP_Disconnect: { - close(); + setTimeOut(1); return; } + default:; // Do nothing + } /* switch(state) */ TcpClient::onReadyToSendData(sourceEvent); @@ -277,15 +269,14 @@ MultipartStream::BodyPart SmtpClient::multipartProducer() { MultipartStream::BodyPart result; - if(outgoingMail->attachments.count()) { + if(!outgoingMail->attachments.isEmpty()) { result = outgoingMail->attachments[0]; + outgoingMail->attachments.remove(0); if(!result.headers->contains(HTTP_HEADER_CONTENT_TRANSFER_ENCODING)) { result.stream = new Base64OutputStream(result.stream); (*result.headers)[HTTP_HEADER_CONTENT_TRANSFER_ENCODING] = _F("base64"); } - - outgoingMail->attachments.remove(0); } return result; @@ -300,7 +291,7 @@ void SmtpClient::sendMailHeaders(MailMessage* mail) mail->stream = new QuotedPrintableOutputStream(mail->stream); } - if(mail->attachments.count()) { + if(!mail->attachments.isEmpty()) { MultipartStream* mStream = new MultipartStream(MultipartStream::Producer(&SmtpClient::multipartProducer, this)); MultipartStream::BodyPart text; text.headers = new HttpHeaders(); @@ -343,12 +334,13 @@ err_t SmtpClient::onReceive(pbuf* buf) pbuf* cur = buf; int parsedBytes = 0; while(cur != nullptr && cur->len > 0) { - parsedBytes += smtpParse((char*)cur->payload, cur->len); + parsedBytes += smtpParse(static_cast(cur->payload), cur->len); cur = cur->next; } if(parsedBytes != buf->tot_len) { - debug_e("Got error: %s:%s", code, message); + debug_e("[SMTP] Got error %s: %s", code, message); + debug_hex(DBG, "SMTP", buf->payload, buf->len); if(!errorCallback || errorCallback(*this, codeValue, message) != 0) { // abort the connection if we cannot handle it. @@ -492,7 +484,6 @@ int SmtpClient::smtpParse(char* buffer, size_t len) case eSMTP_Quitting: { RETURN_ON_ERROR(SMTP_CODE_BYE); - close(); state = eSMTP_Disconnect; break; diff --git a/Sming/Core/Network/SmtpClient.h b/Sming/Core/Network/SmtpClient.h index 0dabf487ae..2b4b4149ad 100644 --- a/Sming/Core/Network/SmtpClient.h +++ b/Sming/Core/Network/SmtpClient.h @@ -90,7 +90,10 @@ using SmtpClientCallback = Delegate authMethods; ObjectQueue mailQ; - char code[4] = {0}; - int codeValue = 0; + char code[4]{0}; + int codeValue{0}; String authChallenge; - char message[SMTP_ERROR_LENGTH + 1] = {0}; - bool isLastLine = false; - uint8_t codeLength = 0; - int options = 0; - MailMessage* outgoingMail = nullptr; - SmtpState state = eSMTP_Banner; - - SmtpClientCallback errorCallback = nullptr; - SmtpClientCallback messageSentCallback = nullptr; + char message[SMTP_ERROR_LENGTH + 1]{0}; + bool isLastLine{false}; + uint8_t codeLength{0}; + int options{0}; + MailMessage* outgoingMail{nullptr}; + SmtpState state{eSMTP_Banner}; + + SmtpClientCallback errorCallback; + SmtpClientCallback messageSentCallback; private: /** diff --git a/Sming/Core/Network/TcpConnection.cpp b/Sming/Core/Network/TcpConnection.cpp index 2b930ba3db..528abb68cd 100644 --- a/Sming/Core/Network/TcpConnection.cpp +++ b/Sming/Core/Network/TcpConnection.cpp @@ -89,15 +89,16 @@ bool TcpConnection::connect(const String& server, int port, bool useSsl) int port; }; DnsLookup* look = new DnsLookup{this, port}; - err_t dnslook = dns_gethostbyname(server.c_str(), &addr, - [](const char* name, LWIP_IP_ADDR_T* ipaddr, void* arg) { - auto dlook = static_cast(arg); - if(dlook != nullptr) { - dlook->con->internalOnDnsResponse(name, ipaddr, dlook->port); - delete dlook; - } - }, - look); + err_t dnslook = dns_gethostbyname( + server.c_str(), &addr, + [](const char* name, LWIP_IP_ADDR_T* ipaddr, void* arg) { + auto dlook = static_cast(arg); + if(dlook != nullptr) { + dlook->con->internalOnDnsResponse(name, ipaddr, dlook->port); + delete dlook; + } + }, + look); if(dnslook == ERR_INPROGRESS) { // Operation pending - see internalOnDnsResponse() return true; diff --git a/Sming/Core/Network/UdpConnection.cpp b/Sming/Core/Network/UdpConnection.cpp index d0daf7d29b..73a4005335 100644 --- a/Sming/Core/Network/UdpConnection.cpp +++ b/Sming/Core/Network/UdpConnection.cpp @@ -119,19 +119,27 @@ void UdpConnection::staticOnReceive(void* arg, struct udp_pcb* pcb, struct pbuf* bool UdpConnection::setMulticast(IpAddress ip) { #if LWIP_MULTICAST_TX_OPTIONS - udp_set_multicast_netif_addr(udp, (ip4_addr_t*)ip); - return true; -#else - return false; + if(udp == nullptr) { + initialize(nullptr); + } + if(udp != nullptr) { + udp_set_multicast_netif_addr(udp, (ip4_addr_t*)ip); + return true; + } #endif + return false; } bool UdpConnection::setMulticastTtl(size_t ttl) { #if LWIP_MULTICAST_TX_OPTIONS - udp_set_multicast_ttl(udp, ttl); - return true; -#else - return false; + if(udp == nullptr) { + initialize(nullptr); + } + if(udp != nullptr) { + udp_set_multicast_ttl(udp, ttl); + return true; + } #endif + return false; } diff --git a/Sming/Core/Network/WebHelpers/base64.cpp b/Sming/Core/Network/WebHelpers/base64.cpp index 945862b4f8..2cf3ab9fdc 100644 --- a/Sming/Core/Network/WebHelpers/base64.cpp +++ b/Sming/Core/Network/WebHelpers/base64.cpp @@ -14,16 +14,17 @@ #include "libb64/cencode.h" #include "libb64/cdecode.h" -// Base-64 encoding produces 3 output bytes for every 2 input bytes -#define MIN_ENCODE_LEN(_in_len) (((_in_len)*3 + 1) / 2) -#define MIN_DECODE_LEN(_in_len) (((_in_len)*2 + 2) / 3) +// Base-64 encodes 6 bits into one character (4 output chars for every 3 input bytes) +#define MIN_ENCODE_LEN(_in_len) (((_in_len)*4 + 2) / 3) +#define MIN_DECODE_LEN(_in_len) (((_in_len)*3 + 3) / 4) int base64_encode(size_t in_len, const unsigned char* in, size_t out_len, char* out) { // Base-64 encoding produces 3 output bytes for every 2 input bytes unsigned min_out_len = MIN_ENCODE_LEN(in_len); - if(out_len < min_out_len) + if(out_len < min_out_len) { return -1; + } base64_encodestate state; base64_init_encodestate(&state, 0); // Don't include any linebreaks @@ -35,12 +36,14 @@ int base64_encode(size_t in_len, const unsigned char* in, size_t out_len, char* String base64_encode(const unsigned char* in, size_t in_len) { String s; - if(!s.setLength(MIN_ENCODE_LEN(in_len))) + if(!s.setLength(MIN_ENCODE_LEN(in_len))) { return nullptr; + } int len = base64_encode(in_len, in, s.length(), s.begin()); - if(len < 0) + if(len < 0) { return nullptr; + } s.setLength(len); return s; @@ -49,8 +52,9 @@ String base64_encode(const unsigned char* in, size_t in_len) /* decode a base64 string in one shot */ int base64_decode(size_t in_len, const char* in, size_t out_len, unsigned char* out) { - if(out_len < MIN_DECODE_LEN(in_len)) + if(out_len < MIN_DECODE_LEN(in_len)) { return -1; + } base64_decodestate _state; base64_init_decodestate(&_state); @@ -60,12 +64,14 @@ int base64_decode(size_t in_len, const char* in, size_t out_len, unsigned char* String base64_decode(const char* in, size_t in_len) { String s; - if(!s.setLength(MIN_DECODE_LEN(in_len))) + if(!s.setLength(MIN_DECODE_LEN(in_len))) { return nullptr; + } int outlen = base64_decode(in_len, in, s.length(), (unsigned char*)s.begin()); - if(outlen < 0) + if(outlen < 0) { return nullptr; + } s.setLength(outlen); return s; diff --git a/Sming/Core/SmingCore.h b/Sming/Core/SmingCore.h index 402665df6c..7420d1ce26 100644 --- a/Sming/Core/SmingCore.h +++ b/Sming/Core/SmingCore.h @@ -57,3 +57,5 @@ #include "DateTime.h" #include "fatfs/ff.h" + +#include diff --git a/Sming/Core/SmingLocale.h b/Sming/Core/SmingLocale.h old mode 100755 new mode 100644 diff --git a/Sming/Core/SmingVersion.h b/Sming/Core/SmingVersion.h index d62d01e2da..363d7ebdb1 100644 --- a/Sming/Core/SmingVersion.h +++ b/Sming/Core/SmingVersion.h @@ -6,7 +6,7 @@ */ #define SMING_MAJOR_VERSION 4 -#define SMING_MINOR_VERSION 2 +#define SMING_MINOR_VERSION 3 #define SMING_PATCH_VERSION 0 #define SMING_PRE_RELEASE "" diff --git a/Sming/Core/Task.h b/Sming/Core/Task.h index a4b0de7c06..1c04dc72e0 100644 --- a/Sming/Core/Task.h +++ b/Sming/Core/Task.h @@ -112,14 +112,15 @@ class Task switch(state) { case State::Suspended: case State::Running: - sleepTimer.initializeMs(interval, - [](void* param) { - auto task = static_cast(param); - task->notify(Notify::Waking); - task->state = State::Running; - task->service(); - }, - this); + sleepTimer.initializeMs( + interval, + [](void* param) { + auto task = static_cast(param); + task->notify(Notify::Waking); + task->state = State::Running; + task->service(); + }, + this); sleepTimer.startOnce(); notify(Notify::Sleeping); state = State::Sleeping; diff --git a/Sming/Libraries/.patches/Adafruit_SSD1306.patch b/Sming/Libraries/.patches/Adafruit_SSD1306.patch index ed74398da7..c3077b4efa 100644 --- a/Sming/Libraries/.patches/Adafruit_SSD1306.patch +++ b/Sming/Libraries/.patches/Adafruit_SSD1306.patch @@ -1,5 +1,5 @@ diff --git a/Adafruit_SSD1306.cpp b/Adafruit_SSD1306.cpp -index 570a335..efcc3eb 100644 +index 570a335..750b750 100644 --- a/Adafruit_SSD1306.cpp +++ b/Adafruit_SSD1306.cpp @@ -32,12 +32,38 @@ All text above, and the splash screen below must be included in any redistributi @@ -150,6 +150,78 @@ index 570a335..efcc3eb 100644 } // clear everything +@@ -564,13 +632,13 @@ void Adafruit_SSD1306::drawFastHLineInternal(int16_t x, int16_t y, int16_t w, ui + if(w <= 0) { return; } + + // set up the pointer for movement through the buffer +- register uint8_t *pBuf = buffer; ++ uint8_t *pBuf = buffer; + // adjust the buffer pointer for the current row + pBuf += ((y/8) * SSD1306_LCDWIDTH); + // and offset x columns in + pBuf += x; + +- register uint8_t mask = 1 << (y&7); ++ uint8_t mask = 1 << (y&7); + + switch (color) + { +@@ -638,27 +706,27 @@ void Adafruit_SSD1306::drawFastVLineInternal(int16_t x, int16_t __y, int16_t __h + } + + // this display doesn't need ints for coordinates, use local byte registers for faster juggling +- register uint8_t y = __y; +- register uint8_t h = __h; ++ uint8_t y = __y; ++ uint8_t h = __h; + + + // set up the pointer for fast movement through the buffer +- register uint8_t *pBuf = buffer; ++ uint8_t *pBuf = buffer; + // adjust the buffer pointer for the current row + pBuf += ((y/8) * SSD1306_LCDWIDTH); + // and offset x columns in + pBuf += x; + + // do the first partial byte, if necessary - this requires some masking +- register uint8_t mod = (y&7); ++ uint8_t mod = (y&7); + if(mod) { + // mask off the high n bits we want to set + mod = 8-mod; + + // note - lookup table results in a nearly 10% performance improvement in fill* functions +- // register uint8_t mask = ~(0xFF >> (mod)); ++ // uint8_t mask = ~(0xFF >> (mod)); + static uint8_t premask[8] = {0x00, 0x80, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC, 0xFE }; +- register uint8_t mask = premask[mod]; ++ uint8_t mask = premask[mod]; + + // adjust the mask if we're not going to reach the end of this byte + if( h < mod) { +@@ -696,7 +764,7 @@ void Adafruit_SSD1306::drawFastVLineInternal(int16_t x, int16_t __y, int16_t __h + } + else { + // store a local value to work with +- register uint8_t val = (color == WHITE) ? 255 : 0; ++ uint8_t val = (color == WHITE) ? 255 : 0; + + do { + // write our value in +@@ -715,10 +783,10 @@ void Adafruit_SSD1306::drawFastVLineInternal(int16_t x, int16_t __y, int16_t __h + if(h) { + mod = h & 7; + // this time we want to mask the low bits of the byte, vs the high bits we did above +- // register uint8_t mask = (1 << mod) - 1; ++ // uint8_t mask = (1 << mod) - 1; + // note - lookup table results in a nearly 10% performance improvement in fill* functions + static uint8_t postmask[8] = {0x00, 0x01, 0x03, 0x07, 0x0F, 0x1F, 0x3F, 0x7F }; +- register uint8_t mask = postmask[mod]; ++ uint8_t mask = postmask[mod]; + switch (color) + { + case WHITE: *pBuf |= mask; break; diff --git a/Adafruit_SSD1306.h b/Adafruit_SSD1306.h index 1162f87..28d783f 100644 --- a/Adafruit_SSD1306.h diff --git a/Sming/Libraries/.patches/arduinoFFT.patch b/Sming/Libraries/.patches/ArduinoFFT.patch similarity index 92% rename from Sming/Libraries/.patches/arduinoFFT.patch rename to Sming/Libraries/.patches/ArduinoFFT.patch index 66fa4a9937..79fb67b3db 100644 --- a/Sming/Libraries/.patches/arduinoFFT.patch +++ b/Sming/Libraries/.patches/ArduinoFFT.patch @@ -1,16 +1,16 @@ diff --git a/src/arduinoFFT.h b/src/arduinoFFT.h -index 73927dc..3436c9d 100644 +index c3c1a0b..8f7e2e4 100644 --- a/src/arduinoFFT.h +++ b/src/arduinoFFT.h -@@ -75,21 +75,11 @@ +@@ -75,7 +75,6 @@ class arduinoFFT { public: /* Constructor */ - arduinoFFT(void); arduinoFFT(double *vReal, double *vImag, uint16_t samples, double samplingFrequency); -- /* Destructor */ -- ~arduinoFFT(void); - /* Functions */ + /* Destructor */ + ~arduinoFFT(void); +@@ -83,13 +82,7 @@ public: uint8_t Revision(void); uint8_t Exponent(uint16_t value); @@ -19,13 +19,13 @@ index 73927dc..3436c9d 100644 - void Compute(double *vReal, double *vImag, uint16_t samples, uint8_t power, uint8_t dir); - void DCRemoval(double *vData, uint16_t samples); - double MajorPeak(double *vD, uint16_t samples, double samplingFrequency); -- void MajorPeak(double *vD, uint16_t samples, double samplingFrequency, double *f, double *v); + void MajorPeak(double *vD, uint16_t samples, double samplingFrequency, double *f, double *v); - void Windowing(double *vData, uint16_t samples, uint8_t windowType, uint8_t dir); void ComplexToMagnitude(); void Compute(uint8_t dir); diff --git a/src/arduinoFFT.cpp b/src/arduinoFFT.cpp -index 790da4f..dcdcfae 100644 +index fb602a9..e038b41 100644 --- a/src/arduinoFFT.cpp +++ b/src/arduinoFFT.cpp @@ -21,11 +21,6 @@ @@ -40,17 +40,7 @@ index 790da4f..dcdcfae 100644 arduinoFFT::arduinoFFT(double *vReal, double *vImag, uint16_t samples, double samplingFrequency) {// Constructor this->_vReal = vReal; -@@ -35,22 +30,11 @@ arduinoFFT::arduinoFFT(double *vReal, double *vImag, uint16_t samples, double sa - this->_power = Exponent(samples); - } - --arduinoFFT::~arduinoFFT(void) --{ --// Destructor --} -- - uint8_t arduinoFFT::Revision(void) - { +@@ -45,12 +40,6 @@ uint8_t arduinoFFT::Revision(void) return(FFT_LIB_REV); } @@ -63,7 +53,7 @@ index 790da4f..dcdcfae 100644 void arduinoFFT::Compute(uint8_t dir) {// Computes in-place complex-to-complex FFT / // Reverse bits / -@@ -115,71 +99,6 @@ void arduinoFFT::Compute(uint8_t dir) +@@ -115,71 +104,6 @@ void arduinoFFT::Compute(uint8_t dir) } } @@ -135,7 +125,7 @@ index 790da4f..dcdcfae 100644 void arduinoFFT::ComplexToMagnitude() { // vM is half the size of vReal and vImag for (uint16_t i = 0; i < this->_samples; i++) { -@@ -187,14 +106,6 @@ void arduinoFFT::ComplexToMagnitude() +@@ -187,14 +111,6 @@ void arduinoFFT::ComplexToMagnitude() } } @@ -150,7 +140,7 @@ index 790da4f..dcdcfae 100644 void arduinoFFT::DCRemoval() { // calculate the mean of vData -@@ -211,23 +122,6 @@ void arduinoFFT::DCRemoval() +@@ -211,23 +127,6 @@ void arduinoFFT::DCRemoval() } } @@ -174,7 +164,7 @@ index 790da4f..dcdcfae 100644 void arduinoFFT::Windowing(uint8_t windowType, uint8_t dir) {// Weighing factors are computed once before multiple use of FFT // The weighing function is symetric; half the weighs are recorded -@@ -285,63 +179,6 @@ void arduinoFFT::Windowing(uint8_t windowType, uint8_t dir) +@@ -285,63 +184,6 @@ void arduinoFFT::Windowing(uint8_t windowType, uint8_t dir) } @@ -238,7 +228,7 @@ index 790da4f..dcdcfae 100644 double arduinoFFT::MajorPeak() { double maxY = 0; -@@ -391,61 +228,8 @@ void arduinoFFT::MajorPeak(double *f, double *v) +@@ -391,58 +233,6 @@ void arduinoFFT::MajorPeak(double *f, double *v) #endif } @@ -294,10 +284,14 @@ index 790da4f..dcdcfae 100644 - #endif -} - + double arduinoFFT::MajorPeakParabola() + { + double maxY = 0; +@@ -492,7 +282,6 @@ void arduinoFFT::Parabola(double x1, double y1, double x2, double y2, double x3, + uint8_t arduinoFFT::Exponent(uint16_t value) { - #warning("This method may not be accessible on future revisions.") // Calculates the base 2 logarithm of a value uint8_t result = 0; while (((value >> result) & 1) != 1) result++; - \ No newline at end of file diff --git a/Sming/Libraries/.patches/Arduino_TensorFlowLite/README.rst b/Sming/Libraries/.patches/Arduino_TensorFlowLite/README.rst deleted file mode 100644 index 514a20a70a..0000000000 --- a/Sming/Libraries/.patches/Arduino_TensorFlowLite/README.rst +++ /dev/null @@ -1,8 +0,0 @@ -Arduino TensorFlow Lite -======================= - -This library runs TensorFlow machine learning models on microcontrollers, allowing you to build AI/ML applications powered by deep learning and neural networks. - -With the included examples, you can recognize speech, detect people using a camera, and recognise "magic wand" gestures using an accelerometer. - -The examples work best with the Arduino Nano 33 BLE Sense board, which has a microphone and accelerometer. diff --git a/Sming/Libraries/Adafruit_GFX/BMPDraw.h b/Sming/Libraries/Adafruit_GFX/BMPDraw.h index 5414d03ad0..4ceb3e0401 100644 --- a/Sming/Libraries/Adafruit_GFX/BMPDraw.h +++ b/Sming/Libraries/Adafruit_GFX/BMPDraw.h @@ -64,7 +64,7 @@ template bool bmpDraw(Adafruit_TFT& tft, String fileName, u uint32_t startTime = millis(); - file_t handle = fileOpen(fileName.c_str(), eFO_ReadOnly); + file_t handle = fileOpen(fileName.c_str(), File::ReadOnly); if(handle < 0) { debug_e("File wasn't found: %s", fileName.c_str()); return false; diff --git a/Sming/Libraries/Adafruit_ILI9341/component.mk b/Sming/Libraries/Adafruit_ILI9341/component.mk index 169766e78e..91e2591dae 100644 --- a/Sming/Libraries/Adafruit_ILI9341/component.mk +++ b/Sming/Libraries/Adafruit_ILI9341/component.mk @@ -10,7 +10,7 @@ else ifeq ($(SMING_ARCH),Esp32) TFT_CS_PIN ?= 18 TFT_DC_PIN ?= 19 TFT_RESET_PIN ?= 21 -else +else ifndef MAKE_DOCS $(warning Arch unsupported) endif diff --git a/Sming/Libraries/ArduCAM/ArduCAMStream.cpp b/Sming/Libraries/ArduCAM/ArduCAMStream.cpp index 91564aa79c..11a491cc14 100644 --- a/Sming/Libraries/ArduCAM/ArduCAMStream.cpp +++ b/Sming/Libraries/ArduCAM/ArduCAMStream.cpp @@ -15,7 +15,7 @@ #define BMPIMAGEOFFSET 66 -const char bmp_header[BMPIMAGEOFFSET] = +const uint8_t bmp_header[BMPIMAGEOFFSET] = { 0x42, 0x4D, 0x36, 0x58, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0xF0, 0x00, 0x00, 0x00, 0x01, 0x00, 0x10, 0x00, 0x03, 0x00, diff --git a/Sming/Libraries/ArduinoFFT b/Sming/Libraries/ArduinoFFT index 566803e9ca..bb99065f9a 160000 --- a/Sming/Libraries/ArduinoFFT +++ b/Sming/Libraries/ArduinoFFT @@ -1 +1 @@ -Subproject commit 566803e9ca2b6ff50895da2791fcfbf80f8fd23a +Subproject commit bb99065f9aad9e5781c5bf9e77613bdd33f0074d diff --git a/Sming/Libraries/ArduinoJson6/include/ArduinoJson.h b/Sming/Libraries/ArduinoJson6/include/ArduinoJson.h index 955857231a..567a80a914 100644 --- a/Sming/Libraries/ArduinoJson6/include/ArduinoJson.h +++ b/Sming/Libraries/ArduinoJson6/include/ArduinoJson.h @@ -256,7 +256,7 @@ template String serialize(const TSource& source, Serializatio template bool saveToFile(const TSource& source, const String& filename, SerializationFormat format = JSON_FORMAT_DEFAULT) { - FileStream stream(filename, eFO_WriteOnly | eFO_CreateNewAlways); + FileStream stream(filename, File::WriteOnly | File::CreateNewAlways); if(!stream.isValid()) { return false; } diff --git a/Sming/Libraries/Arduino_TensorFlowLite b/Sming/Libraries/Arduino_TensorFlowLite index 521b889982..9f00dd38f4 160000 --- a/Sming/Libraries/Arduino_TensorFlowLite +++ b/Sming/Libraries/Arduino_TensorFlowLite @@ -1 +1 @@ -Subproject commit 521b8899822ccce6188d7fe1fb246c06aec9ef17 +Subproject commit 9f00dd38f4329b8e884c417aa1bcfbfb15f47913 diff --git a/Sming/Libraries/CS5460/samples/generic/component.mk b/Sming/Libraries/CS5460/samples/generic/component.mk index 51f1520d50..391ca50c73 100644 --- a/Sming/Libraries/CS5460/samples/generic/component.mk +++ b/Sming/Libraries/CS5460/samples/generic/component.mk @@ -1,2 +1 @@ ARDUINO_LIBRARIES := CS5460 -DISABLE_SPIFFS = 1 diff --git a/Sming/Libraries/DIAL b/Sming/Libraries/DIAL index 712c3d4087..4d54bde7a1 160000 --- a/Sming/Libraries/DIAL +++ b/Sming/Libraries/DIAL @@ -1 +1 @@ -Subproject commit 712c3d4087fca7c90336b559edf912e2bcf9b846 +Subproject commit 4d54bde7a1620b85bf53fcb5670f2321ee562835 diff --git a/Sming/Libraries/GoogleCast b/Sming/Libraries/GoogleCast new file mode 160000 index 0000000000..3e6f1bea43 --- /dev/null +++ b/Sming/Libraries/GoogleCast @@ -0,0 +1 @@ +Subproject commit 3e6f1bea43b56dee27a13a7ab97ff8f3a680303c diff --git a/Sming/Libraries/HueEmulator b/Sming/Libraries/HueEmulator index 20553f6e9e..79d7e79754 160000 --- a/Sming/Libraries/HueEmulator +++ b/Sming/Libraries/HueEmulator @@ -1 +1 @@ -Subproject commit 20553f6e9e223588595e626fc7aa060847df57eb +Subproject commit 79d7e797540519a4fcd9e73a6ca7c6e04715794e diff --git a/Sming/Libraries/MDNS b/Sming/Libraries/MDNS new file mode 160000 index 0000000000..3c1370d47d --- /dev/null +++ b/Sming/Libraries/MDNS @@ -0,0 +1 @@ +Subproject commit 3c1370d47d1deec99074bc9e6ff384998440aff4 diff --git a/Sming/Libraries/ModbusMaster/samples/generic/component.mk b/Sming/Libraries/ModbusMaster/samples/generic/component.mk index 6b0689bce9..2797abfd8b 100644 --- a/Sming/Libraries/ModbusMaster/samples/generic/component.mk +++ b/Sming/Libraries/ModbusMaster/samples/generic/component.mk @@ -1,2 +1 @@ ARDUINO_LIBRARIES := ModbusMaster -DISABLE_SPIFFS = 1 diff --git a/Sming/Libraries/OtaUpgrade/OtaUpgrade/BasicStream.cpp b/Sming/Libraries/OtaUpgrade/OtaUpgrade/BasicStream.cpp index ca549eac12..af28c04518 100644 --- a/Sming/Libraries/OtaUpgrade/OtaUpgrade/BasicStream.cpp +++ b/Sming/Libraries/OtaUpgrade/OtaUpgrade/BasicStream.cpp @@ -13,6 +13,7 @@ #include #include #include +#include extern "C" uint32 user_rf_cal_sector_set(void); @@ -27,29 +28,13 @@ DECLARE_FSTR_ARRAY(AppFlashRegionOffsets, uint32_t); BasicStream::Slot::Slot() { // Get parameters of the slot where the firmware image should be stored. - const rboot_config bootConfig = rboot_get_config(); - uint8_t currentSlot = bootConfig.current_rom; - index = (currentSlot == 0 ? 1 : 0); - address = bootConfig.roms[index]; - size = 0x100000 - (address & 0xFFFFF); + uint8_t currentSlot = rboot_get_current_rom(); + index = (currentSlot == 0) ? 1 : 0; - const auto limitSize = [&](uint32_t otherAddress) { - if(otherAddress > address) { - size = std::min(size, otherAddress - address); - } - }; - - limitSize(flashmem_get_size_bytes()); - limitSize(user_rf_cal_sector_set() * INTERNAL_FLASH_SECTOR_SIZE); - for(uint8_t i = 0; i < bootConfig.count; ++i) { - if(i != currentSlot) { - limitSize(bootConfig.roms[i]); - } - } - - for(auto offset : AppFlashRegionOffsets) { - limitSize(offset); - } + // Lookup slot details from partition table + auto part = Storage::spiFlash->partitions().findOta(index); + address = part.address(); + size = part.size(); } BasicStream::BasicStream() @@ -83,7 +68,7 @@ bool BasicStream::consume(const uint8_t*& data, size_t& size) void BasicStream::setError(Error code) { assert(code != Error::None); - debug_e("Error: %s", errorToString(code).c_str()); + debug_e("Error: %s", toString(code).c_str()); errorCode = code; state = State::Error; } @@ -184,7 +169,7 @@ size_t BasicStream::write(const uint8_t* data, size_t size) break; case State::WriteRom: { - bool ok = rboot_write_flash(&rbootWriteStatus, const_cast(data), std::min(remainingBytes, size)); + bool ok = rboot_write_flash(&rbootWriteStatus, data, std::min(remainingBytes, size)); if(ok) { if(consume(data, size)) { ok = slot.updated = rboot_write_end(&rbootWriteStatus); @@ -225,6 +210,14 @@ size_t BasicStream::write(const uint8_t* data, size_t size) String BasicStream::errorToString(Error code) { + return toString(code); +} + +} // namespace OtaUpgrade + +String toString(OtaUpgrade::BasicStream::Error code) +{ + using Error = OtaUpgrade::BasicStream::Error; switch(code) { case Error::None: return nullptr; @@ -254,5 +247,3 @@ String BasicStream::errorToString(Error code) return F(""); } } - -} // namespace OtaUpgrade diff --git a/Sming/Libraries/OtaUpgrade/OtaUpgrade/BasicStream.h b/Sming/Libraries/OtaUpgrade/OtaUpgrade/BasicStream.h index 995c57034d..303431caae 100644 --- a/Sming/Libraries/OtaUpgrade/OtaUpgrade/BasicStream.h +++ b/Sming/Libraries/OtaUpgrade/OtaUpgrade/BasicStream.h @@ -48,7 +48,7 @@ class BasicStream : public ReadWriteStream /** * @brief Error code values */ - enum Error { + enum class Error { None, ///< No error occured thus far (default value of \c #errorCode if `hasError()` returns false) InvalidFormat, ///< Invalid/unsupported upgrade file format UnsupportedData, ///< Some content of the upgrade file is not supported by this version of OtaUpgradeStream. @@ -67,8 +67,9 @@ class BasicStream : public ReadWriteStream /** @brief Convert error code to string. * @see #errorCode + * @deprecated Use `toString()` global function */ - static String errorToString(Error code); + static String errorToString(Error code) SMING_DEPRECATED; /** @brief Process chunk of upgrade file. * @param data Pointer to chunk of data. @@ -113,11 +114,12 @@ class BasicStream : public ReadWriteStream uint32_t address; uint32_t size; uint8_t index; - bool updated = false; - } slot; + bool updated{false}; + }; + Slot slot; // Instead of RbootOutputStream, the rboot write API is used directly because in a future extension the OTA file may contain data for multiple FLASH regions. - rboot_write_status rbootWriteStatus = {}; + rboot_write_status rbootWriteStatus{}; enum class State { Error, @@ -128,14 +130,14 @@ class BasicStream : public ReadWriteStream VerifyRoms, RomsComplete, }; - State state = State::Header; + State state{State::Header}; #ifdef ENABLE_OTA_SIGNING using Verifier = SignatureVerifier; - static const uint32_t expectedHeaderMagic = OTA_HEADER_MAGIC_SIGNED; + static const uint32_t expectedHeaderMagic{OTA_HEADER_MAGIC_SIGNED}; #else using Verifier = ChecksumVerifier; - static const uint32_t expectedHeaderMagic = OTA_HEADER_MAGIC_NOT_SIGNED; + static const uint32_t expectedHeaderMagic{OTA_HEADER_MAGIC_NOT_SIGNED}; #endif Verifier verifier; @@ -144,9 +146,9 @@ class BasicStream : public ReadWriteStream Verifier::VerificationData verificationData; - size_t remainingBytes = 0; - uint8_t* destinationPtr = nullptr; - uint8_t romIndex = 0; + size_t remainingBytes{0}; + uint8_t* destinationPtr{nullptr}; + uint8_t romIndex{0}; void setupChunk(State nextState, size_t size, void* destination = nullptr) { @@ -183,3 +185,8 @@ class BasicStream : public ReadWriteStream }; } // namespace OtaUpgrade + +/** @brief Convert error code to string. + * @see #errorCode + */ +String toString(OtaUpgrade::BasicStream::Error code); diff --git a/Sming/Libraries/OtaUpgrade/OtaUpgrade/EncryptedStream.cpp b/Sming/Libraries/OtaUpgrade/OtaUpgrade/EncryptedStream.cpp index 80a88457af..fa1c09a0f8 100644 --- a/Sming/Libraries/OtaUpgrade/OtaUpgrade/EncryptedStream.cpp +++ b/Sming/Libraries/OtaUpgrade/OtaUpgrade/EncryptedStream.cpp @@ -46,16 +46,15 @@ size_t EncryptedStream::write(const uint8_t* data, size_t size) case Fragment::ChunkSize: remainingBytes = 1 + chunkSizeMinusOne; - if(buffer == nullptr || bufferSize < remainingBytes) { - free(buffer); - buffer = (uint8_t*)malloc(remainingBytes); - if(buffer == nullptr) { + if(!buffer || bufferSize < remainingBytes) { + buffer.reset(new uint8_t[remainingBytes]); + if(!buffer) { setError(Error::OutOfMemory); break; } bufferSize = remainingBytes; } - fragmentPtr = buffer; + fragmentPtr = buffer.get(); fragment = Fragment::Chunk; break; @@ -63,8 +62,8 @@ size_t EncryptedStream::write(const uint8_t* data, size_t size) unsigned char tag; size_t chiperTextLength = 1 + chunkSizeMinusOne; unsigned long long messageLength = 0; - bool ok = (crypto_secretstream_xchacha20poly1305_pull(&state, buffer, &messageLength, &tag, buffer, - chiperTextLength, nullptr, 0) == 0); + bool ok = (crypto_secretstream_xchacha20poly1305_pull(&state, buffer.get(), &messageLength, &tag, + buffer.get(), chiperTextLength, nullptr, 0) == 0); if(!ok || messageLength > bufferSize) { setError(Error::DecryptionFailed); break; @@ -77,7 +76,7 @@ size_t EncryptedStream::write(const uint8_t* data, size_t size) fragment = Fragment::None; } - BasicStream::write(buffer, static_cast(messageLength)); + BasicStream::write(buffer.get(), size_t(messageLength)); } break; case Fragment::None: diff --git a/Sming/Libraries/OtaUpgrade/OtaUpgrade/EncryptedStream.h b/Sming/Libraries/OtaUpgrade/OtaUpgrade/EncryptedStream.h index 3557c56f82..eec39954c6 100644 --- a/Sming/Libraries/OtaUpgrade/OtaUpgrade/EncryptedStream.h +++ b/Sming/Libraries/OtaUpgrade/OtaUpgrade/EncryptedStream.h @@ -12,6 +12,7 @@ #include "BasicStream.h" #include +#include namespace OtaUpgrade { @@ -20,7 +21,7 @@ namespace OtaUpgrade * * The class processes encrypted firmware upgrade files created by otatool.py. * A buffer is allocated dynamically to fit the largest chunk of the encryption container - * (2kB unless otatool.py was modified). The actual processing of the decrypted data is + * (2kB unless otatool.py was modified). The actual processing of the decrypted data is * defered to #BasicStream. */ class EncryptedStream : public BasicStream @@ -28,11 +29,6 @@ class EncryptedStream : public BasicStream public: EncryptedStream() = default; - ~EncryptedStream() - { - free(buffer); - } - /** @brief Process an arbitrarily sized chunk of an encrypted OTA upgrade file. * @param data Pointer to chunk of data. * @param size Size of chunk pointed to by \a data in bytes. @@ -55,12 +51,12 @@ class EncryptedStream : public BasicStream Chunk, None, }; - Fragment fragment = Fragment::Header; - size_t remainingBytes = sizeof(header); - uint8_t* fragmentPtr = header; - uint8_t* buffer = nullptr; - size_t bufferSize = 0; + Fragment fragment{Fragment::Header}; + size_t remainingBytes{sizeof header}; + uint8_t* fragmentPtr{header}; + std::unique_ptr buffer; + size_t bufferSize{0}; }; } // namespace OtaUpgrade diff --git a/Sming/Libraries/OtaUpgrade/component.mk b/Sming/Libraries/OtaUpgrade/component.mk index 572a4f1b10..d19bec3b07 100644 --- a/Sming/Libraries/OtaUpgrade/component.mk +++ b/Sming/Libraries/OtaUpgrade/component.mk @@ -96,7 +96,7 @@ OTA_DATE_REF := -2208988800000LL _ota-make-build-timestamp: | $(OTA_GENCODE_DIR) $(Q) echo '#include ' > $(OTA_BUILD_TIMESTAMP_SRC) $(Q) echo 'namespace OtaUpgrade {' >> $(OTA_BUILD_TIMESTAMP_SRC) - $(Q) echo 'extern const uint64_t BuildTimestamp PROGMEM = $(shell date +%s%3NLL) - $(OTA_DATE_REF);' >> $(OTA_BUILD_TIMESTAMP_SRC) + $(Q) echo 'extern const uint64_t BuildTimestamp PROGMEM = $(shell $(PYTHON) -c "import time; print('%uLL' % (1000 * time.time()))") - $(OTA_DATE_REF);' >> $(OTA_BUILD_TIMESTAMP_SRC) $(Q) echo '} // namespace OtaUpgrade' >> $(OTA_BUILD_TIMESTAMP_SRC) App-build: _ota-make-build-timestamp diff --git a/Sming/Arch/Esp32/Components/esp_spiffs/.cs b/Sming/Libraries/OtaUpgradeMqtt/.cs similarity index 100% rename from Sming/Arch/Esp32/Components/esp_spiffs/.cs rename to Sming/Libraries/OtaUpgradeMqtt/.cs diff --git a/Sming/Libraries/OtaUpgradeMqtt/README.rst b/Sming/Libraries/OtaUpgradeMqtt/README.rst new file mode 100644 index 0000000000..55386d1f0a --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/README.rst @@ -0,0 +1,155 @@ +OTA Firmware Upgrade via MQTT +============================= + +.. highlight:: c++ + +Introduction +------------ + +This library allows Sming applications to upgrade their firmware Over-The-Air (OTA) using the MQTT protocol. +MTQTT has less overhead compared to HTTP and can be used for faster delivery of application updates. + +Using +----- + +1. Add ``COMPONENT_DEPENDS += OtaUpgradeMqtt`` to your application componenent.mk file. +2. Add these lines to your application:: + + #include + + #if ENABLE_OTA_ADVANCED + #include + #endif + + MqttClient mqtt; + + // Call when IP address has been obtained + void onIp(IpAddress ip, IpAddress mask, IpAddress gateway) + { + // ... + + mqtt.connect(Url(MQTT_URL), "sming"); + + #if ENABLE_OTA_ADVANCED + /* + * The advanced parser suppors all firmware upgrades supported by the `OtaUpgrade` library. + * `OtaUpgrade` library provides firmware signing, firmware encryption and so on. + */ + auto parser = new OtaUpgrade::Mqtt::AdvancedPayloadParser(APP_VERSION_PATCH); + #else + /* + * The command below uses class that stores the firmware directly + * using RbootOutputStream on a location provided by us + */ + auto parser = new OtaUpgrade::Mqtt::RbootPayloadParser(part, APP_VERSION_PATCH); + #endif + + mqtt.setPayloadParser([parser] + (MqttPayloadParserState& state, mqtt_message_t* message, const char* buffer, int length) -> int + { + return parser->parse(state, message, buffer, length); + }); + + String updateTopic = "a/test/u/4.3"; + mqtt.subscribe(updateTopic); + + // ... + } + +See the :sample:`Upgrade` sample application. + +Versioning Principles +--------------------- +To simplify the OTA process we strongly recommend the following versioning principles for your application: + +1. Use `semantic versioning `_. + If your current application version is 4.3.1 then 4 is the major, 3 is the minor and 1 is the patch version number. + +2. Every application firmware knows its version. + +3. An application with the same major and minor version should be compatible for update no matter what the patch number is. + If the new firmware is not compatible then a new minor or major version should be used. + +Theory Of Operation +------------------- +1. On a period of time the application connects to check if there is a new version of the firmware. + In your application this period has to be carefully selected so that OTA updates occur when the device has + enough resources: memory, space on flash, power and time to complete such an update. Also there should be no critical task running at the moment. + Depending on the size of the new firmware and the speed of the connection an update can take 10 to 20 seconds. + +2. The application connects via MQTT to a remote server and subscribes to a special topic. The topic is based on the + application id and its current version. If the current application id is ``test`` and version is ``4.3.1`` then the topic that will be used for OTA is ``a/test/u/4.3``. + +3. If there is a need to support both stable and unstable/nightly builds then the topic name can have `s` or `u` suffix. For example + all stable versions should be published and downloaded from the topic ``a/test/u/4.3/s``. For the unstable ones we can use the topic ``a/test/u/4.3/u``. + If an application is interested in both stable and unstable versions then it can subscribe using the following pattern ``a/test/u/4.3/+``. + +4. The application is waiting for new firmware. When the application is on battery than it makes sense to wait for a limited time and if there is no + message coming back to disconnect. + +Firmware packaging +------------------ +The firmware update must come as one MQTT message. The MQTT protocol allows messages with a maximum size of 268435455 bytes approx 260MB. +This should be perfectly enough for a device that has maximum 1MB available for an application ROM. + +One MQTT message contains: + +- patch version of the firmware +- followed by the firmware data itself + +Based on the :envvar:`ENABLE_OTA_VARINT_VERSION` the patch version can be encoded either using one byte or a `varint `_. +Based on :envvar:`ENABLE_OTA_ADVANCED` the firmware data can be either without any encoding or be signed and encrypted. + +To simplify the packaging this library comes with a tool called ``deployer``. To create a package type the following from your application:: + + make ota-pack OTA_PATCH_VERSION=127 + +Replace 127 with the desired patch version. +If the option ``OTA_PATCH_VERSION`` is omitted from the command line then the patch version will be generated automatically and it will contain the current unix timestamp. + +Once a package is created it can be deployed to the firmware MQTT server using the command below:: + + make ota-deploy MQTT_FIRMWARE_URL=mqtt://relser:relpassword@attachix.com/a/test/u/4.3 + +The ``MQTT_FIRMWARE_URL`` above specifies that + +- protocol is: mqtt without SSL. Allowed values here are ``mqtt`` and ``mqtts``. The latter uses SSL. +- user is: relser +- password is: relpassword +- host is: attachix.com +- path is: /a/test/u/4.3. The path without leading and ending slashes is used to generate the topic name ``a/test/u/4.3``. + +Make sure to replace the MQTT_FIRMWARE_URL value with your MQTT server credentials, host and topic. + +Security +-------- +For additional security a standard SSL/TLS can be used + +1. The communication should be secured using standard SSL. + +2. To prove that the server is the correct one: The MQTT clients should pin the public key fingerprint on the server. + OR have a list of public key fingerprints that are allowed. + +3. To prove that the clients are allowed to connect: Every MQTT client should also have a client certificate that is signed by the server. + +Configuration +------------- + +.. envvar:: ENABLE_OTA_VARINT_VERSION + + Default: 1 (enabled) + + If set to 1 the OTA upgrade mechanism and application will use a `varint `_ + encoding for the patch version. Thus allowing unlimited number of patch versions. Useful for enumerating unstable/nightly releases. + A bit more difficult to read and write but allows for unlimited versions. + + If set to 0 the OTA upgrade mechanism and application will use one byte for the patch version which will limit it to 256 possible patch versions. + Useful for enumarating stable releases. Easier to write and read but limited to 256 versions only. + +.. envvar:: ENABLE_OTA_ADVANCED + + Default: 0 (disabled) + + If set to 1 the library will work with OtaUpgradeStream which supports signature and encryption of the firmware data itself. + See :component:`OtaUpgrade` for details. In the application the AdvancedPayloadParser can be used to do the MQTT message handling. + diff --git a/Sming/Libraries/OtaUpgradeMqtt/api.rst b/Sming/Libraries/OtaUpgradeMqtt/api.rst new file mode 100644 index 0000000000..e9aca95651 --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/api.rst @@ -0,0 +1,8 @@ +OTA Upgrade over MQTT classes +============================= + +.. doxygentypedef:: OtaUpgrade::Mqtt + +.. doxygenclass:: OtaUpgrade::Mqtt::PayloadParser +.. doxygenclass:: OtaUpgrade::Mqtt::RbootPayloadParser +.. doxygenclass:: OtaUpgrade::Mqtt::AdvancedPayloadParser diff --git a/Sming/Libraries/OtaUpgradeMqtt/component.mk b/Sming/Libraries/OtaUpgradeMqtt/component.mk new file mode 100644 index 0000000000..7896ab0797 --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/component.mk @@ -0,0 +1,49 @@ +COMPONENT_SRCDIRS := +COMPONENT_SRCFILES := src/PayloadParser.cpp src/RbootPayloadParser.cpp +COMPONENT_INCDIRS := src/include + +# If enabled (set to 1) then we can use all sofisticated mechanisms to upgrade the firmware using the ``OtaUpgrade`` library. +COMPONENT_VARS := ENABLE_OTA_ADVANCED +ENABLE_OTA_ADVANCED ?= 0 + +ifneq ($(ENABLE_OTA_ADVANCED),0) + COMPONENT_SRCFILES += src/AdvancedPayloadParser.cpp + COMPONENT_DEPENDS := OtaUpgrade +endif + +# If enabled (set to 1) then we can use unlimited number of patch versions +COMPONENT_VARS += ENABLE_OTA_VARINT_VERSION +ENABLE_OTA_VARINT_VERSION ?= 1 + +COMPONENT_CXXFLAGS := -DENABLE_OTA_ADVANCED=$(ENABLE_OTA_ADVANCED) \ + -DENABLE_OTA_VARINT_VERSION=$(ENABLE_OTA_VARINT_VERSION) + +##@Firmware Upgrade + +OTA_TOOLS := $(COMPONENT_PATH)/tools +OTA_DEPLOYMENT_TOOL = $(OTA_TOOLS)/deployer/out/Host/debug/firmware/deployer$(TOOL_EXT) + +$(OTA_DEPLOYMENT_TOOL): + $(Q) $(MAKE) -C $(OTA_TOOLS)/deployer SMING_ARCH=Host ENABLE_CUSTOM_LWIP=2 + + +# SDP = Sming Deployment Pakage +OTA_PACKAGE_EXT =.sdp + +OTA_PATCH_VERSION ?= $(shell date +%s) + +PACKAGE_IN = $(RBOOT_ROM_0_BIN) +ifneq ($(ENABLE_OTA_ADVANCED), 0) + PACKAGE_IN = $(OTA_UPGRADE_FILE) +endif +PACKAGE_OUT = $(PACKAGE_IN)$(OTA_PACKAGE_EXT) + +.PHONY: ota-pack +ota-pack: $(PACKAGE_OUT) ##Creates a deployment package from the current application (use OTA_PATCH_VERSION to specify the version number) + +$(PACKAGE_OUT): $(PACKAGE_IN) $(OTA_DEPLOYMENT_TOOL) + $(Q) $(OTA_DEPLOYMENT_TOOL) pack --debug=0 --nonet -- $(PACKAGE_IN) $(PACKAGE_OUT) $(OTA_PATCH_VERSION) $(ENABLE_OTA_VARINT_VERSION) + +.PHONY: ota-deploy +ota-deploy: $(PACKAGE_OUT) ##Uploads new firmware version of the current application (use MQTT_FIRMWARE_URL to specify the MQTT URL) + $(Q) $(OTA_DEPLOYMENT_TOOL) deploy --debug=0 -- $(PACKAGE_OUT) $(MQTT_FIRMWARE_URL) diff --git a/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/.cproject b/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/.cproject new file mode 100644 index 0000000000..e1450b6031 --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/.cproject @@ -0,0 +1,151 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + make + -f ${ProjDirPath}/Makefile + all + true + true + true + + + make + -f ${ProjDirPath}/Makefile + clean + true + true + true + + + make + -f ${ProjDirPath}/Makefile + flash + true + true + true + + + make + -f ${ProjDirPath}/Makefile + flashonefile + true + true + true + + + make + -f ${ProjDirPath}/Makefile + flashinit + true + true + true + + + make + -f ${ProjDirPath}/Makefile + flashboot + true + true + true + + + make + -f ${ProjDirPath}/Makefile + rebuild + true + true + true + + + + + + + + + + + + + + + + + + + + diff --git a/Sming/Components/mdns/.cs b/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/.cs similarity index 100% rename from Sming/Components/mdns/.cs rename to Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/.cs diff --git a/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/.gitignore b/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/.gitignore new file mode 100644 index 0000000000..a662abf847 --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/.gitignore @@ -0,0 +1 @@ +ota.key diff --git a/samples/Telnet_TCPServer_TCPClient/.project b/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/.project similarity index 95% rename from samples/Telnet_TCPServer_TCPClient/.project rename to Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/.project index c212ceedfb..76f0847014 100644 --- a/samples/Telnet_TCPServer_TCPClient/.project +++ b/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/.project @@ -1,6 +1,6 @@ - Telnet_TCPServer_TCPClient + Ota_Mqtt SmingFramework diff --git a/samples/Telnet_TCPServer_TCPClient/Makefile b/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/Makefile similarity index 100% rename from samples/Telnet_TCPServer_TCPClient/Makefile rename to Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/Makefile diff --git a/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/README.rst b/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/README.rst new file mode 100644 index 0000000000..cc14bab324 --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/README.rst @@ -0,0 +1,88 @@ +OTA over MQTT +============= + +.. highlight:: bash + +Introduction +------------ + +This example demonstrates how you can create an application that updates its firmware Over The Air (OTA) using the MQTT protocol. +This application uses :component:`OtaUpgradeMqtt` and follows the recommended versioning principles. + +Based on :envvar:`ENABLE_OTA_ADVANCED` the firmware data can be either without any encoding or be signed and encrypted. + +Tools +----- +There are two tools that facilitate the packiging and deployment of a new firmware. + +For more information read ``Firmware packaging`` in the documentation of the :component:`OtaUpgradeMqtt` component. + +Security +-------- +Depending on :envvar:`ENABLE_SSL` a standard SSL/TLS can be enabled. This way + +1. The communication between the application and the server will be encrypted using standard SSL. + +2. To prove that the server is the correct one: The MQTT clients should pin the public key fingerprint on the server. + OR have a list of public key fingerprints that are allowed. + +3. Depending on :envvar:`ENABLE_CLIENT_CERTIFICATE` the application can send a client certificate that is signed by the server. + +Configuration +------------- + +.. envvar:: APP_ID + + Default: "test" + + This variable contains the unique application name. + +.. envvar:: APP_VERSION + + Default: not set + + Contains the application major and minor versions separated by comma. Example "4.2". + If not set will use the current major and minor version from Sming. + +.. envvar:: APP_VERSION_PATCH + + Default: not set + + Contains the application patch version as integer. For stable versions you can use 0 until 255. + For unstable versions the current timestamp can be used as a patch version. + +.. envvar:: ENABLE_OTA_VARINT_VERSION + + Default: 1 (enabled) + + If set to 1 the OTA upgrade mechanism and application will use a `varint `_ + encoding for the patch version. Thus allowing unlimited number of patch versions. Useful for enumerating unstable/nightly releases. + A bit more difficult to read and write but allows for unlimited versions. + + If set to 0 the OTA upgrade mechanism and application will use one byte for the patch version which will limit it to 256 possible patch versions. + Useful for enumarating stable releases. Easier to write and read but limited to 256 versions only. + +.. envvar:: ENABLE_OTA_ADVANCED + + Default: 0 (disabled) + + If set to 1 the library will work with OtaUpgradeStream which supports signature and encryption of the firmware data itself. + See :component:`OtaUpgrade` for details. + +.. envvar:: ENABLE_SSL + + Default: unset (disable) + + If set to 1 (highly recommended), OTA upgrade files will be trasnferred securely over TLS/SSL. + +.. envvar:: ENABLE_CLIENT_CERTIFICATE + + Default: 0 (disabled) + + Used in combination with ``ENABLE_SSL``. Set to 1 if the remote server requires the application to authenticate via client certficate. + +.. envvar:: MQTT_URL + + Default: depends on ``ENABLE_SSL`` and ``ENABLE_CLIENT_CERTIFICATE`` values + + Url containing the location of the firmware update MQTT server. diff --git a/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/app/application.cpp b/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/app/application.cpp new file mode 100644 index 0000000000..d93860fc1a --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/app/application.cpp @@ -0,0 +1,154 @@ +#include +#include +#include +#include + +#if ENABLE_OTA_ADVANCED +#include +#endif + +// If you want, you can define WiFi settings globally in Eclipse Environment Variables +#ifndef WIFI_SSID +#define WIFI_SSID "PleaseEnterSSID" // Put your SSID and password here +#define WIFI_PWD "PleaseEnterPass" +#endif + +#ifndef APP_VERSION +#include +#define APP_VERSION MACROQUOTE(SMING_MAJOR_VERSION) "." MACROQUOTE(SMING_MINOR_VERSION) +#define APP_VERSION_PATCH SMING_PATCH_VERSION +#endif + +namespace +{ +MqttClient mqtt; + +#if ENABLE_CLIENT_CERTIFICATE +IMPORT_FSTR(privateKeyData, PROJECT_DIR "/files/private.pem.key.der"); +IMPORT_FSTR(certificateData, PROJECT_DIR "/files/certificate.pem.crt.der"); +#endif + +Storage::Partition findRomPartition(uint8_t slot) +{ + auto part = Storage::spiFlash->partitions().findOta(slot); + if(!part) { + debug_w("Rom slot %d not found", slot); + } + return part; +} + +void otaUpdate() +{ + if(mqtt.isProcessing()) { + Serial.println("There is an update in progress. Refusing to start new update."); + return; + } + + uint8 slot = rboot_get_current_rom(); + if(slot == 0) { + slot = 1; + } else { + slot = 0; + } + + Serial.println("Checking for a new application firmware..."); + + auto part = findRomPartition(slot); + if(!part) { + Serial.println("FAILED: Cannot find application address"); + return; + } + +#ifdef ENABLE_SSL + mqtt.setSslInitHandler([](Ssl::Session& session) { + // These fingerprints change very frequently. + static const Ssl::Fingerprint::Cert::Sha1 sha1Fingerprint PROGMEM = {MQTT_FINGERPRINT_SHA1}; + + // Trust certificate only if it matches the SHA1 fingerprint... + session.validators.pin(sha1Fingerprint); + + // We're using fingerprints, so don't attempt to validate full certificate + session.options.verifyLater = true; + +#if ENABLE_CLIENT_CERTIFICATE + session.keyCert.assign(privateKeyData, certificateData); +#endif + + // Use all supported cipher suites to make a connection + session.cipherSuites = &Ssl::CipherSuites::full; + }); +#endif + + mqtt.connect(Url(MQTT_URL), "sming"); + +#if ENABLE_OTA_ADVANCED + /* + * The advanced parser suppors all firmware upgrades supported by the `OtaUpgrade` library. + * `OtaUpgrade` library provides firmware signing, firmware encryption and so on. + */ + auto parser = new OtaUpgrade::Mqtt::AdvancedPayloadParser(APP_VERSION_PATCH); +#else + /* + * The command below uses class that stores the firmware directly + * using RbootOutputStream on a location provided by us + */ + auto parser = new OtaUpgrade::Mqtt::RbootPayloadParser(part, APP_VERSION_PATCH); +#endif + + mqtt.setPayloadParser([parser](MqttPayloadParserState& state, mqtt_message_t* message, const char* buffer, + int length) -> int { return parser->parse(state, message, buffer, length); }); + + String updateTopic = "a/"; + updateTopic += APP_ID; + updateTopic += "/u/"; + updateTopic += APP_VERSION; + debug_d("Subscribing to topic: %s", updateTopic.c_str()); + mqtt.subscribe(updateTopic); +} + +void showInfo() +{ + Serial.printf(_F("\r\nSDK: v%s\r\n"), system_get_sdk_version()); + Serial.printf(_F("Free Heap: %d\r\n"), system_get_free_heap_size()); + Serial.printf(_F("CPU Frequency: %d MHz\r\n"), system_get_cpu_freq()); + Serial.printf(_F("System Chip ID: %x\r\n"), system_get_chip_id()); + + rboot_config conf = rboot_get_config(); + + debug_d("Count: %d", conf.count); + debug_d("ROM 0: 0x%08x", conf.roms[0]); + debug_d("ROM 1: 0x%08x", conf.roms[1]); + debug_d("ROM 2: 0x%08x", conf.roms[2]); + debug_d("GPIO ROM: %d", conf.gpio_rom); + + Serial.printf(_F("\r\nCurrently running rom %d. Application version: %s\r\n"), conf.current_rom, APP_VERSION); + Serial.println(); +} + +void connectOk(IpAddress ip, IpAddress mask, IpAddress gateway) +{ + /* + This application starts the update right after a successful connecton. + In a real-world application you should start the update procedure + only when the chances of success are high enough. + + For example when there is enough power, free RAM and sufficient time + AND there is no critical task running at the moment. + */ + otaUpdate(); +} + +} // end of anonymous namespace + +void init() +{ + Serial.begin(SERIAL_BAUD_RATE); // 115200 by default + Serial.systemDebugOutput(true); // Debug output to serial + + showInfo(); + + WifiStation.enable(true); + WifiStation.config(WIFI_SSID, WIFI_PWD); + + WifiEvents.onStationGotIP(connectOk); +} diff --git a/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/component.mk b/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/component.mk new file mode 100644 index 0000000000..806c2b46b6 --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/component.mk @@ -0,0 +1,62 @@ +## User configurable settings + +## [Application id and version] ## +# Application id +APP_ID ?= "test" + +# Application version: string containing only the major and minor version separated by comma +# APP_VERSION := "1.2" +# Application patch version: integer containing only the patch version +# APP_VERSION_PATCH := 3 + +## [TLS/SSL settings ] ## +# Uncomment the line below to start using SSL +CONFIG_VARS := ENABLE_SSL +# ENABLE_SSL := Bearssl + +# Set this to one if the remote firmware server requires client certificate +# This option is in effect only when ENABLE_SSL is set +CONFIG_VARS += ENABLE_CLIENT_CERTIFICATE +ENABLE_CLIENT_CERTIFICATE ?= 0 + +## [ Firmware Update Server ] ## +CONFIG_VARS += MQTT_URL +ifeq ($(MQTT_URL),) + MQTT_URL := "mqtt://test.mosquitto.org:1883" + ifdef ENABLE_SSL + ifneq ($(ENABLE_CLIENT_CERTIFICATE),0) + MQTT_URL := "mqtts://test.mosquitto.org:8884" + else + MQTT_URL := "mqtts://test.mosquitto.org:8883" + endif + endif +endif + +# This variable contains the SHA1 fingerprint of the SSL certificate of the MQTT server. +# It is used for certificate pinning. Make sure to change it whenever changing the MQTT_URL +CONFIG_VARS += MQTT_FINGERPRINT_SHA1 +MQTT_FINGERPRINT_SHA1 := "0xEE,0xBC,0x4B,0xF8,0x57,0xE3,0xD3,0xE4,0x07,0x54,0x23,0x1E,0xF0,0xC8,0xA1,0x56,0xE0,0xD3,0x1A,0x1C" + +CONFIG_VARS += ENABLE_OTA_ADVANCED +ENABLE_OTA_ADVANCED ?= 0 + +## End of user configurable settings. Don't change anything below this line + +COMPONENT_DEPENDS := OtaUpgradeMqtt + +## use rboot build mode +RBOOT_ENABLED := 1 + +## Use standard hardware config with two ROM slots and two SPIFFS partitions +HWCONFIG := spiffs-two-roms + +APP_CFLAGS = -DMQTT_URL="\"$(MQTT_URL)"\" \ + -DMQTT_FINGERPRINT_SHA1=$(MQTT_FINGERPRINT_SHA1) \ + -DAPP_ID="\"$(APP_ID)"\" \ + -DENABLE_CLIENT_CERTIFICATE=$(ENABLE_CLIENT_CERTIFICATE) \ + -DENABLE_OTA_ADVANCED=$(ENABLE_OTA_ADVANCED) + +ifneq ($(APP_VERSION),) + APP_CFLAGS += -DAPP_VERSION="\"$(APP_VERSION)"\" \ + -DAPP_VERSION_PATCH=$(APP_VERSION_PATCH) +endif \ No newline at end of file diff --git a/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/files/certificate.pem.crt.der b/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/files/certificate.pem.crt.der new file mode 100644 index 0000000000..05534491e2 Binary files /dev/null and b/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/files/certificate.pem.crt.der differ diff --git a/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/files/private.pem.key.der b/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/files/private.pem.key.der new file mode 100644 index 0000000000..bfd239930f Binary files /dev/null and b/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/files/private.pem.key.der differ diff --git a/Sming/Libraries/OtaUpgradeMqtt/src/AdvancedPayloadParser.cpp b/Sming/Libraries/OtaUpgradeMqtt/src/AdvancedPayloadParser.cpp new file mode 100644 index 0000000000..5aaf30cd97 --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/src/AdvancedPayloadParser.cpp @@ -0,0 +1,40 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * AdvancedPayloadParser.cpp + * + * Created: 2021 - Slavey Karadzhov + * + ****/ + +#include "include/OtaUpgrade/Mqtt/AdvancedPayloadParser.h" + +namespace OtaUpgrade +{ +namespace Mqtt +{ +bool AdvancedPayloadParser::switchRom(const UpdateState& updateState) +{ + auto otaStream = static_cast(updateState.stream); + if(otaStream == nullptr) { + return false; + } + + if(otaStream->hasError()) { + debug_e("Got error: %s", toString(otaStream->errorCode).c_str()); + return false; + } + + return true; +} + +ReadWriteStream* AdvancedPayloadParser::getStorageStream(size_t storageSize) +{ + return new OtaUpgradeStream(); +} + +} // namespace Mqtt +} // namespace OtaUpgrade diff --git a/Sming/Libraries/OtaUpgradeMqtt/src/PayloadParser.cpp b/Sming/Libraries/OtaUpgradeMqtt/src/PayloadParser.cpp new file mode 100644 index 0000000000..56f39ab4e3 --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/src/PayloadParser.cpp @@ -0,0 +1,121 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * PayloadParser.cpp + * + * Created: 2021 - Slavey Karadzhov + * + ****/ + +#include "include/OtaUpgrade/Mqtt/PayloadParser.h" + +namespace OtaUpgrade +{ +namespace Mqtt +{ +int PayloadParser::parse(MqttPayloadParserState& state, mqtt_message_t* message, const char* buffer, int length) +{ + if(message == nullptr) { + debug_e("Invalid MQTT message"); + return ERROR_INVALID_MQTT_MESSAGE; + } + + if(length == MQTT_PAYLOAD_PARSER_START) { + state = MqttPayloadParserState{new UpdateState}; + return 0; + } + + auto updateState = static_cast(state.userData); + if(updateState == nullptr) { + debug_e("Update failed for unknown reason!"); + return ERROR_UNKNOWN_REASON; + } + + if(length == MQTT_PAYLOAD_PARSER_END) { + bool skip = (updateState->stream == nullptr); + if(!skip) { + bool success = switchRom(*updateState); + if(success) { + debug_d("Switching was successful. Restarting..."); + System.restart(); + } else { + debug_e("Switching failed!"); + } + } + delete updateState; + state.userData = nullptr; + return 0; + } + + if(buffer == nullptr) { + debug_e("Invalid MQTT message"); + return ERROR_INVALID_MQTT_MESSAGE; + } + + if(!updateState->started) { + size_t offset = 0; + int patchVersion = getPatchVersion(buffer, length, offset, updateState->version); + state.offset += offset; +#if ENABLE_OTA_VARINT_VERSION + if(patchVersion < 0) { + if(state.offset > allowedVersionBytes) { + debug_e("Invalid patch version."); + return ERROR_INVALID_PATCH_VERSION; + } + return 0; + } +#endif + + updateState->started = true; + if(size_t(patchVersion) < currentPatchVersion) { + // The update is not newer than our current patch version + return 0; + } + + length -= offset; + buffer += offset; + + updateState->stream = getStorageStream(message->common.length - offset); + } + + auto stream = updateState->stream; + if(stream == nullptr) { + return 0; + } + + auto written = stream->write(buffer, length); + return (written - length); +} + +int PayloadParser::getPatchVersion(const char* buffer, int length, size_t& offset, size_t versionStart) +{ + if(buffer == nullptr || length < 1) { + return ERROR_INVALID_PATCH_VERSION; + } + + size_t version = versionStart; +#if ENABLE_OTA_VARINT_VERSION + offset = 0; + int useNextByte = 0; + do { + version += (buffer[offset] & 0x7f); + useNextByte = (buffer[offset++] & 0x80); + } while(useNextByte && (offset < size_t(length))); + + if(useNextByte) { + // all the data is consumed and we still don't have a version number?! + return VERSION_NOT_READY; + } +#else + offset = 1; + version += buffer[0]; +#endif + + return version; +} + +} // namespace Mqtt +} // namespace OtaUpgrade diff --git a/Sming/Libraries/OtaUpgradeMqtt/src/RbootPayloadParser.cpp b/Sming/Libraries/OtaUpgradeMqtt/src/RbootPayloadParser.cpp new file mode 100644 index 0000000000..b18a8fb151 --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/src/RbootPayloadParser.cpp @@ -0,0 +1,40 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * RbootPayloadParser.cpp + * + * Created: 2021 - Slavey Karadzhov + * + ****/ + +#include "include/OtaUpgrade/Mqtt/RbootPayloadParser.h" + +namespace OtaUpgrade +{ +namespace Mqtt +{ +bool RbootPayloadParser::switchRom(const UpdateState& updateState) +{ + uint8_t before = rboot_get_current_rom(); + uint8_t after = (before == 0) ? 1 : 0; + + debug_d("Swapping from rom %u to rom %u.\r\n", before, after); + + return rboot_set_current_rom(after); +} + +ReadWriteStream* RbootPayloadParser::getStorageStream(size_t storageSize) +{ + if(storageSize > part.size()) { + debug_e("The new rom is too big to fit!"); + return nullptr; + } + + return new RbootOutputStream(part.address(), part.size()); +} + +} // namespace Mqtt +} // namespace OtaUpgrade diff --git a/Sming/Libraries/OtaUpgradeMqtt/src/include/OtaUpgrade/Mqtt/AdvancedPayloadParser.h b/Sming/Libraries/OtaUpgradeMqtt/src/include/OtaUpgrade/Mqtt/AdvancedPayloadParser.h new file mode 100644 index 0000000000..1cf01191cc --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/src/include/OtaUpgrade/Mqtt/AdvancedPayloadParser.h @@ -0,0 +1,37 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * AdvancedPayloadParser.h + * + * Created: 2021 - Slavey Karadzhov + * + ****/ + +#pragma once + +#include "PayloadParser.h" +#include + +namespace OtaUpgrade +{ +namespace Mqtt +{ +/** + * This parser allows the processing of firmware data + * that can be encrypted or have signature + */ +class AdvancedPayloadParser : public PayloadParser +{ +public: + using PayloadParser::PayloadParser; + + bool switchRom(const UpdateState& updateState) override; + + ReadWriteStream* getStorageStream(size_t storageSize) override; +}; + +} // namespace Mqtt +} // namespace OtaUpgrade diff --git a/Sming/Libraries/OtaUpgradeMqtt/src/include/OtaUpgrade/Mqtt/PayloadParser.h b/Sming/Libraries/OtaUpgradeMqtt/src/include/OtaUpgrade/Mqtt/PayloadParser.h new file mode 100644 index 0000000000..f246e43292 --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/src/include/OtaUpgrade/Mqtt/PayloadParser.h @@ -0,0 +1,90 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * PayloadParser.h + * + * Created: 2021 - Slavey Karadzhov + * + ****/ + +#pragma once + +#include +#include +#include + +namespace OtaUpgrade +{ +namespace Mqtt +{ +constexpr int8_t VERSION_NOT_READY{-1}; + +constexpr int8_t ERROR_INVALID_MQTT_MESSAGE{-1}; +constexpr int8_t ERROR_INVALID_PATCH_VERSION{-2}; +constexpr int8_t ERROR_UNKNOWN_REASON{-10}; + +class PayloadParser +{ +public: + struct UpdateState { + ReadWriteStream* stream{nullptr}; + bool started{false}; + size_t version{0}; + + ~UpdateState() + { + delete stream; + } + }; + + /** + * @brief + * @param currentPatchVersion + * @param allowedVersionBytes - maximum allowed bytes to be used for describing a patch version + */ + PayloadParser(size_t currentPatchVersion, size_t allowedVersionBytes = 24) + : currentPatchVersion(currentPatchVersion), allowedVersionBytes(allowedVersionBytes) + { + } + + virtual ~PayloadParser() + { + } + + /** + * @brief This method is responsible for switching the rom. + * This method is NOT restarting the system. It will happen lated in the parse method + * @retval true if the switch was successful + */ + virtual bool switchRom(const UpdateState& updateState) = 0; + + /** + * @brief Creates new stream to store the firmware update + * @param size_t storageSize the requested storage size + * + * @retval ReadWriteStream* + */ + virtual ReadWriteStream* getStorageStream(size_t storageSize) = 0; + + /** + * @brief This method takes care to read the incoming MQTT message and pass it to the stream that + * is responsoble for storing the data. + * + * @retval int + * 0 when everything is ok + * <0 when an error has occurred + */ + int parse(MqttPayloadParserState& state, mqtt_message_t* message, const char* buffer, int length); + +private: + int getPatchVersion(const char* buffer, int length, size_t& offset, size_t versionStart = 0); + + size_t currentPatchVersion; + size_t allowedVersionBytes; +}; + +} // namespace Mqtt +} // namespace OtaUpgrade diff --git a/Sming/Libraries/OtaUpgradeMqtt/src/include/OtaUpgrade/Mqtt/RbootPayloadParser.h b/Sming/Libraries/OtaUpgradeMqtt/src/include/OtaUpgrade/Mqtt/RbootPayloadParser.h new file mode 100644 index 0000000000..68e87dd53a --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/src/include/OtaUpgrade/Mqtt/RbootPayloadParser.h @@ -0,0 +1,43 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * RbootPayloadParser.h + * + * Created: 2021 - Slavey Karadzhov + * + ****/ + +#pragma once + +#include "PayloadParser.h" +#include + +namespace OtaUpgrade +{ +namespace Mqtt +{ +/** + * @brief This parser allows the processing of firmware data that is directly stored + * to the flash memory using RbootOutputStream. + */ +class RbootPayloadParser : public PayloadParser +{ +public: + RbootPayloadParser(Storage::Partition part, size_t currentVersion, size_t allowedVersionBytes = 24) + : PayloadParser(currentVersion, allowedVersionBytes), part(part) + { + } + + bool switchRom(const UpdateState& updateState) override; + + ReadWriteStream* getStorageStream(size_t storageSize) override; + +private: + Storage::Partition part; +}; + +} // namespace Mqtt +} // namespace OtaUpgrade diff --git a/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/.cproject b/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/.cproject new file mode 100644 index 0000000000..fecee42f2f --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/.cproject @@ -0,0 +1,97 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + make + + all + true + true + true + + + make + + rebuild + true + true + true + + + make + + flash + true + true + true + + + + diff --git a/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/.project b/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/.project new file mode 100644 index 0000000000..0b94d5c588 --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/.project @@ -0,0 +1,30 @@ + + + FirmwareDeployer + + + SmingFramework + Libraries + Sming + + + + org.eclipse.cdt.managedbuilder.core.genmakebuilder + clean,full,incremental, + + + + + org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder + full,incremental, + + + + + + org.eclipse.cdt.core.cnature + org.eclipse.cdt.core.ccnature + org.eclipse.cdt.managedbuilder.core.managedBuildNature + org.eclipse.cdt.managedbuilder.core.ScannerConfigNature + + diff --git a/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/Makefile b/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/Makefile new file mode 100644 index 0000000000..ff51b6c3a7 --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/Makefile @@ -0,0 +1,9 @@ +##################################################################### +#### Please don't change this file. Use component.mk instead #### +##################################################################### + +ifndef SMING_HOME +$(error SMING_HOME is not set: please configure it as an environment variable) +endif + +include $(SMING_HOME)/project.mk diff --git a/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/README.rst b/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/README.rst new file mode 100644 index 0000000000..5a0f568c0c --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/README.rst @@ -0,0 +1,7 @@ +Deployer +======== + +.. highlight:: bash + +Utility to package and deploy firmware files for upgrade over MQTT +See :library:`OtaUpgradeMqtt` for details. diff --git a/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/app/application.cpp b/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/app/application.cpp new file mode 100644 index 0000000000..f3a513af69 --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/app/application.cpp @@ -0,0 +1,252 @@ +#include +#include + +#ifdef ARCH_HOST +#ifdef __WIN32__ +#include +#endif + +#include +#include +#include +#endif + +// If you want, you can define WiFi settings globally in Eclipse Environment Variables +#ifndef WIFI_SSID +#define WIFI_SSID "PleaseEnterSSID" // Put you SSID and Password here +#define WIFI_PWD "PleaseEnterPass" +#endif + +namespace +{ +MqttClient mqtt; + +#ifdef ARCH_HOST + +size_t charsWriter(const char* buffer, size_t length) +{ + return fwrite(buffer, sizeof(char), length, stdout); +} + +template void print(const T& arg) +{ + String s(arg); + m_nputs(s.c_str(), s.length()); +} + +void println() +{ + m_puts("\r\n"); +} + +template void println(const T& arg) +{ + print(arg); + println(); +} + +/* + * Return number of bytes written, 0 if *any* writes failed + */ +size_t writePatchVersion(size_t patchVersion, bool useVarInt, ReadWriteStream& output) +{ + size_t written = 0; + if(useVarInt) { + while(patchVersion > 0x7f) { + if(output.write(uint8_t(patchVersion | 0x80)) != 1) { + return 0; + } + patchVersion >>= 7; + written++; + } + if(output.write(uint8_t(patchVersion & 0x7f)) != 1) { + return 0; + } + written++; + } else { + written = output.write(uint8_t(patchVersion)); + } + + return written; +} + +static void fileError(IFS::FsBase& fs, const String& filename, const String& operation) +{ + print(F("ERROR: Failed to ")); + print(operation); + print(F(" file '")); + print(filename); + print("': "); + println(fs.getLastErrorString()); +} + +bool pack(const String& inputFileName, const String& outputFileName, size_t patchVersion, bool useVarInt) +{ + HostFileStream input; + if(!input.open(inputFileName)) { + fileError(input, inputFileName, F("open input")); + return false; + } + + HostFileStream output; + if(!output.open(outputFileName, eFO_CreateNewAlways | eFO_WriteOnly)) { + fileError(output, outputFileName, F("open output")); + return false; + } + if(writePatchVersion(patchVersion, useVarInt, output) == 0) { + print(F("writePatchVersion() failed")); + return false; + } + output.copyFrom(&input); + if(input.getLastError() != FS_OK) { + fileError(input, inputFileName, F("read from")); + return false; + } + if(output.getLastError() != FS_OK) { + fileError(output, outputFileName, F("write to")); + return false; + } + + return true; +} + +bool deploy(const String& outputFileName, const String& url) +{ + HostFileStream* output = new HostFileStream(); + if(!output->open(outputFileName)) { + fileError(*output, outputFileName, F("open output")); + return false; + } + + WifiStation.enable(true, false); + WifiStation.config(WIFI_SSID, WIFI_PWD); + WifiAccessPoint.enable(false, false); + WifiEvents.onStationGotIP([url, output](IpAddress ip, IpAddress netmask, IpAddress gateway) { + Url mqttUrl(url); + + mqtt.connect(mqttUrl, "sming"); + mqtt.setConnectedHandler([mqttUrl, output](MqttClient& client, mqtt_message_t* message) -> int { + if(message == nullptr) { + // invalid message received + return 1; + } + + if(message->connack.return_code) { + print(F("ERROR: Connection failed. Reason: ")); + println(mqtt_connect_error_string(mqtt_connect_error_t(message->connack.return_code))); + System.restart(1000); + return 0; + } + + uint8_t retained = 1; + uint8_t QoS = 2; + uint8_t flags = uint8_t(retained + (QoS << 1)); + mqtt.publish(mqttUrl.Path.substring(1), output, flags); + mqtt.setPublishedHandler([](MqttClient& client, mqtt_message_t* message) -> int { + println(F("Firmware uploaded successfully.")); + System.restart(1000); + return 0; + }); + + return 0; + }); + }); + + return true; +} + +void help() +{ + println(); + println(F("Available commands:")); + println(F(" pack fileName.in fileName.out patchVersion Creates a package to be deployed on " + "firmware upgrade server.")); + println(F(" deploy fileName.out mqttUrl Deploys a deployment package to " + "firmware upgrade server.")); + println(); +} + +/* + * Return true to continue execution, false to quit. + */ +bool parseCommands() +{ + auto parameters = commandLine.getParameters(); + if(parameters.count() == 0) { + help(); + return false; + } + + String cmd = parameters[0].text; + + auto checkParameterCount = [&](unsigned minCount, unsigned maxCount) -> bool { + if(parameters.count() < minCount) { + print(F("Insufficient")); + } else if(parameters.count() > maxCount) { + print(F("Too many")); + } else { + return true; + } + + print(F(" parameters for '")); + print(cmd); + println("'."); + return false; + }; + + if(cmd == "pack") { + if(checkParameterCount(4, 5)) { + auto inputFileName = parameters[1].text; + auto outputFileName = parameters[2].text; + auto patchVersion = strtoul(parameters[3].text, nullptr, 0); + bool useVarInt = false; + if(parameters.count() > 4) { + String p(parameters[4].text); + if(p == "0") { + useVarInt = false; + } else if(p == "1") { + useVarInt = true; + } else { + println(F("Invalid setting for useVarInt, must be 1 or 0")); + return false; + } + } + if(!useVarInt && patchVersion > 0xff) { + println(F("Patch version number too large for `useVarInt = 0`")); + } else { + pack(inputFileName, outputFileName, patchVersion, useVarInt); + } + return false; // after packaging the application can be terminated + } + } else if(cmd == "deploy") { + if(checkParameterCount(2, 2)) { + return deploy(parameters[1].text, parameters[2].text); + } + } else { + print(F("ERROR: Unknown command '")); + print(cmd); + println("'."); + } + + help(); + return false; +} + +#endif // ARCH_HOST + +} // namespace + +void init() +{ + Serial.setTxBufferSize(1024); + Serial.begin(SERIAL_BAUD_RATE); + Serial.systemDebugOutput(true); + +#ifdef ARCH_HOST + m_setPuts(charsWriter); + + if(!parseCommands()) { + System.restart(1000); + } +#endif +} diff --git a/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/component.mk b/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/component.mk new file mode 100644 index 0000000000..3069b10758 --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/component.mk @@ -0,0 +1,14 @@ +COMPONENT_DEPENDS := OtaUpgradeMqtt +APP_NAME := deployer + +SMING_ARCH := Host + +##@Firmware Upgrade + +APP=$(TARGET_OUT_0) + +pack: application $(APP) ##Pack firmware files for deployment (HOST_PARAMETERS=inputFile packedFile version 1|0. Example: ) + $(Q) $(APP) pack -- $(HOST_PARAMETERS) + +deploy: application $(APP) ##Deploy firmware files to MQTT server (HOST_PARAMETERS=packedFile MQTTQ_URL. Example MQTT user: ) + $(Q) $(APP) deploy -- $(HOST_PARAMETERS) diff --git a/Sming/Libraries/RapidXML b/Sming/Libraries/RapidXML index 0b56a775b7..cab0af2fda 160000 --- a/Sming/Libraries/RapidXML +++ b/Sming/Libraries/RapidXML @@ -1 +1 @@ -Subproject commit 0b56a775b7b3603631734506be4984f094df3685 +Subproject commit cab0af2fda5d52a365502750db6e6bd01f585053 diff --git a/Sming/Libraries/RingTone b/Sming/Libraries/RingTone index 65ebef1efe..56bdf070f7 160000 --- a/Sming/Libraries/RingTone +++ b/Sming/Libraries/RingTone @@ -1 +1 @@ -Subproject commit 65ebef1efe71f56ee8d01f158420d4ca07a91fe6 +Subproject commit 56bdf070f7e4eebebe15646079aa709af10e248c diff --git a/Sming/Libraries/SmingTest b/Sming/Libraries/SmingTest index b52ffcd329..36913db2ce 160000 --- a/Sming/Libraries/SmingTest +++ b/Sming/Libraries/SmingTest @@ -1 +1 @@ -Subproject commit b52ffcd3297490a39135d50824374d073c4e384b +Subproject commit 36913db2ce3a5be82abb90d2140988307adfc38d diff --git a/Sming/Libraries/TFT_S1D13781 b/Sming/Libraries/TFT_S1D13781 index e83437794c..4940599366 160000 --- a/Sming/Libraries/TFT_S1D13781 +++ b/Sming/Libraries/TFT_S1D13781 @@ -1 +1 @@ -Subproject commit e83437794c493cb4bb1c486aa5181a227ed2408b +Subproject commit 49405993668fc4dfbd51109e836f3eaa7b0b838f diff --git a/Sming/Libraries/UPnP b/Sming/Libraries/UPnP index 066c84cf97..e04f503c15 160000 --- a/Sming/Libraries/UPnP +++ b/Sming/Libraries/UPnP @@ -1 +1 @@ -Subproject commit 066c84cf971652a1d476468d9cd8796f337f54ab +Subproject commit e04f503c151f9cd67d7ee06f156ede583b5f06e5 diff --git a/Sming/Libraries/UPnP-Schema b/Sming/Libraries/UPnP-Schema index 99a5fdbdee..1438502af5 160000 --- a/Sming/Libraries/UPnP-Schema +++ b/Sming/Libraries/UPnP-Schema @@ -1 +1 @@ -Subproject commit 99a5fdbdeed062680809329ffc8c533338d3ceb1 +Subproject commit 1438502af5a922bd04d81f158c6a820815fefe42 diff --git a/Sming/Libraries/flatbuffers/README.rst b/Sming/Libraries/flatbuffers/README.rst new file mode 100644 index 0000000000..54d97a4440 --- /dev/null +++ b/Sming/Libraries/flatbuffers/README.rst @@ -0,0 +1,57 @@ +Flatbuffers +=========== + +.. highlight:: c++ + +FlatBuffers is an efficient cross platform serialization library for C++, C#, C, Go, Java, Kotlin, JavaScript, Lobster, Lua, TypeScript, PHP, Python, Rust and Swift. +It was originally created at Google for game development and other performance-critical applications. + +It is available as Open Source on GitHub under the Apache license, v2 (see LICENSE.txt). + +Using +----- + +Step 1. Add these lines to your application componenent.mk file:: + + COMPONENT_DEPENDS += flatbuffers + +Step 2. Add these lines to your application:: + + #include + +Or directly use the header file generated from the `flatc` compiler. + +Step 3. Basic example:: + + #include "monster.h" + + using namespace MyGame::Example; + + void init() + { + Serial.begin(SERIAL_BAUD_RATE); + + // demonstration how to encode data into a flatbuffer + flatbuffers::FlatBufferBuilder builder; + + auto name = builder.CreateString("Sming Monster"); + + MonsterBuilder monster_builder(builder); + monster_builder.add_name(name); + + auto orc = monster_builder.Finish(); + + // Call `Finish()` to instruct the builder that this monster is complete. + // Note: Regardless of how you created the `orc`, you still need to call + // `Finish()` on the `FlatBufferBuilder`. + builder.Finish(orc); + + // and then decode it + uint8_t *buffer = builder.GetBufferPointer(); + auto monster = GetMonster(buffer); + Serial.printf("Monster name: %s\n", monster->name()->c_str()); + } + +Further reading +--------------- +Take a look at the `official flatbuffers tutorial `_. diff --git a/Sming/Libraries/flatbuffers/component.mk b/Sming/Libraries/flatbuffers/component.mk new file mode 100644 index 0000000000..90560f8b60 --- /dev/null +++ b/Sming/Libraries/flatbuffers/component.mk @@ -0,0 +1,2 @@ +COMPONENT_INCDIRS := src/include +COMPONENT_SUBMODULES := src diff --git a/Sming/Libraries/flatbuffers/samples/Basic/.cproject b/Sming/Libraries/flatbuffers/samples/Basic/.cproject new file mode 100644 index 0000000000..67c056d24e --- /dev/null +++ b/Sming/Libraries/flatbuffers/samples/Basic/.cproject @@ -0,0 +1,151 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + make + -f ${ProjDirPath}/Makefile + all + true + true + true + + + make + -f ${ProjDirPath}/Makefile + clean + true + true + true + + + make + -f ${ProjDirPath}/Makefile + flash + true + true + true + + + make + -f ${ProjDirPath}/Makefile + flashonefile + true + true + true + + + make + -f ${ProjDirPath}/Makefile + flashinit + true + true + true + + + make + -f ${ProjDirPath}/Makefile + flashboot + true + true + true + + + make + -f ${ProjDirPath}/Makefile + rebuild + true + true + true + + + + + + + + + + + + + + + + + + + + diff --git a/Sming/Libraries/flatbuffers/samples/Basic/.project b/Sming/Libraries/flatbuffers/samples/Basic/.project new file mode 100644 index 0000000000..0b15c33502 --- /dev/null +++ b/Sming/Libraries/flatbuffers/samples/Basic/.project @@ -0,0 +1,28 @@ + + + Basic_Flatbuffers + + + SmingFramework + + + + org.eclipse.cdt.managedbuilder.core.genmakebuilder + clean,full,incremental, + + + + + org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder + full,incremental, + + + + + + org.eclipse.cdt.core.cnature + org.eclipse.cdt.core.ccnature + org.eclipse.cdt.managedbuilder.core.managedBuildNature + org.eclipse.cdt.managedbuilder.core.ScannerConfigNature + + diff --git a/Sming/Libraries/flatbuffers/samples/Basic/Makefile b/Sming/Libraries/flatbuffers/samples/Basic/Makefile new file mode 100644 index 0000000000..ff51b6c3a7 --- /dev/null +++ b/Sming/Libraries/flatbuffers/samples/Basic/Makefile @@ -0,0 +1,9 @@ +##################################################################### +#### Please don't change this file. Use component.mk instead #### +##################################################################### + +ifndef SMING_HOME +$(error SMING_HOME is not set: please configure it as an environment variable) +endif + +include $(SMING_HOME)/project.mk diff --git a/Sming/Libraries/flatbuffers/samples/Basic/README.rst b/Sming/Libraries/flatbuffers/samples/Basic/README.rst new file mode 100644 index 0000000000..2a64840bda --- /dev/null +++ b/Sming/Libraries/flatbuffers/samples/Basic/README.rst @@ -0,0 +1,5 @@ +Basic FlatBuffers Sample +======================== + +Basic sample demonstrating encoding and decoding of data using flatbuffers. +For more complicated examples take a look at the `official flatbuffers tutorial `_. diff --git a/tests/HostTests/.cs b/Sming/Libraries/flatbuffers/samples/Basic/app/.cs similarity index 100% rename from tests/HostTests/.cs rename to Sming/Libraries/flatbuffers/samples/Basic/app/.cs diff --git a/Sming/Libraries/flatbuffers/samples/Basic/app/application.cpp b/Sming/Libraries/flatbuffers/samples/Basic/app/application.cpp new file mode 100644 index 0000000000..88c26a56a7 --- /dev/null +++ b/Sming/Libraries/flatbuffers/samples/Basic/app/application.cpp @@ -0,0 +1,30 @@ +#include + +#include "monster.h" + +using namespace MyGame::Sample; + +void init() +{ + Serial.begin(SERIAL_BAUD_RATE); + + // demonstration how to encode data into a flatbuffer + flatbuffers::FlatBufferBuilder builder; + + auto name = builder.CreateString("Sming Monster"); + + MonsterBuilder monster_builder(builder); + monster_builder.add_name(name); + + auto orc = monster_builder.Finish(); + + // Call `Finish()` to instruct the builder that this monster is complete. + // Note: Regardless of how you created the `orc`, you still need to call + // `Finish()` on the `FlatBufferBuilder`. + builder.Finish(orc); + + // and then decode it + uint8_t* buffer = builder.GetBufferPointer(); + auto monster = GetMonster(buffer); + Serial.printf("Monster name: %s\n", monster->name()->c_str()); +} diff --git a/Sming/Libraries/flatbuffers/samples/Basic/component.mk b/Sming/Libraries/flatbuffers/samples/Basic/component.mk new file mode 100644 index 0000000000..3c388da53e --- /dev/null +++ b/Sming/Libraries/flatbuffers/samples/Basic/component.mk @@ -0,0 +1,6 @@ +COMPONENT_DEPENDS := flatbuffers + +CUSTOM_TARGETS += include/monster.h + +include/monster.h: ../../src/samples/monster_generated.h + $(Q) cp $< $@ \ No newline at end of file diff --git a/Sming/Libraries/flatbuffers/samples/Basic/include/monster.h b/Sming/Libraries/flatbuffers/samples/Basic/include/monster.h new file mode 100644 index 0000000000..9b70be4a7a --- /dev/null +++ b/Sming/Libraries/flatbuffers/samples/Basic/include/monster.h @@ -0,0 +1,872 @@ +// automatically generated by the FlatBuffers compiler, do not modify + + +#ifndef FLATBUFFERS_GENERATED_MONSTER_MYGAME_SAMPLE_H_ +#define FLATBUFFERS_GENERATED_MONSTER_MYGAME_SAMPLE_H_ + +#include "flatbuffers/flatbuffers.h" + +namespace MyGame { +namespace Sample { + +struct Vec3; + +struct Monster; +struct MonsterBuilder; +struct MonsterT; + +struct Weapon; +struct WeaponBuilder; +struct WeaponT; + +bool operator==(const Vec3 &lhs, const Vec3 &rhs); +bool operator!=(const Vec3 &lhs, const Vec3 &rhs); +bool operator==(const MonsterT &lhs, const MonsterT &rhs); +bool operator!=(const MonsterT &lhs, const MonsterT &rhs); +bool operator==(const WeaponT &lhs, const WeaponT &rhs); +bool operator!=(const WeaponT &lhs, const WeaponT &rhs); + +inline const flatbuffers::TypeTable *Vec3TypeTable(); + +inline const flatbuffers::TypeTable *MonsterTypeTable(); + +inline const flatbuffers::TypeTable *WeaponTypeTable(); + +enum Color : int8_t { + Color_Red = 0, + Color_Green = 1, + Color_Blue = 2, + Color_MIN = Color_Red, + Color_MAX = Color_Blue +}; + +inline const Color (&EnumValuesColor())[3] { + static const Color values[] = { + Color_Red, + Color_Green, + Color_Blue + }; + return values; +} + +inline const char * const *EnumNamesColor() { + static const char * const names[4] = { + "Red", + "Green", + "Blue", + nullptr + }; + return names; +} + +inline const char *EnumNameColor(Color e) { + if (flatbuffers::IsOutRange(e, Color_Red, Color_Blue)) return ""; + const size_t index = static_cast(e); + return EnumNamesColor()[index]; +} + +enum Equipment : uint8_t { + Equipment_NONE = 0, + Equipment_Weapon = 1, + Equipment_MIN = Equipment_NONE, + Equipment_MAX = Equipment_Weapon +}; + +inline const Equipment (&EnumValuesEquipment())[2] { + static const Equipment values[] = { + Equipment_NONE, + Equipment_Weapon + }; + return values; +} + +inline const char * const *EnumNamesEquipment() { + static const char * const names[3] = { + "NONE", + "Weapon", + nullptr + }; + return names; +} + +inline const char *EnumNameEquipment(Equipment e) { + if (flatbuffers::IsOutRange(e, Equipment_NONE, Equipment_Weapon)) return ""; + const size_t index = static_cast(e); + return EnumNamesEquipment()[index]; +} + +template struct EquipmentTraits { + static const Equipment enum_value = Equipment_NONE; +}; + +template<> struct EquipmentTraits { + static const Equipment enum_value = Equipment_Weapon; +}; + +struct EquipmentUnion { + Equipment type; + void *value; + + EquipmentUnion() : type(Equipment_NONE), value(nullptr) {} + EquipmentUnion(EquipmentUnion&& u) FLATBUFFERS_NOEXCEPT : + type(Equipment_NONE), value(nullptr) + { std::swap(type, u.type); std::swap(value, u.value); } + EquipmentUnion(const EquipmentUnion &); + EquipmentUnion &operator=(const EquipmentUnion &u) + { EquipmentUnion t(u); std::swap(type, t.type); std::swap(value, t.value); return *this; } + EquipmentUnion &operator=(EquipmentUnion &&u) FLATBUFFERS_NOEXCEPT + { std::swap(type, u.type); std::swap(value, u.value); return *this; } + ~EquipmentUnion() { Reset(); } + + void Reset(); + +#ifndef FLATBUFFERS_CPP98_STL + template + void Set(T&& val) { + using RT = typename std::remove_reference::type; + Reset(); + type = EquipmentTraits::enum_value; + if (type != Equipment_NONE) { + value = new RT(std::forward(val)); + } + } +#endif // FLATBUFFERS_CPP98_STL + + static void *UnPack(const void *obj, Equipment type, const flatbuffers::resolver_function_t *resolver); + flatbuffers::Offset Pack(flatbuffers::FlatBufferBuilder &_fbb, const flatbuffers::rehasher_function_t *_rehasher = nullptr) const; + + MyGame::Sample::WeaponT *AsWeapon() { + return type == Equipment_Weapon ? + reinterpret_cast(value) : nullptr; + } + const MyGame::Sample::WeaponT *AsWeapon() const { + return type == Equipment_Weapon ? + reinterpret_cast(value) : nullptr; + } +}; + + +inline bool operator==(const EquipmentUnion &lhs, const EquipmentUnion &rhs) { + if (lhs.type != rhs.type) return false; + switch (lhs.type) { + case Equipment_NONE: { + return true; + } + case Equipment_Weapon: { + return *(reinterpret_cast(lhs.value)) == + *(reinterpret_cast(rhs.value)); + } + default: { + return false; + } + } +} + +inline bool operator!=(const EquipmentUnion &lhs, const EquipmentUnion &rhs) { + return !(lhs == rhs); +} + +bool VerifyEquipment(flatbuffers::Verifier &verifier, const void *obj, Equipment type); +bool VerifyEquipmentVector(flatbuffers::Verifier &verifier, const flatbuffers::Vector> *values, const flatbuffers::Vector *types); + +FLATBUFFERS_MANUALLY_ALIGNED_STRUCT(4) Vec3 FLATBUFFERS_FINAL_CLASS { + private: + float x_; + float y_; + float z_; + + public: + static const flatbuffers::TypeTable *MiniReflectTypeTable() { + return Vec3TypeTable(); + } + Vec3() + : x_(0), + y_(0), + z_(0) { + } + Vec3(float _x, float _y, float _z) + : x_(flatbuffers::EndianScalar(_x)), + y_(flatbuffers::EndianScalar(_y)), + z_(flatbuffers::EndianScalar(_z)) { + } + float x() const { + return flatbuffers::EndianScalar(x_); + } + void mutate_x(float _x) { + flatbuffers::WriteScalar(&x_, _x); + } + float y() const { + return flatbuffers::EndianScalar(y_); + } + void mutate_y(float _y) { + flatbuffers::WriteScalar(&y_, _y); + } + float z() const { + return flatbuffers::EndianScalar(z_); + } + void mutate_z(float _z) { + flatbuffers::WriteScalar(&z_, _z); + } +}; +FLATBUFFERS_STRUCT_END(Vec3, 12); + +inline bool operator==(const Vec3 &lhs, const Vec3 &rhs) { + return + (lhs.x() == rhs.x()) && + (lhs.y() == rhs.y()) && + (lhs.z() == rhs.z()); +} + +inline bool operator!=(const Vec3 &lhs, const Vec3 &rhs) { + return !(lhs == rhs); +} + + +struct MonsterT : public flatbuffers::NativeTable { + typedef Monster TableType; + flatbuffers::unique_ptr pos{}; + int16_t mana = 150; + int16_t hp = 100; + std::string name{}; + std::vector inventory{}; + MyGame::Sample::Color color = MyGame::Sample::Color_Blue; + std::vector> weapons{}; + MyGame::Sample::EquipmentUnion equipped{}; + std::vector path{}; +}; + +inline bool operator==(const MonsterT &lhs, const MonsterT &rhs) { + return + (lhs.pos == rhs.pos) && + (lhs.mana == rhs.mana) && + (lhs.hp == rhs.hp) && + (lhs.name == rhs.name) && + (lhs.inventory == rhs.inventory) && + (lhs.color == rhs.color) && + (lhs.weapons == rhs.weapons) && + (lhs.equipped == rhs.equipped) && + (lhs.path == rhs.path); +} + +inline bool operator!=(const MonsterT &lhs, const MonsterT &rhs) { + return !(lhs == rhs); +} + + +struct Monster FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + typedef MonsterT NativeTableType; + typedef MonsterBuilder Builder; + static const flatbuffers::TypeTable *MiniReflectTypeTable() { + return MonsterTypeTable(); + } + enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { + VT_POS = 4, + VT_MANA = 6, + VT_HP = 8, + VT_NAME = 10, + VT_INVENTORY = 14, + VT_COLOR = 16, + VT_WEAPONS = 18, + VT_EQUIPPED_TYPE = 20, + VT_EQUIPPED = 22, + VT_PATH = 24 + }; + const MyGame::Sample::Vec3 *pos() const { + return GetStruct(VT_POS); + } + MyGame::Sample::Vec3 *mutable_pos() { + return GetStruct(VT_POS); + } + int16_t mana() const { + return GetField(VT_MANA, 150); + } + bool mutate_mana(int16_t _mana) { + return SetField(VT_MANA, _mana, 150); + } + int16_t hp() const { + return GetField(VT_HP, 100); + } + bool mutate_hp(int16_t _hp) { + return SetField(VT_HP, _hp, 100); + } + const flatbuffers::String *name() const { + return GetPointer(VT_NAME); + } + flatbuffers::String *mutable_name() { + return GetPointer(VT_NAME); + } + const flatbuffers::Vector *inventory() const { + return GetPointer *>(VT_INVENTORY); + } + flatbuffers::Vector *mutable_inventory() { + return GetPointer *>(VT_INVENTORY); + } + MyGame::Sample::Color color() const { + return static_cast(GetField(VT_COLOR, 2)); + } + bool mutate_color(MyGame::Sample::Color _color) { + return SetField(VT_COLOR, static_cast(_color), 2); + } + const flatbuffers::Vector> *weapons() const { + return GetPointer> *>(VT_WEAPONS); + } + flatbuffers::Vector> *mutable_weapons() { + return GetPointer> *>(VT_WEAPONS); + } + MyGame::Sample::Equipment equipped_type() const { + return static_cast(GetField(VT_EQUIPPED_TYPE, 0)); + } + const void *equipped() const { + return GetPointer(VT_EQUIPPED); + } + template const T *equipped_as() const; + const MyGame::Sample::Weapon *equipped_as_Weapon() const { + return equipped_type() == MyGame::Sample::Equipment_Weapon ? static_cast(equipped()) : nullptr; + } + void *mutable_equipped() { + return GetPointer(VT_EQUIPPED); + } + const flatbuffers::Vector *path() const { + return GetPointer *>(VT_PATH); + } + flatbuffers::Vector *mutable_path() { + return GetPointer *>(VT_PATH); + } + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + VerifyField(verifier, VT_POS) && + VerifyField(verifier, VT_MANA) && + VerifyField(verifier, VT_HP) && + VerifyOffset(verifier, VT_NAME) && + verifier.VerifyString(name()) && + VerifyOffset(verifier, VT_INVENTORY) && + verifier.VerifyVector(inventory()) && + VerifyField(verifier, VT_COLOR) && + VerifyOffset(verifier, VT_WEAPONS) && + verifier.VerifyVector(weapons()) && + verifier.VerifyVectorOfTables(weapons()) && + VerifyField(verifier, VT_EQUIPPED_TYPE) && + VerifyOffset(verifier, VT_EQUIPPED) && + VerifyEquipment(verifier, equipped(), equipped_type()) && + VerifyOffset(verifier, VT_PATH) && + verifier.VerifyVector(path()) && + verifier.EndTable(); + } + MonsterT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(MonsterT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const; + static flatbuffers::Offset Pack(flatbuffers::FlatBufferBuilder &_fbb, const MonsterT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +}; + +template<> inline const MyGame::Sample::Weapon *Monster::equipped_as() const { + return equipped_as_Weapon(); +} + +struct MonsterBuilder { + typedef Monster Table; + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + void add_pos(const MyGame::Sample::Vec3 *pos) { + fbb_.AddStruct(Monster::VT_POS, pos); + } + void add_mana(int16_t mana) { + fbb_.AddElement(Monster::VT_MANA, mana, 150); + } + void add_hp(int16_t hp) { + fbb_.AddElement(Monster::VT_HP, hp, 100); + } + void add_name(flatbuffers::Offset name) { + fbb_.AddOffset(Monster::VT_NAME, name); + } + void add_inventory(flatbuffers::Offset> inventory) { + fbb_.AddOffset(Monster::VT_INVENTORY, inventory); + } + void add_color(MyGame::Sample::Color color) { + fbb_.AddElement(Monster::VT_COLOR, static_cast(color), 2); + } + void add_weapons(flatbuffers::Offset>> weapons) { + fbb_.AddOffset(Monster::VT_WEAPONS, weapons); + } + void add_equipped_type(MyGame::Sample::Equipment equipped_type) { + fbb_.AddElement(Monster::VT_EQUIPPED_TYPE, static_cast(equipped_type), 0); + } + void add_equipped(flatbuffers::Offset equipped) { + fbb_.AddOffset(Monster::VT_EQUIPPED, equipped); + } + void add_path(flatbuffers::Offset> path) { + fbb_.AddOffset(Monster::VT_PATH, path); + } + explicit MonsterBuilder(flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = flatbuffers::Offset(end); + return o; + } +}; + +inline flatbuffers::Offset CreateMonster( + flatbuffers::FlatBufferBuilder &_fbb, + const MyGame::Sample::Vec3 *pos = 0, + int16_t mana = 150, + int16_t hp = 100, + flatbuffers::Offset name = 0, + flatbuffers::Offset> inventory = 0, + MyGame::Sample::Color color = MyGame::Sample::Color_Blue, + flatbuffers::Offset>> weapons = 0, + MyGame::Sample::Equipment equipped_type = MyGame::Sample::Equipment_NONE, + flatbuffers::Offset equipped = 0, + flatbuffers::Offset> path = 0) { + MonsterBuilder builder_(_fbb); + builder_.add_path(path); + builder_.add_equipped(equipped); + builder_.add_weapons(weapons); + builder_.add_inventory(inventory); + builder_.add_name(name); + builder_.add_pos(pos); + builder_.add_hp(hp); + builder_.add_mana(mana); + builder_.add_equipped_type(equipped_type); + builder_.add_color(color); + return builder_.Finish(); +} + +inline flatbuffers::Offset CreateMonsterDirect( + flatbuffers::FlatBufferBuilder &_fbb, + const MyGame::Sample::Vec3 *pos = 0, + int16_t mana = 150, + int16_t hp = 100, + const char *name = nullptr, + const std::vector *inventory = nullptr, + MyGame::Sample::Color color = MyGame::Sample::Color_Blue, + const std::vector> *weapons = nullptr, + MyGame::Sample::Equipment equipped_type = MyGame::Sample::Equipment_NONE, + flatbuffers::Offset equipped = 0, + const std::vector *path = nullptr) { + auto name__ = name ? _fbb.CreateString(name) : 0; + auto inventory__ = inventory ? _fbb.CreateVector(*inventory) : 0; + auto weapons__ = weapons ? _fbb.CreateVector>(*weapons) : 0; + auto path__ = path ? _fbb.CreateVectorOfStructs(*path) : 0; + return MyGame::Sample::CreateMonster( + _fbb, + pos, + mana, + hp, + name__, + inventory__, + color, + weapons__, + equipped_type, + equipped, + path__); +} + +flatbuffers::Offset CreateMonster(flatbuffers::FlatBufferBuilder &_fbb, const MonsterT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); + +struct WeaponT : public flatbuffers::NativeTable { + typedef Weapon TableType; + std::string name{}; + int16_t damage = 0; +}; + +inline bool operator==(const WeaponT &lhs, const WeaponT &rhs) { + return + (lhs.name == rhs.name) && + (lhs.damage == rhs.damage); +} + +inline bool operator!=(const WeaponT &lhs, const WeaponT &rhs) { + return !(lhs == rhs); +} + + +struct Weapon FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + typedef WeaponT NativeTableType; + typedef WeaponBuilder Builder; + static const flatbuffers::TypeTable *MiniReflectTypeTable() { + return WeaponTypeTable(); + } + enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { + VT_NAME = 4, + VT_DAMAGE = 6 + }; + const flatbuffers::String *name() const { + return GetPointer(VT_NAME); + } + flatbuffers::String *mutable_name() { + return GetPointer(VT_NAME); + } + int16_t damage() const { + return GetField(VT_DAMAGE, 0); + } + bool mutate_damage(int16_t _damage) { + return SetField(VT_DAMAGE, _damage, 0); + } + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + VerifyOffset(verifier, VT_NAME) && + verifier.VerifyString(name()) && + VerifyField(verifier, VT_DAMAGE) && + verifier.EndTable(); + } + WeaponT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(WeaponT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const; + static flatbuffers::Offset Pack(flatbuffers::FlatBufferBuilder &_fbb, const WeaponT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +}; + +struct WeaponBuilder { + typedef Weapon Table; + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + void add_name(flatbuffers::Offset name) { + fbb_.AddOffset(Weapon::VT_NAME, name); + } + void add_damage(int16_t damage) { + fbb_.AddElement(Weapon::VT_DAMAGE, damage, 0); + } + explicit WeaponBuilder(flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = flatbuffers::Offset(end); + return o; + } +}; + +inline flatbuffers::Offset CreateWeapon( + flatbuffers::FlatBufferBuilder &_fbb, + flatbuffers::Offset name = 0, + int16_t damage = 0) { + WeaponBuilder builder_(_fbb); + builder_.add_name(name); + builder_.add_damage(damage); + return builder_.Finish(); +} + +inline flatbuffers::Offset CreateWeaponDirect( + flatbuffers::FlatBufferBuilder &_fbb, + const char *name = nullptr, + int16_t damage = 0) { + auto name__ = name ? _fbb.CreateString(name) : 0; + return MyGame::Sample::CreateWeapon( + _fbb, + name__, + damage); +} + +flatbuffers::Offset CreateWeapon(flatbuffers::FlatBufferBuilder &_fbb, const WeaponT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); + +inline MonsterT *Monster::UnPack(const flatbuffers::resolver_function_t *_resolver) const { + auto _o = std::unique_ptr(new MonsterT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); +} + +inline void Monster::UnPackTo(MonsterT *_o, const flatbuffers::resolver_function_t *_resolver) const { + (void)_o; + (void)_resolver; + { auto _e = pos(); if (_e) _o->pos = flatbuffers::unique_ptr(new MyGame::Sample::Vec3(*_e)); } + { auto _e = mana(); _o->mana = _e; } + { auto _e = hp(); _o->hp = _e; } + { auto _e = name(); if (_e) _o->name = _e->str(); } + { auto _e = inventory(); if (_e) { _o->inventory.resize(_e->size()); std::copy(_e->begin(), _e->end(), _o->inventory.begin()); } } + { auto _e = color(); _o->color = _e; } + { auto _e = weapons(); if (_e) { _o->weapons.resize(_e->size()); for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->weapons[_i] = flatbuffers::unique_ptr(_e->Get(_i)->UnPack(_resolver)); } } } + { auto _e = equipped_type(); _o->equipped.type = _e; } + { auto _e = equipped(); if (_e) _o->equipped.value = MyGame::Sample::EquipmentUnion::UnPack(_e, equipped_type(), _resolver); } + { auto _e = path(); if (_e) { _o->path.resize(_e->size()); for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->path[_i] = *_e->Get(_i); } } } +} + +inline flatbuffers::Offset Monster::Pack(flatbuffers::FlatBufferBuilder &_fbb, const MonsterT* _o, const flatbuffers::rehasher_function_t *_rehasher) { + return CreateMonster(_fbb, _o, _rehasher); +} + +inline flatbuffers::Offset CreateMonster(flatbuffers::FlatBufferBuilder &_fbb, const MonsterT *_o, const flatbuffers::rehasher_function_t *_rehasher) { + (void)_rehasher; + (void)_o; + struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const MonsterT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; + auto _pos = _o->pos ? _o->pos.get() : 0; + auto _mana = _o->mana; + auto _hp = _o->hp; + auto _name = _o->name.empty() ? 0 : _fbb.CreateString(_o->name); + auto _inventory = _o->inventory.size() ? _fbb.CreateVector(_o->inventory) : 0; + auto _color = _o->color; + auto _weapons = _o->weapons.size() ? _fbb.CreateVector> (_o->weapons.size(), [](size_t i, _VectorArgs *__va) { return CreateWeapon(*__va->__fbb, __va->__o->weapons[i].get(), __va->__rehasher); }, &_va ) : 0; + auto _equipped_type = _o->equipped.type; + auto _equipped = _o->equipped.Pack(_fbb); + auto _path = _o->path.size() ? _fbb.CreateVectorOfStructs(_o->path) : 0; + return MyGame::Sample::CreateMonster( + _fbb, + _pos, + _mana, + _hp, + _name, + _inventory, + _color, + _weapons, + _equipped_type, + _equipped, + _path); +} + +inline WeaponT *Weapon::UnPack(const flatbuffers::resolver_function_t *_resolver) const { + auto _o = std::unique_ptr(new WeaponT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); +} + +inline void Weapon::UnPackTo(WeaponT *_o, const flatbuffers::resolver_function_t *_resolver) const { + (void)_o; + (void)_resolver; + { auto _e = name(); if (_e) _o->name = _e->str(); } + { auto _e = damage(); _o->damage = _e; } +} + +inline flatbuffers::Offset Weapon::Pack(flatbuffers::FlatBufferBuilder &_fbb, const WeaponT* _o, const flatbuffers::rehasher_function_t *_rehasher) { + return CreateWeapon(_fbb, _o, _rehasher); +} + +inline flatbuffers::Offset CreateWeapon(flatbuffers::FlatBufferBuilder &_fbb, const WeaponT *_o, const flatbuffers::rehasher_function_t *_rehasher) { + (void)_rehasher; + (void)_o; + struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const WeaponT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; + auto _name = _o->name.empty() ? 0 : _fbb.CreateString(_o->name); + auto _damage = _o->damage; + return MyGame::Sample::CreateWeapon( + _fbb, + _name, + _damage); +} + +inline bool VerifyEquipment(flatbuffers::Verifier &verifier, const void *obj, Equipment type) { + switch (type) { + case Equipment_NONE: { + return true; + } + case Equipment_Weapon: { + auto ptr = reinterpret_cast(obj); + return verifier.VerifyTable(ptr); + } + default: return true; + } +} + +inline bool VerifyEquipmentVector(flatbuffers::Verifier &verifier, const flatbuffers::Vector> *values, const flatbuffers::Vector *types) { + if (!values || !types) return !values && !types; + if (values->size() != types->size()) return false; + for (flatbuffers::uoffset_t i = 0; i < values->size(); ++i) { + if (!VerifyEquipment( + verifier, values->Get(i), types->GetEnum(i))) { + return false; + } + } + return true; +} + +inline void *EquipmentUnion::UnPack(const void *obj, Equipment type, const flatbuffers::resolver_function_t *resolver) { + switch (type) { + case Equipment_Weapon: { + auto ptr = reinterpret_cast(obj); + return ptr->UnPack(resolver); + } + default: return nullptr; + } +} + +inline flatbuffers::Offset EquipmentUnion::Pack(flatbuffers::FlatBufferBuilder &_fbb, const flatbuffers::rehasher_function_t *_rehasher) const { + switch (type) { + case Equipment_Weapon: { + auto ptr = reinterpret_cast(value); + return CreateWeapon(_fbb, ptr, _rehasher).Union(); + } + default: return 0; + } +} + +inline EquipmentUnion::EquipmentUnion(const EquipmentUnion &u) : type(u.type), value(nullptr) { + switch (type) { + case Equipment_Weapon: { + value = new MyGame::Sample::WeaponT(*reinterpret_cast(u.value)); + break; + } + default: + break; + } +} + +inline void EquipmentUnion::Reset() { + switch (type) { + case Equipment_Weapon: { + auto ptr = reinterpret_cast(value); + delete ptr; + break; + } + default: break; + } + value = nullptr; + type = Equipment_NONE; +} + +inline const flatbuffers::TypeTable *ColorTypeTable() { + static const flatbuffers::TypeCode type_codes[] = { + { flatbuffers::ET_CHAR, 0, 0 }, + { flatbuffers::ET_CHAR, 0, 0 }, + { flatbuffers::ET_CHAR, 0, 0 } + }; + static const flatbuffers::TypeFunction type_refs[] = { + MyGame::Sample::ColorTypeTable + }; + static const char * const names[] = { + "Red", + "Green", + "Blue" + }; + static const flatbuffers::TypeTable tt = { + flatbuffers::ST_ENUM, 3, type_codes, type_refs, nullptr, nullptr, names + }; + return &tt; +} + +inline const flatbuffers::TypeTable *EquipmentTypeTable() { + static const flatbuffers::TypeCode type_codes[] = { + { flatbuffers::ET_SEQUENCE, 0, -1 }, + { flatbuffers::ET_SEQUENCE, 0, 0 } + }; + static const flatbuffers::TypeFunction type_refs[] = { + MyGame::Sample::WeaponTypeTable + }; + static const char * const names[] = { + "NONE", + "Weapon" + }; + static const flatbuffers::TypeTable tt = { + flatbuffers::ST_UNION, 2, type_codes, type_refs, nullptr, nullptr, names + }; + return &tt; +} + +inline const flatbuffers::TypeTable *Vec3TypeTable() { + static const flatbuffers::TypeCode type_codes[] = { + { flatbuffers::ET_FLOAT, 0, -1 }, + { flatbuffers::ET_FLOAT, 0, -1 }, + { flatbuffers::ET_FLOAT, 0, -1 } + }; + static const int64_t values[] = { 0, 4, 8, 12 }; + static const char * const names[] = { + "x", + "y", + "z" + }; + static const flatbuffers::TypeTable tt = { + flatbuffers::ST_STRUCT, 3, type_codes, nullptr, nullptr, values, names + }; + return &tt; +} + +inline const flatbuffers::TypeTable *MonsterTypeTable() { + static const flatbuffers::TypeCode type_codes[] = { + { flatbuffers::ET_SEQUENCE, 0, 0 }, + { flatbuffers::ET_SHORT, 0, -1 }, + { flatbuffers::ET_SHORT, 0, -1 }, + { flatbuffers::ET_STRING, 0, -1 }, + { flatbuffers::ET_BOOL, 0, -1 }, + { flatbuffers::ET_UCHAR, 1, -1 }, + { flatbuffers::ET_CHAR, 0, 1 }, + { flatbuffers::ET_SEQUENCE, 1, 2 }, + { flatbuffers::ET_UTYPE, 0, 3 }, + { flatbuffers::ET_SEQUENCE, 0, 3 }, + { flatbuffers::ET_SEQUENCE, 1, 0 } + }; + static const flatbuffers::TypeFunction type_refs[] = { + MyGame::Sample::Vec3TypeTable, + MyGame::Sample::ColorTypeTable, + MyGame::Sample::WeaponTypeTable, + MyGame::Sample::EquipmentTypeTable + }; + static const char * const names[] = { + "pos", + "mana", + "hp", + "name", + "friendly", + "inventory", + "color", + "weapons", + "equipped_type", + "equipped", + "path" + }; + static const flatbuffers::TypeTable tt = { + flatbuffers::ST_TABLE, 11, type_codes, type_refs, nullptr, nullptr, names + }; + return &tt; +} + +inline const flatbuffers::TypeTable *WeaponTypeTable() { + static const flatbuffers::TypeCode type_codes[] = { + { flatbuffers::ET_STRING, 0, -1 }, + { flatbuffers::ET_SHORT, 0, -1 } + }; + static const char * const names[] = { + "name", + "damage" + }; + static const flatbuffers::TypeTable tt = { + flatbuffers::ST_TABLE, 2, type_codes, nullptr, nullptr, nullptr, names + }; + return &tt; +} + +inline const MyGame::Sample::Monster *GetMonster(const void *buf) { + return flatbuffers::GetRoot(buf); +} + +inline const MyGame::Sample::Monster *GetSizePrefixedMonster(const void *buf) { + return flatbuffers::GetSizePrefixedRoot(buf); +} + +inline Monster *GetMutableMonster(void *buf) { + return flatbuffers::GetMutableRoot(buf); +} + +inline bool VerifyMonsterBuffer( + flatbuffers::Verifier &verifier) { + return verifier.VerifyBuffer(nullptr); +} + +inline bool VerifySizePrefixedMonsterBuffer( + flatbuffers::Verifier &verifier) { + return verifier.VerifySizePrefixedBuffer(nullptr); +} + +inline void FinishMonsterBuffer( + flatbuffers::FlatBufferBuilder &fbb, + flatbuffers::Offset root) { + fbb.Finish(root); +} + +inline void FinishSizePrefixedMonsterBuffer( + flatbuffers::FlatBufferBuilder &fbb, + flatbuffers::Offset root) { + fbb.FinishSizePrefixed(root); +} + +inline flatbuffers::unique_ptr UnPackMonster( + const void *buf, + const flatbuffers::resolver_function_t *res = nullptr) { + return flatbuffers::unique_ptr(GetMonster(buf)->UnPack(res)); +} + +inline flatbuffers::unique_ptr UnPackSizePrefixedMonster( + const void *buf, + const flatbuffers::resolver_function_t *res = nullptr) { + return flatbuffers::unique_ptr(GetSizePrefixedMonster(buf)->UnPack(res)); +} + +} // namespace Sample +} // namespace MyGame + +#endif // FLATBUFFERS_GENERATED_MONSTER_MYGAME_SAMPLE_H_ diff --git a/Sming/Libraries/flatbuffers/src b/Sming/Libraries/flatbuffers/src new file mode 160000 index 0000000000..fc4fffea41 --- /dev/null +++ b/Sming/Libraries/flatbuffers/src @@ -0,0 +1 @@ +Subproject commit fc4fffea41f5b682198edfc72eca536d33fd848c diff --git a/Sming/Libraries/flatbuffers/src.patch b/Sming/Libraries/flatbuffers/src.patch new file mode 100644 index 0000000000..e88e5e4661 --- /dev/null +++ b/Sming/Libraries/flatbuffers/src.patch @@ -0,0 +1,17 @@ +diff --git a/include/flatbuffers/base.h b/include/flatbuffers/base.h +index 54a51aa..ffc2d7d 100644 +--- a/include/flatbuffers/base.h ++++ b/include/flatbuffers/base.h +@@ -32,11 +32,7 @@ + #include + #include + +-#if defined(ARDUINO) && !defined(ARDUINOSTL_M_H) +- #include +-#else +- #include +-#endif ++#include + + #include + #include diff --git a/Sming/Libraries/modbusino/samples/generic/component.mk b/Sming/Libraries/modbusino/samples/generic/component.mk index 4163e3ffed..068e9f51a6 100644 --- a/Sming/Libraries/modbusino/samples/generic/component.mk +++ b/Sming/Libraries/modbusino/samples/generic/component.mk @@ -1,5 +1,4 @@ ARDUINO_LIBRARIES := modbusino -DISABLE_SPIFFS = 1 CONFIG_VARS += MB_SLAVE_ADDR MB_SLAVE_ADDR ?= 1 diff --git a/Sming/Libraries/nanopb/README.rst b/Sming/Libraries/nanopb/README.rst new file mode 100644 index 0000000000..5a70236598 --- /dev/null +++ b/Sming/Libraries/nanopb/README.rst @@ -0,0 +1,56 @@ +Nano Protocol-Buffer +==================== + +Introduction +------------ + +This component adds support for `Nano Protocol-Buffer `_ implementation. + + Nanopb is a small code-size `Protocol Buffers `_ implementation in ansi C. It is especially suitable for use in microcontrollers, but fits any memory restricted system. + + +C file generation from Proto files +---------------------------------- + +Once this component is installed you can use it to add Nano Protocol-Buffer support to your project and generate C and header files from Proto files. +One possible way to call the generator is to go to the directory where the proto file is located and run the generator. As shown below:: + + make -C $SMING_HOME fetch nano-pb + cd + python $SMING_HOME/Libraries/nanopb/nanopb/generator/nanopb_generator.py .proto + +After the generator tool is run you will have newly generated C and header files that can be used in your Sming application. + +Using +----- + +1. Add ``COMPONENT_DEPENDS += nanopb`` to your application componenent.mk file. +2. Add these lines to your application:: + + #include + // The line below should be replaced with the generated header file + #include "cast_channel.pb.h" + +3. Example:: + + #include + #include "cast_channel.pb.h" + + void doSomething(const uint8_t* data, size_t length) + { + // ... + + extensions_api_cast_channel_CastMessage message = extensions_api_cast_channel_CastMessage_init_default; + + message.protocol_version = extensions_api_cast_channel_CastMessage_ProtocolVersion_CASTV2_1_0; + message.source_id.funcs.encode = &pbEncodeData; + message.source_id.arg = new PbData(sourceId); + message.destination_id.funcs.encode = &pbEncodeData; + message.destination_id.arg = new PbData(destinationId); + message.nameSpace.funcs.encode = &pbEncodeData; + message.nameSpace.arg = new PbData(ns); + message.payload_type = extensions_api_cast_channel_CastMessage_PayloadType_STRING; + message.payload_utf8.funcs.encode = &pbEncodeData; + message.payload_utf8.arg = new PbData((uint8_t*)data, length); + // ... + } \ No newline at end of file diff --git a/Sming/Libraries/nanopb/component.mk b/Sming/Libraries/nanopb/component.mk new file mode 100644 index 0000000000..28bb9259eb --- /dev/null +++ b/Sming/Libraries/nanopb/component.mk @@ -0,0 +1,4 @@ +COMPONENT_SRCDIRS := nanopb src +COMPONENT_INCDIRS := nanopb src/include + +COMPONENT_SUBMODULES += nanopb \ No newline at end of file diff --git a/Sming/Libraries/nanopb/nanopb b/Sming/Libraries/nanopb/nanopb new file mode 160000 index 0000000000..049485ff55 --- /dev/null +++ b/Sming/Libraries/nanopb/nanopb @@ -0,0 +1 @@ +Subproject commit 049485ff557178f646d573eca3bd647f543b760b diff --git a/Sming/Libraries/nanopb/requirements.txt b/Sming/Libraries/nanopb/requirements.txt new file mode 100644 index 0000000000..b0c79cc0ec --- /dev/null +++ b/Sming/Libraries/nanopb/requirements.txt @@ -0,0 +1 @@ +protobuf diff --git a/Sming/Libraries/nanopb/src/Callback.cpp b/Sming/Libraries/nanopb/src/Callback.cpp new file mode 100644 index 0000000000..76794000d0 --- /dev/null +++ b/Sming/Libraries/nanopb/src/Callback.cpp @@ -0,0 +1,42 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * Callback.cpp + * + ****/ + +#include "include/Protobuf/Callback.h" +#include +#include + +// See: https://iam777.tistory.com/538 + +namespace Protobuf +{ +bool OutputCallback::encode(pb_ostream_t* stream, const pb_field_t* field) +{ + if(!pb_encode_tag_for_field(stream, field)) { + return false; + } + + return pb_encode_string(stream, data, length); +} + +bool InputCallback::decode(pb_istream_t* stream, const pb_field_t* field) +{ + size_t available = stream->bytes_left; + auto new_buf = realloc(data, length + available); + if(new_buf == nullptr) { + return false; + } + data = static_cast(new_buf); + auto old_length = length; + length += available; + + return pb_read(stream, &data[old_length], available); +} + +} // namespace Protobuf diff --git a/Sming/Libraries/nanopb/src/Stream.cpp b/Sming/Libraries/nanopb/src/Stream.cpp new file mode 100644 index 0000000000..2b44e3b4c1 --- /dev/null +++ b/Sming/Libraries/nanopb/src/Stream.cpp @@ -0,0 +1,46 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * Stream.cpp + * + ****/ + +#include "include/Protobuf/Stream.h" + +// See: https://iam777.tistory.com/538 + +namespace Protobuf +{ +bool InputStream::decode(const pb_msgdesc_t* fields, void* dest_struct) +{ + int avail = stream.available(); + if(avail <= 0) { + return false; + } + pb_istream_t is{}; + is.callback = [](pb_istream_t* stream, pb_byte_t* buf, size_t count) -> bool { + auto self = static_cast(stream->state); + assert(self != nullptr); + size_t read = self->stream.readBytes(reinterpret_cast(buf), count); + return true; + }; + is.state = this; + is.bytes_left = size_t(avail); + is.errmsg = nullptr; + return pb_decode(&is, fields, dest_struct); +} + +size_t OutputStream::encode(const pb_msgdesc_t* fields, const void* src_struct) +{ + pb_ostream_t os{}; + os.state = const_cast(this); + os.callback = buf_write; + os.max_size = SIZE_MAX; + + return pb_encode(&os, fields, src_struct) ? os.bytes_written : 0; +} + +} // namespace Protobuf diff --git a/Sming/Libraries/nanopb/src/include/Protobuf.h b/Sming/Libraries/nanopb/src/include/Protobuf.h new file mode 100644 index 0000000000..b8a1b57fcf --- /dev/null +++ b/Sming/Libraries/nanopb/src/include/Protobuf.h @@ -0,0 +1,23 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * Protobuf.h + * + ****/ + +#pragma once + +#include "Protobuf/Stream.h" +#include "Protobuf/Callback.h" + +namespace Protobuf +{ +inline size_t getEncodeSize(const pb_msgdesc_t* fields, const void* src_struct) +{ + return DummyOutputStream().encode(fields, src_struct); +} + +} // namespace Protobuf diff --git a/Sming/Libraries/nanopb/src/include/Protobuf/Callback.h b/Sming/Libraries/nanopb/src/include/Protobuf/Callback.h new file mode 100644 index 0000000000..dad64b7665 --- /dev/null +++ b/Sming/Libraries/nanopb/src/include/Protobuf/Callback.h @@ -0,0 +1,123 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * Callback.h + * + ****/ + +#pragma once + +#include +#include + +namespace Protobuf +{ +/** + * @brief Base class to manage handling of pb_callback_t instances + */ +class Callback +{ +public: + Callback(pb_callback_t& cb) : cb(cb) + { + cb.arg = this; + } + + virtual ~Callback() + { + free(data); + cb.arg = nullptr; + cb.funcs.encode = nullptr; + cb.funcs.decode = nullptr; + } + + bool operator==(const String& s) const + { + return s.equals(reinterpret_cast(data), length); + } + + bool operator!=(const String& s) const + { + return !operator==(s); + } + + uint8_t* getData() + { + return data; + } + + size_t getLength() + { + return length; + } + + explicit operator String() const + { + return String(reinterpret_cast(data), length); + } + +protected: + pb_callback_t& cb; + uint8_t* data{nullptr}; + size_t length{0}; +}; + +/** + * @brief Handles input data decoding + */ +class InputCallback : public Callback +{ +public: + /** + * @brief Construct a decoding instance + */ + InputCallback(pb_callback_t& cb) : Callback(cb) + { + cb.funcs.decode = static_decode; + } + +protected: + static bool static_decode(pb_istream_t* stream, const pb_field_t* field, void** arg) + { + auto self = static_cast(*arg); + return self ? self->decode(stream, field) : false; + } + bool decode(pb_istream_t* stream, const pb_field_t* field); +}; + +/** + * @brief Handles output data encoding + */ +class OutputCallback : public Callback +{ +public: + /** + * @brief Construct an encoding instance + * @param data + * @param len + */ + OutputCallback(pb_callback_t& cb, const void* data, size_t len) : Callback(cb) + { + cb.funcs.encode = static_encode; + this->data = new uint8_t[len]; + memcpy(this->data, data, len); + length = len; + } + + OutputCallback(pb_callback_t& cb, const String& value) : OutputCallback(cb, value.c_str(), value.length()) + { + } + +protected: + static bool static_encode(pb_ostream_t* stream, const pb_field_t* field, void* const* arg) + { + auto self = static_cast(*arg); + return self ? self->encode(stream, field) : false; + } + bool encode(pb_ostream_t* stream, const pb_field_t* field); +}; + +} // namespace Protobuf diff --git a/Sming/Libraries/nanopb/src/include/Protobuf/Stream.h b/Sming/Libraries/nanopb/src/include/Protobuf/Stream.h new file mode 100644 index 0000000000..6e7cfbbcab --- /dev/null +++ b/Sming/Libraries/nanopb/src/include/Protobuf/Stream.h @@ -0,0 +1,90 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * Stream.h + * + ****/ + +#pragma once + +#include +#include +#include +#include +#include + +namespace Protobuf +{ +class Stream +{ +public: + ~Stream() + { + } +}; + +class InputStream : public Stream +{ +public: + InputStream(IDataSourceStream& source) : stream(source) + { + } + + bool decode(const pb_msgdesc_t* fields, void* dest_struct); + + String getErrorString() const + { + return errmsg; + } + +private: + const char* errmsg{nullptr}; + IDataSourceStream& stream; +}; + +class OutputStream : public Stream +{ +public: + size_t encode(const pb_msgdesc_t* fields, const void* src_struct); + +protected: + static bool buf_write(pb_ostream_t* stream, const pb_byte_t* buf, size_t count) + { + auto self = static_cast(stream->state); + assert(self != nullptr); + return self->write(buf, count); + } + virtual bool write(const pb_byte_t* buf, size_t count) = 0; +}; + +class DummyOutputStream : public OutputStream +{ +protected: + bool write(const pb_byte_t* buf, size_t count) override + { + return true; + } +}; + +class TcpClientOutputStream : public OutputStream +{ +public: + TcpClientOutputStream(TcpClient& client) : client(client) + { + } + +protected: + bool write(const pb_byte_t* buf, size_t count) override + { + int err = client.send(reinterpret_cast(buf), count); + // debug_i("client.send(%u): %d", count, err); + return err > 0; + } + + TcpClient& client; +}; + +} // namespace Protobuf diff --git a/Sming/Makefile b/Sming/Makefile index ee0c49f580..a64374e5ca 100644 --- a/Sming/Makefile +++ b/Sming/Makefile @@ -22,7 +22,7 @@ all: ##@Cleaning .PHONY: dist-clean -dist-clean: submodules-clean samples-clean docs-clean ##Clean everything for all arch/build types +dist-clean: submodules-clean samples-clean docs-clean tests-clean ##Clean everything for all arch/build types -$(Q) rm -rf out .PHONY: clean @@ -103,8 +103,8 @@ docs: submodules ##Build the Sming documentation # For integration testing both samples and tests are moved outside of the repo. SMING_PROJECTS_DIR ?= $(abspath $(SMING_HOME)/..) -SAMPLES_DIR := $(call FixPath,$(SMING_PROJECTS_DIR)/samples) -TESTS_DIR := $(call FixPath,$(SMING_PROJECTS_DIR)/tests) +SMING_PROJECTS_DIR := $(call FixPath, $(SMING_PROJECTS_DIR)) +SAMPLES_DIR := $(SMING_PROJECTS_DIR)/samples SAMPLE_NAMES = $(shell ls -1 $(SAMPLES_DIR)) @@ -112,17 +112,44 @@ SAMPLE_NAMES = $(shell ls -1 $(SAMPLES_DIR)) samples: | $(SAMPLE_NAMES) ##Build all sample applications $(SAMPLE_NAMES): - $(Q) $(MAKE) --no-print-directory -C $(SAMPLES_DIR)/$@ + @printf "\n\n** Building ${@F} **\n\n" + $(Q) $(MAKE) --no-print-directory -C $(SAMPLES_DIR)/$@ PIP_ARGS=-q python-requirements all + +# Build component samples +.PHONY: component-samples +component-samples: submodules ##Build all samples contained in components + $(Q) $(MAKE) --no-print-directory build-component-samples + +# Marks sample as build complete for faster re-testing +BUILT_SUFFIX := /out/.built +COMPONENT_SAMPLE_TARGETS = $(addsuffix $(BUILT_SUFFIX),$(wildcard $(foreach c,$(COMPONENT_SEARCH_DIRS),$c/*/samples/*))) + +.PHONY: build-component-samples +build-component-samples: $(COMPONENT_SAMPLE_TARGETS) + +$(COMPONENT_SAMPLE_TARGETS): + @printf "\n\n** Building $(notdir $(@:$(BUILT_SUFFIX)=)) **\n\n" + $(Q) $(MAKE) --no-print-directory -C $(@:$(BUILT_SUFFIX)=) PIP_ARGS=-q python-requirements all + $(Q) touch $@ -TESTS_COMPLETED = $(addsuffix /.complete,$(call ListSubDirs,$(TESTS_DIR))) PHONY: tests -tests: $(TESTS_COMPLETED) ##Build and run all test applications +tests: submodules ##Build and run all tests + $(Q) $(MAKE) --no-print-directory build-and-run-tests + +COMPLETED_SUFFIX := /.complete +COMPONENT_TESTS = $(call dirx,$(wildcard $(foreach c,$(COMPONENT_SEARCH_DIRS),$c/*/test/component.mk))) +TESTS_COMPLETED = $(addsuffix $(COMPLETED_SUFFIX),$(wildcard $(SMING_PROJECTS_DIR)/tests/* $(COMPONENT_TESTS))) + +PHONY: build-and-run-tests +build-and-run-tests: $(TESTS_COMPLETED) ##Build and run all test applications $(TESTS_COMPLETED): + @printf "\n\n** Building $(notdir $(@D)) **\n\n" $(Q) $(MAKE) -C $(@D) execute $(Q) touch $@ + ##@Cleaning .PHONY: samples-clean @@ -130,19 +157,32 @@ samples-clean: ##Clean all sample applications (all arch/build types) @echo Cleaning all samples... -$(Q) cd $(SAMPLES_DIR) && rm -rf $(addsuffix /out,$(SAMPLE_NAMES)) +.PHONY: component-samples-clean +component-samples-clean: ##Clean all component samples + @echo Cleaning all component samples... + -$(Q) rm -rf $(dir $(COMPONENT_SAMPLE_TARGETS)) + CLEAN_TESTS := $(TESTS_COMPLETED:complete=clean) .PHONY: tests-clean -tests-clean: $(CLEAN_TESTS) ##Clean all test applications (all arch/build types) +tests-clean: $(wildcard $(CLEAN_TESTS)) ##Clean all test applications (all arch/build types) -$(Q) rm -f $(TESTS_COMPLETED) .PHONY: $(CLEAN_TESTS) $(CLEAN_TESTS): @echo Cleaning '$(@D)' - $(Q) $(MAKE) -C $(@D) clean + -$(Q) $(MAKE) -C $(@D) clean ##@Tools +CACHE_VARS += PIP_ARGS +PIP_ARGS ?= +.PHONY: python-requirements +python-requirements: ##Install Python requirements for framework via pip (use PIP_ARGS=... for additional options) + @echo Installing Python requirements... + $(Q) $(PYTHON) -m pip install $(PIP_ARGS) -r $(SMING_HOME)/../Tools/requirements.txt + + # Recursive wildcard search # $1 -> list of directories # $2 -> file extensions filters (using % as wildcard) diff --git a/Sming/Platform/AccessPoint.h b/Sming/Platform/AccessPoint.h index 9a0f475dda..9fb52210f8 100644 --- a/Sming/Platform/AccessPoint.h +++ b/Sming/Platform/AccessPoint.h @@ -81,6 +81,20 @@ class AccessPointClass */ String getMAC(char sep = '\0') const; + /** @brief Set Access Point MAC address + * @param addr The new MAC address + * @retval bool true on success + * + * Must be called from `init()` before activating Access Point. + * Espressif place certain limitations on MAC addresses: + * + * Bit 0 of the first byte of the MAC address can not be 1. For example: + * + * OK: "1a:XX:XX:XX:XX:XX" + * NOT OK: "15:XX:XX:XX:XX:XX" + */ + virtual bool setMacAddress(const MacAddress& addr) const = 0; + /** @brief Get WiFi AP network mask * @retval IpAddress WiFi AP network mask */ diff --git a/Sming/Platform/Station.cpp b/Sming/Platform/Station.cpp index c10f14f70b..350e0ad723 100644 --- a/Sming/Platform/Station.cpp +++ b/Sming/Platform/Station.cpp @@ -10,7 +10,18 @@ #include "Station.h" -/* StationClass */ +String toString(WpsStatus status) +{ + switch(status) { +#define XX(name) \ + case WpsStatus::name: \ + return F(#name); + WPS_STATUS_MAP(XX) +#undef XX + default: + return F("Unknown_") + unsigned(status); + } +} bool StationClass::isConnected() const { diff --git a/Sming/Platform/Station.h b/Sming/Platform/Station.h index 9a3138d94e..19e68d40da 100644 --- a/Sming/Platform/Station.h +++ b/Sming/Platform/Station.h @@ -63,14 +63,25 @@ struct SmartConfigEventInfo { MacAddress bssid; ///< AP BSSID }; +#define WPS_STATUS_MAP(XX) \ + XX(Success) \ + XX(Failed) \ + XX(Timeout) \ + XX(WEP) + /// WiFi WPS callback status -enum WpsStatus { - eWPS_Success = 0, - eWPS_Failed, - eWPS_Timeout, - eWPS_WEP, +enum class WpsStatus { +#define XX(name) name, + WPS_STATUS_MAP(XX) +#undef XX }; +#define XX(name) constexpr WpsStatus eWPS_##name{WpsStatus::name}; +WPS_STATUS_MAP(XX) +#undef XX + +String toString(WpsStatus status); + /** * @brief Scan complete handler function */ @@ -185,6 +196,20 @@ class StationClass */ String getMAC(char sep = '\0') const; + /** @brief Set WiFi station MAC address + * @param addr The new MAC address + * @retval bool true on success + * + * Must be called from `init()` before activating station. + * Espressif place certain limitations on MAC addresses: + * + * Bit 0 of the first byte of the MAC address can not be 1. For example: + * + * OK: "1a:XX:XX:XX:XX:XX" + * NOT OK: "15:XX:XX:XX:XX:XX" + */ + virtual bool setMacAddress(const MacAddress& addr) const = 0; + /** @brief Get WiFi station network mask * @retval IpAddress WiFi station network mask */ diff --git a/Sming/README.rst b/Sming/README.rst index f199106129..bc32255df2 100644 --- a/Sming/README.rst +++ b/Sming/README.rst @@ -18,7 +18,7 @@ Serial Communications This will recompile your application to use the revised baud rate. Note that this will change the default speed used for both flashing and serial comms. - See also :component-esp8266:`esptool` and :component:`terminal` for further details. + See also :component:`esptool` and :component:`terminal` for further details. The default rate for serial ports is 115200 baud. You can change it like this:: diff --git a/Sming/Services/CommandProcessing/CommandHandler.h b/Sming/Services/CommandProcessing/CommandHandler.h index eac66a7950..03c4419981 100644 --- a/Sming/Services/CommandProcessing/CommandHandler.h +++ b/Sming/Services/CommandProcessing/CommandHandler.h @@ -162,7 +162,11 @@ class CommandHandler VerboseMode verboseMode = VERBOSE; String currentPrompt; +#ifdef ARCH_HOST + char currentEOL = '\n'; +#else char currentEOL = '\r'; +#endif String currentWelcomeMessage; }; diff --git a/Sming/Wiring/WHashMap.h b/Sming/Wiring/WHashMap.h index 62366230b1..9eeb9bc6b4 100644 --- a/Sming/Wiring/WHashMap.h +++ b/Sming/Wiring/WHashMap.h @@ -30,6 +30,7 @@ #include #include +#include /** * @brief HashMap class template diff --git a/Sming/build.mk b/Sming/build.mk index 98f2889422..6ee11a70a9 100644 --- a/Sming/build.mk +++ b/Sming/build.mk @@ -27,6 +27,9 @@ UNAME := $(shell uname -s) ifneq ($(filter MINGW32_NT%,$(UNAME)),) UNAME := Windows TOOL_EXT := .exe + # May be required by some applications (e.g. openssl) + HOME ?= $(USERPROFILE) + export HOME else ifneq ($(filter CYGWIN%,$(UNAME)),) # Cygwin Detected UNAME := Linux @@ -280,20 +283,22 @@ endef # Display variable and list values, e.g. $(call PrintVariable,LIBS) # $1 -> Name of variable containing values +# $2 -> (optional) tag to use instead of variable name define PrintVariable - $(info $1) + $(info $(if $2,$2,$1):) $(foreach item,$($1),$(info - $(item))) endef define PrintVariableSorted - $(info $1) + $(info $(if $2,$2,$1):) $(foreach item,$(sort $($1)),$(info - $(value item))) endef # Display list of variable references with their values e.g. $(call PrintVariableRefs,DEBUG_VARS) # $1 -> Name of variable containing list of variable names +# $2 -> (optional) tag to use instead of variable name define PrintVariableRefs - $(info $1) + $(info $(if $2,$2,$1):) $(foreach item,$(sort $($1)),$(info - $(item) = $(value $(item))) ) endef diff --git a/Sming/building.rst b/Sming/building.rst index 4904002d34..d6557b053d 100644 --- a/Sming/building.rst +++ b/Sming/building.rst @@ -127,6 +127,13 @@ To switch to a different build architecture, for example: To inspect the current build configuration, type ``make list-config``. +Hardware configuration +~~~~~~~~~~~~~~~~~~~~~~ + +The appropriate hardware configuration should be selected in the +project's component.mk file. Use one of the standard configurations +or create your own. See :ref:`hardware_config`. + Configuration variables ~~~~~~~~~~~~~~~~~~~~~~~ @@ -140,7 +147,7 @@ sessions, and will override any values set in your project’s - Type ``make SPIFF_BIN=test-rom`` to build the project and (if enabled) create a SPIFFS image file called ``test-rom.bin`` -- Type ``make flash COM_PORT=COM4 SPI_MODE=dio SPI_SIZE=4M`` to flash +- Type ``make flash COM_PORT=COM4`` to flash the project and ``test-rom`` SPIFFS image using the provided flash memory settings - Next time you type ``make flash``, the same settings will be used, no @@ -149,7 +156,7 @@ sessions, and will override any values set in your project’s A separate cache is maintained for each build type (arch + release/debug). For example: -- Type ``make SMING_RELASE=1 list-config`` to switch to release build +- Type ``make SMING_RELEASE=1 list-config`` to switch to release build and display the active configuration Type ``make config-clean`` to clear all caches and revert to defaults. @@ -686,10 +693,8 @@ clean and easy to follow. Components can be rebuilt and cleaned individually. For example: -- ``make spiffs-build`` runs the Component ‘make’ for spiffs, which - contains the spiffs library and spiffy tool. -- ``make spiffs-clean`` removes all intermediate build files for the - Component +- ``make spiffs-build`` runs the Component ‘make’ for spiffs, which contains the SPIFFS library. +- ``make spiffs-clean`` removes all intermediate build files for the Component - ``make spiffs-rebuild`` cleans and then re-builds the Component By default, a regular ``make`` performs an incremental build on the diff --git a/Sming/component-wrapper.mk b/Sming/component-wrapper.mk index 81af7e389e..ae7d2c4cfd 100644 --- a/Sming/component-wrapper.mk +++ b/Sming/component-wrapper.mk @@ -15,6 +15,7 @@ all: $(error Internal makefile) include $(SMING_HOME)/build.mk +include $(PROJECT_DIR)/$(OUT_BASE)/hwconfig.mk # Makefile runs in the build directory COMPONENT_BUILD_DIR := $(CURDIR) diff --git a/Sming/component.mk b/Sming/component.mk index 62a9f18a22..4d3ba173fa 100644 --- a/Sming/component.mk +++ b/Sming/component.mk @@ -13,9 +13,11 @@ COMPONENT_INCDIRS := \ . COMPONENT_DEPENDS := \ + Storage \ sming-arch \ FlashString \ spiffs \ + IFS \ http-parser \ libb64 \ ws_parser \ @@ -147,8 +149,25 @@ GLOBAL_CFLAGS += -DSTRING_OBJECT_SIZE=$(STRING_OBJECT_SIZE) ##@Flashing .PHONY: flashinit -flashinit: $(ESPTOOL) $(FLASH_INIT_DATA) | $(FW_BASE) ##Erase your device's flash memory and reset system configuration area to defaults +flashinit: $(ESPTOOL) | $(FW_BASE) ##Erase your device's flash memory $(info Flash init data default and blank data) - $(info DISABLE_SPIFFS = $(DISABLE_SPIFFS)) $(Q) $(EraseFlash) - $(call WriteFlash,$(FLASH_INIT_CHUNKS)) + +.PHONY: flashboot +flashboot: $(FLASH_BOOT_LOADER) kill_term ##Write just the Bootloader + $(call WriteFlash,$(FLASH_BOOT_CHUNKS)) + +.PHONY: flashapp +flashapp: all kill_term ##Write just the application image(s) + $(Q) $(call CheckPartitionChunks,$(FLASH_APP_CHUNKS)) + $(call WriteFlash,$(FLASH_APP_CHUNKS)) + +.PHONY: flash +flash: all kill_term ##Write the boot loader and all defined partition images + $(Q) $(call CheckPartitionChunks,$(FLASH_PARTITION_CHUNKS)) + $(call WriteFlash,$(FLASH_BOOT_CHUNKS) $(FLASH_MAP_CHUNK) $(FLASH_PARTITION_CHUNKS)) +ifeq ($(ENABLE_GDB), 1) + $(GDB_CMDLINE) +else ifneq ($(SMING_ARCH),Host) + $(TERMINAL) +endif diff --git a/Sming/options.json b/Sming/options.json new file mode 100644 index 0000000000..a713d4bc65 --- /dev/null +++ b/Sming/options.json @@ -0,0 +1,26 @@ +{ + "4m": { + "description": "Set Flash size to 4MByte", + "devices": { + "spiFlash": { + "size": "4M" + } + } + }, + "spiffs": { + "description": "Add standard SPIFFS partition", + "partitions": { + "spiffs0": { + "address": "0x200000", + "size": "512K", + "type": "data", + "subtype": "spiffs", + "filename": "$(SPIFF_BIN_OUT)", + "build": { + "target": "spiffsgen", + "files": "$(SPIFF_FILES)" + } + } + } + } +} \ No newline at end of file diff --git a/Sming/project.mk b/Sming/project.mk index b7ee31eed9..7cf647de90 100644 --- a/Sming/project.mk +++ b/Sming/project.mk @@ -55,6 +55,7 @@ CACHE_VARS := # Use PROJECT_DIR to identify the project source directory, from where this makefile must be included DEBUG_VARS += PROJECT_DIR PROJECT_DIR := $(CURDIR) +export PROJECT_DIR ifeq ($(MAKELEVEL),0) $(info ) @@ -234,6 +235,9 @@ ifndef MAKE_CLEAN -include $(CONFIG_CACHE_FILE) endif +# Also export debug variables here for access by external tools (e.g. python) +CONFIG_DEBUG_FILE := $(OUT_BASE)/debug.mk + # Append standard search directories to any defined by the application ALL_SEARCH_DIRS := $(call FixPath,$(abspath $(COMPONENT_SEARCH_DIRS))) COMPONENTS_EXTRA_INCDIR += $(ALL_SEARCH_DIRS) @@ -318,6 +322,8 @@ CMP_App_VARS := $(CONFIG_VARS) CMP_App_ALL_VARS := $(CONFIG_VARS) $(foreach c,$(COMPONENTS),$(eval $(call ParseComponentLibs,$c))) +$(PartitionCreateTargets) + export COMPONENTS_EXTRA_INCDIR export APPCODE export APP_CFLAGS @@ -336,7 +342,7 @@ COMPONENT_DIRS := $(foreach d,$(ALL_SEARCH_DIRS),$(wildcard $d/*)) %/component.mk: @if [ -f $(@D)/../.patches/$(notdir $(@D))/component.mk ]; then \ echo Patching $(abspath $(@D)/../.patches/$(notdir $(@D))/component.mk); \ - cp -u $(@D)/../.patches/$(notdir $(@D))/component.mk $@; \ + cp $(@D)/../.patches/$(notdir $(@D))/component.mk $@; \ fi SUBMODULES_FOUND = $(wildcard $(SUBMODULES:=/.submodule)) @@ -483,7 +489,7 @@ cs-dev: ##Apply coding style to all files changed from current upstream develop $(SMING_MAKE) $@ .PHONY: gdb -gdb: kill_term ##Run the debugger console +gdb: ##Run the debugger console $(GDB_CMDLINE) .PHONY: fetch @@ -512,6 +518,28 @@ else $(Q) $(PYTHON) -m pip install $(PIP_ARGS) $(foreach reqfile,$(PYTHON_REQUIREMENTS),-r $(reqfile)) endif +##@IDE Tools + +.PHONY: ide-vscode +ide-vscode: checkdirs submodules ##Create/update vscode project configuration + $(Q) $(MAKE) --no-print-directory ide-vscode-update + +export CLI_TARGET_OPTIONS +export HOST_PARAMETERS + +.PHONY: ide-vscode-update +ide-vscode-update: + $(Q) CXX=$(CXX) \ + SMING_HOME=$(SMING_HOME) \ + ESP_HOME=$(ESP_HOME) \ + IDF_PATH=$(IDF_PATH) \ + IDF_TOOLS_PATH=$(IDF_TOOLS_PATH) \ + SMING_ARCH=$(SMING_ARCH) \ + GDB=$(GDB) \ + COM_SPEED_GDB=$(COM_SPEED_GDB) \ + WSL_ROOT=$(WSL_ROOT) \ + $(PYTHON) $(SMING_HOME)/../Tools/vscode/setup.py + ##@Testing # OTA Server @@ -520,7 +548,19 @@ SERVER_OTA_PORT ?= 9999 .PHONY: otaserver otaserver: all ##Launch a simple python HTTP server for testing OTA updates $(info Starting OTA server for TESTING) - $(Q) cd $(FW_BASE) && $(PYTHON) -m SimpleHTTPServer $(SERVER_OTA_PORT) + $(Q) cd $(FW_BASE) && $(PYTHON) -m http.server $(SERVER_OTA_PORT) + + +# +.PHONY: tcp-serial-redirect +tcp-serial-redirect: ##Redirect COM port to TCP port + $(info Starting serial redirector) +ifdef WSL_ROOT + $(Q) cmd.exe /c start /MIN python3 $(WSL_ROOT)/$(SMING_HOME)/../Tools/tcp_serial_redirect.py $(COM_PORT) $(COM_SPEED_SERIAL) +else + $(Q) gnome-terminal -- bash -c "$(PYTHON) $(SMING_HOME)/../Tools/tcp_serial_redirect.py $(COM_PORT) $(COM_SPEED_SERIAL)" +endif + ##@Help @@ -628,4 +668,5 @@ endef ifndef MAKE_CLEAN CACHED_VAR_NAMES := $(sort $(CACHED_VAR_NAMES) $(CONFIG_VARS) $(RELINK_VARS) $(CACHE_VARS)) $(eval $(call WriteConfigCache,$(CONFIG_CACHE_FILE),CACHED_VAR_NAMES)) +$(eval $(call WriteCacheValues,$(CONFIG_DEBUG_FILE),$(sort $(DEBUG_VARS)))) endif diff --git a/Sming/spiffs.hw b/Sming/spiffs.hw new file mode 100644 index 0000000000..fff0598752 --- /dev/null +++ b/Sming/spiffs.hw @@ -0,0 +1,7 @@ +{ + "name": "Single SPIFFS partition", + "base_config": "standard-4m", + "options": [ + "spiffs" + ] +} \ No newline at end of file diff --git a/Sming/standard-4m.hw b/Sming/standard-4m.hw new file mode 100644 index 0000000000..1aa31486df --- /dev/null +++ b/Sming/standard-4m.hw @@ -0,0 +1,7 @@ +{ + "name": "Standard config with 4M flash", + "base_config": "standard", + "options": [ + "4m" + ] +} \ No newline at end of file diff --git a/Sming/Arch/Esp8266/Tools/docker/README.md b/Tools/Docker/README.md similarity index 100% rename from Sming/Arch/Esp8266/Tools/docker/README.md rename to Tools/Docker/README.md diff --git a/Tools/Docker/cli/Dockerfile b/Tools/Docker/cli/Dockerfile new file mode 100644 index 0000000000..4a49c5cd20 --- /dev/null +++ b/Tools/Docker/cli/Dockerfile @@ -0,0 +1,43 @@ +FROM ubuntu + +# ------------------------------------------------------------------------------ +# Set Environment +# ------------------------------------------------------------------------------ + +# Common +ENV SMING_HOME /opt/Sming/Sming +ENV PYTHON /usr/bin/python3 + +# Esp8266 +#ENV UDK_ROOT /opt/esp-open-sdk +ENV EQT_ROOT /opt/esp-quick-toolchain +ENV ESP_HOME $EQT_ROOT + +# Esp32 +ENV IDF_PATH /opt/esp-idf +ENV IDF_TOOLS_PATH /opt/esp32 +ENV ESP32_PYTHON_PATH $PYTHON + +# ------------------------------------------------------------------------------ +# Pre-requisites +# ------------------------------------------------------------------------------ + +RUN apt-get -y update \ + && DEBIAN_FRONTEND=noninteractive \ + TZ=Europe/London \ + apt-get install -y \ + git \ + sudo \ + tzdata + +# ------------------------------------------------------------------------------ +# Fetch Sming and install tools +# ------------------------------------------------------------------------------ + +ARG SMING_BRANCH=develop +ARG SMING_REPO=SmingHub/Sming +ARG INSTALL_ARGS=all + +RUN git clone -b $SMING_BRANCH -- https://github.com/$SMING_REPO $(readlink -m $SMING_HOME/..) + +RUN $SMING_HOME/../Tools/install.sh $INSTALL_ARGS diff --git a/Tools/Docker/cli/docker-compose.yml b/Tools/Docker/cli/docker-compose.yml new file mode 100644 index 0000000000..15c63a5d56 --- /dev/null +++ b/Tools/Docker/cli/docker-compose.yml @@ -0,0 +1,16 @@ +sming-cli: + + build: . + +# Uncomment the lines below if you want to use the source code in your host computer +# volumes: +# - ../../Sming/:/workspace/Sming/ + + ports: + - "10080:80" + +# Modify the line below if you serial adapter is mapped to a different port +# devices: +# - "/dev/ttyUSB0:/dev/ttyUSB0" + + privileged: true diff --git a/Tools/Docker/ide/Dockerfile b/Tools/Docker/ide/Dockerfile new file mode 100644 index 0000000000..bc046b46a3 --- /dev/null +++ b/Tools/Docker/ide/Dockerfile @@ -0,0 +1,42 @@ +# ------------------------------------ +# Fast dockerized development environment +# for the Sming Framework: https://github.com/SmingHub/Sming.git +# ------------------------------------ +FROM docker_sming-cli +MAINTAINER Slavey Karadzhov "slav@attachix.com" + +# ------------------------------------------------------------------------------ +# Install Cloud9 and Supervisor +# ------------------------------------------------------------------------------ + +RUN apt-get update -y \ + && apt-get install -y \ + apache2-utils \ + libxml2-dev \ + locales-all \ + npm \ + nodejs \ + sshfs \ + supervisor \ + tmux + +RUN git clone https://github.com/c9/core.git /cloud9 \ + && curl -s -L https://raw.githubusercontent.com/c9/install/master/install.sh | bash \ + && /cloud9/scripts/install-sdk.sh \ + && sed -i -e 's_127.0.0.1_0.0.0.0_g' /cloud9/configs/standalone.js \ + && mkdir -p /var/log/supervisor + +ADD supervisord.conf /etc/ + +# VOLUME /workspace + +EXPOSE 80 + +SHELL ["/bin/bash", "-c"] + +CMD ["/usr/bin/supervisord", "--nodaemon", "--configuration", "/etc/supervisord.conf"] + +COPY assets/welcome.html /cloud9/plugins/c9.ide.welcome/welcome.html +COPY assets/welcome.js /cloud9/plugins/c9.ide.welcome/welcome.js + +ENTRYPOINT /usr/bin/supervisord diff --git a/Tools/Docker/ide/LICENSE b/Tools/Docker/ide/LICENSE new file mode 100644 index 0000000000..725e164f24 --- /dev/null +++ b/Tools/Docker/ide/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2017 Slavey Karadzhov +Copyright (c) 2014 Kevin Delfour + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Sming/Arch/Esp8266/Tools/docker/assets/welcome.html b/Tools/Docker/ide/assets/welcome.html similarity index 64% rename from Sming/Arch/Esp8266/Tools/docker/assets/welcome.html rename to Tools/Docker/ide/assets/welcome.html index eae672a9f8..1dbce867cf 100644 --- a/Sming/Arch/Esp8266/Tools/docker/assets/welcome.html +++ b/Tools/Docker/ide/assets/welcome.html @@ -6,7 +6,7 @@