Sketch emulation on host (#5342)

* WIP compile examples on host with 'make examples'

* WIP bufferize tcp input

* WIP Makefile

* WIP network to rework, tcp/udp to factorize, udp addresses broken

* minor changes to the core

* WIP basic udp working

* WIP mdns

* WIP mcast receiving, not sending

* WIP mdns OK

* beta version

* SSL + doc

* update travis host test command

* licenses

* typo

* doc: arduino builder is not around: declare functions before calling them

* fix with latest SSL PR, compile in 32 bits mode

* fix make clean

* make -m32 optional

* 32bits compiler ability tester

* WIP

* WIP (fix 1 vtable error, still another one to hunt with using spiffs)

* example astyle

* fix os_printf_plus

* load / save mock spiffs

* fix style

* fix using spiffs/mock

* don't mess ram

* update doc

* remove leftover

* optimization -Os except for CI, rename ARCH32 to FORCE32

* revert useless cast (not even compiled)

* remove unused function

* use proper type for pointer arithmetics

* makefile: sketch object and cpp file moved to bin/ directories
easier to clean, and IDE don't like them

* changes for review

* make use of %zd

* less verbose makefile by default (option)

* update readme

cores/esp8266/Esp.cpp

@@ -22,7 +22,7 @@
 #include "flash_utils.h"
 #include "eboot_command.h"
 #include <memory>
-#include "interrupts.h"
+#include <interrupts.h>
 #include "MD5Builder.h"
 #include "umm_malloc/umm_malloc.h"
 #include "cont.h"
@@ -165,6 +165,7 @@ void EspClass::restart(void)
 uint16_t EspClass::getVcc(void)
 {
     InterruptLock lock;
+    (void)lock;
     return system_get_vdd33();
 }
 

cores/esp8266/FSImpl.h

@@ -62,6 +62,7 @@ class DirImpl {
 
 class FSImpl {
 public:
+    virtual ~FSImpl () { }
     virtual bool begin() = 0;
     virtual void end() = 0;
     virtual bool format() = 0;

cores/esp8266/Updater.cpp

@@ -1,8 +1,8 @@
 #include "Updater.h"
 #include "Arduino.h"
 #include "eboot_command.h"
-#include "interrupts.h"
-#include "esp8266_peri.h"
+#include <interrupts.h>
+#include <esp8266_peri.h>
 
 //#define DEBUG_UPDATER Serial
 
@@ -84,21 +84,21 @@ bool UpdaterClass::begin(size_t size, int command, int ledPin, uint8_t ledOn) {
 
   wifi_set_sleep_type(NONE_SLEEP_T);
 
-  uint32_t updateStartAddress = 0;
+  uintptr_t updateStartAddress = 0;
   if (command == U_FLASH) {
     //size of current sketch rounded to a sector
-    uint32_t currentSketchSize = (ESP.getSketchSize() + FLASH_SECTOR_SIZE - 1) & (~(FLASH_SECTOR_SIZE - 1));
+    size_t currentSketchSize = (ESP.getSketchSize() + FLASH_SECTOR_SIZE - 1) & (~(FLASH_SECTOR_SIZE - 1));
     //address of the end of the space available for sketch and update
-    uint32_t updateEndAddress = (uint32_t)&_SPIFFS_start - 0x40200000;
+    uintptr_t updateEndAddress = (uintptr_t)&_SPIFFS_start - 0x40200000;
     //size of the update rounded to a sector
-    uint32_t roundedSize = (size + FLASH_SECTOR_SIZE - 1) & (~(FLASH_SECTOR_SIZE - 1));
+    size_t roundedSize = (size + FLASH_SECTOR_SIZE - 1) & (~(FLASH_SECTOR_SIZE - 1));
     //address where we will start writing the update
     updateStartAddress = (updateEndAddress > roundedSize)? (updateEndAddress - roundedSize) : 0;
 
 #ifdef DEBUG_UPDATER
-        DEBUG_UPDATER.printf("[begin] roundedSize:       0x%08X (%d)\n", roundedSize, roundedSize);
-        DEBUG_UPDATER.printf("[begin] updateEndAddress:  0x%08X (%d)\n", updateEndAddress, updateEndAddress);
-        DEBUG_UPDATER.printf("[begin] currentSketchSize: 0x%08X (%d)\n", currentSketchSize, currentSketchSize);
+        DEBUG_UPDATER.printf("[begin] roundedSize:       0x%08zX (%zd)\n", roundedSize, roundedSize);
+        DEBUG_UPDATER.printf("[begin] updateEndAddress:  0x%08zX (%zd)\n", updateEndAddress, updateEndAddress);
+        DEBUG_UPDATER.printf("[begin] currentSketchSize: 0x%08zX (%zd)\n", currentSketchSize, currentSketchSize);
 #endif
 
     //make sure that the size of both sketches is less than the total space (updateEndAddress)
@@ -108,7 +108,7 @@ bool UpdaterClass::begin(size_t size, int command, int ledPin, uint8_t ledOn) {
     }
   }
   else if (command == U_SPIFFS) {
-     updateStartAddress = (uint32_t)&_SPIFFS_start - 0x40200000;
+     updateStartAddress = (uintptr_t)&_SPIFFS_start - 0x40200000;
   }
   else {
     // unknown command
@@ -133,7 +133,7 @@ bool UpdaterClass::begin(size_t size, int command, int ledPin, uint8_t ledOn) {
 #ifdef DEBUG_UPDATER
   DEBUG_UPDATER.printf("[begin] _startAddress:     0x%08X (%d)\n", _startAddress, _startAddress);
   DEBUG_UPDATER.printf("[begin] _currentAddress:   0x%08X (%d)\n", _currentAddress, _currentAddress);
-  DEBUG_UPDATER.printf("[begin] _size:             0x%08X (%d)\n", _size, _size);
+  DEBUG_UPDATER.printf("[begin] _size:             0x%08zX (%zd)\n", _size, _size);
 #endif
 
   _md5.begin();
@@ -159,7 +159,7 @@ bool UpdaterClass::end(bool evenIfRemaining){
 
   if(hasError() || (!isFinished() && !evenIfRemaining)){
 #ifdef DEBUG_UPDATER
-    DEBUG_UPDATER.printf("premature end: res:%u, pos:%u/%u\n", getError(), progress(), _size);
+    DEBUG_UPDATER.printf("premature end: res:%u, pos:%zu/%zu\n", getError(), progress(), _size);
 #endif
 
     _reset();
@@ -199,10 +199,10 @@ bool UpdaterClass::end(bool evenIfRemaining){
     eboot_command_write(&ebcmd);
 
 #ifdef DEBUG_UPDATER
-    DEBUG_UPDATER.printf("Staged: address:0x%08X, size:0x%08X\n", _startAddress, _size);
+    DEBUG_UPDATER.printf("Staged: address:0x%08X, size:0x%08zX\n", _startAddress, _size);
   }
   else if (_command == U_SPIFFS) {
-    DEBUG_UPDATER.printf("SPIFFS: address:0x%08X, size:0x%08X\n", _startAddress, _size);
+    DEBUG_UPDATER.printf("SPIFFS: address:0x%08X, size:0x%08zX\n", _startAddress, _size);
 #endif
   }
 

cores/esp8266/spiffs_api.cpp

@@ -109,6 +109,7 @@ bool isSpiffsFilenameValid(const char* name)
 }
 
 // these symbols should be defined in the linker script for each flash layout
+#ifndef CORE_MOCK
 #ifdef ARDUINO
 extern "C" uint32_t _SPIFFS_start;
 extern "C" uint32_t _SPIFFS_end;
@@ -131,6 +132,7 @@ FS SPIFFS = FS(FSImplPtr(new SPIFFSImpl(
                              SPIFFS_PHYS_PAGE,
                              SPIFFS_PHYS_BLOCK,
                              SPIFFS_MAX_OPEN_FILES)));
-#endif
+#endif // ARDUINO
+#endif // !CORE_MOCK
 
 #endif

cores/esp8266/spiffs_api.h

@@ -220,7 +220,7 @@ class SPIFFSImpl : public FSImpl
         size_t cacheBufSize = SPIFFS_buffer_bytes_for_cache(&_fs, _maxOpenFds);
 
         if (!_workBuf) {
-            DEBUGV("SPIFFSImpl: allocating %d+%d+%d=%d bytes\r\n",
+            DEBUGV("SPIFFSImpl: allocating %zd+%zd+%zd=%zd bytes\r\n",
                    workBufSize, fdsBufSize, cacheBufSize,
                    workBufSize + fdsBufSize + cacheBufSize);
             _workBuf.reset(new uint8_t[workBufSize]);

libraries/DNSServer/src/DNSServer.cpp

@@ -10,7 +10,7 @@
 
 DNSServer::DNSServer()
 {
-  _ttl = htonl(60);
+  _ttl = lwip_htonl(60);
   _errorReplyCode = DNSReplyCode::NonExistentDomain;
 }
 
@@ -35,7 +35,7 @@ void DNSServer::setErrorReplyCode(const DNSReplyCode &replyCode)
 
 void DNSServer::setTTL(const uint32_t &ttl)
 {
-  _ttl = htonl(ttl);
+  _ttl = lwip_htonl(ttl);
 }
 
 void DNSServer::stop()
@@ -81,7 +81,7 @@ void DNSServer::processNextRequest()
 
 bool DNSServer::requestIncludesOnlyOneQuestion(const DNSHeader* dnsHeader)
 {
-  return ntohs(dnsHeader->QDCount) == 1 &&
+  return lwip_ntohs(dnsHeader->QDCount) == 1 &&
          dnsHeader->ANCount == 0 &&
          dnsHeader->NSCount == 0 &&
          dnsHeader->ARCount == 0;

libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp

@@ -734,7 +734,7 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size)
         free(buff);
 
         if(size && (int) size != bytesWritten) {
-            DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] Stream payload bytesWritten %d and size %d mismatch!.\n", bytesWritten, size);
+            DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] Stream payload bytesWritten %d and size %zd mismatch!.\n", bytesWritten, size);
             DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] ERROR SEND PAYLOAD FAILED!");
             return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED);
         } else {

libraries/ESP8266WebServer/src/ESP8266WebServer.cpp

@@ -438,12 +438,9 @@ void ESP8266WebServer::sendContent(const String& content) {
   const char * footer = "\r\n";
   size_t len = content.length();
   if(_chunked) {
-    char * chunkSize = (char *)malloc(11);
-    if(chunkSize){
-      sprintf(chunkSize, "%x%s", len, footer);
-      _currentClientWrite(chunkSize, strlen(chunkSize));
-      free(chunkSize);
-    }
+    char chunkSize[11];
+    sprintf(chunkSize, "%zx\r\n", len);
+    _currentClientWrite(chunkSize, strlen(chunkSize));
   }
   _currentClientWrite(content.c_str(), len);
   if(_chunked){
@@ -461,12 +458,9 @@ void ESP8266WebServer::sendContent_P(PGM_P content) {
 void ESP8266WebServer::sendContent_P(PGM_P content, size_t size) {
   const char * footer = "\r\n";
   if(_chunked) {
-    char * chunkSize = (char *)malloc(11);
-    if(chunkSize){
-      sprintf(chunkSize, "%x%s", size, footer);
-      _currentClientWrite(chunkSize, strlen(chunkSize));
-      free(chunkSize);
-    }
+    char chunkSize[11];
+    sprintf(chunkSize, "%zx\r\n", size);
+    _currentClientWrite(chunkSize, strlen(chunkSize));
   }
   _currentClientWrite_P(content, size);
   if(_chunked){

libraries/ESP8266WiFi/examples/udp/udp.ino

@@ -0,0 +1,80 @@
+/*
+  UDPSendReceive.pde:
+  This sketch receives UDP message strings, prints them to the serial port
+  and sends an "acknowledge" string back to the sender
+
+  A Processing sketch is included at the end of file that can be used to send
+  and received messages for testing with a computer.
+
+  created 21 Aug 2010
+  by Michael Margolis
+
+  This code is in the public domain.
+
+  adapted from Ethernet library examples
+*/
+
+
+#include <ESP8266WiFi.h>
+#include <WiFiUdp.h>
+
+#define SSID "ssid"
+#define PSK  "psk"
+
+unsigned int localPort = 8888;      // local port to listen on
+
+// buffers for receiving and sending data
+char packetBuffer[UDP_TX_PACKET_MAX_SIZE]; //buffer to hold incoming packet,
+char  ReplyBuffer[] = "acknowledged\r\n";       // a string to send back
+
+WiFiUDP Udp;
+
+void setup() {
+  Serial.begin(115200);
+  WiFi.mode(WIFI_STA);
+  WiFi.begin(SSID, PSK);
+  while (WiFi.status() != WL_CONNECTED) {
+    Serial.print('.');
+    delay(500);
+  }
+  Serial.print("Connected! IP address: ");
+  Serial.println(WiFi.localIP());
+  Serial.printf("UDP server on port %d\n", localPort);
+  Udp.begin(localPort);
+}
+
+void loop() {
+  // if there's data available, read a packet
+  int packetSize = Udp.parsePacket();
+  if (packetSize) {
+    Serial.print("Received packet of size ");
+    Serial.println(packetSize);
+    Serial.print("From ");
+    IPAddress remote = Udp.remoteIP();
+    for (int i = 0; i < 4; i++) {
+      Serial.print(remote[i], DEC);
+      if (i < 3) {
+        Serial.print(".");
+      }
+    }
+    Serial.print(", port ");
+    Serial.println(Udp.remotePort());
+
+    // read the packet into packetBufffer
+    Udp.read(packetBuffer, UDP_TX_PACKET_MAX_SIZE);
+    Serial.println("Contents:");
+    Serial.println(packetBuffer);
+
+    // send a reply, to the IP address and port that sent us the packet we received
+    Udp.beginPacket(Udp.remoteIP(), Udp.remotePort());
+    Udp.write(ReplyBuffer);
+    Udp.endPacket();
+  }
+  delay(10);
+}
+
+/*
+  test (shell/netcat):
+  ---------------
+	  nc -u 192.168.esp.address 8888
+*/

libraries/ESP8266WiFi/src/BearSSLHelpers.cpp

@@ -826,6 +826,8 @@ bool X509List::append(const uint8_t *derCert, size_t derLen) {
   return true;
 }
 
+#if !CORE_MOCK
+
 // Second stack thunked helpers
 make_stack_thunk(br_ssl_engine_recvapp_ack);
 make_stack_thunk(br_ssl_engine_recvapp_buf);
@@ -836,4 +838,6 @@ make_stack_thunk(br_ssl_engine_sendapp_buf);
 make_stack_thunk(br_ssl_engine_sendrec_ack);
 make_stack_thunk(br_ssl_engine_sendrec_buf);
 
+#endif
+
 };

libraries/ESP8266WiFi/src/WiFiClient.cpp

@@ -38,7 +38,7 @@ extern "C"
 #include "lwip/tcp.h"
 #include "lwip/inet.h"
 #include "lwip/netif.h"
-#include "include/ClientContext.h"
+#include <include/ClientContext.h>
 #include "c_types.h"
 
 uint16_t WiFiClient::_localPort = 0;

libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.cpp

@@ -40,10 +40,12 @@ extern "C" {
 #include "lwip/tcp.h"
 #include "lwip/inet.h"
 #include "lwip/netif.h"
-#include "include/ClientContext.h"
+#include <include/ClientContext.h>
 #include "c_types.h"
 #include "coredecls.h"
 
+#if !CORE_MOCK
+
 // The BearSSL thunks in use for now
 #define br_ssl_engine_recvapp_ack thunk_br_ssl_engine_recvapp_ack
 #define br_ssl_engine_recvapp_buf thunk_br_ssl_engine_recvapp_buf
@@ -54,6 +56,8 @@ extern "C" {
 #define br_ssl_engine_sendrec_ack thunk_br_ssl_engine_sendrec_ack
 #define br_ssl_engine_sendrec_buf thunk_br_ssl_engine_sendrec_buf
 
+#endif
+
 namespace BearSSL {
 
 void WiFiClientSecure::_clear() {
@@ -1377,6 +1381,21 @@ bool WiFiClientSecure::loadPrivateKey(Stream& stream, size_t size) {
 // SSL debugging which should focus on the WiFiClientBearSSL objects.
 
 extern "C" {
+
+#if CORE_MOCK
+
+  void br_esp8266_stack_proxy_init(uint8_t *space, uint16_t size) {
+    (void)space;
+    (void)size;
+  }
+  void _BearSSLCheckStack(const char *fcn, const char *file, int line) {
+    (void)fcn;
+    (void)file;
+    (void)line;
+  }
+
+#else // !CORE_MOCK
+
   extern size_t br_esp8266_stack_proxy_usage();
 
   void _BearSSLCheckStack(const char *fcn, const char *file, int line) {
@@ -1386,7 +1405,7 @@ extern "C" {
     int freeheap = ESP.getFreeHeap();
     static int laststack, lastheap, laststack2;
     if ((laststack != freestack) || (lastheap != freeheap) || (laststack2 != (int)br_esp8266_stack_proxy_usage())) {
-      Serial.printf("%s:%s(%d): FREESTACK=%d, STACK2USAGE=%d, FREEHEAP=%d\n", file, fcn, line, freestack, br_esp8266_stack_proxy_usage(), freeheap);
+      Serial.printf("%s:%s(%d): FREESTACK=%d, STACK2USAGE=%zd, FREEHEAP=%d\n", file, fcn, line, freestack, br_esp8266_stack_proxy_usage(), freeheap);
       if (freestack < 256) {
         Serial.printf("!!! Out of main stack space\n");
       }
@@ -1405,6 +1424,8 @@ extern "C" {
     }
   }
 
+#endif // !CORE_MOCK
+
   void _BearSSLSerialPrint(const char *str) {
     static int cnt = 0;
     Serial.printf("%s", str);

libraries/ESP8266WiFi/src/WiFiServer.cpp

@@ -35,7 +35,7 @@ extern "C" {
 #include "lwip/opt.h"
 #include "lwip/tcp.h"
 #include "lwip/inet.h"
-#include "include/ClientContext.h"
+#include <include/ClientContext.h>
 
 WiFiServer::WiFiServer(IPAddress addr, uint16_t port)
 : _port(port)

libraries/ESP8266WiFi/src/WiFiServerSecureBearSSL.cpp

@@ -34,7 +34,7 @@ extern "C" {
 #include "lwip/opt.h"
 #include "lwip/tcp.h"
 #include "lwip/inet.h"
-#include "include/ClientContext.h"
+#include <include/ClientContext.h>
 #include "WiFiServerSecureBearSSL.h"
 
 namespace BearSSL {