Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Continue boot on bad PSRAM #1558

Merged
merged 8 commits into from
Dec 12, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions code/components/jomjol_helper/Helper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ static const char* TAG = "HELPER";

using namespace std;

unsigned int systemStatus = 0;

sdmmc_cid_t SDCardCid;
sdmmc_csd_t SDCardCsd;

Expand Down Expand Up @@ -770,6 +772,38 @@ string getMac(void) {
return macFormated;
}


void setSystemStatusFlag(SystemStatusFlag_t flag) {
systemStatus = systemStatus | flag; // set bit

char buf[20];
snprintf(buf, sizeof(buf), "0x%08X", getSystemStatus());
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "New System Status: " + std::string(buf));
}

void clearSystemStatusFlag(SystemStatusFlag_t flag) {
systemStatus = systemStatus | ~flag; // clear bit

char buf[20];
snprintf(buf, sizeof(buf), "0x%08X", getSystemStatus());
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "New System Status: " + std::string(buf));
}

int getSystemStatus(void) {
return systemStatus;
}

bool isSetSystemStatusFlag(SystemStatusFlag_t flag) {
//ESP_LOGE(TAG, "Flag (0x%08X) is set (0x%08X): %d", flag, systemStatus , ((systemStatus & flag) == flag));

if ((systemStatus & flag) == flag) {
return true;
}
else {
return false;
}
}

string getResetReason(void) {
std::string reasonText;

Expand Down
21 changes: 21 additions & 0 deletions code/components/jomjol_helper/Helper.h
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,27 @@ string getSDCardCapacity();
string getSDCardSectorSize();

string getMac(void);


/* Error bit fields
One bit per error
Make sure it matches https://github.com/jomjol/AI-on-the-edge-device/wiki/Error-Codes */
enum SystemStatusFlag_t { // One bit per error
// First Byte
SYSTEM_STATUS_PSRAM_BAD = 1 << 0, // 4, Critical Error
SYSTEM_STATUS_HEAP_TOO_SMALL = 1 << 1, // 8, Critical Error
SYSTEM_STATUS_CAM_BAD = 1 << 2, // 1, Critical Error

// Second Byte
SYSTEM_STATUS_CAM_FB_BAD = 1 << (0+8), // 2, Flow still might work
SYSTEM_STATUS_NTP_BAD = 1 << (1+8), // 16, Flow will work but time will be wrong
};

void setSystemStatusFlag(SystemStatusFlag_t flag);
void clearSystemStatusFlag(SystemStatusFlag_t flag);
int getSystemStatus(void);
bool isSetSystemStatusFlag(SystemStatusFlag_t flag);

string getResetReason(void);

const char* get404(void);
5 changes: 3 additions & 2 deletions code/main/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -65,5 +65,6 @@ FILE(GLOB_RECURSE app_sources ${CMAKE_SOURCE_DIR}/main/*.*)
# idf_component_register(SRCS ${app_sources})

idf_component_register(SRCS ${app_sources}
INCLUDE_DIRS ".")

INCLUDE_DIRS "."
# REQUIRES esp_psram) # comming in IDF 5.0
)
75 changes: 51 additions & 24 deletions code/main/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@

#include "driver/gpio.h"
#include "sdkconfig.h"
//#include "esp_psram.h" // Comming in IDF 5.0, see https://docs.espressif.com/projects/esp-idf/en/v5.0-beta1/esp32/migration-guides/release-5.x/system.html?highlight=esp_psram_get_size
#include "spiram.h"
#include "esp_spiram.h"

// SD-Card ////////////////////
#include "nvs_flash.h"
Expand Down Expand Up @@ -120,7 +123,7 @@ bool Init_NVS_SDCard()
return true;
}

void task_NoSDBlink(void *pvParameter)
void task_MainInitError_blink(void *pvParameter)
{
gpio_pad_select_gpio(BLINK_GPIO);
gpio_set_direction(BLINK_GPIO, GPIO_MODE_OUTPUT);
Expand All @@ -145,7 +148,6 @@ void task_NoSDBlink(void *pvParameter)
extern "C" void app_main(void)
{
TickType_t xDelay;
bool initSucessful = true;

ESP_LOGI(TAG, "\n\n\n\n\n"); // Add mark on log to see when it restarted

Expand All @@ -158,9 +160,9 @@ extern "C" void app_main(void)

if (!Init_NVS_SDCard())
{
xTaskCreate(&task_NoSDBlink, "task_NoSDBlink", configMINIMAL_STACK_SIZE * 64, NULL, tskIDLE_PRIORITY+1, NULL);
return;
};
xTaskCreate(&task_MainInitError_blink, "task_MainInitError_blink", configMINIMAL_STACK_SIZE * 64, NULL, tskIDLE_PRIORITY+1, NULL);
return; // No way to continue without SD-Card!
}

string versionFormated = "Branch: '" + std::string(GIT_BRANCH) + \
"', Revision: " + std::string(GIT_REV) +", Date/Time: " + std::string(BUILD_TIME) + \
Expand Down Expand Up @@ -214,6 +216,7 @@ extern "C" void app_main(void)

if (!setup_time()) {
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "NTP Initialization failed!");
setSystemStatusFlag(SYSTEM_STATUS_NTP_BAD);
}

setBootTime();
Expand All @@ -229,13 +232,32 @@ extern "C" void app_main(void)
std::string zw = gettimestring("%Y%m%d-%H%M%S");
ESP_LOGD(TAG, "time %s", zw.c_str());

/* Check if PSRAM can be initalized */
esp_err_t ret;
ret = esp_spiram_init();
if (ret == ESP_FAIL) { // Failed to init PSRAM, most likely not available or broken
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Failed to initialize PSRAM (" + std::to_string(ret) + ")!");
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Either your device misses the PSRAM chip or it is broken!");
setSystemStatusFlag(SYSTEM_STATUS_PSRAM_BAD);
}
else { // PSRAM init ok
/* Check if PSRAM provides at least 4 MB */
size_t psram_size = esp_spiram_get_size();
// size_t psram_size = esp_psram_get_size(); // comming in IDF 5.0
LogFile.WriteToFile(ESP_LOG_INFO, TAG, "The device has " + std::to_string(psram_size/1024/1024) + " MBytes of PSRAM");
if (psram_size < (4*1024*1024)) { // PSRAM is below 4 MBytes
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "At least 4 MBytes are required!");
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Does the device really have a 4 Mbytes PSRAM?");
setSystemStatusFlag(SYSTEM_STATUS_PSRAM_BAD);
}
}

/* Check available Heap memory */
size_t _hsize = getESPHeapSize();
if (_hsize < 4000000) // Check for a bit less than 4 MB (but clearly over 2 MB)
{
std::string _zws = "Not enough PSRAM available. Expected around 4 MBytes - available: " + std::to_string((float)_hsize/1024/1024) + " MBytes!";
_zws = _zws + "\nEither not initialized, too small (2 MByte only) or not present at all. Firmware cannot start!!";
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, _zws);
} else { // Bad Camera Status, retry init
if (_hsize < 4000000) { // Check available Heap memory for a bit less than 4 MB (a test on a good device showed 4187558 bytes to be available)
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Not enough Heap memory available. Expected around 4 MBytes, but only " + std::to_string(_hsize) + " Bytes are available! That is not enough for this firmware!");
setSystemStatusFlag(SYSTEM_STATUS_HEAP_TOO_SMALL);
} else { // Heap memory is ok
if (camStatus != ESP_OK) {
LogFile.WriteToFile(ESP_LOG_WARN, TAG, "Failed to initialize camera module, retrying...");

Expand All @@ -249,16 +271,16 @@ extern "C" void app_main(void)
if (camStatus != ESP_OK) {
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Failed to initialize camera module!");
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Check that your camera module is working and connected properly!");
initSucessful = false;
setSystemStatusFlag(SYSTEM_STATUS_CAM_BAD);
}
} else { // Test Camera
camera_fb_t * fb = esp_camera_fb_get();
if (!fb) {
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Camera Framebuffer cannot be initialized!");
/* Easiest would be to simply restart here and try again,
how ever there seem to be systems where it fails at startup but still work corectly later.
Therefore we treat it still as successed!
//initSucessful = false; */
Therefore we treat it still as successed! */
setSystemStatusFlag(SYSTEM_STATUS_CAM_FB_BAD);
}
else {
esp_camera_fb_return(fb);
Expand All @@ -267,8 +289,6 @@ extern "C" void app_main(void)
}
}



xDelay = 2000 / portTICK_PERIOD_MS;
ESP_LOGD(TAG, "main: sleep for: %ldms", (long) xDelay*10);
vTaskDelay( xDelay );
Expand All @@ -287,17 +307,24 @@ extern "C" void app_main(void)
ESP_LOGD(TAG, "vor reg server main");
register_server_main_uri(server, "/sdcard");

if (initSucessful) {

/* Testing */
//setSystemStatusFlag(SYSTEM_STATUS_CAM_FB_BAD);
//setSystemStatusFlag(SYSTEM_STATUS_PSRAM_BAD);

/* Main Init has successed or only an error which allows to continue operation */
if (getSystemStatus() == 0) { // No error flag is set
LogFile.WriteToFile(ESP_LOG_INFO, TAG, "Initialization completed successfully!");
ESP_LOGD(TAG, "vor do autostart");
TFliteDoAutoStart();
}
else { // Initialization failed
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Initialization failed. Will restart in 5 minutes!");
vTaskDelay(60*4000 / portTICK_RATE_MS); // Wait 4 minutes to give time to do an OTA or fetch the log
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Initialization failed. Will restart in 1 minute!");
vTaskDelay(60*1000 / portTICK_RATE_MS); // Wait 1 minute to give time to do an OTA or fetch the log
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Initialization failed. Will restart now!");
doReboot();
else if (isSetSystemStatusFlag(SYSTEM_STATUS_CAM_FB_BAD) || // Non critical errors occured, we try to continue...
isSetSystemStatusFlag(SYSTEM_STATUS_NTP_BAD)) {
LogFile.WriteToFile(ESP_LOG_WARN, TAG, "Initialization completed with errors, but trying to continue...");
ESP_LOGD(TAG, "vor do autostart");
TFliteDoAutoStart();
}
else { // Any other error is critical and makes running the flow impossible.
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Initialization failed. Not starting flows!");
}
}
26 changes: 23 additions & 3 deletions code/main/server_main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -230,9 +230,29 @@ esp_err_t hello_main_handler(httpd_req_t *req)
}
}

if (filetosend == "/sdcard/html/index.html" && isSetupModusActive()) {
ESP_LOGD(TAG, "System is in setup mode --> index.html --> setup.html");
filetosend = "/sdcard/html/setup.html";
if (filetosend == "/sdcard/html/index.html") {
if (isSetSystemStatusFlag(SYSTEM_STATUS_PSRAM_BAD) || // Initialization failed with crritical errors!
isSetSystemStatusFlag(SYSTEM_STATUS_CAM_BAD)) {
LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "We have a critical error, not serving main page!");

char buf[20];
std::string message = "<h1>AI on the Edge Device</h1><b>We have one or more critical errors:</b><br>";

for (int i = 0; i < 32; i++) {
if (isSetSystemStatusFlag((SystemStatusFlag_t)(1<<i))) {
snprintf(buf, sizeof(buf), "0x%08X", 1<<i);
message += std::string(buf) + "<br>";
}
}

message += "<br>Please check <a href=\"https://github.com/jomjol/AI-on-the-edge-device/wiki/Error-Codes\" target=_blank>github.com/jomjol/AI-on-the-edge-device/wiki/Error-Codes</a> for more information!";
httpd_resp_send_err(req, HTTPD_500_INTERNAL_SERVER_ERROR, message.c_str());
return ESP_FAIL;
}
else if (isSetupModusActive()) {
ESP_LOGD(TAG, "System is in setup mode --> index.html --> setup.html");
filetosend = "/sdcard/html/setup.html";
}
}

ESP_LOGD(TAG, "Filename: %s", filename);
Expand Down
1 change: 1 addition & 0 deletions code/sdkconfig.defaults
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ CONFIG_SPIRAM_MEMTEST=y
CONFIG_SPIRAM_MALLOC_ALWAYSINTERNAL=16384
CONFIG_SPIRAM_MALLOC_RESERVE_INTERNAL=40960
CONFIG_SPIRAM_CACHE_WORKAROUND=y
CONFIG_SPIRAM_IGNORE_NOTFOUND=y

CONFIG_ESP_INT_WDT_TIMEOUT_MS=300

Expand Down