Skip to content

Commit f0291e4

Browse files
trentgilltrentgill
authored
bytecode cross-compiled on dev machine (#481)
* using cli tools to convert to c header files * add string lengths for ii files too * bytecode cross-compilation & loading working * cleanup makefile and document lua bc cross-compilation process * fix dependencies in makefile * remove lua 5.2 compatibility. unused anyway * tracking lua globals for env clear * fix buffer name when loading scripts * support crow which doesn't have require * lua env no longer loads some useless modules
1 parent 862b013 commit f0291e4

File tree

10 files changed

+110
-117
lines changed

10 files changed

+110
-117
lines changed

‎Makefile

+23-9
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ LUAS=submodules/lua/src
1212
BOOTLOADER=submodules/dfu-stm32f7
1313
BUILD_DIR := build
1414
PRJ_DIR=crow
15+
LUAC_CROSS=util/luacc
1516

1617
CC=arm-none-eabi-gcc
1718
LD=arm-none-eabi-gcc
@@ -53,7 +54,8 @@ CFLAGS += $(MCFLAGS)
5354
CFLAGS += $(OPTIMIZE)
5455
CFLAGS += $(DEFS) -I. -I./ $(STM32_INCLUDES)
5556
CFLAGS += -fsingle-precision-constant -Wdouble-promotion
56-
CFLAGS += -DLUA_32BITS -DLUA_COMPAT_5_2
57+
CFLAGS += -DLUA_32BITS
58+
# CFLAGS += -DLUA_32BITS -DLUA_COMPAT_5_2
5759
CFLAGS += -fno-common
5860
CFLAGS += -DVERSION=\"$(GIT_VERSION)\"
5961
CFLAGS += -ffunction-sections -fdata-sections # provides majority of LTO binary size reduction
@@ -141,19 +143,19 @@ $(II_TARGET): util/ii_lua_module.lua
141143

142144
$(BUILD_DIR)/ii_%.lua: $(II_SRCD)/%.lua util/ii_lua_module.lua | $(BUILD_DIR)
143145
@lua util/ii_lua_module.lua $< $@
144-
@echo lua $@
146+
@echo "ii-lua-module $< -> $@"
145147

146148
$(BUILD_DIR)/iihelp.lua: $(II_SRC) util/ii_lua_help.lua | $(BUILD_DIR)
147149
@lua util/ii_lua_help.lua $(II_SRCD) $@
148-
@echo lua $@
150+
@echo "ii-lua-help $@"
149151

150152
$(BUILD_DIR)/ii_c_layer.h: $(II_SRC) util/ii_c_layer.lua | $(BUILD_DIR)
151153
@lua util/ii_c_layer.lua $(II_SRCD) $@
152-
@echo lua $@
154+
@echo "ii-c-layer $@"
153155

154156
$(BUILD_DIR)/ii_lualink.h: $(II_SRC) util/ii_lualinker.lua | $(BUILD_DIR)
155157
@lua util/ii_lualinker.lua $(II_SRCD) $@
156-
@echo lua $@
158+
@echo "ii-lualinker $@"
157159

158160

159161
### destination sources
@@ -180,6 +182,7 @@ LUA_SRC += $(II_TARGET)
180182

181183
LUA_PP = $(LUA_SRC:%.lua=%.lua.h)
182184
LUA_PP: $(LUA_SRC)
185+
@echo "pre-compiling lua sources to bytecode wrapped in c headers"
183186

184187
LUACORE_OBJS= lapi.o lcode.o lctype.o ldebug.o ldo.o ldump.o lfunc.o lgc.o llex.o \
185188
lmem.o lobject.o lopcodes.o lparser.o lstate.o lstring.o ltable.o \
@@ -194,6 +197,8 @@ OBJS = $(SRC:%.c=$(OBJDIR)/%.o)
194197
OBJS += $(addprefix $(LUAS)/,$(LUACORE_OBJS) $(LUALIB_OBJS) )
195198
OBJS += Startup.o
196199

200+
$(OBJS): $(LUA_PP)
201+
197202
# specific objects that require built dependencies (ii)
198203
$(OBJDIR)/lib/l_bootstrap.o: $(LUA_PP) $(BUILD_DIR)/ii_lualink.h
199204
# $(OBJDIR)/lib/lualink.o: $(LUA_PP) $(BUILD_DIR)/ii_lualink.h
@@ -301,10 +306,19 @@ zip: $(BIN) $(TARGET).dfu
301306
@echo f2l $< "->" $@
302307
@$(FENNEL) --compile $< > $@
303308

304-
%.lua.h: %.lua util/l2h.lua
305-
@luac -p $<
306-
@echo l2h $< "->" $@
307-
@lua util/l2h.lua $<
309+
# a bunch of gnarly make-functions to massage the intermediate stage filenames
310+
# everything goes into /build now, and we have to save output of LUAC_CROSS into
311+
# named files for (xxd -i) to build include files with valid names
312+
# could be avoided by a more complicated pass in (sed), but this was easier
313+
# 1. cross-compile all .lua files into .lc bytecode for stm32-arm-cortex-m7 format
314+
# 2. wrap the .lc binary files into .h headers with auto-generated names
315+
# 3. add const qualifiers to headers to satisfy C99 struct initializer requirement
316+
317+
%.lua.h: %.lua $(BUILD_DIR)
318+
@echo l2h $< "->" $(addprefix $(BUILD_DIR)/, $(notdir $(subst .lua.h,.h,$@)))
319+
@$(LUAC_CROSS) -s -o $(addprefix $(BUILD_DIR)/, $(notdir $(subst .lua,.lc,$<))) $<
320+
@xxd -i $(addprefix $(BUILD_DIR)/, $(notdir $(subst .lua,.lc,$<))) $(addprefix $(BUILD_DIR)/, $(notdir $(subst .lua.h,.h,$@)))
321+
@sed -i 's/unsigned int/const unsigned int/g' $(addprefix $(BUILD_DIR)/, $(notdir $(subst .lua.h,.h,$@)))
308322

309323
Startup.o: $(STARTUP)
310324
@$(CC) $(CFLAGS) -c $< -o $@

‎lib/l_bootstrap.c

+57-84
Original file line numberDiff line numberDiff line change
@@ -4,46 +4,46 @@
44

55
#include "l_crowlib.h"
66

7-
#include "lua/crowlib.lua.h"
8-
#include "lua/asl.lua.h"
9-
#include "lua/asllib.lua.h"
10-
#include "lua/clock.lua.h"
11-
#include "lua/metro.lua.h"
12-
#include "lua/public.lua.h"
13-
#include "lua/input.lua.h"
14-
#include "lua/output.lua.h"
15-
#include "lua/ii.lua.h"
16-
#include "build/iihelp.lua.h" // generated lua stub for loading i2c modules
17-
#include "lua/calibrate.lua.h"
18-
#include "lua/sequins.lua.h"
19-
#include "lua/quote.lua.h"
20-
#include "lua/timeline.lua.h"
21-
#include "lua/hotswap.lua.h"
7+
// Lua libs wrapped in C-headers
8+
#include "build/crowlib.h"
9+
#include "build/asl.h"
10+
#include "build/asllib.h"
11+
#include "build/clock.h"
12+
#include "build/metro.h"
13+
#include "build/public.h"
14+
#include "build/input.h"
15+
#include "build/output.h"
16+
#include "build/ii.h"
17+
#include "build/iihelp.h" // generated lua stub for loading i2c modules
18+
#include "build/calibrate.h"
19+
#include "build/sequins.h"
20+
#include "build/quote.h"
21+
#include "build/timeline.h"
22+
#include "build/hotswap.h"
2223

2324
#include "build/ii_lualink.h" // generated C header for linking to lua
2425

25-
static int _writer(lua_State *L, const void *p, size_t sz, void *ud);
26-
static int _load_chunk(lua_State* L, const char* code, int strip);
2726
static int _open_lib( lua_State *L, const struct lua_lib_locator* lib, const char* name );
27+
static void lua_full_gc(lua_State* L);
2828

2929
// mark the 3rd arg 'false' if you need to debug that library
3030
const struct lua_lib_locator Lua_libs[] =
31-
{ { "lua_crowlib" , lua_crowlib , true}
32-
, { "lua_asl" , lua_asl , true}
33-
, { "lua_asllib" , lua_asllib , true}
34-
, { "lua_clock" , lua_clock , true}
35-
, { "lua_metro" , lua_metro , true}
36-
, { "lua_input" , lua_input , true}
37-
, { "lua_output" , lua_output , true}
38-
, { "lua_public" , lua_public , true}
39-
, { "lua_ii" , lua_ii , true}
40-
, { "build_iihelp" , build_iihelp , true}
41-
, { "lua_calibrate" , lua_calibrate , true}
42-
, { "lua_sequins" , lua_sequins , true}
43-
, { "lua_quote" , lua_quote , true}
44-
, { "lua_timeline" , lua_timeline , true}
45-
, { "lua_hotswap" , lua_hotswap , true}
46-
, { NULL , NULL , true}
31+
{ { "lua_crowlib" , build_crowlib_lc , true, build_crowlib_lc_len}
32+
, { "lua_asl" , build_asl_lc , true, build_asl_lc_len}
33+
, { "lua_asllib" , build_asllib_lc , true, build_asllib_lc_len}
34+
, { "lua_clock" , build_clock_lc , true, build_clock_lc_len}
35+
, { "lua_metro" , build_metro_lc , true, build_metro_lc_len}
36+
, { "lua_input" , build_input_lc , true, build_input_lc_len}
37+
, { "lua_output" , build_output_lc , true, build_output_lc_len}
38+
, { "lua_public" , build_public_lc , true, build_public_lc_len}
39+
, { "lua_ii" , build_ii_lc , true, build_ii_lc_len}
40+
, { "build_iihelp" , build_iihelp_lc , true, build_iihelp_lc_len}
41+
, { "lua_calibrate" , build_calibrate_lc , true, build_calibrate_lc_len}
42+
, { "lua_sequins" , build_sequins_lc , true, build_sequins_lc_len}
43+
, { "lua_quote" , build_quote_lc , true, build_quote_lc_len}
44+
, { "lua_timeline" , build_timeline_lc , true, build_timeline_lc_len}
45+
, { "lua_hotswap" , build_hotswap_lc , true, build_hotswap_lc_len}
46+
, { NULL , NULL , true, 0}
4747
};
4848

4949

@@ -69,9 +69,18 @@ void l_bootstrap_init(lua_State* L){
6969
// crowlib C extensions
7070
l_crowlib_init(L);
7171

72+
// track all user-created globals
73+
luaL_dostring(L,
74+
"_user={}\n"
75+
"local function trace(t,k,v)\n"
76+
"_user[k]=true\n"
77+
"rawset(t,k,v)\n"
78+
"end\n"
79+
"setmetatable(_G,{ __newindex = trace })\n"
80+
);
81+
7282
// perform two full garbage collection cycles for full cleanup
73-
lua_gc(L, LUA_GCCOLLECT, 1);
74-
lua_gc(L, LUA_GCCOLLECT, 1);
83+
lua_full_gc(L);
7584
}
7685

7786

@@ -92,19 +101,22 @@ int l_bootstrap_dofile(lua_State* L)
92101
default:{ cname[p++] = l_name[i]; } break;
93102
}
94103
}
104+
// goto fail; // no match was found, so error out (silently?)
105+
95106
strcomplete:
96107
lua_pop( L, 1 );
97108
switch( _open_lib( L, Lua_libs, cname ) ){
98109
case -1: goto fail;
99-
case 1: return 1;
110+
case 1: lua_full_gc(L); return 1;
100111
default: break;
101112
}
102113
switch( _open_lib( L, Lua_ii_libs, cname ) ){
103114
case -1: goto fail;
104-
case 1: return 1;
115+
case 1: lua_full_gc(L); return 1;
105116
default: break;
106117
}
107118
printf("can't open library: %s\n", (char*)cname);
119+
108120
fail:
109121
lua_pushnil(L);
110122
return 1;
@@ -117,58 +129,14 @@ int l_bootstrap_dofile(lua_State* L)
117129

118130
/////////// private defns
119131

120-
// this is a somewhat arbitrary size. must be big enough for the biggest library. 8kB was too small
121-
#define SIZED_STRING_LEN 0x4000 // (16kB)
122-
struct sized_string{
123-
char data[SIZED_STRING_LEN];
124-
int len;
125-
};
126-
127-
#define UNUSED(x) (void)(sizeof(x))
128-
static int _writer(lua_State *L, const void *p, size_t sz, void *ud)
129-
{
130-
UNUSED(L);
131-
struct sized_string* chunkstr = ud; /// need explicit cast?
132-
if(chunkstr->len + sz >= SIZED_STRING_LEN){
133-
printf("chunkstr too small.\n");
134-
return 1;
135-
}
136-
memcpy(&chunkstr->data[chunkstr->len], p, sz);
137-
chunkstr->len += sz;
138-
return 0;
139-
}
140-
#undef UNUSED
141-
142-
static int _load_chunk(lua_State* L, const char* code, int strip)
143-
{
144-
int retval = 0;
145-
struct sized_string chunkstr = {.len = 0};
146-
{ // scope lua_State to destroy it asap
147-
lua_State* LL=luaL_newstate();
148-
if( !LL ){ printf("luaL_newstate failed\n"); return 1; }
149-
if( luaL_loadstring(LL, code) ){
150-
printf("loadstring error\n");
151-
retval = 1;
152-
goto close_LL;
153-
}
154-
if( lua_dump(LL, _writer, &chunkstr, strip) ){
155-
printf("dump error\n");
156-
retval = 1;
157-
goto close_LL;
158-
}
159-
close_LL:
160-
lua_close(LL);
161-
}
162-
luaL_loadbuffer(L, chunkstr.data, chunkstr.len, chunkstr.data); // load our compiled chunk
163-
return retval;
164-
}
165-
166132
static int _open_lib( lua_State *L, const struct lua_lib_locator* lib, const char* name )
167133
{
168134
uint8_t i = 0;
169135
while( lib[i].addr_of_luacode != NULL ){
170136
if( !strcmp( name, lib[i].name ) ){ // if the strings match
171-
if( _load_chunk(L, lib[i].addr_of_luacode, lib[i].stripped) ){
137+
if( luaL_loadbuffer(L, (const char*)lib[i].addr_of_luacode
138+
, lib[i].len
139+
, lib[i].name) ){
172140
printf("can't load library: %s\n", (char*)lib[i].name );
173141
printf( "%s\n", (char*)lua_tostring( L, -1 ) );
174142
lua_pop( L, 1 );
@@ -186,3 +154,8 @@ static int _open_lib( lua_State *L, const struct lua_lib_locator* lib, const cha
186154
}
187155
return 0; // not found
188156
}
157+
158+
static void lua_full_gc(lua_State* L){
159+
lua_gc(L, LUA_GCCOLLECT, 1);
160+
lua_gc(L, LUA_GCCOLLECT, 1);
161+
}

‎lib/lualink.c

+18-9
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,9 @@
3939
#define WATCHDOG_FREQ 0x100000 // ~1s how often we run the watchdog
4040
#define WATCHDOG_COUNT 2 // how many watchdogs before 'frozen'
4141

42+
4243
// Basic crow script
43-
#include "lua/First.lua.h"
44+
#include "build/First.h"
4445

4546
// Private prototypes
4647
static void Lua_linkctolua( lua_State* L );
@@ -80,10 +81,17 @@ lua_State* Lua_Init(void)
8081
luaL_openlibs(L);
8182
Lua_linkctolua(L);
8283
l_bootstrap_init(L); // redefine dofile(), print(), load crowlib
83-
// Lua_eval(L, lua_bootstrap
84-
// , strlen(lua_bootstrap)
85-
// , "=lib"
86-
// );
84+
return L;
85+
}
86+
87+
lua_State* Lua_ReInit_Environment(lua_State* L){
88+
// clear user-created globals
89+
luaL_dostring(L, "for k,_ in pairs(_user) do\n"
90+
"_G[k] = nil\n"
91+
"end\n"
92+
"_G._user = {}\n");
93+
lua_gc(L, LUA_GCCOLLECT, 1);
94+
lua_gc(L, LUA_GCCOLLECT, 1);
8795
return L;
8896
}
8997

@@ -99,14 +107,15 @@ lua_State* Lua_Reset( void )
99107
}
100108
events_clear();
101109
clock_cancel_coro_all();
102-
Lua_DeInit();
103-
return Lua_Init();
110+
return Lua_ReInit_Environment(L);
111+
// Lua_DeInit();
112+
// return Lua_Init();
104113
}
105114

106115
void Lua_load_default_script( void )
107116
{
108-
Lua_eval(L, lua_First
109-
, strlen(lua_First)
117+
Lua_eval(L, (const char*)build_First_lc
118+
, build_First_lc_len
110119
, "=First.lua"
111120
);
112121
}

‎lib/lualink.h

+2-1
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,9 @@
99
typedef void (*ErrorHandler_t)(char* error_message);
1010
struct lua_lib_locator{
1111
const char* name;
12-
const char* addr_of_luacode;
12+
const unsigned char* addr_of_luacode;
1313
const bool stripped;
14+
const unsigned int len;
1415
};
1516

1617
extern volatile int CPU_count; // count from main.c

‎lib/repl.c

+1-10
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ typedef enum{ REPL_normal
1818
// global variables
1919
lua_State* Lua;
2020
L_repl_mode repl_mode = REPL_normal;
21-
char* new_script;
21+
char new_script[USER_SCRIPT_SIZE]; // static alloc avoids malloc failures
2222
uint16_t new_script_len;
2323
static bool running_from_mem;
2424
static char running_script_name[64];
@@ -48,7 +48,6 @@ void REPL_init( lua_State* lua )
4848
printf("failed to load user script\n");
4949
Caw_send_luachunk("failed to load user script");
5050
}
51-
free(new_script);
5251
break;
5352
}
5453
case USERSCRIPT_Clear:
@@ -97,7 +96,6 @@ void REPL_upload( int flash )
9796
} else {
9897
Caw_send_luachunk("User script evaluation failed.");
9998
}
100-
free(new_script);
10199
}
102100
repl_mode = REPL_normal;
103101
}
@@ -194,7 +192,6 @@ static void REPL_receive_script( char* buf, uint32_t len, ErrorHandler_t errfn )
194192
if( new_script_len + len >= USER_SCRIPT_SIZE ){
195193
Caw_send_luachunk("!ERROR! Script is too long.");
196194
repl_mode = REPL_discard;
197-
free(new_script);
198195
} else {
199196
memcpy( &new_script[new_script_len], buf, len );
200197
new_script_len += len;
@@ -203,12 +200,6 @@ static void REPL_receive_script( char* buf, uint32_t len, ErrorHandler_t errfn )
203200

204201
static bool REPL_new_script_buffer( uint32_t len )
205202
{
206-
new_script = calloc( sizeof(char), len);
207-
if( new_script == NULL ){
208-
printf("malloc failed. REPL\n");
209-
Caw_send_luachunk("!ERROR! Out of memory.");
210-
return false;
211-
}
212203
new_script_len = 0; // reset counter as the buffer is empty
213204
return true;
214205
}

‎lua/hotswap.lua

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
--- hotswap library
22

33
--- globals are available on crow, otherwise require for norns
4+
local require = require or function() end
45
local s = sequins or require 'lib/sequins'
56
local tl = timeline or require 'lib/timeline'
67

0 commit comments

Comments
 (0)