From 895cc032f0039337b5b4c78300ee9c34d514ca32 Mon Sep 17 00:00:00 2001 From: umbynos Date: Thu, 15 Jul 2021 17:59:25 +0200 Subject: [PATCH 01/37] update bufferflow_timedraw as bufferflow_timed --- bufferflow_timedraw.go | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/bufferflow_timedraw.go b/bufferflow_timedraw.go index b20f27f48..019ed5af4 100644 --- a/bufferflow_timedraw.go +++ b/bufferflow_timedraw.go @@ -12,36 +12,46 @@ type BufferflowTimedRaw struct { Port string Output chan []byte Input chan string + done chan bool ticker *time.Ticker } var ( bufferedOutputRaw []byte + sPortRaw string ) func (b *BufferflowTimedRaw) Init() { - log.Println("Initting timed buffer flow (output once every 16ms)") - - go func() { - for data := range b.Input { - bufferedOutputRaw = append(bufferedOutputRaw, []byte(data)...) - } - }() + log.Println("Initting timed buffer raw flow (output once every 16ms)") + bufferedOutputRaw = nil + sPortRaw = "" go func() { b.ticker = time.NewTicker(16 * time.Millisecond) - for _ = range b.ticker.C { - if len(bufferedOutputRaw) != 0 { - m := SpPortMessageRaw{b.Port, bufferedOutputRaw} - buf, _ := json.Marshal(m) - // data is now encoded in base64 format - // need a decoder on the other side - b.Output <- []byte(buf) - bufferedOutputRaw = nil + b.done = make(chan bool) + Loop: + for { + select { + case data := <-b.Input: + bufferedOutputRaw = append(bufferedOutputRaw, []byte(data)...) + sPortRaw = b.Port + case <-b.ticker.C: + if bufferedOutputRaw != nil { + m := SpPortMessageRaw{sPortRaw, bufferedOutputRaw} + buf, _ := json.Marshal(m) + // data is now encoded in base64 format + // need a decoder on the other side + b.Output <- []byte(buf) + bufferedOutputRaw = nil + sPortRaw = "" + } + case <-b.done: + break Loop } } - }() + close(b.Input) + }() } func (b *BufferflowTimedRaw) BlockUntilReady(cmd string, id string) (bool, bool) { From 05ee6f52ec725fe6829f5b5e404e3d96d2ef4c8e Mon Sep 17 00:00:00 2001 From: umbynos Date: Thu, 15 Jul 2021 17:59:38 +0200 Subject: [PATCH 02/37] remove old commands --- hub.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hub.go b/hub.go index 46aa9f679..e78e1b35b 100755 --- a/hub.go +++ b/hub.go @@ -45,7 +45,7 @@ func (h *hub) run() { h.connections[c] = true // send supported commands c.send <- []byte("{\"Version\" : \"" + version + "\"} ") - c.send <- []byte("{\"Commands\" : [\"list\", \"open [portName] [baud] [bufferAlgorithm (optional)]\", \"send [portName] [cmd]\", \"sendnobuf [portName] [cmd]\", \"close [portName]\", \"bufferalgorithms\", \"baudrates\", \"restart\", \"exit\", \"program [portName] [board:name] [$path/to/filename/without/extension]\", \"programfromurl [portName] [board:name] [urlToHexFile]\"]} ") + c.send <- []byte("{\"Commands\" : [\"list\", \"open [portName] [baud] [bufferAlgorithm (optional)]\", \"send [portName] [cmd]\", \"sendnobuf [portName] [cmd]\", \"close [portName]\", \"restart\", \"exit\", \"program [portName] [board:name] [$path/to/filename/without/extension]\", \"programfromurl [portName] [board:name] [urlToHexFile]\"]} ") c.send <- []byte("{\"Hostname\" : \"" + *hostname + "\"} ") c.send <- []byte("{\"OS\" : \"" + runtime.GOOS + "\"} ") case c := <-h.unregister: From 6a50f87272509daf48104b38dc2699b3391eaa3d Mon Sep 17 00:00:00 2001 From: Umberto Baldi Date: Mon, 19 Jul 2021 16:25:31 +0200 Subject: [PATCH 03/37] remove utf8 decoding with timedraw buffer type --- serialport.go | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/serialport.go b/serialport.go index c7e574a7a..18b0f36ca 100755 --- a/serialport.go +++ b/serialport.go @@ -92,7 +92,7 @@ type SpPortMessageRaw struct { D []byte // the data, i.e. G0 X0 Y0 } -func (p *serport) reader() { +func (p *serport) reader(buftype string) { //var buf bytes.Buffer ch := make([]byte, 1024) @@ -120,21 +120,24 @@ func (p *serport) reader() { // read can return legitimate bytes as well as an error // so process the bytes if n > 0 if n > 0 { - //log.Print("Read " + strconv.Itoa(n) + " bytes ch: " + string(ch)) + log.Print("Read " + strconv.Itoa(n) + " bytes ch: " + string(ch)) data := "" - - for i, w := 0, 0; i < n; i += w { - runeValue, width := utf8.DecodeRune(ch[i:n]) - if runeValue == utf8.RuneError { - buffered_ch.Write(append(ch[i:n])) - break - } - if i == n { - buffered_ch.Reset() + if buftype == "timedraw" { + data = string(ch[:n]) + } else { + for i, w := 0, 0; i < n; i += w { + runeValue, width := utf8.DecodeRune(ch[i:n]) + if runeValue == utf8.RuneError { + buffered_ch.Write(append(ch[i:n])) + break + } + if i == n { + buffered_ch.Reset() + } + data += string(runeValue) + w = width } - data += string(runeValue) - w = width } //log.Print("The data i will convert to json is:") @@ -344,7 +347,7 @@ func spHandlerOpen(portname string, baud int, buftype string) { go p.writerBuffered() // this is thread to send to serial port regardless of block go p.writerNoBuf() - p.reader() + p.reader(buftype) spListDual(false) spList(false) From e80400b7ddbbc2e8f34f1e6701b55102c3a99289 Mon Sep 17 00:00:00 2001 From: Umberto Baldi Date: Mon, 19 Jul 2021 18:34:47 +0200 Subject: [PATCH 04/37] binary support (WIP) --- bufferflow.go | 3 +- bufferflow_default.go | 4 ++ bufferflow_timed.go | 6 +- bufferflow_timedbinary.go | 114 ++++++++++++++++++++++++++++++++++++++ bufferflow_timedraw.go | 4 ++ serialport.go | 8 ++- 6 files changed, 136 insertions(+), 3 deletions(-) create mode 100644 bufferflow_timedbinary.go diff --git a/bufferflow.go b/bufferflow.go index 42ee9e9b4..565e173b7 100644 --- a/bufferflow.go +++ b/bufferflow.go @@ -5,7 +5,7 @@ import ( //"time" ) -var availableBufferAlgorithms = []string{"default", "timed", "timedraw"} +var availableBufferAlgorithms = []string{"default", "timed", "timedraw", "timedbinary"} type BufferMsg struct { Cmd string @@ -19,6 +19,7 @@ type Bufferflow interface { Init() BlockUntilReady(cmd string, id string) (bool, bool) // implement this method //JustQueue(cmd string, id string) bool // implement this method + OnIncomingDataBinary(data []byte) OnIncomingData(data string) // implement this method ClearOutSemaphore() // implement this method BreakApartCommands(cmd string) []string // implement this method diff --git a/bufferflow_default.go b/bufferflow_default.go index 7f37ac3d2..4bfe89b9f 100644 --- a/bufferflow_default.go +++ b/bufferflow_default.go @@ -20,6 +20,10 @@ func (b *BufferflowDefault) BlockUntilReady(cmd string, id string) (bool, bool) return true, false } +// not implemented, we are gonna use OnIncomingData +func (b *BufferflowDefault) OnIncomingDataBinary(data []byte) { +} + func (b *BufferflowDefault) OnIncomingData(data string) { //log.Printf("OnIncomingData() start. data:%v\n", data) } diff --git a/bufferflow_timed.go b/bufferflow_timed.go index d0be90227..6b7a0af14 100644 --- a/bufferflow_timed.go +++ b/bufferflow_timed.go @@ -18,7 +18,7 @@ type BufferflowTimed struct { var ( bufferedOutput string - sPort string + sPort string ) func (b *BufferflowTimed) Init() { @@ -61,6 +61,10 @@ func (b *BufferflowTimed) BlockUntilReady(cmd string, id string) (bool, bool) { return true, false } +// not implemented, we are gonna use OnIncomingData +func (b *BufferflowTimed) OnIncomingDataBinary(data []byte) { +} + func (b *BufferflowTimed) OnIncomingData(data string) { b.Input <- data } diff --git a/bufferflow_timedbinary.go b/bufferflow_timedbinary.go new file mode 100644 index 000000000..adaadde60 --- /dev/null +++ b/bufferflow_timedbinary.go @@ -0,0 +1,114 @@ +package main + +import ( + "encoding/json" + "time" + + log "github.com/sirupsen/logrus" +) + +type BufferflowTimedBinary struct { + Name string + Port string + Output chan []byte + Input chan []byte + done chan bool + ticker *time.Ticker +} + +var ( + bufferedOutputBinary []byte + sPortBinary string +) + +func (b *BufferflowTimedBinary) Init() { + log.Println("Initting timed buffer binary flow (output once every 16ms)") + bufferedOutputBinary = nil + sPortBinary = "" + + go func() { + b.ticker = time.NewTicker(16 * time.Millisecond) + b.done = make(chan bool) + Loop: + for { + select { + case data := <-b.Input: + bufferedOutputBinary = append(bufferedOutputBinary, []byte(data)...) + sPortBinary = b.Port + case <-b.ticker.C: + if bufferedOutputBinary != nil { + m := SpPortMessageRaw{sPortBinary, bufferedOutputBinary} + buf, _ := json.Marshal(m) + b.Output <- []byte(buf) + bufferedOutputBinary = nil + sPortBinary = "" + } + case <-b.done: + break Loop + } + } + + close(b.Input) + }() +} + +func (b *BufferflowTimedBinary) BlockUntilReady(cmd string, id string) (bool, bool) { + //log.Printf("BlockUntilReady() start\n") + return true, false +} + +func (b *BufferflowTimedBinary) OnIncomingDataBinary(data []byte) { + b.Input <- data +} + +// not implemented, we are gonna use OnIncomingDataBinary +func (b *BufferflowTimedBinary) OnIncomingData(data string) { +} + +// Clean out b.sem so it can truly block +func (b *BufferflowTimedBinary) ClearOutSemaphore() { +} + +func (b *BufferflowTimedBinary) BreakApartCommands(cmd string) []string { + return []string{cmd} +} + +func (b *BufferflowTimedBinary) Pause() { + return +} + +func (b *BufferflowTimedBinary) Unpause() { + return +} + +func (b *BufferflowTimedBinary) SeeIfSpecificCommandsShouldSkipBuffer(cmd string) bool { + return false +} + +func (b *BufferflowTimedBinary) SeeIfSpecificCommandsShouldPauseBuffer(cmd string) bool { + return false +} + +func (b *BufferflowTimedBinary) SeeIfSpecificCommandsShouldUnpauseBuffer(cmd string) bool { + return false +} + +func (b *BufferflowTimedBinary) SeeIfSpecificCommandsShouldWipeBuffer(cmd string) bool { + return false +} + +func (b *BufferflowTimedBinary) SeeIfSpecificCommandsReturnNoResponse(cmd string) bool { + return false +} + +func (b *BufferflowTimedBinary) ReleaseLock() { +} + +func (b *BufferflowTimedBinary) IsBufferGloballySendingBackIncomingData() bool { + return true +} + +func (b *BufferflowTimedBinary) Close() { + b.ticker.Stop() + close(b.Input) +} diff --git a/bufferflow_timedraw.go b/bufferflow_timedraw.go index 019ed5af4..5f94cf047 100644 --- a/bufferflow_timedraw.go +++ b/bufferflow_timedraw.go @@ -59,6 +59,10 @@ func (b *BufferflowTimedRaw) BlockUntilReady(cmd string, id string) (bool, bool) return true, false } +// not implemented, we are gonna use OnIncomingData +func (b *BufferflowTimedRaw) OnIncomingDataBinary(data []byte) { +} + func (b *BufferflowTimedRaw) OnIncomingData(data string) { b.Input <- data } diff --git a/serialport.go b/serialport.go index 18b0f36ca..87cddfe87 100755 --- a/serialport.go +++ b/serialport.go @@ -148,7 +148,11 @@ func (p *serport) reader(buftype string) { // writes to the serialport. each bufferflow type will decide // this on its own based on its logic, i.e. tinyg vs grbl vs others //p.b.bufferwatcher..OnIncomingData(data) - p.bufferwatcher.OnIncomingData(data) + if buftype == "timedbinary" { + p.bufferwatcher.OnIncomingDataBinary(ch[:n]) + } else { + p.bufferwatcher.OnIncomingData(data) + } // see if the OnIncomingData handled the broadcast back // to the user. this option was added in case the OnIncomingData wanted @@ -330,6 +334,8 @@ func spHandlerOpen(portname string, baud int, buftype string) { bw = &BufferflowTimed{Name: "timed", Port: portname, Output: h.broadcastSys, Input: make(chan string)} } else if buftype == "timedraw" { bw = &BufferflowTimedRaw{Name: "timedraw", Port: portname, Output: h.broadcastSys, Input: make(chan string)} + } else if buftype == "timedbinary" { + bw = &BufferflowTimedBinary{Name: "timedbinary", Port: portname, Output: h.broadcastSys, Input: make(chan []byte)} } else { bw = &BufferflowDefault{Port: portname} } From eb2b3f3b2af7c85fa89e28d4648d9aebf1aec821 Mon Sep 17 00:00:00 2001 From: umbynos Date: Tue, 20 Jul 2021 14:59:15 +0200 Subject: [PATCH 05/37] use switch case --- hub.go | 2 +- serialport.go | 45 +++++++++++++++++++++++---------------------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/hub.go b/hub.go index e78e1b35b..c5d5bdde5 100755 --- a/hub.go +++ b/hub.go @@ -127,7 +127,7 @@ func checkCmd(m []byte) { } // pass in buffer type now as string. if user does not // ask for a buffer type pass in empty string - bufferAlgorithm := "" + bufferAlgorithm := "default" // use the default buffer if none is specified if len(args) > 3 { // cool. we got a buffer type request buftype := strings.Replace(args[3], "\n", "", -1) diff --git a/serialport.go b/serialport.go index 87cddfe87..a83f715b0 100755 --- a/serialport.go +++ b/serialport.go @@ -103,7 +103,7 @@ func (p *serport) reader(buftype string) { n, err := p.portIo.Read(ch) - //if we detect that port is closing, break out o this for{} loop. + //if we detect that port is closing, break out of this for{} loop. if p.isClosing { strmsg := "Shutting down reader on " + p.portConf.Name log.Println(strmsg) @@ -118,16 +118,20 @@ func (p *serport) reader(buftype string) { } // read can return legitimate bytes as well as an error - // so process the bytes if n > 0 + // so process the n bytes if n > 0 if n > 0 { log.Print("Read " + strconv.Itoa(n) + " bytes ch: " + string(ch)) data := "" - if buftype == "timedraw" { + switch buftype { + case "timedraw", "timed": // data is handled differently inside BufferFlowTimed (bufferedOutput is string) and BufferFlowTimedRaw (bufferedOutputRaw is []byte) data = string(ch[:n]) - } else { + p.bufferwatcher.OnIncomingData(data) + case "timedbinary": + p.bufferwatcher.OnIncomingDataBinary(ch[:n]) + case "default": for i, w := 0, 0; i < n; i += w { - runeValue, width := utf8.DecodeRune(ch[i:n]) + runeValue, width := utf8.DecodeRune(ch[i:n]) // try to decode the first i bytes in the buffer (UTF8 runes do not have a fixed lenght) if runeValue == utf8.RuneError { buffered_ch.Write(append(ch[i:n])) break @@ -138,20 +142,14 @@ func (p *serport) reader(buftype string) { data += string(runeValue) w = width } - } - - //log.Print("The data i will convert to json is:") - //log.Print(data) - - // give the data to our bufferflow so it can do it's work - // to read/translate the data to see if it wants to block - // writes to the serialport. each bufferflow type will decide - // this on its own based on its logic, i.e. tinyg vs grbl vs others - //p.b.bufferwatcher..OnIncomingData(data) - if buftype == "timedbinary" { - p.bufferwatcher.OnIncomingDataBinary(ch[:n]) - } else { + // give the data to our bufferflow so it can do it's work + // to read/translate the data to see if it wants to block + // writes to the serialport. each bufferflow type will decide + // this on its own based on its logic, i.e. tinyg vs grbl vs others + //p.b.bufferwatcher..OnIncomingData(data) p.bufferwatcher.OnIncomingData(data) + default: + log.Panicf("unknown buffer type %s", buftype) } // see if the OnIncomingData handled the broadcast back @@ -330,14 +328,17 @@ func spHandlerOpen(portname string, baud int, buftype string) { var bw Bufferflow - if buftype == "timed" { + switch buftype { + case "timed": bw = &BufferflowTimed{Name: "timed", Port: portname, Output: h.broadcastSys, Input: make(chan string)} - } else if buftype == "timedraw" { + case "timedraw": bw = &BufferflowTimedRaw{Name: "timedraw", Port: portname, Output: h.broadcastSys, Input: make(chan string)} - } else if buftype == "timedbinary" { + case "timedbinary": bw = &BufferflowTimedBinary{Name: "timedbinary", Port: portname, Output: h.broadcastSys, Input: make(chan []byte)} - } else { + case "default": bw = &BufferflowDefault{Port: portname} + default: + log.Panicf("unknown buffer type: %s", buftype) } bw.Init() From 1cd174aacf7d8b47750bcb5e84516c1f394781e8 Mon Sep 17 00:00:00 2001 From: umbynos Date: Mon, 1 Mar 2021 17:45:10 +0100 Subject: [PATCH 06/37] fixed test deps --- poetry.lock | 90 +++++++++++++++++++++++++++++++++++--------------- pyproject.toml | 8 +++-- 2 files changed, 69 insertions(+), 29 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0d51c0ae4..3882a7458 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2,7 +2,7 @@ name = "atomicwrites" version = "1.4.0" description = "Atomic file writes." -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" @@ -10,7 +10,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" name = "attrs" version = "20.3.0" description = "Classes Without Boilerplate" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" @@ -24,7 +24,7 @@ tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (> name = "certifi" version = "2020.12.5" description = "Python package for providing Mozilla's CA Bundle." -category = "dev" +category = "main" optional = false python-versions = "*" @@ -32,7 +32,7 @@ python-versions = "*" name = "chardet" version = "4.0.0" description = "Universal encoding detector for Python 2 and 3" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" @@ -40,7 +40,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" name = "colorama" version = "0.4.4" description = "Cross-platform colored terminal text." -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" @@ -48,7 +48,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" name = "idna" version = "2.10" description = "Internationalized Domain Names in Applications (IDNA)" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" @@ -56,7 +56,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" name = "iniconfig" version = "1.1.1" description = "iniconfig: brain-dead simple config-ini parsing" -category = "dev" +category = "main" optional = false python-versions = "*" @@ -64,26 +64,34 @@ python-versions = "*" name = "invoke" version = "1.5.0" description = "Pythonic task execution" -category = "dev" +category = "main" optional = false python-versions = "*" [[package]] name = "packaging" -version = "20.8" +version = "20.9" description = "Core utilities for Python packages" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.dependencies] pyparsing = ">=2.0.2" +[[package]] +name = "pathlib" +version = "1.0.1" +description = "Object-oriented filesystem paths" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "pluggy" version = "0.13.1" description = "plugin and hook calling mechanisms for python" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" @@ -105,7 +113,7 @@ test = ["ipaddress", "mock", "unittest2", "enum34", "pywin32", "wmi"] name = "py" version = "1.10.0" description = "library with cross-python path, ini-parsing, io, code, log facilities" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" @@ -113,15 +121,15 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" name = "pyparsing" version = "2.4.7" description = "Python parsing module" -category = "dev" +category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "pytest" -version = "6.2.1" +version = "6.2.2" description = "pytest: simple powerful testing with Python" -category = "dev" +category = "main" optional = false python-versions = ">=3.6" @@ -142,7 +150,7 @@ testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xm name = "requests" version = "2.25.1" description = "Python HTTP for Humans." -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" @@ -156,19 +164,27 @@ urllib3 = ">=1.21.1,<1.27" security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] +[[package]] +name = "six" +version = "1.15.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" + [[package]] name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" -category = "dev" +category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "urllib3" -version = "1.26.2" +version = "1.26.3" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" @@ -177,10 +193,21 @@ brotli = ["brotlipy (>=0.6.0)"] secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +[[package]] +name = "websocket-client" +version = "0.57.0" +description = "WebSocket client for Python. hybi13 is supported." +category = "main" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +six = "*" + [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "d4024a50235f771811ce457845b6cfef87950b8eb07a8a6e3ad10005482c8a05" +content-hash = "a5bef322fa63f7462def1e5a0122aefe8e37507ec3b9f9c99fbd672e2024a656" [metadata.files] atomicwrites = [ @@ -217,8 +244,11 @@ invoke = [ {file = "invoke-1.5.0.tar.gz", hash = "sha256:f0c560075b5fb29ba14dad44a7185514e94970d1b9d57dcd3723bec5fed92650"}, ] packaging = [ - {file = "packaging-20.8-py2.py3-none-any.whl", hash = "sha256:24e0da08660a87484d1602c30bb4902d74816b6985b93de36926f5bc95741858"}, - {file = "packaging-20.8.tar.gz", hash = "sha256:78598185a7008a470d64526a8059de9aaa449238f280fc9eb6b13ba6c4109093"}, + {file = "packaging-20.9-py2.py3-none-any.whl", hash = "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a"}, + {file = "packaging-20.9.tar.gz", hash = "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5"}, +] +pathlib = [ + {file = "pathlib-1.0.1.tar.gz", hash = "sha256:6940718dfc3eff4258203ad5021090933e5c04707d5ca8cc9e73c94a7894ea9f"}, ] pluggy = [ {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, @@ -263,18 +293,26 @@ pyparsing = [ {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, ] pytest = [ - {file = "pytest-6.2.1-py3-none-any.whl", hash = "sha256:1969f797a1a0dbd8ccf0fecc80262312729afea9c17f1d70ebf85c5e76c6f7c8"}, - {file = "pytest-6.2.1.tar.gz", hash = "sha256:66e419b1899bc27346cb2c993e12c5e5e8daba9073c1fbce33b9807abc95c306"}, + {file = "pytest-6.2.2-py3-none-any.whl", hash = "sha256:b574b57423e818210672e07ca1fa90aaf194a4f63f3ab909a2c67ebb22913839"}, + {file = "pytest-6.2.2.tar.gz", hash = "sha256:9d1edf9e7d0b84d72ea3dbcdfd22b35fb543a5e8f2a60092dd578936bf63d7f9"}, ] requests = [ {file = "requests-2.25.1-py2.py3-none-any.whl", hash = "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"}, {file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"}, ] +six = [ + {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, + {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, +] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] urllib3 = [ - {file = "urllib3-1.26.2-py2.py3-none-any.whl", hash = "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473"}, - {file = "urllib3-1.26.2.tar.gz", hash = "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08"}, + {file = "urllib3-1.26.3-py2.py3-none-any.whl", hash = "sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80"}, + {file = "urllib3-1.26.3.tar.gz", hash = "sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73"}, +] +websocket-client = [ + {file = "websocket_client-0.57.0-py2.py3-none-any.whl", hash = "sha256:0fc45c961324d79c781bab301359d5a1b00b13ad1b10415a4780229ef71a5549"}, + {file = "websocket_client-0.57.0.tar.gz", hash = "sha256:d735b91d6d1692a6a181f2a8c9e0238e5f6373356f561bb9dc4c7af36f452010"}, ] diff --git a/pyproject.toml b/pyproject.toml index c3d576b7c..3e06d5261 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,11 +8,13 @@ license = "GPLv2" [tool.poetry.dependencies] python = "^3.9" psutil = "^5.8.0" - -[tool.poetry.dev-dependencies] -pytest = "^6.2.1" +websocket_client = "^0.57.0" +pytest = "^6.2.2" requests = "^2.25.1" invoke = "^1.5.0" +pathlib = "^1.0.1" + +[tool.poetry.dev-dependencies] [build-system] requires = ["poetry-core>=1.0.0"] From 48cc6950e48e69a50477c3be0d0038e9075cca2e Mon Sep 17 00:00:00 2001 From: umbynos Date: Thu, 4 Mar 2021 10:44:10 +0100 Subject: [PATCH 07/37] =?UTF-8?q?socketio=20test=20connection=20is=20worki?= =?UTF-8?q?ng=20=F0=9F=8E=89=20(with=20the=20correct=20python-socketio=20v?= =?UTF-8?q?ersion)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- poetry.lock | 78 +++++++++++++++++++++++++++++++++++------------- pyproject.toml | 3 +- test/conftest.py | 10 ++++++- test/test_ws.py | 11 +++++++ 4 files changed, 80 insertions(+), 22 deletions(-) create mode 100644 test/test_ws.py diff --git a/poetry.lock b/poetry.lock index 3882a7458..1261fb68e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,11 @@ +[[package]] +name = "asyncio" +version = "3.4.3" +description = "reference implementation of PEP 3156" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "atomicwrites" version = "1.4.0" @@ -30,11 +38,11 @@ python-versions = "*" [[package]] name = "chardet" -version = "4.0.0" +version = "3.0.4" description = "Universal encoding detector for Python 2 and 3" category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = "*" [[package]] name = "colorama" @@ -146,6 +154,37 @@ toml = "*" [package.extras] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] +[[package]] +name = "python-engineio" +version = "3.14.2" +description = "Engine.IO server" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +six = ">=1.9.0" + +[package.extras] +asyncio_client = ["aiohttp (>=3.4)"] +client = ["requests (>=2.21.0)", "websocket-client (>=0.54.0)"] + +[[package]] +name = "python-socketio" +version = "4.6.1" +description = "Socket.IO server" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +python-engineio = ">=3.13.0,<4" +six = ">=1.9.0" + +[package.extras] +asyncio_client = ["aiohttp (>=3.4)", "websockets (>=7.0)"] +client = ["requests (>=2.21.0)", "websocket-client (>=0.54.0)"] + [[package]] name = "requests" version = "2.25.1" @@ -193,23 +232,18 @@ brotli = ["brotlipy (>=0.6.0)"] secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] -[[package]] -name = "websocket-client" -version = "0.57.0" -description = "WebSocket client for Python. hybi13 is supported." -category = "main" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[package.dependencies] -six = "*" - [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "a5bef322fa63f7462def1e5a0122aefe8e37507ec3b9f9c99fbd672e2024a656" +content-hash = "168545733833f6a77146c9976e6ead33d1d85eefa70685180b7c7843671b8db0" [metadata.files] +asyncio = [ + {file = "asyncio-3.4.3-cp33-none-win32.whl", hash = "sha256:b62c9157d36187eca799c378e572c969f0da87cd5fc42ca372d92cdb06e7e1de"}, + {file = "asyncio-3.4.3-cp33-none-win_amd64.whl", hash = "sha256:c46a87b48213d7464f22d9a497b9eef8c1928b68320a2fa94240f969f6fec08c"}, + {file = "asyncio-3.4.3-py3-none-any.whl", hash = "sha256:c4d18b22701821de07bd6aea8b53d21449ec0ec5680645e5317062ea21817d2d"}, + {file = "asyncio-3.4.3.tar.gz", hash = "sha256:83360ff8bc97980e4ff25c964c7bd3923d333d177aa4f7fb736b019f26c7cb41"}, +] atomicwrites = [ {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, @@ -223,8 +257,8 @@ certifi = [ {file = "certifi-2020.12.5.tar.gz", hash = "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c"}, ] chardet = [ - {file = "chardet-4.0.0-py2.py3-none-any.whl", hash = "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"}, - {file = "chardet-4.0.0.tar.gz", hash = "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa"}, + {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"}, + {file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"}, ] colorama = [ {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, @@ -296,6 +330,14 @@ pytest = [ {file = "pytest-6.2.2-py3-none-any.whl", hash = "sha256:b574b57423e818210672e07ca1fa90aaf194a4f63f3ab909a2c67ebb22913839"}, {file = "pytest-6.2.2.tar.gz", hash = "sha256:9d1edf9e7d0b84d72ea3dbcdfd22b35fb543a5e8f2a60092dd578936bf63d7f9"}, ] +python-engineio = [ + {file = "python-engineio-3.14.2.tar.gz", hash = "sha256:eab4553f2804c1ce97054c8b22cf0d5a9ab23128075248b97e1a5b2f29553085"}, + {file = "python_engineio-3.14.2-py2.py3-none-any.whl", hash = "sha256:5a9e6086d192463b04a1428ff1f85b6ba631bbb19d453b144ffc04f530542b84"}, +] +python-socketio = [ + {file = "python-socketio-4.6.1.tar.gz", hash = "sha256:cd1f5aa492c1eb2be77838e837a495f117e17f686029ebc03d62c09e33f4fa10"}, + {file = "python_socketio-4.6.1-py2.py3-none-any.whl", hash = "sha256:5a21da53fdbdc6bb6c8071f40e13d100e0b279ad997681c2492478e06f370523"}, +] requests = [ {file = "requests-2.25.1-py2.py3-none-any.whl", hash = "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"}, {file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"}, @@ -312,7 +354,3 @@ urllib3 = [ {file = "urllib3-1.26.3-py2.py3-none-any.whl", hash = "sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80"}, {file = "urllib3-1.26.3.tar.gz", hash = "sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73"}, ] -websocket-client = [ - {file = "websocket_client-0.57.0-py2.py3-none-any.whl", hash = "sha256:0fc45c961324d79c781bab301359d5a1b00b13ad1b10415a4780229ef71a5549"}, - {file = "websocket_client-0.57.0.tar.gz", hash = "sha256:d735b91d6d1692a6a181f2a8c9e0238e5f6373356f561bb9dc4c7af36f452010"}, -] diff --git a/pyproject.toml b/pyproject.toml index 3e06d5261..ca7672636 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,11 +8,12 @@ license = "GPLv2" [tool.poetry.dependencies] python = "^3.9" psutil = "^5.8.0" -websocket_client = "^0.57.0" pytest = "^6.2.2" requests = "^2.25.1" invoke = "^1.5.0" pathlib = "^1.0.1" +asyncio = "^3.4.3" +python-socketio = "^4" [tool.poetry.dev-dependencies] diff --git a/test/conftest.py b/test/conftest.py index fd50856ec..733eddb91 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -7,7 +7,8 @@ import pytest from invoke import Local from invoke.context import Context - +import socketio as io +import asyncio @pytest.fixture(scope="function") def agent(pytestconfig): @@ -43,3 +44,10 @@ def agent(pytestconfig): @pytest.fixture(scope="session") def base_url(): return "http://127.0.0.1:8991" + +@pytest.fixture(scope="session") +def socketio(base_url, data=""): + sio = io.Client() + sio.connect(base_url) + yield sio + sio.disconnect() diff --git a/test/test_ws.py b/test/test_ws.py new file mode 100644 index 000000000..90f18504d --- /dev/null +++ b/test/test_ws.py @@ -0,0 +1,11 @@ +import asyncio +import socketio as io +import time + +def test_ws_connection(agent, socketio): + print('my sid is', socketio.sid) + assert socketio.sid is not None + +def test_list(agent, socketio): + socketio.emit('command', 'list') + From fc1222ef930cd37d5eb9683b1807c6c7a87c0df6 Mon Sep 17 00:00:00 2001 From: umbynos Date: Wed, 21 Jul 2021 11:24:23 +0200 Subject: [PATCH 08/37] add callback to capture returned message, add new test for serial --- test/test_ws.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/test/test_ws.py b/test/test_ws.py index 90f18504d..4d1b28fba 100644 --- a/test/test_ws.py +++ b/test/test_ws.py @@ -1,5 +1,4 @@ -import asyncio -import socketio as io +import socketio import time def test_ws_connection(agent, socketio): @@ -7,5 +6,19 @@ def test_ws_connection(agent, socketio): assert socketio.sid is not None def test_list(agent, socketio): + socketio.on('message', message_handler) socketio.emit('command', 'list') - + time.sleep(.1) + +def test__open_serial_default(agent, socketio): + socketio.on('message', message_handler) + socketio.emit('command', 'open /dev/ttyACM0 9600') + time.sleep(.1) + socketio.emit('command', 'send /dev/ttyACM0 /"ciao/"') + time.sleep(.1) + socketio.emit('command', 'close /dev/ttyACM0') + time.sleep(.1) + + +def message_handler(msg): + print('Received message: ', msg) \ No newline at end of file From 9c7d3f8a6fa1f205ae3677d6c3ffbc41a138c7da Mon Sep 17 00:00:00 2001 From: umbynos Date: Wed, 21 Jul 2021 13:23:06 +0200 Subject: [PATCH 09/37] fix tests: "socketio.exceptions.ConnectionError: Connection refused by the server" --- test/conftest.py | 4 ++-- test/test_ws.py | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 733eddb91..9f9c0389d 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -45,8 +45,8 @@ def agent(pytestconfig): def base_url(): return "http://127.0.0.1:8991" -@pytest.fixture(scope="session") -def socketio(base_url, data=""): +@pytest.fixture(scope="function") +def socketio(base_url, agent): sio = io.Client() sio.connect(base_url) yield sio diff --git a/test/test_ws.py b/test/test_ws.py index 4d1b28fba..d4f67056e 100644 --- a/test/test_ws.py +++ b/test/test_ws.py @@ -1,16 +1,15 @@ -import socketio import time -def test_ws_connection(agent, socketio): +def test_ws_connection(socketio): print('my sid is', socketio.sid) assert socketio.sid is not None -def test_list(agent, socketio): +def test_list(socketio): socketio.on('message', message_handler) socketio.emit('command', 'list') time.sleep(.1) -def test__open_serial_default(agent, socketio): +def test__open_serial_default(socketio): socketio.on('message', message_handler) socketio.emit('command', 'open /dev/ttyACM0 9600') time.sleep(.1) From dfcc14e8a61f536934f484f56caab0af1b631b30 Mon Sep 17 00:00:00 2001 From: umbynos Date: Fri, 23 Jul 2021 10:49:09 +0200 Subject: [PATCH 10/37] minor optimizations: data and buf are already an array of bytes --- bufferflow_timedbinary.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bufferflow_timedbinary.go b/bufferflow_timedbinary.go index adaadde60..6ac689706 100644 --- a/bufferflow_timedbinary.go +++ b/bufferflow_timedbinary.go @@ -33,13 +33,13 @@ func (b *BufferflowTimedBinary) Init() { for { select { case data := <-b.Input: - bufferedOutputBinary = append(bufferedOutputBinary, []byte(data)...) + bufferedOutputBinary = append(bufferedOutputBinary, data...) sPortBinary = b.Port case <-b.ticker.C: if bufferedOutputBinary != nil { m := SpPortMessageRaw{sPortBinary, bufferedOutputBinary} buf, _ := json.Marshal(m) - b.Output <- []byte(buf) + b.Output <- buf bufferedOutputBinary = nil sPortBinary = "" } From b7f70f19ef5b59ac14bc0294c9e9fb0a8a45357a Mon Sep 17 00:00:00 2001 From: umbynos Date: Fri, 23 Jul 2021 10:52:47 +0200 Subject: [PATCH 11/37] enhanced a bit how the logic of the serial works --- serialport.go | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/serialport.go b/serialport.go index a83f715b0..339d1fd12 100755 --- a/serialport.go +++ b/serialport.go @@ -95,12 +95,11 @@ type SpPortMessageRaw struct { func (p *serport) reader(buftype string) { //var buf bytes.Buffer - ch := make([]byte, 1024) timeCheckOpen := time.Now() var buffered_ch bytes.Buffer for { - + ch := make([]byte, 1024) //a new array of bytes is initilized everytime because we pass it (as a pointer) in a channel, it can be improved n, err := p.portIo.Read(ch) //if we detect that port is closing, break out of this for{} loop. @@ -111,16 +110,11 @@ func (p *serport) reader(buftype string) { break } - if err == nil { - ch = append(buffered_ch.Bytes(), ch[:n]...) - n += len(buffered_ch.Bytes()) - buffered_ch.Reset() - } - // read can return legitimate bytes as well as an error - // so process the n bytes if n > 0 - if n > 0 { - log.Print("Read " + strconv.Itoa(n) + " bytes ch: " + string(ch)) + // so process the n bytes red, if n > 0 + if n > 0 && err == nil { + + log.Print("Read " + strconv.Itoa(n) + " bytes ch: " + string(ch[:n])) data := "" switch buftype { @@ -129,11 +123,15 @@ func (p *serport) reader(buftype string) { p.bufferwatcher.OnIncomingData(data) case "timedbinary": p.bufferwatcher.OnIncomingDataBinary(ch[:n]) - case "default": + case "default": // the bufferbuftype is actually called default 🤷‍♂️ + // save the left out bytes for the next iteration due to UTF-8 encoding + ch = append(buffered_ch.Bytes(), ch[:n]...) // TODO ch is not handled correctly: doing this way it's length is messed up. Use ch2 + n += len(buffered_ch.Bytes()) + buffered_ch.Reset() for i, w := 0, 0; i < n; i += w { runeValue, width := utf8.DecodeRune(ch[i:n]) // try to decode the first i bytes in the buffer (UTF8 runes do not have a fixed lenght) if runeValue == utf8.RuneError { - buffered_ch.Write(append(ch[i:n])) + buffered_ch.Write(ch[i:n]) break } if i == n { @@ -146,7 +144,6 @@ func (p *serport) reader(buftype string) { // to read/translate the data to see if it wants to block // writes to the serialport. each bufferflow type will decide // this on its own based on its logic, i.e. tinyg vs grbl vs others - //p.b.bufferwatcher..OnIncomingData(data) p.bufferwatcher.OnIncomingData(data) default: log.Panicf("unknown buffer type %s", buftype) From c86bb4f382543748b3f970eda5559599e8857b75 Mon Sep 17 00:00:00 2001 From: umbynos Date: Fri, 23 Jul 2021 10:55:14 +0200 Subject: [PATCH 12/37] enhance a lot test on serial communication (with different buffer types) The tests should be skipped on the CI (no board connected) --- test/common.py | 8 +++ test/test_ws.py | 138 +++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 test/common.py diff --git a/test/common.py b/test/common.py new file mode 100644 index 000000000..9d05ce26b --- /dev/null +++ b/test/common.py @@ -0,0 +1,8 @@ +import os + +def running_on_ci(): + """ + Returns whether the program is running on a CI environment + """ + val = os.getenv("GITHUB_WORKFLOW") + return val is not None diff --git a/test/test_ws.py b/test/test_ws.py index d4f67056e..0d49e42c0 100644 --- a/test/test_ws.py +++ b/test/test_ws.py @@ -1,23 +1,159 @@ import time +import json +import base64 +import pytest + +from common import running_on_ci +message = [] def test_ws_connection(socketio): print('my sid is', socketio.sid) assert socketio.sid is not None def test_list(socketio): + global message socketio.on('message', message_handler) socketio.emit('command', 'list') time.sleep(.1) + # print (message) + assert "list" in message[0] + assert "Ports" in message[1] + assert "Network" in message[2] +# NOTE run the following tests on linux with a board connected to the PC and with this sketch on it: https://gist.github.com/Protoneer/96db95bfb87c3befe46e +@pytest.mark.skipif( + running_on_ci(), + reason="VMs have no serial ports", +) def test__open_serial_default(socketio): + global message + message = [] socketio.on('message', message_handler) socketio.emit('command', 'open /dev/ttyACM0 9600') + time.sleep(.1) # give time to message to be filled + assert "\"IsOpen\": true" in message[2] + socketio.emit('command', 'send /dev/ttyACM0 /"ciao/"') + time.sleep(.1) + assert "send /dev/ttyACM0 /\"ciao/\"" in message[3] + assert "ciao" in extract_serial_data(message) + + # test with a lot of emoji: they can be messed up + message = [] # reinitialize the message buffer + socketio.emit('command', 'send /dev/ttyACM0 /"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/"') time.sleep(.1) + assert "send /dev/ttyACM0 /\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in message[0] + emoji_output = extract_serial_data(message) + assert "/\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in emoji_output # this is failing because of UTF8 encoding problems + message = [] + socketio.emit('command', 'close /dev/ttyACM0') + time.sleep(.1) + # print (message) + assert "\"IsOpen\": false," in message[8] + +@pytest.mark.skipif( + running_on_ci(), + reason="VMs have no serial ports", +) +def test__open_serial_timed(socketio): + global message + message = [] + socketio.on('message', message_handler) + socketio.emit('command', 'open /dev/ttyACM0 9600 timed') + time.sleep(.1) # give time to message to be filled + assert "\"IsOpen\": true" in message[2] socketio.emit('command', 'send /dev/ttyACM0 /"ciao/"') time.sleep(.1) + assert "send /dev/ttyACM0 /\"ciao/\"" in message[3] + assert "ciao" in extract_serial_data(message) + + # test with a lot of emoji: usually they get messed up + message = [] # reinitialize the message buffer + socketio.emit('command', 'send /dev/ttyACM0 /"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/"') + time.sleep(.1) + assert "send /dev/ttyACM0 /\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in message[0] + emoji_output = extract_serial_data(message) + assert "/\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in emoji_output + message = [] socketio.emit('command', 'close /dev/ttyACM0') time.sleep(.1) + # print (message) + assert "\"IsOpen\": false," in message[8] +@pytest.mark.skipif( + running_on_ci(), + reason="VMs have no serial ports", +) +def test__open_serial_timedraw(socketio): + global message + message = [] + socketio.on('message', message_handler) + socketio.emit('command', 'open /dev/ttyACM0 9600 timedraw') + time.sleep(.1) # give time to message to be filled + assert "\"IsOpen\": true" in message[2] + socketio.emit('command', 'send /dev/ttyACM0 /"ciao/"') + time.sleep(.1) + assert "send /dev/ttyACM0 /\"ciao/\"" in message[3] + assert "ciao" in decode_output(extract_serial_data(message)) + # test with a lot of emoji: usually they get messed up + message = [] # reinitialize the message buffer + socketio.emit('command', 'send /dev/ttyACM0 /"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/"') + time.sleep(.1) + assert "send /dev/ttyACM0 /\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in message[0] + # print (message) + assert "/\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in decode_output(extract_serial_data(message)) + socketio.emit('command', 'close /dev/ttyACM0') + time.sleep(.1) + # print (message) + assert "\"IsOpen\": false," in message[10] + +@pytest.mark.skipif( + running_on_ci(), + reason="VMs have no serial ports", +) +def test__open_serial_timedbinary(socketio): + global message + message = [] + socketio.on('message', message_handler) + socketio.emit('command', 'open /dev/ttyACM0 9600 timedbinary') + time.sleep(1) # give time to message to be filled + assert "\"IsOpen\": true" in message[2] + socketio.emit('command', 'send /dev/ttyACM0 /"ciao/"') + time.sleep(.1) + assert "send /dev/ttyACM0 /\"ciao/\"" in message[3] + assert "ciao" in decode_output(extract_serial_data(message)) + + # test with a lot of emoji: usually they get messed up + message = [] # reinitialize the message buffer + socketio.emit('command', 'send /dev/ttyACM0 /"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/"') + time.sleep(.1) + assert "send /dev/ttyACM0 /\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in message[0] + assert "/\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in decode_output(extract_serial_data(message)) + socketio.emit('command', 'close /dev/ttyACM0') + time.sleep(.1) + # print (message) + assert "\"IsOpen\": false," in message[10] + + +# callback called by socketio when a message is received def message_handler(msg): - print('Received message: ', msg) \ No newline at end of file + # print('Received message: ', msg) + global message + message.append(msg) + +# helper function used to extract serial data from it's json representation +# NOTE make sure to pass a clean message (maybe reinitialize the message global var before populating it) +def extract_serial_data(msg): + serial_data = "" + for i in msg: + if "{\"P\"" in i: + # print (json.loads(i)["D"]) + serial_data+=json.loads(i)["D"] + # print("serialdata:"+serial_data) + return serial_data + +def decode_output(raw_output): + # print(raw_output) + base64_bytes = raw_output.encode('ascii') #encode rawoutput message into a bytes-like object + output_bytes = base64.b64decode(base64_bytes) + return output_bytes.decode('utf-8') From 099a57564b5529ca503fc0f669fc0f483ec77f2f Mon Sep 17 00:00:00 2001 From: umbynos Date: Fri, 23 Jul 2021 12:09:58 +0200 Subject: [PATCH 13/37] =?UTF-8?q?update=20and=20enhance=20commands=20outpu?= =?UTF-8?q?t=20(the=20space=20in=20front=20of=20`<`=20and=20`>`=20is=20req?= =?UTF-8?q?uired)=20=F0=9F=A4=B7=E2=80=8D=E2=99=82=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hub.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/hub.go b/hub.go index c5d5bdde5..5103789e8 100755 --- a/hub.go +++ b/hub.go @@ -38,6 +38,22 @@ var h = hub{ connections: make(map[*connection]bool), } +const commands = "{\"Commands\": [" + + "\"list\", " + + "\"open < portName > < baud > [bufferAlgorithm: ({default}, timed, timedraw, timedbinary)]\", " + + "\"send < portName > < cmd >\", " + + "\"sendnobuf < portName > < cmd >\", " + + "\"close < portName >\", " + + "\"restart\", " + + "\"exit\", " + + "\"killupload\", " + + "\"downloadtool < tool > < toolVersion: {latest} > < pack: {arduino} > < behaviour: {keep} >\", " + + "\"log\", " + + "\"memorystats\", " + + "\"gc\", " + + "\"hostname\", " + + "\"version\"]} " + func (h *hub) run() { for { select { @@ -45,7 +61,7 @@ func (h *hub) run() { h.connections[c] = true // send supported commands c.send <- []byte("{\"Version\" : \"" + version + "\"} ") - c.send <- []byte("{\"Commands\" : [\"list\", \"open [portName] [baud] [bufferAlgorithm (optional)]\", \"send [portName] [cmd]\", \"sendnobuf [portName] [cmd]\", \"close [portName]\", \"restart\", \"exit\", \"program [portName] [board:name] [$path/to/filename/without/extension]\", \"programfromurl [portName] [board:name] [urlToHexFile]\"]} ") + c.send <- []byte(commands) c.send <- []byte("{\"Hostname\" : \"" + *hostname + "\"} ") c.send <- []byte("{\"OS\" : \"" + runtime.GOOS + "\"} ") case c := <-h.unregister: From b41d0cc9a47c7174c9891c595f40a8812d7b3cc1 Mon Sep 17 00:00:00 2001 From: umbynos Date: Thu, 29 Jul 2021 16:34:02 +0200 Subject: [PATCH 14/37] increased sleeptime, remove harcoded message[i]: should work on different systems --- test/test_ws.py | 99 +++++++++++++++++++++++++------------------------ 1 file changed, 51 insertions(+), 48 deletions(-) diff --git a/test/test_ws.py b/test/test_ws.py index 0d49e42c0..889c7786a 100644 --- a/test/test_ws.py +++ b/test/test_ws.py @@ -14,125 +14,128 @@ def test_list(socketio): global message socketio.on('message', message_handler) socketio.emit('command', 'list') - time.sleep(.1) - # print (message) - assert "list" in message[0] - assert "Ports" in message[1] - assert "Network" in message[2] + time.sleep(.2) + print (message) + assert any("list" in i for i in message) + assert any("Ports" in i for i in message) + assert any("Network" in i for i in message) # NOTE run the following tests on linux with a board connected to the PC and with this sketch on it: https://gist.github.com/Protoneer/96db95bfb87c3befe46e @pytest.mark.skipif( running_on_ci(), reason="VMs have no serial ports", ) -def test__open_serial_default(socketio): +def test_open_serial_default(socketio): + time.sleep(.2) global message message = [] socketio.on('message', message_handler) socketio.emit('command', 'open /dev/ttyACM0 9600') - time.sleep(.1) # give time to message to be filled - assert "\"IsOpen\": true" in message[2] + time.sleep(.5) # give time to message to be filled + assert any("\"IsOpen\": true" in i for i in message) socketio.emit('command', 'send /dev/ttyACM0 /"ciao/"') - time.sleep(.1) - assert "send /dev/ttyACM0 /\"ciao/\"" in message[3] + time.sleep(.2) + assert any("send /dev/ttyACM0 /\"ciao/\"" in i for i in message) assert "ciao" in extract_serial_data(message) # test with a lot of emoji: they can be messed up - message = [] # reinitialize the message buffer + # message = [] # reinitialize the message buffer socketio.emit('command', 'send /dev/ttyACM0 /"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/"') - time.sleep(.1) - assert "send /dev/ttyACM0 /\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in message[0] + time.sleep(.2) + print (message) + assert any("send /dev/ttyACM0 /\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in i for i in message) emoji_output = extract_serial_data(message) - assert "/\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in emoji_output # this is failing because of UTF8 encoding problems + assert "/\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in emoji_output # this could be failing because of UTF8 encoding problems message = [] socketio.emit('command', 'close /dev/ttyACM0') - time.sleep(.1) - # print (message) - assert "\"IsOpen\": false," in message[8] + time.sleep(.2) + assert any("\"IsOpen\": false," in i for i in message) @pytest.mark.skipif( running_on_ci(), reason="VMs have no serial ports", ) -def test__open_serial_timed(socketio): +def test_open_serial_timed(socketio): + time.sleep(.2) global message message = [] socketio.on('message', message_handler) socketio.emit('command', 'open /dev/ttyACM0 9600 timed') - time.sleep(.1) # give time to message to be filled - assert "\"IsOpen\": true" in message[2] + time.sleep(.5) # give time to message to be filled + print(message) + assert any("\"IsOpen\": true" in i for i in message) socketio.emit('command', 'send /dev/ttyACM0 /"ciao/"') - time.sleep(.1) - assert "send /dev/ttyACM0 /\"ciao/\"" in message[3] + time.sleep(.2) + assert any("send /dev/ttyACM0 /\"ciao/\"" in i for i in message) assert "ciao" in extract_serial_data(message) # test with a lot of emoji: usually they get messed up message = [] # reinitialize the message buffer socketio.emit('command', 'send /dev/ttyACM0 /"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/"') - time.sleep(.1) - assert "send /dev/ttyACM0 /\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in message[0] - emoji_output = extract_serial_data(message) - assert "/\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in emoji_output + time.sleep(.2) + assert any("send /dev/ttyACM0 /\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in i for i in message) + assert "/\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in extract_serial_data(message) message = [] socketio.emit('command', 'close /dev/ttyACM0') - time.sleep(.1) + time.sleep(.2) # print (message) - assert "\"IsOpen\": false," in message[8] + assert any("\"IsOpen\": false," in i for i in message) @pytest.mark.skipif( running_on_ci(), reason="VMs have no serial ports", ) -def test__open_serial_timedraw(socketio): +def test_open_serial_timedraw(socketio): global message message = [] socketio.on('message', message_handler) socketio.emit('command', 'open /dev/ttyACM0 9600 timedraw') - time.sleep(.1) # give time to message to be filled - assert "\"IsOpen\": true" in message[2] + time.sleep(.5) # give time to message to be filled + assert any("\"IsOpen\": true" in i for i in message) socketio.emit('command', 'send /dev/ttyACM0 /"ciao/"') - time.sleep(.1) - assert "send /dev/ttyACM0 /\"ciao/\"" in message[3] + time.sleep(.2) + assert any("send /dev/ttyACM0 /\"ciao/\"" in i for i in message) assert "ciao" in decode_output(extract_serial_data(message)) # test with a lot of emoji: usually they get messed up message = [] # reinitialize the message buffer socketio.emit('command', 'send /dev/ttyACM0 /"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/"') - time.sleep(.1) - assert "send /dev/ttyACM0 /\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in message[0] + time.sleep(.2) + assert any("send /dev/ttyACM0 /\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in i for i in message) # print (message) assert "/\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in decode_output(extract_serial_data(message)) socketio.emit('command', 'close /dev/ttyACM0') - time.sleep(.1) + time.sleep(.2) # print (message) - assert "\"IsOpen\": false," in message[10] + assert any("\"IsOpen\": false," in i for i in message) @pytest.mark.skipif( running_on_ci(), reason="VMs have no serial ports", ) -def test__open_serial_timedbinary(socketio): +def test_open_serial_timedbinary(socketio): global message message = [] socketio.on('message', message_handler) socketio.emit('command', 'open /dev/ttyACM0 9600 timedbinary') - time.sleep(1) # give time to message to be filled - assert "\"IsOpen\": true" in message[2] + time.sleep(.5) # give time to message to be filled + assert any("\"IsOpen\": true" in i for i in message) socketio.emit('command', 'send /dev/ttyACM0 /"ciao/"') - time.sleep(.1) - assert "send /dev/ttyACM0 /\"ciao/\"" in message[3] + time.sleep(.2) + assert any("send /dev/ttyACM0 /\"ciao/\"" in i for i in message) + print (message) assert "ciao" in decode_output(extract_serial_data(message)) # test with a lot of emoji: usually they get messed up message = [] # reinitialize the message buffer socketio.emit('command', 'send /dev/ttyACM0 /"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/"') - time.sleep(.1) - assert "send /dev/ttyACM0 /\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in message[0] + time.sleep(.2) + assert any("send /dev/ttyACM0 /\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in i for i in message) assert "/\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in decode_output(extract_serial_data(message)) socketio.emit('command', 'close /dev/ttyACM0') - time.sleep(.1) + time.sleep(.2) # print (message) - assert "\"IsOpen\": false," in message[10] + assert any("\"IsOpen\": false," in i for i in message) # callback called by socketio when a message is received @@ -147,9 +150,9 @@ def extract_serial_data(msg): serial_data = "" for i in msg: if "{\"P\"" in i: - # print (json.loads(i)["D"]) + print (json.loads(i)["D"]) serial_data+=json.loads(i)["D"] - # print("serialdata:"+serial_data) + print("serialdata:"+serial_data) return serial_data def decode_output(raw_output): From fcaa53c4839f8fe910980037093553c68667c3f3 Mon Sep 17 00:00:00 2001 From: umbynos Date: Thu, 29 Jul 2021 17:06:44 +0200 Subject: [PATCH 15/37] generalize the tests --- test/test_ws.py | 127 ++++++++++++++++++------------------------------ 1 file changed, 46 insertions(+), 81 deletions(-) diff --git a/test/test_ws.py b/test/test_ws.py index 889c7786a..a09d36f7e 100644 --- a/test/test_ws.py +++ b/test/test_ws.py @@ -26,115 +26,80 @@ def test_list(socketio): reason="VMs have no serial ports", ) def test_open_serial_default(socketio): - time.sleep(.2) - global message - message = [] - socketio.on('message', message_handler) - socketio.emit('command', 'open /dev/ttyACM0 9600') - time.sleep(.5) # give time to message to be filled - assert any("\"IsOpen\": true" in i for i in message) - socketio.emit('command', 'send /dev/ttyACM0 /"ciao/"') - time.sleep(.2) - assert any("send /dev/ttyACM0 /\"ciao/\"" in i for i in message) - assert "ciao" in extract_serial_data(message) + general_test_serial(socketio, "default") - # test with a lot of emoji: they can be messed up - # message = [] # reinitialize the message buffer - socketio.emit('command', 'send /dev/ttyACM0 /"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/"') - time.sleep(.2) - print (message) - assert any("send /dev/ttyACM0 /\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in i for i in message) - emoji_output = extract_serial_data(message) - assert "/\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in emoji_output # this could be failing because of UTF8 encoding problems - message = [] - socketio.emit('command', 'close /dev/ttyACM0') - time.sleep(.2) - assert any("\"IsOpen\": false," in i for i in message) @pytest.mark.skipif( running_on_ci(), reason="VMs have no serial ports", ) def test_open_serial_timed(socketio): - time.sleep(.2) - global message - message = [] - socketio.on('message', message_handler) - socketio.emit('command', 'open /dev/ttyACM0 9600 timed') - time.sleep(.5) # give time to message to be filled - print(message) - assert any("\"IsOpen\": true" in i for i in message) - socketio.emit('command', 'send /dev/ttyACM0 /"ciao/"') - time.sleep(.2) - assert any("send /dev/ttyACM0 /\"ciao/\"" in i for i in message) - assert "ciao" in extract_serial_data(message) + general_test_serial(socketio, "timed") - # test with a lot of emoji: usually they get messed up - message = [] # reinitialize the message buffer - socketio.emit('command', 'send /dev/ttyACM0 /"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/"') - time.sleep(.2) - assert any("send /dev/ttyACM0 /\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in i for i in message) - assert "/\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in extract_serial_data(message) - message = [] - socketio.emit('command', 'close /dev/ttyACM0') - time.sleep(.2) - # print (message) - assert any("\"IsOpen\": false," in i for i in message) @pytest.mark.skipif( running_on_ci(), reason="VMs have no serial ports", ) def test_open_serial_timedraw(socketio): - global message - message = [] - socketio.on('message', message_handler) - socketio.emit('command', 'open /dev/ttyACM0 9600 timedraw') - time.sleep(.5) # give time to message to be filled - assert any("\"IsOpen\": true" in i for i in message) - socketio.emit('command', 'send /dev/ttyACM0 /"ciao/"') - time.sleep(.2) - assert any("send /dev/ttyACM0 /\"ciao/\"" in i for i in message) - assert "ciao" in decode_output(extract_serial_data(message)) + general_test_serial(socketio, "timedraw") - # test with a lot of emoji: usually they get messed up - message = [] # reinitialize the message buffer - socketio.emit('command', 'send /dev/ttyACM0 /"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/"') - time.sleep(.2) - assert any("send /dev/ttyACM0 /\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in i for i in message) - # print (message) - assert "/\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in decode_output(extract_serial_data(message)) - socketio.emit('command', 'close /dev/ttyACM0') - time.sleep(.2) - # print (message) - assert any("\"IsOpen\": false," in i for i in message) @pytest.mark.skipif( running_on_ci(), reason="VMs have no serial ports", ) def test_open_serial_timedbinary(socketio): + general_test_serial(socketio, "timedbinary") + + +def general_test_serial(socketio, buffertype): + port = "/dev/ttyACM0" global message message = [] + #in message var we will find the "response" socketio.on('message', message_handler) - socketio.emit('command', 'open /dev/ttyACM0 9600 timedbinary') - time.sleep(.5) # give time to message to be filled + #open a new serial connection with the specified buffertype, if buffertype s empty it will use the default one + socketio.emit('command', 'open ' + port + ' 9600 ' + buffertype) + # give time to the message var to be filled + time.sleep(.5) + print(message) + # the serial connection should be open now assert any("\"IsOpen\": true" in i for i in message) - socketio.emit('command', 'send /dev/ttyACM0 /"ciao/"') - time.sleep(.2) - assert any("send /dev/ttyACM0 /\"ciao/\"" in i for i in message) - print (message) - assert "ciao" in decode_output(extract_serial_data(message)) - # test with a lot of emoji: usually they get messed up - message = [] # reinitialize the message buffer - socketio.emit('command', 'send /dev/ttyACM0 /"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/"') + #test with string + # send the string "ciao" using the serial connection + socketio.emit('command', 'send ' + port + ' /"ciao/"') + time.sleep(1) + print(message) + # check if the send command has been registered + assert any("send " + port + " /\"ciao/\"" in i for i in message) + #check if message has been sent back by the connected board + if buffertype in ("timedbinary", "timedraw"): + output = decode_output(extract_serial_data(message)) + elif buffertype in ("default", "timed"): + output = extract_serial_data(message) + assert "ciao" in output + + #test with emoji + message = [] # reinitialize the message buffer to have a clean situation + # send a lot of emoji: they can be messed up + socketio.emit('command', 'send ' + port + ' /"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/"') time.sleep(.2) - assert any("send /dev/ttyACM0 /\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in i for i in message) - assert "/\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in decode_output(extract_serial_data(message)) - socketio.emit('command', 'close /dev/ttyACM0') + print(message) + # check if the send command has been registered + assert any("send " + port + " /\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in i for i in message) + if buffertype in ("timedbinary", "timedraw"): + output = decode_output(extract_serial_data(message)) + elif buffertype in ("default", "timed"): + output = extract_serial_data(message) + assert "/\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in output + + #finally close the serial port + socketio.emit('command', 'close ' + port) time.sleep(.2) - # print (message) + print (message) + #check if port has been closed assert any("\"IsOpen\": false," in i for i in message) From 43451a31f4907d3664cb04df38af4217459f5eb8 Mon Sep 17 00:00:00 2001 From: Umberto Baldi <34278123+umbynos@users.noreply.github.com> Date: Mon, 2 Aug 2021 10:53:08 +0200 Subject: [PATCH 16/37] Apply suggestions from code review Co-authored-by: per1234 --- serialport.go | 4 ++-- test/test_ws.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/serialport.go b/serialport.go index 339d1fd12..8f20d108e 100755 --- a/serialport.go +++ b/serialport.go @@ -125,11 +125,11 @@ func (p *serport) reader(buftype string) { p.bufferwatcher.OnIncomingDataBinary(ch[:n]) case "default": // the bufferbuftype is actually called default 🤷‍♂️ // save the left out bytes for the next iteration due to UTF-8 encoding - ch = append(buffered_ch.Bytes(), ch[:n]...) // TODO ch is not handled correctly: doing this way it's length is messed up. Use ch2 + ch = append(buffered_ch.Bytes(), ch[:n]...) // TODO ch is not handled correctly: doing this way its length is messed up. Use ch2 n += len(buffered_ch.Bytes()) buffered_ch.Reset() for i, w := 0, 0; i < n; i += w { - runeValue, width := utf8.DecodeRune(ch[i:n]) // try to decode the first i bytes in the buffer (UTF8 runes do not have a fixed lenght) + runeValue, width := utf8.DecodeRune(ch[i:n]) // try to decode the first i bytes in the buffer (UTF8 runes do not have a fixed length) if runeValue == utf8.RuneError { buffered_ch.Write(ch[i:n]) break diff --git a/test/test_ws.py b/test/test_ws.py index a09d36f7e..0d9a29aba 100644 --- a/test/test_ws.py +++ b/test/test_ws.py @@ -59,7 +59,7 @@ def general_test_serial(socketio, buffertype): message = [] #in message var we will find the "response" socketio.on('message', message_handler) - #open a new serial connection with the specified buffertype, if buffertype s empty it will use the default one + #open a new serial connection with the specified buffertype, if buffertype is empty it will use the default one socketio.emit('command', 'open ' + port + ' 9600 ' + buffertype) # give time to the message var to be filled time.sleep(.5) @@ -109,7 +109,7 @@ def message_handler(msg): global message message.append(msg) -# helper function used to extract serial data from it's json representation +# helper function used to extract serial data from its JSON representation # NOTE make sure to pass a clean message (maybe reinitialize the message global var before populating it) def extract_serial_data(msg): serial_data = "" From 15af2e4e4159e323e7a898559f315cb54b4e2e02 Mon Sep 17 00:00:00 2001 From: umbynos Date: Mon, 2 Aug 2021 11:07:36 +0200 Subject: [PATCH 17/37] add sketch used for testing --- test/test_ws.py | 2 +- test/testdata/SerialEcho.ino | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 test/testdata/SerialEcho.ino diff --git a/test/test_ws.py b/test/test_ws.py index 0d9a29aba..29832066a 100644 --- a/test/test_ws.py +++ b/test/test_ws.py @@ -20,7 +20,7 @@ def test_list(socketio): assert any("Ports" in i for i in message) assert any("Network" in i for i in message) -# NOTE run the following tests on linux with a board connected to the PC and with this sketch on it: https://gist.github.com/Protoneer/96db95bfb87c3befe46e +# NOTE run the following tests on linux with a board connected to the PC and with the sketch found in test/testdata/SerialEcho.ino on it @pytest.mark.skipif( running_on_ci(), reason="VMs have no serial ports", diff --git a/test/testdata/SerialEcho.ino b/test/testdata/SerialEcho.ino new file mode 100644 index 000000000..efb7c59b7 --- /dev/null +++ b/test/testdata/SerialEcho.ino @@ -0,0 +1,18 @@ +int incomingByte = 0; // for incoming serial data + +void setup() { + Serial.begin(9600); // opens serial port, sets data rate to 9600 bps +} + +void loop() { + // send data only when you receive data: + if (Serial.available() > 0) { + + // read the incoming byte: + incomingByte = Serial.read(); + + // say what you got: + Serial.print((char)incomingByte); + } + +} From 575efa1d1be3c18abff62afdec592df72ca133d8 Mon Sep 17 00:00:00 2001 From: Silvano Cerza Date: Tue, 3 Aug 2021 13:07:02 +0200 Subject: [PATCH 18/37] Fix panic closing closed channel --- hub.go | 62 +++++++++++++++++++++++----------------------------------- 1 file changed, 24 insertions(+), 38 deletions(-) diff --git a/hub.go b/hub.go index 5103789e8..0ba80bcbd 100755 --- a/hub.go +++ b/hub.go @@ -54,6 +54,27 @@ const commands = "{\"Commands\": [" + "\"hostname\", " + "\"version\"]} " +func (h *hub) unregisterConnection(c *connection) { + if _, contains := h.connections[c]; !contains { + return + } + delete(h.connections, c) + close(c.send) +} + +func (h *hub) sendToRegisteredConnections(data []byte) { + for c := range h.connections { + select { + case c.send <- data: + //log.Print("did broadcast to ") + //log.Print(c.ws.RemoteAddr()) + //c.send <- []byte("hello world") + default: + h.unregisterConnection(c) + } + } +} + func (h *hub) run() { for { select { @@ -65,49 +86,14 @@ func (h *hub) run() { c.send <- []byte("{\"Hostname\" : \"" + *hostname + "\"} ") c.send <- []byte("{\"OS\" : \"" + runtime.GOOS + "\"} ") case c := <-h.unregister: - delete(h.connections, c) - // put close in func cuz it was creating panics and want - // to isolate - func() { - // this method can panic if websocket gets disconnected - // from users browser and we see we need to unregister a couple - // of times, i.e. perhaps from incoming data from serial triggering - // an unregister. (NOT 100% sure why seeing c.send be closed twice here) - defer func() { - if e := recover(); e != nil { - log.Println("Got panic: ", e) - } - }() - close(c.send) - }() + h.unregisterConnection(c) case m := <-h.broadcast: if len(m) > 0 { checkCmd(m) - - for c := range h.connections { - select { - case c.send <- m: - //log.Print("did broadcast to ") - //log.Print(c.ws.RemoteAddr()) - //c.send <- []byte("hello world") - default: - delete(h.connections, c) - close(c.send) - } - } + h.sendToRegisteredConnections(m) } case m := <-h.broadcastSys: - for c := range h.connections { - select { - case c.send <- m: - //log.Print("did broadcast to ") - //log.Print(c.ws.RemoteAddr()) - //c.send <- []byte("hello world") - default: - delete(h.connections, c) - close(c.send) - } - } + h.sendToRegisteredConnections(m) } } } From 2d787339b89fb087da7709b828a1a45c3c862572 Mon Sep 17 00:00:00 2001 From: Umberto Baldi Date: Tue, 3 Aug 2021 15:42:13 +0200 Subject: [PATCH 19/37] apply suggestions --- hub.go | 33 ++++++++++--------- test/common.py | 3 +- test/testdata/{ => SerialEcho}/SerialEcho.ino | 0 3 files changed, 19 insertions(+), 17 deletions(-) rename test/testdata/{ => SerialEcho}/SerialEcho.ino (100%) diff --git a/hub.go b/hub.go index 0ba80bcbd..6554acf1a 100755 --- a/hub.go +++ b/hub.go @@ -38,21 +38,24 @@ var h = hub{ connections: make(map[*connection]bool), } -const commands = "{\"Commands\": [" + - "\"list\", " + - "\"open < portName > < baud > [bufferAlgorithm: ({default}, timed, timedraw, timedbinary)]\", " + - "\"send < portName > < cmd >\", " + - "\"sendnobuf < portName > < cmd >\", " + - "\"close < portName >\", " + - "\"restart\", " + - "\"exit\", " + - "\"killupload\", " + - "\"downloadtool < tool > < toolVersion: {latest} > < pack: {arduino} > < behaviour: {keep} >\", " + - "\"log\", " + - "\"memorystats\", " + - "\"gc\", " + - "\"hostname\", " + - "\"version\"]} " +const commands = `{ + "Commands": [ + "list", + "open < portName > < baud > [bufferAlgorithm: ({default}, timed, timedraw, timedbinary)]", + "send < portName > < cmd >", + "sendnobuf < portName > < cmd >", + "close < portName >", + "restart", + "exit", + "killupload", + "downloadtool < tool > < toolVersion: {latest} > < pack: {arduino} > < behaviour: {keep} >", + "log", + "memorystats", + "gc", + "hostname", + "version" + ] +}` func (h *hub) unregisterConnection(c *connection) { if _, contains := h.connections[c]; !contains { diff --git a/test/common.py b/test/common.py index 9d05ce26b..34e16b523 100644 --- a/test/common.py +++ b/test/common.py @@ -4,5 +4,4 @@ def running_on_ci(): """ Returns whether the program is running on a CI environment """ - val = os.getenv("GITHUB_WORKFLOW") - return val is not None + return 'GITHUB_WORKFLOW' in os.environ diff --git a/test/testdata/SerialEcho.ino b/test/testdata/SerialEcho/SerialEcho.ino similarity index 100% rename from test/testdata/SerialEcho.ino rename to test/testdata/SerialEcho/SerialEcho.ino From 8f9ff203f81a32bf82a0764fecd7e807004f28c5 Mon Sep 17 00:00:00 2001 From: Umberto Baldi Date: Tue, 3 Aug 2021 17:30:06 +0200 Subject: [PATCH 20/37] Partially revert #e80400b7ddbbc2e8f34f1e6701b55102c3a99289 --- bufferflow.go | 1 - bufferflow_default.go | 4 ---- bufferflow_timed.go | 4 ---- bufferflow_timedbinary.go | 5 +---- bufferflow_timedraw.go | 4 ---- serialport.go | 4 +--- 6 files changed, 2 insertions(+), 20 deletions(-) diff --git a/bufferflow.go b/bufferflow.go index 565e173b7..b3132be00 100644 --- a/bufferflow.go +++ b/bufferflow.go @@ -19,7 +19,6 @@ type Bufferflow interface { Init() BlockUntilReady(cmd string, id string) (bool, bool) // implement this method //JustQueue(cmd string, id string) bool // implement this method - OnIncomingDataBinary(data []byte) OnIncomingData(data string) // implement this method ClearOutSemaphore() // implement this method BreakApartCommands(cmd string) []string // implement this method diff --git a/bufferflow_default.go b/bufferflow_default.go index 4bfe89b9f..7f37ac3d2 100644 --- a/bufferflow_default.go +++ b/bufferflow_default.go @@ -20,10 +20,6 @@ func (b *BufferflowDefault) BlockUntilReady(cmd string, id string) (bool, bool) return true, false } -// not implemented, we are gonna use OnIncomingData -func (b *BufferflowDefault) OnIncomingDataBinary(data []byte) { -} - func (b *BufferflowDefault) OnIncomingData(data string) { //log.Printf("OnIncomingData() start. data:%v\n", data) } diff --git a/bufferflow_timed.go b/bufferflow_timed.go index 6b7a0af14..9ae0453a2 100644 --- a/bufferflow_timed.go +++ b/bufferflow_timed.go @@ -61,10 +61,6 @@ func (b *BufferflowTimed) BlockUntilReady(cmd string, id string) (bool, bool) { return true, false } -// not implemented, we are gonna use OnIncomingData -func (b *BufferflowTimed) OnIncomingDataBinary(data []byte) { -} - func (b *BufferflowTimed) OnIncomingData(data string) { b.Input <- data } diff --git a/bufferflow_timedbinary.go b/bufferflow_timedbinary.go index 6ac689706..d58a7058b 100644 --- a/bufferflow_timedbinary.go +++ b/bufferflow_timedbinary.go @@ -57,12 +57,9 @@ func (b *BufferflowTimedBinary) BlockUntilReady(cmd string, id string) (bool, bo return true, false } -func (b *BufferflowTimedBinary) OnIncomingDataBinary(data []byte) { - b.Input <- data -} - // not implemented, we are gonna use OnIncomingDataBinary func (b *BufferflowTimedBinary) OnIncomingData(data string) { + b.Input <- []byte(data) } // Clean out b.sem so it can truly block diff --git a/bufferflow_timedraw.go b/bufferflow_timedraw.go index 5f94cf047..019ed5af4 100644 --- a/bufferflow_timedraw.go +++ b/bufferflow_timedraw.go @@ -59,10 +59,6 @@ func (b *BufferflowTimedRaw) BlockUntilReady(cmd string, id string) (bool, bool) return true, false } -// not implemented, we are gonna use OnIncomingData -func (b *BufferflowTimedRaw) OnIncomingDataBinary(data []byte) { -} - func (b *BufferflowTimedRaw) OnIncomingData(data string) { b.Input <- data } diff --git a/serialport.go b/serialport.go index 8f20d108e..82f1ddc10 100755 --- a/serialport.go +++ b/serialport.go @@ -118,11 +118,9 @@ func (p *serport) reader(buftype string) { data := "" switch buftype { - case "timedraw", "timed": // data is handled differently inside BufferFlowTimed (bufferedOutput is string) and BufferFlowTimedRaw (bufferedOutputRaw is []byte) + case "timedraw", "timed", "timedbinary": data = string(ch[:n]) p.bufferwatcher.OnIncomingData(data) - case "timedbinary": - p.bufferwatcher.OnIncomingDataBinary(ch[:n]) case "default": // the bufferbuftype is actually called default 🤷‍♂️ // save the left out bytes for the next iteration due to UTF-8 encoding ch = append(buffered_ch.Bytes(), ch[:n]...) // TODO ch is not handled correctly: doing this way its length is messed up. Use ch2 From 0bbb45b537e3e902472904c574755dfd8ef6aa01 Mon Sep 17 00:00:00 2001 From: umbynos Date: Wed, 4 Aug 2021 13:07:28 +0200 Subject: [PATCH 21/37] =?UTF-8?q?=F0=9F=A7=B9(cleanup)=20and=20?= =?UTF-8?q?=F0=9F=9B=A0=EF=B8=8F(refactoring)=20of=20bufferflow=20stuff?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bufferflow.go | 38 ++-------------- bufferflow_default.go | 48 +++----------------- bufferflow_timed.go | 91 ++++++++++++------------------------- bufferflow_timedbinary.go | 95 ++++++++++++--------------------------- bufferflow_timedraw.go | 93 ++++++++++++-------------------------- serialport.go | 8 ++-- 6 files changed, 100 insertions(+), 273 deletions(-) diff --git a/bufferflow.go b/bufferflow.go index b3132be00..4d2d2d452 100644 --- a/bufferflow.go +++ b/bufferflow.go @@ -1,11 +1,6 @@ package main -import ( -//"log" -//"time" -) - -var availableBufferAlgorithms = []string{"default", "timed", "timedraw", "timedbinary"} +// availableBufferAlgorithms = {"default", "timed", "timedraw", "timedbinary"} type BufferMsg struct { Cmd string @@ -18,32 +13,7 @@ type BufferMsg struct { type Bufferflow interface { Init() BlockUntilReady(cmd string, id string) (bool, bool) // implement this method - //JustQueue(cmd string, id string) bool // implement this method - OnIncomingData(data string) // implement this method - ClearOutSemaphore() // implement this method - BreakApartCommands(cmd string) []string // implement this method - Pause() // implement this method - Unpause() // implement this method - SeeIfSpecificCommandsShouldSkipBuffer(cmd string) bool // implement this method - SeeIfSpecificCommandsShouldPauseBuffer(cmd string) bool // implement this method - SeeIfSpecificCommandsShouldUnpauseBuffer(cmd string) bool // implement this method - SeeIfSpecificCommandsShouldWipeBuffer(cmd string) bool // implement this method - SeeIfSpecificCommandsReturnNoResponse(cmd string) bool // implement this method - ReleaseLock() // implement this method - IsBufferGloballySendingBackIncomingData() bool // implement this method - Close() // implement this method -} - -/*data packets returned to client*/ -type DataCmdComplete struct { - Cmd string - Id string - P string - BufSize int `json:"-"` - D string `json:"-"` -} - -type DataPerLine struct { - P string - D string + OnIncomingData(data string) // implement this method + IsBufferGloballySendingBackIncomingData() bool // implement this method + Close() // implement this method } diff --git a/bufferflow_default.go b/bufferflow_default.go index 7f37ac3d2..bd174b1b9 100644 --- a/bufferflow_default.go +++ b/bufferflow_default.go @@ -5,11 +5,14 @@ import ( ) type BufferflowDefault struct { - Name string - Port string + port string } -var () +func NewBufferflowDefault(port string) *BufferflowDefault { + return &BufferflowDefault{ + port: port, + } +} func (b *BufferflowDefault) Init() { log.Println("Initting default buffer flow (which means no buffering)") @@ -24,45 +27,6 @@ func (b *BufferflowDefault) OnIncomingData(data string) { //log.Printf("OnIncomingData() start. data:%v\n", data) } -// Clean out b.sem so it can truly block -func (b *BufferflowDefault) ClearOutSemaphore() { -} - -func (b *BufferflowDefault) BreakApartCommands(cmd string) []string { - return []string{cmd} -} - -func (b *BufferflowDefault) Pause() { - return -} - -func (b *BufferflowDefault) Unpause() { - return -} - -func (b *BufferflowDefault) SeeIfSpecificCommandsShouldSkipBuffer(cmd string) bool { - return false -} - -func (b *BufferflowDefault) SeeIfSpecificCommandsShouldPauseBuffer(cmd string) bool { - return false -} - -func (b *BufferflowDefault) SeeIfSpecificCommandsShouldUnpauseBuffer(cmd string) bool { - return false -} - -func (b *BufferflowDefault) SeeIfSpecificCommandsShouldWipeBuffer(cmd string) bool { - return false -} - -func (b *BufferflowDefault) SeeIfSpecificCommandsReturnNoResponse(cmd string) bool { - return false -} - -func (b *BufferflowDefault) ReleaseLock() { -} - func (b *BufferflowDefault) IsBufferGloballySendingBackIncomingData() bool { return false } diff --git a/bufferflow_timed.go b/bufferflow_timed.go index 9ae0453a2..966aa9025 100644 --- a/bufferflow_timed.go +++ b/bufferflow_timed.go @@ -8,49 +8,53 @@ import ( ) type BufferflowTimed struct { - Name string - Port string - Output chan []byte - Input chan string - done chan bool - ticker *time.Ticker + port string + output chan []byte + input chan string + done chan bool + ticker *time.Ticker + sPort string + bufferedOutput string } -var ( - bufferedOutput string - sPort string -) +func NewBufferflowTimed(port string, output chan []byte) *BufferflowTimed { + return &BufferflowTimed{ + port: port, + output: output, + input: make(chan string), + done: make(chan bool), + ticker: time.NewTicker(16 * time.Millisecond), + sPort: "", + bufferedOutput: "", + } +} func (b *BufferflowTimed) Init() { log.Println("Initting timed buffer flow (output once every 16ms)") - bufferedOutput = "" - sPort = "" go func() { - b.ticker = time.NewTicker(16 * time.Millisecond) - b.done = make(chan bool) Loop: for { select { - case data := <-b.Input: - bufferedOutput = bufferedOutput + data - sPort = b.Port + case data := <-b.input: + b.bufferedOutput = b.bufferedOutput + data + b.sPort = b.port case <-b.ticker.C: - if bufferedOutput != "" { - m := SpPortMessage{sPort, bufferedOutput} + if b.bufferedOutput != "" { + m := SpPortMessage{b.sPort, b.bufferedOutput} buf, _ := json.Marshal(m) // data is now encoded in base64 format // need a decoder on the other side - b.Output <- []byte(buf) - bufferedOutput = "" - sPort = "" + b.output <- []byte(buf) + b.bufferedOutput = "" + b.sPort = "" } case <-b.done: break Loop } } - close(b.Input) + close(b.input) }() @@ -62,46 +66,7 @@ func (b *BufferflowTimed) BlockUntilReady(cmd string, id string) (bool, bool) { } func (b *BufferflowTimed) OnIncomingData(data string) { - b.Input <- data -} - -// Clean out b.sem so it can truly block -func (b *BufferflowTimed) ClearOutSemaphore() { -} - -func (b *BufferflowTimed) BreakApartCommands(cmd string) []string { - return []string{cmd} -} - -func (b *BufferflowTimed) Pause() { - return -} - -func (b *BufferflowTimed) Unpause() { - return -} - -func (b *BufferflowTimed) SeeIfSpecificCommandsShouldSkipBuffer(cmd string) bool { - return false -} - -func (b *BufferflowTimed) SeeIfSpecificCommandsShouldPauseBuffer(cmd string) bool { - return false -} - -func (b *BufferflowTimed) SeeIfSpecificCommandsShouldUnpauseBuffer(cmd string) bool { - return false -} - -func (b *BufferflowTimed) SeeIfSpecificCommandsShouldWipeBuffer(cmd string) bool { - return false -} - -func (b *BufferflowTimed) SeeIfSpecificCommandsReturnNoResponse(cmd string) bool { - return false -} - -func (b *BufferflowTimed) ReleaseLock() { + b.input <- data } func (b *BufferflowTimed) IsBufferGloballySendingBackIncomingData() bool { diff --git a/bufferflow_timedbinary.go b/bufferflow_timedbinary.go index d58a7058b..59eecfd91 100644 --- a/bufferflow_timedbinary.go +++ b/bufferflow_timedbinary.go @@ -8,47 +8,50 @@ import ( ) type BufferflowTimedBinary struct { - Name string - Port string - Output chan []byte - Input chan []byte - done chan bool - ticker *time.Ticker -} - -var ( + port string + output chan []byte + input chan []byte + done chan bool + ticker *time.Ticker bufferedOutputBinary []byte sPortBinary string -) +} + +func NewBufferflowTimedBinary(port string, output chan []byte) *BufferflowTimedBinary { + return &BufferflowTimedBinary{ + port: port, + output: output, + input: make(chan []byte), + done: make(chan bool), + ticker: time.NewTicker(16 * time.Millisecond), + bufferedOutputBinary: nil, + sPortBinary: "", + } +} func (b *BufferflowTimedBinary) Init() { log.Println("Initting timed buffer binary flow (output once every 16ms)") - bufferedOutputBinary = nil - sPortBinary = "" - go func() { - b.ticker = time.NewTicker(16 * time.Millisecond) - b.done = make(chan bool) Loop: for { select { - case data := <-b.Input: - bufferedOutputBinary = append(bufferedOutputBinary, data...) - sPortBinary = b.Port + case data := <-b.input: + b.bufferedOutputBinary = append(b.bufferedOutputBinary, data...) + b.sPortBinary = b.port case <-b.ticker.C: - if bufferedOutputBinary != nil { - m := SpPortMessageRaw{sPortBinary, bufferedOutputBinary} + if b.bufferedOutputBinary != nil { + m := SpPortMessageRaw{b.sPortBinary, b.bufferedOutputBinary} buf, _ := json.Marshal(m) - b.Output <- buf - bufferedOutputBinary = nil - sPortBinary = "" + b.output <- buf + b.bufferedOutputBinary = nil + b.sPortBinary = "" } case <-b.done: break Loop } } - close(b.Input) + close(b.input) }() } @@ -57,48 +60,8 @@ func (b *BufferflowTimedBinary) BlockUntilReady(cmd string, id string) (bool, bo return true, false } -// not implemented, we are gonna use OnIncomingDataBinary func (b *BufferflowTimedBinary) OnIncomingData(data string) { - b.Input <- []byte(data) -} - -// Clean out b.sem so it can truly block -func (b *BufferflowTimedBinary) ClearOutSemaphore() { -} - -func (b *BufferflowTimedBinary) BreakApartCommands(cmd string) []string { - return []string{cmd} -} - -func (b *BufferflowTimedBinary) Pause() { - return -} - -func (b *BufferflowTimedBinary) Unpause() { - return -} - -func (b *BufferflowTimedBinary) SeeIfSpecificCommandsShouldSkipBuffer(cmd string) bool { - return false -} - -func (b *BufferflowTimedBinary) SeeIfSpecificCommandsShouldPauseBuffer(cmd string) bool { - return false -} - -func (b *BufferflowTimedBinary) SeeIfSpecificCommandsShouldUnpauseBuffer(cmd string) bool { - return false -} - -func (b *BufferflowTimedBinary) SeeIfSpecificCommandsShouldWipeBuffer(cmd string) bool { - return false -} - -func (b *BufferflowTimedBinary) SeeIfSpecificCommandsReturnNoResponse(cmd string) bool { - return false -} - -func (b *BufferflowTimedBinary) ReleaseLock() { + b.input <- []byte(data) } func (b *BufferflowTimedBinary) IsBufferGloballySendingBackIncomingData() bool { @@ -107,5 +70,5 @@ func (b *BufferflowTimedBinary) IsBufferGloballySendingBackIncomingData() bool { func (b *BufferflowTimedBinary) Close() { b.ticker.Stop() - close(b.Input) + close(b.input) } diff --git a/bufferflow_timedraw.go b/bufferflow_timedraw.go index 019ed5af4..a67d2d6c0 100644 --- a/bufferflow_timedraw.go +++ b/bufferflow_timedraw.go @@ -8,49 +8,53 @@ import ( ) type BufferflowTimedRaw struct { - Name string - Port string - Output chan []byte - Input chan string - done chan bool - ticker *time.Ticker -} - -var ( + port string + output chan []byte + input chan string + done chan bool + ticker *time.Ticker bufferedOutputRaw []byte sPortRaw string -) +} + +func NewBufferflowTimedRaw(port string, output chan []byte) *BufferflowTimedRaw { + return &BufferflowTimedRaw{ + port: port, + output: output, + input: make(chan string), + done: make(chan bool), + ticker: time.NewTicker(16 * time.Millisecond), + bufferedOutputRaw: nil, + sPortRaw: "", + } +} func (b *BufferflowTimedRaw) Init() { log.Println("Initting timed buffer raw flow (output once every 16ms)") - bufferedOutputRaw = nil - sPortRaw = "" go func() { - b.ticker = time.NewTicker(16 * time.Millisecond) - b.done = make(chan bool) Loop: for { select { - case data := <-b.Input: - bufferedOutputRaw = append(bufferedOutputRaw, []byte(data)...) - sPortRaw = b.Port + case data := <-b.input: + b.bufferedOutputRaw = append(b.bufferedOutputRaw, []byte(data)...) + b.sPortRaw = b.port case <-b.ticker.C: - if bufferedOutputRaw != nil { - m := SpPortMessageRaw{sPortRaw, bufferedOutputRaw} + if b.bufferedOutputRaw != nil { + m := SpPortMessageRaw{b.sPortRaw, b.bufferedOutputRaw} buf, _ := json.Marshal(m) // data is now encoded in base64 format // need a decoder on the other side - b.Output <- []byte(buf) - bufferedOutputRaw = nil - sPortRaw = "" + b.output <- []byte(buf) + b.bufferedOutputRaw = nil + b.sPortRaw = "" } case <-b.done: break Loop } } - close(b.Input) + close(b.input) }() } @@ -60,46 +64,7 @@ func (b *BufferflowTimedRaw) BlockUntilReady(cmd string, id string) (bool, bool) } func (b *BufferflowTimedRaw) OnIncomingData(data string) { - b.Input <- data -} - -// Clean out b.sem so it can truly block -func (b *BufferflowTimedRaw) ClearOutSemaphore() { -} - -func (b *BufferflowTimedRaw) BreakApartCommands(cmd string) []string { - return []string{cmd} -} - -func (b *BufferflowTimedRaw) Pause() { - return -} - -func (b *BufferflowTimedRaw) Unpause() { - return -} - -func (b *BufferflowTimedRaw) SeeIfSpecificCommandsShouldSkipBuffer(cmd string) bool { - return false -} - -func (b *BufferflowTimedRaw) SeeIfSpecificCommandsShouldPauseBuffer(cmd string) bool { - return false -} - -func (b *BufferflowTimedRaw) SeeIfSpecificCommandsShouldUnpauseBuffer(cmd string) bool { - return false -} - -func (b *BufferflowTimedRaw) SeeIfSpecificCommandsShouldWipeBuffer(cmd string) bool { - return false -} - -func (b *BufferflowTimedRaw) SeeIfSpecificCommandsReturnNoResponse(cmd string) bool { - return false -} - -func (b *BufferflowTimedRaw) ReleaseLock() { + b.input <- data } func (b *BufferflowTimedRaw) IsBufferGloballySendingBackIncomingData() bool { @@ -108,5 +73,5 @@ func (b *BufferflowTimedRaw) IsBufferGloballySendingBackIncomingData() bool { func (b *BufferflowTimedRaw) Close() { b.ticker.Stop() - close(b.Input) + close(b.input) } diff --git a/serialport.go b/serialport.go index 82f1ddc10..c84bfd926 100755 --- a/serialport.go +++ b/serialport.go @@ -325,13 +325,13 @@ func spHandlerOpen(portname string, baud int, buftype string) { switch buftype { case "timed": - bw = &BufferflowTimed{Name: "timed", Port: portname, Output: h.broadcastSys, Input: make(chan string)} + bw = NewBufferflowTimed(portname, h.broadcastSys) case "timedraw": - bw = &BufferflowTimedRaw{Name: "timedraw", Port: portname, Output: h.broadcastSys, Input: make(chan string)} + bw = NewBufferflowTimedRaw(portname, h.broadcastSys) case "timedbinary": - bw = &BufferflowTimedBinary{Name: "timedbinary", Port: portname, Output: h.broadcastSys, Input: make(chan []byte)} + bw = NewBufferflowTimedBinary(portname, h.broadcastSys) case "default": - bw = &BufferflowDefault{Port: portname} + bw = NewBufferflowDefault(portname) default: log.Panicf("unknown buffer type: %s", buftype) } From 5e2ad37f417d8761f238405e7dc2455435d35648 Mon Sep 17 00:00:00 2001 From: umbynos Date: Wed, 4 Aug 2021 16:37:06 +0200 Subject: [PATCH 22/37] extract code in helper function and uniform the code reintroduce the closing of input channel (it's required) --- bufferflow_timed.go | 47 +++++++++++++++++++-------------------- bufferflow_timedbinary.go | 45 ++++++++++++++++++++----------------- bufferflow_timedraw.go | 46 ++++++++++++++++++++------------------ 3 files changed, 72 insertions(+), 66 deletions(-) diff --git a/bufferflow_timed.go b/bufferflow_timed.go index 966aa9025..9c654c18d 100644 --- a/bufferflow_timed.go +++ b/bufferflow_timed.go @@ -31,33 +31,32 @@ func NewBufferflowTimed(port string, output chan []byte) *BufferflowTimed { func (b *BufferflowTimed) Init() { log.Println("Initting timed buffer flow (output once every 16ms)") + go b.consumeInput() +} - go func() { - Loop: - for { - select { - case data := <-b.input: - b.bufferedOutput = b.bufferedOutput + data - b.sPort = b.port - case <-b.ticker.C: - if b.bufferedOutput != "" { - m := SpPortMessage{b.sPort, b.bufferedOutput} - buf, _ := json.Marshal(m) - // data is now encoded in base64 format - // need a decoder on the other side - b.output <- []byte(buf) - b.bufferedOutput = "" - b.sPort = "" - } - case <-b.done: - break Loop +func (b *BufferflowTimed) consumeInput() { +Loop: + for { + select { + case data := <-b.input: // use the buffer and append data to it + b.bufferedOutput = b.bufferedOutput + data + b.sPort = b.port + case <-b.ticker.C: // after 16ms send the buffered output message + if b.bufferedOutput != "" { + m := SpPortMessage{b.sPort, b.bufferedOutput} + buf, _ := json.Marshal(m) + // data is now encoded in base64 format + // need a decoder on the other side + b.output <- buf + // reset the buffer and the port + b.bufferedOutput = "" + b.sPort = "" } + case <-b.done: + break Loop //this is required, a simple break statement would only exit the innermost switch statement } - - close(b.input) - - }() - + } + close(b.input) } func (b *BufferflowTimed) BlockUntilReady(cmd string, id string) (bool, bool) { diff --git a/bufferflow_timedbinary.go b/bufferflow_timedbinary.go index 59eecfd91..5e18c720f 100644 --- a/bufferflow_timedbinary.go +++ b/bufferflow_timedbinary.go @@ -31,28 +31,32 @@ func NewBufferflowTimedBinary(port string, output chan []byte) *BufferflowTimedB func (b *BufferflowTimedBinary) Init() { log.Println("Initting timed buffer binary flow (output once every 16ms)") - go func() { - Loop: - for { - select { - case data := <-b.input: - b.bufferedOutputBinary = append(b.bufferedOutputBinary, data...) - b.sPortBinary = b.port - case <-b.ticker.C: - if b.bufferedOutputBinary != nil { - m := SpPortMessageRaw{b.sPortBinary, b.bufferedOutputBinary} - buf, _ := json.Marshal(m) - b.output <- buf - b.bufferedOutputBinary = nil - b.sPortBinary = "" - } - case <-b.done: - break Loop + go b.consumeInput() +} + +func (b *BufferflowTimedBinary) consumeInput() { +Loop: + for { + select { + case data := <-b.input: // use the buffer and append data to it + b.bufferedOutputBinary = append(b.bufferedOutputBinary, data...) + b.sPortBinary = b.port + case <-b.ticker.C: // after 16ms send the buffered output message + if b.bufferedOutputBinary != nil { + m := SpPortMessageRaw{b.sPortBinary, b.bufferedOutputBinary} + buf, _ := json.Marshal(m) + // data is now encoded in base64 format + // need a decoder on the other side + b.output <- buf + // reset the buffer and the port + b.bufferedOutputBinary = nil + b.sPortBinary = "" } + case <-b.done: + break Loop //this is required, a simple break statement would only exit the innermost switch statement } - - close(b.input) - }() + } + close(b.input) } func (b *BufferflowTimedBinary) BlockUntilReady(cmd string, id string) (bool, bool) { @@ -70,5 +74,6 @@ func (b *BufferflowTimedBinary) IsBufferGloballySendingBackIncomingData() bool { func (b *BufferflowTimedBinary) Close() { b.ticker.Stop() + b.done <- true close(b.input) } diff --git a/bufferflow_timedraw.go b/bufferflow_timedraw.go index a67d2d6c0..6078ca85f 100644 --- a/bufferflow_timedraw.go +++ b/bufferflow_timedraw.go @@ -31,31 +31,32 @@ func NewBufferflowTimedRaw(port string, output chan []byte) *BufferflowTimedRaw func (b *BufferflowTimedRaw) Init() { log.Println("Initting timed buffer raw flow (output once every 16ms)") + go b.consumeInput() +} - go func() { - Loop: - for { - select { - case data := <-b.input: - b.bufferedOutputRaw = append(b.bufferedOutputRaw, []byte(data)...) - b.sPortRaw = b.port - case <-b.ticker.C: - if b.bufferedOutputRaw != nil { - m := SpPortMessageRaw{b.sPortRaw, b.bufferedOutputRaw} - buf, _ := json.Marshal(m) - // data is now encoded in base64 format - // need a decoder on the other side - b.output <- []byte(buf) - b.bufferedOutputRaw = nil - b.sPortRaw = "" - } - case <-b.done: - break Loop +func (b *BufferflowTimedRaw) consumeInput() { +Loop: + for { + select { + case data := <-b.input: // use the buffer and append data to it + b.bufferedOutputRaw = append(b.bufferedOutputRaw, []byte(data)...) + b.sPortRaw = b.port + case <-b.ticker.C: // after 16ms send the buffered output message + if b.bufferedOutputRaw != nil { + m := SpPortMessageRaw{b.sPortRaw, b.bufferedOutputRaw} + buf, _ := json.Marshal(m) + // data is now encoded in base64 format + // need a decoder on the other side + b.output <- buf + // reset the buffer and the port + b.bufferedOutputRaw = nil + b.sPortRaw = "" } + case <-b.done: + break Loop //this is required, a simple break statement would only exit the innermost switch statement } - - close(b.input) - }() + } + close(b.input) } func (b *BufferflowTimedRaw) BlockUntilReady(cmd string, id string) (bool, bool) { @@ -73,5 +74,6 @@ func (b *BufferflowTimedRaw) IsBufferGloballySendingBackIncomingData() bool { func (b *BufferflowTimedRaw) Close() { b.ticker.Stop() + b.done <- true close(b.input) } From 5d0bd27f28ef6bb1b98a2cdadf9a4860447e1094 Mon Sep 17 00:00:00 2001 From: umbynos Date: Wed, 4 Aug 2021 17:44:47 +0200 Subject: [PATCH 23/37] optimize the handling of data coming from the serial port --- serialport.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/serialport.go b/serialport.go index c84bfd926..433cfb5ef 100755 --- a/serialport.go +++ b/serialport.go @@ -98,9 +98,10 @@ func (p *serport) reader(buftype string) { timeCheckOpen := time.Now() var buffered_ch bytes.Buffer + serialBuffer := make([]byte, 1024) for { - ch := make([]byte, 1024) //a new array of bytes is initilized everytime because we pass it (as a pointer) in a channel, it can be improved - n, err := p.portIo.Read(ch) + n, err := p.portIo.Read(serialBuffer) + bufferPart := serialBuffer[:n] //if we detect that port is closing, break out of this for{} loop. if p.isClosing { @@ -114,22 +115,22 @@ func (p *serport) reader(buftype string) { // so process the n bytes red, if n > 0 if n > 0 && err == nil { - log.Print("Read " + strconv.Itoa(n) + " bytes ch: " + string(ch[:n])) + log.Print("Read " + strconv.Itoa(n) + " bytes ch: " + string(bufferPart[:n])) data := "" switch buftype { case "timedraw", "timed", "timedbinary": - data = string(ch[:n]) + data = string(bufferPart[:n]) p.bufferwatcher.OnIncomingData(data) case "default": // the bufferbuftype is actually called default 🤷‍♂️ // save the left out bytes for the next iteration due to UTF-8 encoding - ch = append(buffered_ch.Bytes(), ch[:n]...) // TODO ch is not handled correctly: doing this way its length is messed up. Use ch2 + bufferPart = append(buffered_ch.Bytes(), bufferPart[:n]...) n += len(buffered_ch.Bytes()) buffered_ch.Reset() for i, w := 0, 0; i < n; i += w { - runeValue, width := utf8.DecodeRune(ch[i:n]) // try to decode the first i bytes in the buffer (UTF8 runes do not have a fixed length) + runeValue, width := utf8.DecodeRune(bufferPart[i:n]) // try to decode the first i bytes in the buffer (UTF8 runes do not have a fixed length) if runeValue == utf8.RuneError { - buffered_ch.Write(ch[i:n]) + buffered_ch.Write(bufferPart[i:n]) break } if i == n { From f3d5dca4b1f177934c62c264ad7c302ea4c5d395 Mon Sep 17 00:00:00 2001 From: umbynos Date: Wed, 4 Aug 2021 17:49:17 +0200 Subject: [PATCH 24/37] =?UTF-8?q?uniform=20default=20bufferflow=20and=20?= =?UTF-8?q?=F0=9F=A7=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bufferflow.go | 1 - bufferflow_default.go | 40 ++++++++++++++++++++++++++++++++-------- bufferflow_timed.go | 4 ---- bufferflow_timedraw.go | 4 ---- serialport.go | 38 +++++--------------------------------- 5 files changed, 37 insertions(+), 50 deletions(-) diff --git a/bufferflow.go b/bufferflow.go index 4d2d2d452..05b6ce663 100644 --- a/bufferflow.go +++ b/bufferflow.go @@ -14,6 +14,5 @@ type Bufferflow interface { Init() BlockUntilReady(cmd string, id string) (bool, bool) // implement this method OnIncomingData(data string) // implement this method - IsBufferGloballySendingBackIncomingData() bool // implement this method Close() // implement this method } diff --git a/bufferflow_default.go b/bufferflow_default.go index bd174b1b9..9374983e2 100644 --- a/bufferflow_default.go +++ b/bufferflow_default.go @@ -1,21 +1,47 @@ package main import ( + "encoding/json" + log "github.com/sirupsen/logrus" ) type BufferflowDefault struct { - port string + port string + output chan []byte + input chan string + done chan bool } -func NewBufferflowDefault(port string) *BufferflowDefault { +func NewBufferflowDefault(port string, output chan []byte) *BufferflowDefault { return &BufferflowDefault{ - port: port, + port: port, + output: output, + input: make(chan string), + done: make(chan bool), } } func (b *BufferflowDefault) Init() { log.Println("Initting default buffer flow (which means no buffering)") + go b.consumeInput() +} + +func (b *BufferflowDefault) consumeInput() { +Loop: + for { + select { + case data := <-b.input: + m := SpPortMessage{b.port, data} + message, _ := json.Marshal(m) + // data is now encoded in base64 format + // need a decoder on the other side + b.output <- message + case <-b.done: + break Loop //this is required, a simple break statement would only exit the innermost switch statement + } + } + close(b.input) // close the input channel at the end of the computation } func (b *BufferflowDefault) BlockUntilReady(cmd string, id string) (bool, bool) { @@ -24,12 +50,10 @@ func (b *BufferflowDefault) BlockUntilReady(cmd string, id string) (bool, bool) } func (b *BufferflowDefault) OnIncomingData(data string) { - //log.Printf("OnIncomingData() start. data:%v\n", data) -} - -func (b *BufferflowDefault) IsBufferGloballySendingBackIncomingData() bool { - return false + b.input <- data } func (b *BufferflowDefault) Close() { + b.done <- true + close(b.done) } diff --git a/bufferflow_timed.go b/bufferflow_timed.go index 9c654c18d..06b5f19e1 100644 --- a/bufferflow_timed.go +++ b/bufferflow_timed.go @@ -68,10 +68,6 @@ func (b *BufferflowTimed) OnIncomingData(data string) { b.input <- data } -func (b *BufferflowTimed) IsBufferGloballySendingBackIncomingData() bool { - return true -} - func (b *BufferflowTimed) Close() { b.ticker.Stop() b.done <- true diff --git a/bufferflow_timedraw.go b/bufferflow_timedraw.go index 6078ca85f..0cb82b35c 100644 --- a/bufferflow_timedraw.go +++ b/bufferflow_timedraw.go @@ -68,10 +68,6 @@ func (b *BufferflowTimedRaw) OnIncomingData(data string) { b.input <- data } -func (b *BufferflowTimedRaw) IsBufferGloballySendingBackIncomingData() bool { - return true -} - func (b *BufferflowTimedRaw) Close() { b.ticker.Stop() b.done <- true diff --git a/serialport.go b/serialport.go index 433cfb5ef..93592f5ae 100755 --- a/serialport.go +++ b/serialport.go @@ -2,7 +2,6 @@ package main import ( "bytes" - "encoding/json" "io" "strconv" "time" @@ -121,6 +120,10 @@ func (p *serport) reader(buftype string) { switch buftype { case "timedraw", "timed", "timedbinary": data = string(bufferPart[:n]) + // give the data to our bufferflow so it can do it's work + // to read/translate the data to see if it wants to block + // writes to the serialport. each bufferflow type will decide + // this on its own based on its logic p.bufferwatcher.OnIncomingData(data) case "default": // the bufferbuftype is actually called default 🤷‍♂️ // save the left out bytes for the next iteration due to UTF-8 encoding @@ -139,41 +142,10 @@ func (p *serport) reader(buftype string) { data += string(runeValue) w = width } - // give the data to our bufferflow so it can do it's work - // to read/translate the data to see if it wants to block - // writes to the serialport. each bufferflow type will decide - // this on its own based on its logic, i.e. tinyg vs grbl vs others p.bufferwatcher.OnIncomingData(data) default: log.Panicf("unknown buffer type %s", buftype) } - - // see if the OnIncomingData handled the broadcast back - // to the user. this option was added in case the OnIncomingData wanted - // to do something fancier or implementation specific, i.e. TinyG Buffer - // actually sends back data on a perline basis rather than our method - // where we just send the moment we get it. the reason for this is that - // the browser was sometimes getting back packets out of order which - // of course would screw things up when parsing - - if p.bufferwatcher.IsBufferGloballySendingBackIncomingData() == false { - //m := SpPortMessage{"Alice", "Hello"} - m := SpPortMessage{p.portConf.Name, data} - //log.Print("The m obj struct is:") - //log.Print(m) - - //b, err := json.MarshalIndent(m, "", "\t") - b, err := json.Marshal(m) - if err != nil { - log.Println(err) - h.broadcastSys <- []byte("Error creating json on " + p.portConf.Name + " " + - err.Error() + " The data we were trying to convert is: " + string(ch[:n])) - break - } - //log.Print("Printing out json byte data...") - //log.Print(string(b)) - h.broadcastSys <- b - } } // double check that we got characters in the buffer @@ -332,7 +304,7 @@ func spHandlerOpen(portname string, baud int, buftype string) { case "timedbinary": bw = NewBufferflowTimedBinary(portname, h.broadcastSys) case "default": - bw = NewBufferflowDefault(portname) + bw = NewBufferflowDefault(portname, h.broadcastSys) default: log.Panicf("unknown buffer type: %s", buftype) } From 5db1975d22193b68fabe8547cf79c2605cc329ce Mon Sep 17 00:00:00 2001 From: umbynos Date: Wed, 4 Aug 2021 17:57:14 +0200 Subject: [PATCH 25/37] forgot to fix this in #621 --- bufferflow_timedbinary.go | 2 +- bufferflow_timedraw.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bufferflow_timedbinary.go b/bufferflow_timedbinary.go index 5e18c720f..355d39146 100644 --- a/bufferflow_timedbinary.go +++ b/bufferflow_timedbinary.go @@ -75,5 +75,5 @@ func (b *BufferflowTimedBinary) IsBufferGloballySendingBackIncomingData() bool { func (b *BufferflowTimedBinary) Close() { b.ticker.Stop() b.done <- true - close(b.input) + close(b.done) } diff --git a/bufferflow_timedraw.go b/bufferflow_timedraw.go index 0cb82b35c..1e87ad81a 100644 --- a/bufferflow_timedraw.go +++ b/bufferflow_timedraw.go @@ -71,5 +71,5 @@ func (b *BufferflowTimedRaw) OnIncomingData(data string) { func (b *BufferflowTimedRaw) Close() { b.ticker.Stop() b.done <- true - close(b.input) + close(b.done) } From 37cf9978e4f53fd642493b18d8616fbd33468a1a Mon Sep 17 00:00:00 2001 From: umbynos Date: Thu, 5 Aug 2021 12:26:00 +0200 Subject: [PATCH 26/37] =?UTF-8?q?apply=20suggestions=20from=20code=20revie?= =?UTF-8?q?w=20=E2=9C=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bufferflow.go | 10 ---------- bufferflow_default.go | 4 ++-- bufferflow_timed.go | 4 ++-- bufferflow_timedbinary.go | 8 ++------ bufferflow_timedraw.go | 4 ++-- hub.go | 7 ++++--- 6 files changed, 12 insertions(+), 25 deletions(-) diff --git a/bufferflow.go b/bufferflow.go index 05b6ce663..8f17cf592 100644 --- a/bufferflow.go +++ b/bufferflow.go @@ -1,15 +1,5 @@ package main -// availableBufferAlgorithms = {"default", "timed", "timedraw", "timedbinary"} - -type BufferMsg struct { - Cmd string - Port string - TriggeringResponse string - //Desc string - //Desc string -} - type Bufferflow interface { Init() BlockUntilReady(cmd string, id string) (bool, bool) // implement this method diff --git a/bufferflow_default.go b/bufferflow_default.go index 9374983e2..74a71f9d4 100644 --- a/bufferflow_default.go +++ b/bufferflow_default.go @@ -8,12 +8,12 @@ import ( type BufferflowDefault struct { port string - output chan []byte + output chan<- []byte input chan string done chan bool } -func NewBufferflowDefault(port string, output chan []byte) *BufferflowDefault { +func NewBufferflowDefault(port string, output chan<- []byte) *BufferflowDefault { return &BufferflowDefault{ port: port, output: output, diff --git a/bufferflow_timed.go b/bufferflow_timed.go index 06b5f19e1..6c759de4b 100644 --- a/bufferflow_timed.go +++ b/bufferflow_timed.go @@ -9,7 +9,7 @@ import ( type BufferflowTimed struct { port string - output chan []byte + output chan<- []byte input chan string done chan bool ticker *time.Ticker @@ -17,7 +17,7 @@ type BufferflowTimed struct { bufferedOutput string } -func NewBufferflowTimed(port string, output chan []byte) *BufferflowTimed { +func NewBufferflowTimed(port string, output chan<- []byte) *BufferflowTimed { return &BufferflowTimed{ port: port, output: output, diff --git a/bufferflow_timedbinary.go b/bufferflow_timedbinary.go index 355d39146..451c9ec0e 100644 --- a/bufferflow_timedbinary.go +++ b/bufferflow_timedbinary.go @@ -9,7 +9,7 @@ import ( type BufferflowTimedBinary struct { port string - output chan []byte + output chan<- []byte input chan []byte done chan bool ticker *time.Ticker @@ -17,7 +17,7 @@ type BufferflowTimedBinary struct { sPortBinary string } -func NewBufferflowTimedBinary(port string, output chan []byte) *BufferflowTimedBinary { +func NewBufferflowTimedBinary(port string, output chan<- []byte) *BufferflowTimedBinary { return &BufferflowTimedBinary{ port: port, output: output, @@ -68,10 +68,6 @@ func (b *BufferflowTimedBinary) OnIncomingData(data string) { b.input <- []byte(data) } -func (b *BufferflowTimedBinary) IsBufferGloballySendingBackIncomingData() bool { - return true -} - func (b *BufferflowTimedBinary) Close() { b.ticker.Stop() b.done <- true diff --git a/bufferflow_timedraw.go b/bufferflow_timedraw.go index 1e87ad81a..671444e47 100644 --- a/bufferflow_timedraw.go +++ b/bufferflow_timedraw.go @@ -9,7 +9,7 @@ import ( type BufferflowTimedRaw struct { port string - output chan []byte + output chan<- []byte input chan string done chan bool ticker *time.Ticker @@ -17,7 +17,7 @@ type BufferflowTimedRaw struct { sPortRaw string } -func NewBufferflowTimedRaw(port string, output chan []byte) *BufferflowTimedRaw { +func NewBufferflowTimedRaw(port string, output chan<- []byte) *BufferflowTimedRaw { return &BufferflowTimedRaw{ port: port, output: output, diff --git a/hub.go b/hub.go index 6554acf1a..1497ace8a 100755 --- a/hub.go +++ b/hub.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "fmt" "io" "os" "runtime" @@ -84,10 +85,10 @@ func (h *hub) run() { case c := <-h.register: h.connections[c] = true // send supported commands - c.send <- []byte("{\"Version\" : \"" + version + "\"} ") + c.send <- []byte(fmt.Sprintf(`{"Version" : "%s"} `, version)) c.send <- []byte(commands) - c.send <- []byte("{\"Hostname\" : \"" + *hostname + "\"} ") - c.send <- []byte("{\"OS\" : \"" + runtime.GOOS + "\"} ") + c.send <- []byte(fmt.Sprintf(`{"Hostname" : "%s"} `, *hostname)) + c.send <- []byte(fmt.Sprintf(`{"OS" : "%s"} `, runtime.GOOS)) case c := <-h.unregister: h.unregisterConnection(c) case m := <-h.broadcast: From 724de590dc512a9835457c35f52c543d5eb22510 Mon Sep 17 00:00:00 2001 From: umbynos Date: Thu, 5 Aug 2021 12:41:18 +0200 Subject: [PATCH 27/37] remove timedbinary: it's the same as timedraw except for the casting --- bufferflow_timedbinary.go | 75 --------------------------------------- serialport.go | 4 +-- test/test_ws.py | 12 ++----- 3 files changed, 3 insertions(+), 88 deletions(-) delete mode 100644 bufferflow_timedbinary.go diff --git a/bufferflow_timedbinary.go b/bufferflow_timedbinary.go deleted file mode 100644 index 451c9ec0e..000000000 --- a/bufferflow_timedbinary.go +++ /dev/null @@ -1,75 +0,0 @@ -package main - -import ( - "encoding/json" - "time" - - log "github.com/sirupsen/logrus" -) - -type BufferflowTimedBinary struct { - port string - output chan<- []byte - input chan []byte - done chan bool - ticker *time.Ticker - bufferedOutputBinary []byte - sPortBinary string -} - -func NewBufferflowTimedBinary(port string, output chan<- []byte) *BufferflowTimedBinary { - return &BufferflowTimedBinary{ - port: port, - output: output, - input: make(chan []byte), - done: make(chan bool), - ticker: time.NewTicker(16 * time.Millisecond), - bufferedOutputBinary: nil, - sPortBinary: "", - } -} - -func (b *BufferflowTimedBinary) Init() { - log.Println("Initting timed buffer binary flow (output once every 16ms)") - go b.consumeInput() -} - -func (b *BufferflowTimedBinary) consumeInput() { -Loop: - for { - select { - case data := <-b.input: // use the buffer and append data to it - b.bufferedOutputBinary = append(b.bufferedOutputBinary, data...) - b.sPortBinary = b.port - case <-b.ticker.C: // after 16ms send the buffered output message - if b.bufferedOutputBinary != nil { - m := SpPortMessageRaw{b.sPortBinary, b.bufferedOutputBinary} - buf, _ := json.Marshal(m) - // data is now encoded in base64 format - // need a decoder on the other side - b.output <- buf - // reset the buffer and the port - b.bufferedOutputBinary = nil - b.sPortBinary = "" - } - case <-b.done: - break Loop //this is required, a simple break statement would only exit the innermost switch statement - } - } - close(b.input) -} - -func (b *BufferflowTimedBinary) BlockUntilReady(cmd string, id string) (bool, bool) { - //log.Printf("BlockUntilReady() start\n") - return true, false -} - -func (b *BufferflowTimedBinary) OnIncomingData(data string) { - b.input <- []byte(data) -} - -func (b *BufferflowTimedBinary) Close() { - b.ticker.Stop() - b.done <- true - close(b.done) -} diff --git a/serialport.go b/serialport.go index 93592f5ae..745e3b239 100755 --- a/serialport.go +++ b/serialport.go @@ -118,7 +118,7 @@ func (p *serport) reader(buftype string) { data := "" switch buftype { - case "timedraw", "timed", "timedbinary": + case "timedraw", "timed": data = string(bufferPart[:n]) // give the data to our bufferflow so it can do it's work // to read/translate the data to see if it wants to block @@ -301,8 +301,6 @@ func spHandlerOpen(portname string, baud int, buftype string) { bw = NewBufferflowTimed(portname, h.broadcastSys) case "timedraw": bw = NewBufferflowTimedRaw(portname, h.broadcastSys) - case "timedbinary": - bw = NewBufferflowTimedBinary(portname, h.broadcastSys) case "default": bw = NewBufferflowDefault(portname, h.broadcastSys) default: diff --git a/test/test_ws.py b/test/test_ws.py index 29832066a..59abbb62e 100644 --- a/test/test_ws.py +++ b/test/test_ws.py @@ -45,14 +45,6 @@ def test_open_serial_timedraw(socketio): general_test_serial(socketio, "timedraw") -@pytest.mark.skipif( - running_on_ci(), - reason="VMs have no serial ports", -) -def test_open_serial_timedbinary(socketio): - general_test_serial(socketio, "timedbinary") - - def general_test_serial(socketio, buffertype): port = "/dev/ttyACM0" global message @@ -75,7 +67,7 @@ def general_test_serial(socketio, buffertype): # check if the send command has been registered assert any("send " + port + " /\"ciao/\"" in i for i in message) #check if message has been sent back by the connected board - if buffertype in ("timedbinary", "timedraw"): + if buffertype == "timedraw": output = decode_output(extract_serial_data(message)) elif buffertype in ("default", "timed"): output = extract_serial_data(message) @@ -89,7 +81,7 @@ def general_test_serial(socketio, buffertype): print(message) # check if the send command has been registered assert any("send " + port + " /\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in i for i in message) - if buffertype in ("timedbinary", "timedraw"): + if buffertype == "timedraw": output = decode_output(extract_serial_data(message)) elif buffertype in ("default", "timed"): output = extract_serial_data(message) From 402a848eb49efa9a9c087514b812265c354e25e3 Mon Sep 17 00:00:00 2001 From: Silvano Cerza Date: Fri, 6 Aug 2021 10:38:19 +0200 Subject: [PATCH 28/37] Escape html commands string --- hub.go | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/hub.go b/hub.go index 1497ace8a..1cd07e9f1 100755 --- a/hub.go +++ b/hub.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "fmt" + "html" "io" "os" "runtime" @@ -40,22 +41,22 @@ var h = hub{ } const commands = `{ - "Commands": [ - "list", - "open < portName > < baud > [bufferAlgorithm: ({default}, timed, timedraw, timedbinary)]", - "send < portName > < cmd >", - "sendnobuf < portName > < cmd >", - "close < portName >", - "restart", - "exit", - "killupload", - "downloadtool < tool > < toolVersion: {latest} > < pack: {arduino} > < behaviour: {keep} >", - "log", - "memorystats", - "gc", - "hostname", - "version" - ] + "Commands": [ + "list", + "open [bufferAlgorithm: ({default}, timed, timedraw, timedbinary)]", + "send ", + "sendnobuf ", + "close ", + "restart", + "exit", + "killupload", + "downloadtool ", + "log", + "memorystats", + "gc", + "hostname", + "version" + ] }` func (h *hub) unregisterConnection(c *connection) { @@ -86,7 +87,7 @@ func (h *hub) run() { h.connections[c] = true // send supported commands c.send <- []byte(fmt.Sprintf(`{"Version" : "%s"} `, version)) - c.send <- []byte(commands) + c.send <- []byte(html.EscapeString(commands)) c.send <- []byte(fmt.Sprintf(`{"Hostname" : "%s"} `, *hostname)) c.send <- []byte(fmt.Sprintf(`{"OS" : "%s"} `, runtime.GOOS)) case c := <-h.unregister: From d5228ec762ff7824c823bf4dcf6861a4e2696d27 Mon Sep 17 00:00:00 2001 From: umbynos Date: Fri, 6 Aug 2021 14:55:28 +0200 Subject: [PATCH 29/37] forgot to remove timed_binary --- hub.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hub.go b/hub.go index 1cd07e9f1..0ee05555b 100755 --- a/hub.go +++ b/hub.go @@ -43,7 +43,7 @@ var h = hub{ const commands = `{ "Commands": [ "list", - "open [bufferAlgorithm: ({default}, timed, timedraw, timedbinary)]", + "open [bufferAlgorithm: ({default}, timed, timedraw)]", "send ", "sendnobuf ", "close ", From bcf00236f632992013ad7f1c8aa77d6a7c83ab18 Mon Sep 17 00:00:00 2001 From: Umberto Baldi Date: Mon, 9 Aug 2021 11:44:34 +0200 Subject: [PATCH 30/37] remove useless id field (was unused) --- serial.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/serial.go b/serial.go index 38057ff6c..0c7902d53 100755 --- a/serial.go +++ b/serial.go @@ -15,7 +15,6 @@ type writeRequest struct { p *serport d string buffer bool - id string } type serialhub struct { @@ -84,12 +83,12 @@ func (sh *serialhub) run() { close(p.sendNoBuf) case wr := <-sh.write: // if user sent in the commands as one text mode line - write(wr, "") + write(wr) } } } -func write(wr writeRequest, id string) { +func write(wr writeRequest) { if wr.buffer { //log.Println("Send was normal send, so sending to wr.p.sendBuffered") wr.p.sendBuffered <- wr.d From d077ded06d168b1454b9b16ad1fa4d218fc60cae Mon Sep 17 00:00:00 2001 From: Umberto Baldi Date: Mon, 9 Aug 2021 12:52:09 +0200 Subject: [PATCH 31/37] remove useless channel done & other stuff --- serialport.go | 46 ++-------------------------------------------- 1 file changed, 2 insertions(+), 44 deletions(-) diff --git a/serialport.go b/serialport.go index 745e3b239..313cba23e 100755 --- a/serialport.go +++ b/serialport.go @@ -12,19 +12,8 @@ import ( ) type SerialConfig struct { - Name string - Baud int - - // Size int // 0 get translated to 8 - // Parity SomeNewTypeToGetCorrectDefaultOf_None - // StopBits SomeNewTypeToGetCorrectDefaultOf_1 - - // RTSFlowControl bool - // DTRFlowControl bool - // XONFlowControl bool - - // CRLFTranslate bool - // TimeoutStuff int + Name string + Baud int RtsOn bool DtrOn bool } @@ -34,8 +23,6 @@ type serport struct { portConf *SerialConfig portIo io.ReadWriteCloser - done chan bool // signals the end of this request - // Keep track of whether we're being actively closed // just so we don't show scary error messages isClosing bool @@ -57,30 +44,6 @@ type serport struct { bufferwatcher Bufferflow } -type Cmd struct { - data string - id string - skippedBuffer bool - willHandleCompleteResponse bool -} - -type CmdComplete struct { - Cmd string - Id string - P string - BufSize int `json:"-"` - D string `json:"-"` -} - -type qwReport struct { - Cmd string - QCnt int - Id string - D string `json:"-"` - Buf string `json:"-"` - P string -} - type SpPortMessage struct { P string // the port, i.e. com22 D string // the data, i.e. G0 X0 Y0 @@ -93,7 +56,6 @@ type SpPortMessageRaw struct { func (p *serport) reader(buftype string) { - //var buf bytes.Buffer timeCheckOpen := time.Now() var buffered_ch bytes.Buffer @@ -324,10 +286,6 @@ func spHandlerOpen(portname string, baud int, buftype string) { spListDual(false) spList(false) - - //go p.reader() - //p.done = make(chan bool) - //<-p.done } func spHandlerClose(p *serport) { From d704dd280615bc6a70bd7e8c8666eecd01f214ae Mon Sep 17 00:00:00 2001 From: Umberto Baldi Date: Tue, 10 Aug 2021 11:31:19 +0200 Subject: [PATCH 32/37] =?UTF-8?q?make=20sendNoBuf=20more=20general:=20will?= =?UTF-8?q?=20be=20used=20later=20=F0=9F=98=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- serialport.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/serialport.go b/serialport.go index 313cba23e..e46df3f88 100755 --- a/serialport.go +++ b/serialport.go @@ -36,7 +36,7 @@ type serport struct { sendBuffered chan string // unbuffered channel of outbound messages that bypass internal serial port buffer - sendNoBuf chan string + sendNoBuf chan []byte // Do we have an extra channel/thread to watch our buffer? BufferType string @@ -180,7 +180,7 @@ func (p *serport) writerBuffered() { } else { // send to the non-buffered serial port writer //log.Println("About to send to p.sendNoBuf channel") - p.sendNoBuf <- data + p.sendNoBuf <- []byte(data) } } msgstr := "writerBuffered just got closed. make sure you make a new one. port:" + p.portConf.Name @@ -204,7 +204,7 @@ func (p *serport) writerNoBuf() { // FINALLY, OF ALL THE CODE IN THIS PROJECT // WE TRULY/FINALLY GET TO WRITE TO THE SERIAL PORT! - n2, err := p.portIo.Write([]byte(data)) + n2, err := p.portIo.Write(data) log.Print("Just wrote ", n2, " bytes to serial: ", string(data)) if err != nil { @@ -254,7 +254,7 @@ func spHandlerOpen(portname string, baud int, buftype string) { log.Print("Opened port successfully") //p := &serport{send: make(chan []byte, 256), portConf: conf, portIo: sp} // we can go up to 256,000 lines of gcode in the buffer - p := &serport{sendBuffered: make(chan string, 256000), sendNoBuf: make(chan string), portConf: conf, portIo: sp, BufferType: buftype} + p := &serport{sendBuffered: make(chan string, 256000), sendNoBuf: make(chan []byte), portConf: conf, portIo: sp, BufferType: buftype} var bw Bufferflow From 791b03e93311999544fa800ff069892617912625 Mon Sep 17 00:00:00 2001 From: Umberto Baldi Date: Wed, 11 Aug 2021 13:18:11 +0200 Subject: [PATCH 33/37] add `sendraw` command to send base64 encoded bytes, add tests (for send raw and for open/close port) --- go.sum | 1 + hub.go | 5 +-- serial.go | 28 ++++++------ serialport.go | 49 ++++++++++++++++++++- test/conftest.py | 18 ++++++++ test/test_ws.py | 109 ++++++++++++++++++++++++++++++++++++++--------- 6 files changed, 172 insertions(+), 38 deletions(-) diff --git a/go.sum b/go.sum index 43f085b20..786092344 100644 --- a/go.sum +++ b/go.sum @@ -391,6 +391,7 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= +golang.org/x/tools v0.0.0-20191216052735-49a3e744a425 h1:VvQyQJN0tSuecqgcIxMWnnfG5kSmgy9KZR9sW3W5QeA= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/hub.go b/hub.go index 0ee05555b..c141a6cd8 100755 --- a/hub.go +++ b/hub.go @@ -44,8 +44,7 @@ const commands = `{ "Commands": [ "list", "open [bufferAlgorithm: ({default}, timed, timedraw)]", - "send ", - "sendnobuf ", + "(send, sendnobuf, sendraw) ", "close ", "restart", "exit", @@ -160,7 +159,7 @@ func checkCmd(m []byte) { }() } else if strings.HasPrefix(sl, "send") { - // will catch send and sendnobuf + // will catch send and sendnobuf and sendraw go spWrite(s) } else if strings.HasPrefix(sl, "list") { go spList(false) diff --git a/serial.go b/serial.go index 0c7902d53..0bfd77873 100755 --- a/serial.go +++ b/serial.go @@ -14,7 +14,7 @@ import ( type writeRequest struct { p *serport d string - buffer bool + buffer string } type serialhub struct { @@ -89,13 +89,15 @@ func (sh *serialhub) run() { } func write(wr writeRequest) { - if wr.buffer { - //log.Println("Send was normal send, so sending to wr.p.sendBuffered") + switch wr.buffer { + case "send": wr.p.sendBuffered <- wr.d - } else { - //log.Println("Send was sendnobuf, so sending to wr.p.sendNoBuf") - wr.p.sendNoBuf <- wr.d + case "sendnobuf": + wr.p.sendNoBuf <- []byte(wr.d) + case "sendraw": + wr.p.sendRaw <- wr.d } + // no default since we alredy verified in spWrite() } // spList broadcasts a Json representation of the ports found @@ -274,13 +276,13 @@ func spWrite(arg string) { var wr writeRequest wr.p = myport - // see if args[0] is send or sendnobuf - if args[0] != "sendnobuf" { - // we were just given a "send" so buffer it - wr.buffer = true - } else { - //log.Println("sendnobuf specified so wr.buffer is false") - wr.buffer = false + // see if args[0] is send or sendnobuf or sendraw + switch args[0] { + case "send", "sendnobuf", "sendraw": + wr.buffer = args[0] + default: + spErr("Unsupported send command:" + args[0] + ". Please specify a valid one") + return } // include newline or not in the write? that is the question. diff --git a/serialport.go b/serialport.go index e46df3f88..f29d90dee 100755 --- a/serialport.go +++ b/serialport.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "encoding/base64" "io" "strconv" "time" @@ -38,6 +39,9 @@ type serport struct { // unbuffered channel of outbound messages that bypass internal serial port buffer sendNoBuf chan []byte + // channel containing raw base64 encoded binary data (outbound messages) + sendRaw chan string + // Do we have an extra channel/thread to watch our buffer? BufferType string //bufferwatcher *BufferflowDummypause @@ -222,6 +226,46 @@ func (p *serport) writerNoBuf() { spList(false) } +// this method runs as its own thread because it's instantiated +// as a "go" method. so if it blocks inside, it is ok +func (p *serport) writerRaw() { + // this method can panic if user closes serial port and something is + // in BlockUntilReady() and then a send occurs on p.sendNoBuf + + defer func() { + if e := recover(); e != nil { + log.Println("Got panic: ", e) + } + }() + + // this for loop blocks on p.sendRaw until that channel + // sees something come in + for data := range p.sendRaw { + + // Decode stuff + sDec, err := base64.StdEncoding.DecodeString(data) + if err != nil { + log.Println("Decoding error:", err) + } + log.Println(string(sDec)) + + // we want to block here if we are being asked to pause. + goodToGo, _ := p.bufferwatcher.BlockUntilReady(string(data), "") + + if goodToGo == false { + log.Println("We got back from BlockUntilReady() but apparently we must cancel this cmd") + // since we won't get a buffer decrement in p.sendNoBuf, we must do it here + p.itemsInBuffer-- + } else { + // send to the non-buffered serial port writer + p.sendNoBuf <- sDec + } + } + msgstr := "writerRaw just got closed. make sure you make a new one. port:" + p.portConf.Name + log.Println(msgstr) + h.broadcastSys <- []byte(msgstr) +} + func spHandlerOpen(portname string, baud int, buftype string) { log.Print("Inside spHandler") @@ -254,7 +298,7 @@ func spHandlerOpen(portname string, baud int, buftype string) { log.Print("Opened port successfully") //p := &serport{send: make(chan []byte, 256), portConf: conf, portIo: sp} // we can go up to 256,000 lines of gcode in the buffer - p := &serport{sendBuffered: make(chan string, 256000), sendNoBuf: make(chan []byte), portConf: conf, portIo: sp, BufferType: buftype} + p := &serport{sendBuffered: make(chan string, 256000), sendNoBuf: make(chan []byte), sendRaw: make(chan string), portConf: conf, portIo: sp, BufferType: buftype} var bw Bufferflow @@ -282,6 +326,9 @@ func spHandlerOpen(portname string, baud int, buftype string) { go p.writerBuffered() // this is thread to send to serial port regardless of block go p.writerNoBuf() + // this is thread to send to serial port but with base64 decoding + go p.writerRaw() + p.reader(buftype) spListDual(false) diff --git a/test/conftest.py b/test/conftest.py index 9f9c0389d..e46026e59 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -51,3 +51,21 @@ def socketio(base_url, agent): sio.connect(base_url) yield sio sio.disconnect() + +@pytest.fixture(scope="session") +def serial_port(): + return "/dev/ttyACM0" # maybe this could be enhanced by calling arduino-cli + +@pytest.fixture(scope="session") +def baudrate(): + return "9600" + +# open_port cannot be coced as a fixture because of the buffertype parameter + +# at the end of the test closes the serial port +@pytest.fixture(scope="function") +def close_port(socketio, serial_port): + yield socketio + socketio.emit('command', 'close ' + serial_port) + time.sleep(.5) + diff --git a/test/test_ws.py b/test/test_ws.py index 59abbb62e..2afdb1d84 100644 --- a/test/test_ws.py +++ b/test/test_ws.py @@ -20,39 +20,84 @@ def test_list(socketio): assert any("Ports" in i for i in message) assert any("Network" in i for i in message) -# NOTE run the following tests on linux with a board connected to the PC and with the sketch found in test/testdata/SerialEcho.ino on it +# NOTE run the following tests with a board connected to the PC @pytest.mark.skipif( running_on_ci(), reason="VMs have no serial ports", ) -def test_open_serial_default(socketio): - general_test_serial(socketio, "default") +def test_open_serial_default(socketio, serial_port, baudrate): + general_open_serial(socketio, serial_port, baudrate, "default") @pytest.mark.skipif( running_on_ci(), reason="VMs have no serial ports", ) -def test_open_serial_timed(socketio): - general_test_serial(socketio, "timed") +def test_open_serial_timed(socketio, serial_port, baudrate): + general_open_serial(socketio, serial_port, baudrate, "timed") @pytest.mark.skipif( running_on_ci(), reason="VMs have no serial ports", ) -def test_open_serial_timedraw(socketio): - general_test_serial(socketio, "timedraw") +def test_open_serial_timedraw(socketio, serial_port, baudrate): + general_open_serial(socketio, serial_port, baudrate, "timedraw") -def general_test_serial(socketio, buffertype): - port = "/dev/ttyACM0" +# NOTE run the following tests with a board connected to the PC and with the sketch found in test/testdata/SerialEcho.ino on it be sure to change serial_address in conftest.py +@pytest.mark.skipif( + running_on_ci(), + reason="VMs have no serial ports", +) +def test_send_serial_default(socketio, close_port, serial_port, baudrate): + general_send_serial(socketio, close_port, serial_port, baudrate, "default") + + +@pytest.mark.skipif( + running_on_ci(), + reason="VMs have no serial ports", +) +def test_send_serial_timed(socketio, close_port, serial_port, baudrate): + general_send_serial(socketio, close_port, serial_port, baudrate, "timed") + + +@pytest.mark.skipif( + running_on_ci(), + reason="VMs have no serial ports", +) +def test_send_serial_timedraw(socketio, close_port, serial_port, baudrate): + general_send_serial(socketio, close_port, serial_port, baudrate, "timedraw") + + +def general_open_serial(socketio, serial_port, baudrate, buffertype): + global message + message = [] + # in message var we will find the "response" + socketio.on('message', message_handler) + socketio.emit('command', 'open ' + serial_port + ' ' + baudrate + ' ' + buffertype) + # give time to the message var to be filled + time.sleep(.5) + print(message) + # the serial connection should be open now + assert any("\"IsOpen\": true" in i for i in message) + + # close the serial port + socketio.emit('command', 'close ' + serial_port) + time.sleep(.2) + print (message) + #check if port has been closed + assert any("\"IsOpen\": false," in i for i in message) + + + +def general_send_serial(socketio, close_port, serial_port, baudrate, buffertype): global message message = [] #in message var we will find the "response" socketio.on('message', message_handler) #open a new serial connection with the specified buffertype, if buffertype is empty it will use the default one - socketio.emit('command', 'open ' + port + ' 9600 ' + buffertype) + socketio.emit('command', 'open ' + serial_port + ' ' + baudrate + ' ' + buffertype) # give time to the message var to be filled time.sleep(.5) print(message) @@ -61,11 +106,11 @@ def general_test_serial(socketio, buffertype): #test with string # send the string "ciao" using the serial connection - socketio.emit('command', 'send ' + port + ' /"ciao/"') + socketio.emit('command', 'send ' + serial_port + ' /"ciao/"') time.sleep(1) print(message) # check if the send command has been registered - assert any("send " + port + " /\"ciao/\"" in i for i in message) + assert any("send " + serial_port + " /\"ciao/\"" in i for i in message) #check if message has been sent back by the connected board if buffertype == "timedraw": output = decode_output(extract_serial_data(message)) @@ -76,23 +121,45 @@ def general_test_serial(socketio, buffertype): #test with emoji message = [] # reinitialize the message buffer to have a clean situation # send a lot of emoji: they can be messed up - socketio.emit('command', 'send ' + port + ' /"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/"') - time.sleep(.2) + socketio.emit('command', 'send ' + serial_port + ' /"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/"') + time.sleep(.5) print(message) # check if the send command has been registered - assert any("send " + port + " /\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in i for i in message) + assert any("send " + serial_port + " /\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in i for i in message) if buffertype == "timedraw": output = decode_output(extract_serial_data(message)) elif buffertype in ("default", "timed"): output = extract_serial_data(message) assert "/\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in output + # the serial connection is closed by close_port() fixture: even if in case of test failure - #finally close the serial port - socketio.emit('command', 'close ' + port) - time.sleep(.2) - print (message) - #check if port has been closed - assert any("\"IsOpen\": false," in i for i in message) + +def test_sendraw_serial(socketio, close_port, serial_port, baudrate): + global message + message = [] + #in message var we will find the "response" + socketio.on('message', message_handler) + #open a new serial connection with the specified buffertype, if buffertype is empty it will use the default one + socketio.emit('command', 'open ' + serial_port + ' ' + baudrate + ' timedraw') + # give time to the message var to be filled + time.sleep(.5) + print(message) + # the serial connection should be open now + assert any("\"IsOpen\": true" in i for i in message) + + #test with bytes + integers = [1, 2, 3, 4, 5] + bytes_array=bytearray(integers) + encoded_integers = base64.b64encode(bytes_array).decode('ascii') + socketio.emit('command', 'sendraw ' + serial_port + ' ' + encoded_integers) + time.sleep(1) + print(message) + # check if the send command has been registered + assert any(("sendraw " + serial_port + ' ' + encoded_integers) in i for i in message) + #check if message has been sent back by the connected board + output = extract_serial_data(message) # TODO use decode_output() + print (output) + assert encoded_integers in output # callback called by socketio when a message is received From 8dadd0cbedb918ba56847df7727c36e86b6bd077 Mon Sep 17 00:00:00 2001 From: Umberto Baldi Date: Wed, 11 Aug 2021 13:24:44 +0200 Subject: [PATCH 34/37] forgot to skip test_sendraw_serial on CI --- test/test_ws.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/test_ws.py b/test/test_ws.py index 2afdb1d84..072fcde4f 100644 --- a/test/test_ws.py +++ b/test/test_ws.py @@ -133,7 +133,10 @@ def general_send_serial(socketio, close_port, serial_port, baudrate, buffertype) assert "/\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in output # the serial connection is closed by close_port() fixture: even if in case of test failure - +@pytest.mark.skipif( + running_on_ci(), + reason="VMs have no serial ports", +) def test_sendraw_serial(socketio, close_port, serial_port, baudrate): global message message = [] From 500a1ee3ac522ad50ee84d3cd6432dacc47b38d6 Mon Sep 17 00:00:00 2001 From: Umberto Baldi Date: Thu, 12 Aug 2021 11:54:24 +0200 Subject: [PATCH 35/37] update comments --- bufferflow_default.go | 2 -- bufferflow_timed.go | 2 -- bufferflow_timedraw.go | 3 +-- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/bufferflow_default.go b/bufferflow_default.go index 74a71f9d4..715c9443e 100644 --- a/bufferflow_default.go +++ b/bufferflow_default.go @@ -34,8 +34,6 @@ Loop: case data := <-b.input: m := SpPortMessage{b.port, data} message, _ := json.Marshal(m) - // data is now encoded in base64 format - // need a decoder on the other side b.output <- message case <-b.done: break Loop //this is required, a simple break statement would only exit the innermost switch statement diff --git a/bufferflow_timed.go b/bufferflow_timed.go index 6c759de4b..d1950775c 100644 --- a/bufferflow_timed.go +++ b/bufferflow_timed.go @@ -45,8 +45,6 @@ Loop: if b.bufferedOutput != "" { m := SpPortMessage{b.sPort, b.bufferedOutput} buf, _ := json.Marshal(m) - // data is now encoded in base64 format - // need a decoder on the other side b.output <- buf // reset the buffer and the port b.bufferedOutput = "" diff --git a/bufferflow_timedraw.go b/bufferflow_timedraw.go index 671444e47..20a2a0b8b 100644 --- a/bufferflow_timedraw.go +++ b/bufferflow_timedraw.go @@ -45,8 +45,7 @@ Loop: if b.bufferedOutputRaw != nil { m := SpPortMessageRaw{b.sPortRaw, b.bufferedOutputRaw} buf, _ := json.Marshal(m) - // data is now encoded in base64 format - // need a decoder on the other side + // since bufferedOutputRaw is a []byte is base64-encoded by json.Marshal() function automatically b.output <- buf // reset the buffer and the port b.bufferedOutputRaw = nil From a86a61ace3e3aaed13f05965e1e878e1f0facd59 Mon Sep 17 00:00:00 2001 From: Umberto Baldi Date: Thu, 12 Aug 2021 13:03:01 +0200 Subject: [PATCH 36/37] refactor tests --- test/conftest.py | 15 +++++- test/test_ws.py | 126 +++++++++++++++++++++++------------------------ 2 files changed, 76 insertions(+), 65 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index e46026e59..3f1b185f2 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -8,7 +8,6 @@ from invoke import Local from invoke.context import Context import socketio as io -import asyncio @pytest.fixture(scope="function") def agent(pytestconfig): @@ -69,3 +68,17 @@ def close_port(socketio, serial_port): socketio.emit('command', 'close ' + serial_port) time.sleep(.5) + +@pytest.fixture(scope="function") +def message(socketio): + global message + message = [] + #in message var we will find the "response" + socketio.on('message', message_handler) + return message + +# callback called by socketio when a message is received +def message_handler(msg): + # print('Received message: ', msg) + global message + message.append(msg) \ No newline at end of file diff --git a/test/test_ws.py b/test/test_ws.py index 072fcde4f..8a7143e74 100644 --- a/test/test_ws.py +++ b/test/test_ws.py @@ -6,13 +6,13 @@ from common import running_on_ci message = [] + def test_ws_connection(socketio): print('my sid is', socketio.sid) assert socketio.sid is not None -def test_list(socketio): - global message - socketio.on('message', message_handler) + +def test_list(socketio, message): socketio.emit('command', 'list') time.sleep(.2) print (message) @@ -20,29 +20,30 @@ def test_list(socketio): assert any("Ports" in i for i in message) assert any("Network" in i for i in message) + # NOTE run the following tests with a board connected to the PC @pytest.mark.skipif( running_on_ci(), reason="VMs have no serial ports", ) -def test_open_serial_default(socketio, serial_port, baudrate): - general_open_serial(socketio, serial_port, baudrate, "default") +def test_open_serial_default(socketio, serial_port, baudrate, message): + general_open_serial(socketio, serial_port, baudrate, message, "default") @pytest.mark.skipif( running_on_ci(), reason="VMs have no serial ports", ) -def test_open_serial_timed(socketio, serial_port, baudrate): - general_open_serial(socketio, serial_port, baudrate, "timed") +def test_open_serial_timed(socketio, serial_port, baudrate, message): + general_open_serial(socketio, serial_port, baudrate, message, "timed") @pytest.mark.skipif( running_on_ci(), reason="VMs have no serial ports", ) -def test_open_serial_timedraw(socketio, serial_port, baudrate): - general_open_serial(socketio, serial_port, baudrate, "timedraw") +def test_open_serial_timedraw(socketio, serial_port, baudrate, message): + general_open_serial(socketio, serial_port, baudrate, message, "timedraw") # NOTE run the following tests with a board connected to the PC and with the sketch found in test/testdata/SerialEcho.ino on it be sure to change serial_address in conftest.py @@ -50,39 +51,53 @@ def test_open_serial_timedraw(socketio, serial_port, baudrate): running_on_ci(), reason="VMs have no serial ports", ) -def test_send_serial_default(socketio, close_port, serial_port, baudrate): - general_send_serial(socketio, close_port, serial_port, baudrate, "default") +def test_send_serial_default(socketio, close_port, serial_port, baudrate, message): + general_send_serial(socketio, close_port, serial_port, baudrate, message, "default") @pytest.mark.skipif( running_on_ci(), reason="VMs have no serial ports", ) -def test_send_serial_timed(socketio, close_port, serial_port, baudrate): - general_send_serial(socketio, close_port, serial_port, baudrate, "timed") +def test_send_serial_timed(socketio, close_port, serial_port, baudrate, message): + general_send_serial(socketio, close_port, serial_port, baudrate, message, "timed") @pytest.mark.skipif( running_on_ci(), reason="VMs have no serial ports", ) -def test_send_serial_timedraw(socketio, close_port, serial_port, baudrate): - general_send_serial(socketio, close_port, serial_port, baudrate, "timedraw") +def test_send_serial_timedraw(socketio, close_port, serial_port, baudrate, message): + general_send_serial(socketio, close_port, serial_port, baudrate, message, "timedraw") -def general_open_serial(socketio, serial_port, baudrate, buffertype): - global message - message = [] - # in message var we will find the "response" - socketio.on('message', message_handler) - socketio.emit('command', 'open ' + serial_port + ' ' + baudrate + ' ' + buffertype) - # give time to the message var to be filled - time.sleep(.5) - print(message) - # the serial connection should be open now - assert any("\"IsOpen\": true" in i for i in message) +@pytest.mark.skipif( + running_on_ci(), + reason="VMs have no serial ports", +) +def test_send_emoji_serial_default(socketio, close_port, serial_port, baudrate, message): + general_send_emoji_serial(socketio, close_port, serial_port, baudrate, message, "default") + + +@pytest.mark.skipif( + running_on_ci(), + reason="VMs have no serial ports", +) +def test_send_emoji_serial_timed(socketio, close_port, serial_port, baudrate, message): + general_send_emoji_serial(socketio, close_port, serial_port, baudrate, message, "timed") + + +@pytest.mark.skipif( + running_on_ci(), + reason="VMs have no serial ports", +) +def test_send_emoji_serial_timedraw(socketio, close_port, serial_port, baudrate, message): + general_send_emoji_serial(socketio, close_port, serial_port, baudrate, message, "timedraw") - # close the serial port + +def general_open_serial(socketio, serial_port, baudrate, message, buffertype): + open_serial_port(socketio, serial_port, baudrate, message, buffertype) + # test the closing of the serial port, we are gonna use close_port for the other tests socketio.emit('command', 'close ' + serial_port) time.sleep(.2) print (message) @@ -90,39 +105,28 @@ def general_open_serial(socketio, serial_port, baudrate, buffertype): assert any("\"IsOpen\": false," in i for i in message) - -def general_send_serial(socketio, close_port, serial_port, baudrate, buffertype): - global message - message = [] - #in message var we will find the "response" - socketio.on('message', message_handler) - #open a new serial connection with the specified buffertype, if buffertype is empty it will use the default one - socketio.emit('command', 'open ' + serial_port + ' ' + baudrate + ' ' + buffertype) - # give time to the message var to be filled - time.sleep(.5) - print(message) - # the serial connection should be open now - assert any("\"IsOpen\": true" in i for i in message) - - #test with string +def general_send_serial(socketio, close_port, serial_port, baudrate, message, buffertype): + open_serial_port(socketio, serial_port, baudrate, message, buffertype) # send the string "ciao" using the serial connection - socketio.emit('command', 'send ' + serial_port + ' /"ciao/"') + socketio.emit('command', 'send ' + serial_port + ' ciao') time.sleep(1) print(message) # check if the send command has been registered - assert any("send " + serial_port + " /\"ciao/\"" in i for i in message) + assert any("send " + serial_port + " ciao" in i for i in message) #check if message has been sent back by the connected board if buffertype == "timedraw": output = decode_output(extract_serial_data(message)) elif buffertype in ("default", "timed"): output = extract_serial_data(message) assert "ciao" in output + # the serial connection is closed by close_port() fixture: even if in case of test failure - #test with emoji - message = [] # reinitialize the message buffer to have a clean situation + +def general_send_emoji_serial(socketio, close_port, serial_port, baudrate, message, buffertype): + open_serial_port(socketio, serial_port, baudrate, message, buffertype) # send a lot of emoji: they can be messed up socketio.emit('command', 'send ' + serial_port + ' /"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/"') - time.sleep(.5) + time.sleep(1) print(message) # check if the send command has been registered assert any("send " + serial_port + " /\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in i for i in message) @@ -133,23 +137,23 @@ def general_send_serial(socketio, close_port, serial_port, baudrate, buffertype) assert "/\"🧀🧀🧀🧀🧀🧀🧀🧀🧀🧀/\"" in output # the serial connection is closed by close_port() fixture: even if in case of test failure -@pytest.mark.skipif( - running_on_ci(), - reason="VMs have no serial ports", -) -def test_sendraw_serial(socketio, close_port, serial_port, baudrate): - global message - message = [] - #in message var we will find the "response" - socketio.on('message', message_handler) - #open a new serial connection with the specified buffertype, if buffertype is empty it will use the default one - socketio.emit('command', 'open ' + serial_port + ' ' + baudrate + ' timedraw') + +def open_serial_port(socketio, serial_port, baudrate, message, buffertype): + #open a new serial connection with the specified buffertype + socketio.emit('command', 'open ' + serial_port + ' ' + baudrate + ' ' + buffertype) # give time to the message var to be filled time.sleep(.5) print(message) # the serial connection should be open now assert any("\"IsOpen\": true" in i for i in message) + +@pytest.mark.skipif( + running_on_ci(), + reason="VMs have no serial ports", +) +def test_sendraw_serial(socketio, close_port, serial_port, baudrate, message): + open_serial_port(socketio, serial_port, baudrate, message, "timedraw") #test with bytes integers = [1, 2, 3, 4, 5] bytes_array=bytearray(integers) @@ -165,12 +169,6 @@ def test_sendraw_serial(socketio, close_port, serial_port, baudrate): assert encoded_integers in output -# callback called by socketio when a message is received -def message_handler(msg): - # print('Received message: ', msg) - global message - message.append(msg) - # helper function used to extract serial data from its JSON representation # NOTE make sure to pass a clean message (maybe reinitialize the message global var before populating it) def extract_serial_data(msg): From a21ae9e4019f88b88df742353e782dc2285a3ed1 Mon Sep 17 00:00:00 2001 From: Umberto Baldi Date: Mon, 16 Aug 2021 13:21:05 +0200 Subject: [PATCH 37/37] remove BlockUntilReady because it was unused --- bufferflow.go | 5 ++--- bufferflow_default.go | 5 ----- bufferflow_timed.go | 5 ----- bufferflow_timedraw.go | 5 ----- serialport.go | 28 ++++++---------------------- 5 files changed, 8 insertions(+), 40 deletions(-) diff --git a/bufferflow.go b/bufferflow.go index 8f17cf592..07d34698b 100644 --- a/bufferflow.go +++ b/bufferflow.go @@ -2,7 +2,6 @@ package main type Bufferflow interface { Init() - BlockUntilReady(cmd string, id string) (bool, bool) // implement this method - OnIncomingData(data string) // implement this method - Close() // implement this method + OnIncomingData(data string) // implement this method + Close() // implement this method } diff --git a/bufferflow_default.go b/bufferflow_default.go index 715c9443e..2d6d6d281 100644 --- a/bufferflow_default.go +++ b/bufferflow_default.go @@ -42,11 +42,6 @@ Loop: close(b.input) // close the input channel at the end of the computation } -func (b *BufferflowDefault) BlockUntilReady(cmd string, id string) (bool, bool) { - //log.Printf("BlockUntilReady() start\n") - return true, false -} - func (b *BufferflowDefault) OnIncomingData(data string) { b.input <- data } diff --git a/bufferflow_timed.go b/bufferflow_timed.go index d1950775c..b89427f37 100644 --- a/bufferflow_timed.go +++ b/bufferflow_timed.go @@ -57,11 +57,6 @@ Loop: close(b.input) } -func (b *BufferflowTimed) BlockUntilReady(cmd string, id string) (bool, bool) { - //log.Printf("BlockUntilReady() start\n") - return true, false -} - func (b *BufferflowTimed) OnIncomingData(data string) { b.input <- data } diff --git a/bufferflow_timedraw.go b/bufferflow_timedraw.go index 20a2a0b8b..ab238bfe8 100644 --- a/bufferflow_timedraw.go +++ b/bufferflow_timedraw.go @@ -58,11 +58,6 @@ Loop: close(b.input) } -func (b *BufferflowTimedRaw) BlockUntilReady(cmd string, id string) (bool, bool) { - //log.Printf("BlockUntilReady() start\n") - return true, false -} - func (b *BufferflowTimedRaw) OnIncomingData(data string) { b.input <- data } diff --git a/serialport.go b/serialport.go index f29d90dee..b427e1d2e 100755 --- a/serialport.go +++ b/serialport.go @@ -174,18 +174,10 @@ func (p *serport) writerBuffered() { // sees something come in for data := range p.sendBuffered { - // we want to block here if we are being asked to pause. - goodToGo, _ := p.bufferwatcher.BlockUntilReady(string(data), "") - - if goodToGo == false { - log.Println("We got back from BlockUntilReady() but apparently we must cancel this cmd") - // since we won't get a buffer decrement in p.sendNoBuf, we must do it here - p.itemsInBuffer-- - } else { - // send to the non-buffered serial port writer - //log.Println("About to send to p.sendNoBuf channel") - p.sendNoBuf <- []byte(data) - } + // send to the non-buffered serial port writer + //log.Println("About to send to p.sendNoBuf channel") + p.sendNoBuf <- []byte(data) + } msgstr := "writerBuffered just got closed. make sure you make a new one. port:" + p.portConf.Name log.Println(msgstr) @@ -249,17 +241,9 @@ func (p *serport) writerRaw() { } log.Println(string(sDec)) - // we want to block here if we are being asked to pause. - goodToGo, _ := p.bufferwatcher.BlockUntilReady(string(data), "") + // send to the non-buffered serial port writer + p.sendNoBuf <- sDec - if goodToGo == false { - log.Println("We got back from BlockUntilReady() but apparently we must cancel this cmd") - // since we won't get a buffer decrement in p.sendNoBuf, we must do it here - p.itemsInBuffer-- - } else { - // send to the non-buffered serial port writer - p.sendNoBuf <- sDec - } } msgstr := "writerRaw just got closed. make sure you make a new one. port:" + p.portConf.Name log.Println(msgstr)