feat(encoding): improve binary encoding format

encoding_test.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/qedus/osmpbf"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEncodingSimple(t *testing.T) {
+
+	var node = &osmpbf.Node{ID: 100, Lat: -50, Lon: 77}
+	var expectedBytes = []byte{0xc0, 0x49, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x40, 0x53, 0x40, 0x0, 0x0, 0x0, 0x0, 0x0}
+	var expectedLatlon = map[string]string{"lon": "77.0000000", "lat": "-50.0000000"}
+
+	// encode
+	var stringid, byteval = nodeToBytes(node)
+	assert.Equal(t, "100", stringid)
+	assert.Equal(t, expectedBytes, byteval)
+
+	// decode
+	var latlon = bytesToLatLon(byteval)
+	assert.Equal(t, expectedLatlon, latlon)
+}
+
+func TestEncodingFloatPrecision(t *testing.T) {
+
+	var node = &osmpbf.Node{ID: 100, Lat: -50.555555555, Lon: 77.777777777}
+	var expectedBytes = []byte{0xc0, 0x49, 0x47, 0x1c, 0x71, 0xc5, 0xeb, 0x6, 0x40, 0x53, 0x71, 0xc7, 0x1c, 0x70, 0xf1, 0x51}
+	var expectedLatlon = map[string]string{"lon": "77.7777778", "lat": "-50.5555556"}
+
+	// encode
+	var stringid, byteval = nodeToBytes(node)
+	assert.Equal(t, "100", stringid)
+	assert.Equal(t, expectedBytes, byteval)
+
+	// decode
+	var latlon = bytesToLatLon(byteval)
+	assert.Equal(t, expectedLatlon, latlon)
+}
+
+func TestEncodingBitmaskValues(t *testing.T) {
+
+	var tags = map[string]string{"entrance": "main", "wheelchair": "yes"}
+	var node = &osmpbf.Node{ID: 100, Lat: -50, Lon: 77, Tags: tags}
+	var expectedBytes = []byte{0xc0, 0x49, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x40, 0x53, 0x40, 0x0, 0x0, 0x0, 0x0, 0x0, 0xa0}
+	var expectedLatlon = map[string]string{"lon": "77.0000000", "lat": "-50.0000000", "entrance": "2", "wheelchair": "2"}
+
+	// encode
+	var stringid, byteval = nodeToBytes(node)
+	assert.Equal(t, "100", stringid)
+	assert.Equal(t, expectedBytes, byteval)
+
+	// decode
+	var latlon = bytesToLatLon(byteval)
+	assert.Equal(t, expectedLatlon, latlon)
+}

pbf2json.go

@@ -1,19 +1,23 @@
 package main
 
-import "encoding/json"
-import "fmt"
-import "flag"
-import "bytes"
-import "os"
-import "log"
-import "io"
-
-import "runtime"
-import "strings"
-import "strconv"
-import "github.com/qedus/osmpbf"
-import "github.com/syndtr/goleveldb/leveldb"
-import "github.com/paulmach/go.geo"
+import (
+	"bytes"
+	"encoding/binary"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"math"
+	"os"
+	"runtime"
+	"strconv"
+	"strings"
+
+	"github.com/paulmach/go.geo"
+	"github.com/qedus/osmpbf"
+	"github.com/syndtr/goleveldb/leveldb"
+)
 
 type settings struct {
 	PbfPath    string
@@ -238,7 +242,7 @@ func isWheelchairAccessibleNode(node *osmpbf.Node) uint8 {
 
 // write to leveldb immediately
 func cacheStore(db *leveldb.DB, node *osmpbf.Node) {
-	id, val := formatLevelDB(node)
+	id, val := nodeToBytes(node)
 	err := db.Put([]byte(id), []byte(val), nil)
 	if err != nil {
 		log.Fatal(err)
@@ -247,7 +251,7 @@ func cacheStore(db *leveldb.DB, node *osmpbf.Node) {
 
 // queue a leveldb write in a batch
 func cacheQueue(batch *leveldb.Batch, node *osmpbf.Node) {
-	id, val := formatLevelDB(node)
+	id, val := nodeToBytes(node)
 	batch.Put([]byte(id), []byte(val))
 }
 
@@ -273,47 +277,58 @@ func cacheLookup(db *leveldb.DB, way *osmpbf.Way) ([]map[string]string, error) {
 			return container, err
 		}
 
-		s := string(data)
-		spl := strings.Split(s, ":")
+		container = append(container, bytesToLatLon(data))
+	}
 
-		latlon := make(map[string]string)
-		lat, lon := spl[0], spl[1]
-		latlon["lat"] = lat
-		latlon["lon"] = lon
+	return container, nil
+}
 
-		// check for third & fourth fields which indicate an entrance
-		// and the level of wheelchair accessibility
-		if len(spl) == 4 {
-			latlon["entrance"] = spl[2]
-			latlon["wheelchair"] = spl[3]
-		}
+// decode bytes to a 'latlon' type object
+func bytesToLatLon(data []byte) map[string]string {
 
-		container = append(container, latlon)
+	var latlon = make(map[string]string)
 
-	}
+	var lat64 = math.Float64frombits(binary.BigEndian.Uint64(data[0:8]))
+	var lon64 = math.Float64frombits(binary.BigEndian.Uint64(data[8:16]))
+	latlon["lat"] = strconv.FormatFloat(lat64, 'f', 7, 64)
+	latlon["lon"] = strconv.FormatFloat(lon64, 'f', 7, 64)
 
-	return container, nil
+	// check for the bitmask byte which indicates things like an
+	// entrance and the level of wheelchair accessibility
+	if len(data) > 16 {
+		latlon["entrance"] = fmt.Sprintf("%d", (data[16]&0xC0)>>6)
+		latlon["wheelchair"] = fmt.Sprintf("%d", (data[16]&0x30)>>4)
+	}
 
-	// fmt.Println(way.NodeIDs)
-	// fmt.Println(container)
-	// os.Exit(1)
+	return latlon
 }
 
-func formatLevelDB(node *osmpbf.Node) (id string, val []byte) {
-
-	stringid := strconv.FormatInt(node.ID, 10)
+// encode a node as bytes (between 16 & 17 bytes used)
+func nodeToBytes(node *osmpbf.Node) (string, []byte) {
 
 	var bufval bytes.Buffer
-	bufval.WriteString(strconv.FormatFloat(node.Lat, 'f', 7, 64))
-	bufval.WriteString(":")
-	bufval.WriteString(strconv.FormatFloat(node.Lon, 'f', 7, 64))
 
+	// encode lat/lon as two 64 bit floats packed in to 16 bytes
+	var bufLatLon = make([]byte, 16)
+	binary.BigEndian.PutUint64(bufLatLon[:8], math.Float64bits(node.Lat))
+	binary.BigEndian.PutUint64(bufLatLon[8:], math.Float64bits(node.Lon))
+	bufval.Write(bufLatLon)
+
+	// generate a bitmask for relevant tag features
 	var isEntrance = isEntranceNode(node)
 	if isEntrance > 0 {
-		bufval.WriteString(fmt.Sprintf(":%d:%d", isEntrance, isWheelchairAccessibleNode(node)))
+		// leftmost two bits are for the entrance, next two bits are accessibility
+		// remaining 4 rightmost bits are reserved for future use.
+		var bitmask = isEntrance << 6
+		var isWheelchairAccessible = isWheelchairAccessibleNode(node)
+		if isWheelchairAccessible > 0 {
+			bitmask |= isWheelchairAccessible << 4
+		}
+		bufval.WriteByte(bitmask)
 	}
 
-	byteval := []byte(bufval.String())
+	stringid := strconv.FormatInt(node.ID, 10)
+	byteval := bufval.Bytes()
 
 	return stringid, byteval
 }