diff --git a/build/pbf2json.darwin-x64 b/build/pbf2json.darwin-x64 index 46b79d5..64ab666 100755 Binary files a/build/pbf2json.darwin-x64 and b/build/pbf2json.darwin-x64 differ diff --git a/build/pbf2json.linux-arm b/build/pbf2json.linux-arm index 1ef912c..06783c6 100755 Binary files a/build/pbf2json.linux-arm and b/build/pbf2json.linux-arm differ diff --git a/build/pbf2json.linux-x64 b/build/pbf2json.linux-x64 index f67b6d0..135e112 100755 Binary files a/build/pbf2json.linux-x64 and b/build/pbf2json.linux-x64 differ diff --git a/build/pbf2json.win32-x64 b/build/pbf2json.win32-x64 index d72a0e0..16ba5f2 100755 Binary files a/build/pbf2json.win32-x64 and b/build/pbf2json.win32-x64 differ diff --git a/encoding_test.go b/encoding_test.go new file mode 100644 index 0000000..62a78d2 --- /dev/null +++ b/encoding_test.go @@ -0,0 +1,57 @@ +package main + +import ( + "testing" + + "github.com/qedus/osmpbf" + "github.com/stretchr/testify/assert" +) + +func TestEncodingSimple(t *testing.T) { + + var node = &osmpbf.Node{ID: 100, Lat: -50, Lon: 77} + var expectedBytes = []byte{0xc0, 0x49, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x40, 0x53, 0x40, 0x0, 0x0, 0x0, 0x0, 0x0} + var expectedLatlon = map[string]string{"lon": "77.0000000", "lat": "-50.0000000"} + + // encode + var stringid, byteval = nodeToBytes(node) + assert.Equal(t, "100", stringid) + assert.Equal(t, expectedBytes, byteval) + + // decode + var latlon = bytesToLatLon(byteval) + assert.Equal(t, expectedLatlon, latlon) +} + +func TestEncodingFloatPrecision(t *testing.T) { + + var node = &osmpbf.Node{ID: 100, Lat: -50.555555555, Lon: 77.777777777} + var expectedBytes = []byte{0xc0, 0x49, 0x47, 0x1c, 0x71, 0xc5, 0xeb, 0x6, 0x40, 0x53, 0x71, 0xc7, 0x1c, 0x70, 0xf1, 0x51} + var expectedLatlon = map[string]string{"lon": "77.7777778", "lat": "-50.5555556"} + + // encode + var stringid, byteval = nodeToBytes(node) + assert.Equal(t, "100", stringid) + assert.Equal(t, expectedBytes, byteval) + + // decode + var latlon = bytesToLatLon(byteval) + assert.Equal(t, expectedLatlon, latlon) +} + +func TestEncodingBitmaskValues(t *testing.T) { + + var tags = map[string]string{"entrance": "main", "wheelchair": "yes"} + var node = &osmpbf.Node{ID: 100, Lat: -50, Lon: 77, Tags: tags} + var expectedBytes = []byte{0xc0, 0x49, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x40, 0x53, 0x40, 0x0, 0x0, 0x0, 0x0, 0x0, 0xa0} + var expectedLatlon = map[string]string{"lon": "77.0000000", "lat": "-50.0000000", "entrance": "2", "wheelchair": "2"} + + // encode + var stringid, byteval = nodeToBytes(node) + assert.Equal(t, "100", stringid) + assert.Equal(t, expectedBytes, byteval) + + // decode + var latlon = bytesToLatLon(byteval) + assert.Equal(t, expectedLatlon, latlon) +} diff --git a/pbf2json.go b/pbf2json.go index e5f5f4b..23191c3 100644 --- a/pbf2json.go +++ b/pbf2json.go @@ -1,19 +1,23 @@ package main -import "encoding/json" -import "fmt" -import "flag" -import "bytes" -import "os" -import "log" -import "io" - -import "runtime" -import "strings" -import "strconv" -import "github.com/qedus/osmpbf" -import "github.com/syndtr/goleveldb/leveldb" -import "github.com/paulmach/go.geo" +import ( + "bytes" + "encoding/binary" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "math" + "os" + "runtime" + "strconv" + "strings" + + "github.com/paulmach/go.geo" + "github.com/qedus/osmpbf" + "github.com/syndtr/goleveldb/leveldb" +) type settings struct { PbfPath string @@ -238,7 +242,7 @@ func isWheelchairAccessibleNode(node *osmpbf.Node) uint8 { // write to leveldb immediately func cacheStore(db *leveldb.DB, node *osmpbf.Node) { - id, val := formatLevelDB(node) + id, val := nodeToBytes(node) err := db.Put([]byte(id), []byte(val), nil) if err != nil { log.Fatal(err) @@ -247,7 +251,7 @@ func cacheStore(db *leveldb.DB, node *osmpbf.Node) { // queue a leveldb write in a batch func cacheQueue(batch *leveldb.Batch, node *osmpbf.Node) { - id, val := formatLevelDB(node) + id, val := nodeToBytes(node) batch.Put([]byte(id), []byte(val)) } @@ -273,47 +277,58 @@ func cacheLookup(db *leveldb.DB, way *osmpbf.Way) ([]map[string]string, error) { return container, err } - s := string(data) - spl := strings.Split(s, ":") + container = append(container, bytesToLatLon(data)) + } - latlon := make(map[string]string) - lat, lon := spl[0], spl[1] - latlon["lat"] = lat - latlon["lon"] = lon + return container, nil +} - // check for third & fourth fields which indicate an entrance - // and the level of wheelchair accessibility - if len(spl) == 4 { - latlon["entrance"] = spl[2] - latlon["wheelchair"] = spl[3] - } +// decode bytes to a 'latlon' type object +func bytesToLatLon(data []byte) map[string]string { - container = append(container, latlon) + var latlon = make(map[string]string) - } + var lat64 = math.Float64frombits(binary.BigEndian.Uint64(data[0:8])) + var lon64 = math.Float64frombits(binary.BigEndian.Uint64(data[8:16])) + latlon["lat"] = strconv.FormatFloat(lat64, 'f', 7, 64) + latlon["lon"] = strconv.FormatFloat(lon64, 'f', 7, 64) - return container, nil + // check for the bitmask byte which indicates things like an + // entrance and the level of wheelchair accessibility + if len(data) > 16 { + latlon["entrance"] = fmt.Sprintf("%d", (data[16]&0xC0)>>6) + latlon["wheelchair"] = fmt.Sprintf("%d", (data[16]&0x30)>>4) + } - // fmt.Println(way.NodeIDs) - // fmt.Println(container) - // os.Exit(1) + return latlon } -func formatLevelDB(node *osmpbf.Node) (id string, val []byte) { - - stringid := strconv.FormatInt(node.ID, 10) +// encode a node as bytes (between 16 & 17 bytes used) +func nodeToBytes(node *osmpbf.Node) (string, []byte) { var bufval bytes.Buffer - bufval.WriteString(strconv.FormatFloat(node.Lat, 'f', 7, 64)) - bufval.WriteString(":") - bufval.WriteString(strconv.FormatFloat(node.Lon, 'f', 7, 64)) + // encode lat/lon as two 64 bit floats packed in to 16 bytes + var bufLatLon = make([]byte, 16) + binary.BigEndian.PutUint64(bufLatLon[:8], math.Float64bits(node.Lat)) + binary.BigEndian.PutUint64(bufLatLon[8:], math.Float64bits(node.Lon)) + bufval.Write(bufLatLon) + + // generate a bitmask for relevant tag features var isEntrance = isEntranceNode(node) if isEntrance > 0 { - bufval.WriteString(fmt.Sprintf(":%d:%d", isEntrance, isWheelchairAccessibleNode(node))) + // leftmost two bits are for the entrance, next two bits are accessibility + // remaining 4 rightmost bits are reserved for future use. + var bitmask = isEntrance << 6 + var isWheelchairAccessible = isWheelchairAccessibleNode(node) + if isWheelchairAccessible > 0 { + bitmask |= isWheelchairAccessible << 4 + } + bufval.WriteByte(bitmask) } - byteval := []byte(bufval.String()) + stringid := strconv.FormatInt(node.ID, 10) + byteval := bufval.Bytes() return stringid, byteval }