Skip to content

Commit

Permalink
Merge pull request #15 from LdDl/new-export-import
Browse files Browse the repository at this point in the history 
Improve export/import
  • Loading branch information
@LdDl
LdDl authored Mar 29, 2021
2 parents e3d1382 + 828d7b4 commit e850b66
Show file tree
Hide file tree
Showing 6 changed files with 257 additions and 78 deletions.
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
benchmark_graph.csv
/osm_data
/data/export_pgrouting.csv
/data/export_pgrouting_vertices.csv
/data/export_pgrouting_contractions.csv
new.txt
old.txt
cmd/osm2ch/*.csv
Expand Down
15 changes: 9 additions & 6 deletions ROADMAP.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,17 @@ New ideas, thought about needed features will be store in this file.
* Initial core
* Import file of specific format **Done as CSV**
* Export file of specific format **Done as CSV**
* Turn Restricted Shortest Path extension for CH-algorithm **Propably not modify algorithm, but graph. UPD: done with osm2ch - https://github.com/LdDl/osm2ch#osm2ch**
* Thoughts and discussions about OSM graph and extensions **Need some ideas about parsing and preparing. UPD: osm2ch - https://github.com/LdDl/osm2ch#osm2ch**
* Map matcher as another project **WIP (now it is in local git storage)** Here is the one https://github.com/LdDl/horizon
* Bring OSM parser **WIP It's done in poor way. PRs are welcome**
* Bring OSM restrictions **WIP It's done in poor way. PRs are welcome**
* Turn Restricted Shortest Path extension for CH-algorithm **Propably not modify algorithm, but graph. Excluded: done with osm2ch - https://github.com/LdDl/osm2ch#osm2ch**
* Thoughts and discussions about OSM graph and extensions **Need some ideas about parsing and preparing. Excluded: osm2ch - https://github.com/LdDl/osm2ch#osm2ch**
* Map matcher as another project **Here is the one https://github.com/LdDl/horizon**
* Bring OSM parser **Excluded**
* Bring OSM restrictions **Excluded**
* OneTwoMany function (contraction hierarchies) **Done, may be some bench comparisons**
* Replace int with int64 (OSM purposes) **Done**
* Separate benchmarks to BENCHMARK.md **Done**
* Better CSV format or another format (JSON / binary). **W.I.P. Splitting single file to multiple**

### Planned
* Better CSV format or another format (JSON / binary)
* Better heuristics for calculationg importance of each vertex.
* Max-cost path finder.
* N-best shortest pathes.
146 changes: 109 additions & 37 deletions export.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,20 +5,33 @@ import (
"fmt"
"os"
"strconv"
"strings"
)

// ExportToFile Exports graph to file of CSV-format
// Header of main CSV-file:
// Header of edges CSV-file:
// from_vertex_id - int64, ID of source vertex
// to_vertex_id - int64, ID of target vertex
// f_internal - int64, Internal ID of source vertex
// t_internal - int64, Internal ID of target vertex
// weight - float64, Weight of an edge
// Header of vertices CSV-file:
// vertex_id - int64, ID of vertex
// internal_id - int64, Internal ID of vertex
// order_pos - int, Position of vertex in hierarchies (evaluted by library)
// importance - int, Importance of vertex in graph (evaluted by library)
// Header of contractios CSV-file:
// from_vertex_id - int64, ID of source vertex
// to_vertex_id - int64, ID of arget vertex
// f_internal - int64, Internal ID of source vertex
// t_internal - int64, Internal ID of target vertex
// weight - float64, Weight of an edge
// via_vertex_id - int64, ID of vertex through which the contraction exists (-1 if no contraction)
// v_internal - int64, Internal ID of vertex through which the contraction exists (-1 if no contraction)
// via_vertex_id - int64, ID of vertex through which the contraction exists
// v_internal - int64, Internal ID of vertex through which the contraction exists
func (graph *Graph) ExportToFile(fname string) error {

file, err := os.Create(fname)
fnamePart := strings.Split(fname, ".csv") // to guarantee proper filename and its extension
file, err := os.Create(fnamePart[0] + ".csv")
if err != nil {
return err
}
Expand All @@ -27,7 +40,35 @@ func (graph *Graph) ExportToFile(fname string) error {
writer := csv.NewWriter(file)
defer writer.Flush()
writer.Comma = ';'
err = writer.Write([]string{"from_vertex_id", "to_vertex_id", "f_internal", "t_internal", "weight", "via_vertex_id", "v_internal"})
err = writer.Write([]string{"from_vertex_id", "to_vertex_id", "f_internal", "t_internal", "weight"})
if err != nil {
return err
}

fileVertices, err := os.Create(fnamePart[0] + "_vertices.csv")
if err != nil {
return err
}
defer fileVertices.Close()

writerVertices := csv.NewWriter(fileVertices)
defer writerVertices.Flush()
writerVertices.Comma = ';'
err = writerVertices.Write([]string{"vertex_id", "internal_id", "order_pos", "importance"})
if err != nil {
return err
}

fileContractions, err := os.Create(fnamePart[0] + "_contractions.csv")
if err != nil {
return err
}
defer fileContractions.Close()

writerContractions := csv.NewWriter(fileContractions)
defer writerContractions.Flush()
writerContractions.Comma = ';'
err = writerContractions.Write([]string{"from_vertex_id", "to_vertex_id", "f_internal", "t_internal", "weight", "via_vertex_id", "v_internal"})
if err != nil {
return err
}
Expand All @@ -36,55 +77,86 @@ func (graph *Graph) ExportToFile(fname string) error {
currentVertexExternal := graph.Vertices[i].Label
currentVertexInternal := graph.Vertices[i].vertexNum

// Write reference information about vertex
err = writerVertices.Write([]string{
fmt.Sprintf("%d", currentVertexExternal),
fmt.Sprintf("%d", currentVertexInternal),
fmt.Sprintf("%d", graph.Vertices[i].orderPos),
fmt.Sprintf("%d", graph.Vertices[i].importance),
})
if err != nil {
return err
}

// Write reference information about "incoming" adjacent vertices
incomingNeighbors := graph.Vertices[i].inEdges
incomingCosts := graph.Vertices[i].inECost
for j := 0; j < len(incomingNeighbors); j++ {
fromVertexExternal := graph.Vertices[incomingNeighbors[j]].Label
fromVertexInternal := incomingNeighbors[j]
cost := incomingCosts[j]
isContractExternal := int64(-1)
isContractInternal := int64(-1)
if v, ok := graph.contracts[fromVertexInternal][currentVertexInternal]; ok {
isContractExternal = graph.Vertices[v].Label
isContractInternal = v
}
err = writer.Write([]string{
fmt.Sprintf("%d", fromVertexExternal),
fmt.Sprintf("%d", currentVertexExternal),
fmt.Sprintf("%d", fromVertexInternal),
fmt.Sprintf("%d", currentVertexInternal),
strconv.FormatFloat(cost, 'f', -1, 64),
fmt.Sprintf("%d", isContractExternal),
fmt.Sprintf("%d", isContractInternal),
})
if err != nil {
return err
isContractExternal := graph.Vertices[v].Label
isContractInternal := v
err = writerContractions.Write([]string{
fmt.Sprintf("%d", fromVertexExternal),
fmt.Sprintf("%d", currentVertexExternal),
fmt.Sprintf("%d", fromVertexInternal),
fmt.Sprintf("%d", currentVertexInternal),
strconv.FormatFloat(cost, 'f', -1, 64),
fmt.Sprintf("%d", isContractExternal),
fmt.Sprintf("%d", isContractInternal),
})
if err != nil {
return err
}
} else {
err = writer.Write([]string{
fmt.Sprintf("%d", fromVertexExternal),
fmt.Sprintf("%d", currentVertexExternal),
fmt.Sprintf("%d", fromVertexInternal),
fmt.Sprintf("%d", currentVertexInternal),
strconv.FormatFloat(cost, 'f', -1, 64),
})
if err != nil {
return err
}
}
}

// Write reference information about "outcoming" adjacent vertices
outcomingNeighbors := graph.Vertices[i].outEdges
outcomingCosts := graph.Vertices[i].outECost
for j := 0; j < len(outcomingNeighbors); j++ {
toVertexExternal := graph.Vertices[outcomingNeighbors[j]].Label
toVertexInternal := outcomingNeighbors[j]
cost := outcomingCosts[j]
isContractExternal := int64(-1)
isContractInternal := int64(-1)
if v, ok := graph.contracts[currentVertexInternal][toVertexInternal]; ok {
isContractExternal = graph.Vertices[v].Label
isContractInternal = v
}
err = writer.Write([]string{
fmt.Sprintf("%d", currentVertexExternal),
fmt.Sprintf("%d", toVertexExternal),
fmt.Sprintf("%d", currentVertexInternal),
fmt.Sprintf("%d", toVertexInternal),
strconv.FormatFloat(cost, 'f', -1, 64),
fmt.Sprintf("%d", isContractExternal),
fmt.Sprintf("%d", isContractInternal),
})
if err != nil {
return err
isContractExternal := graph.Vertices[v].Label
isContractInternal := v
err = writerContractions.Write([]string{
fmt.Sprintf("%d", currentVertexExternal),
fmt.Sprintf("%d", toVertexExternal),
fmt.Sprintf("%d", currentVertexInternal),
fmt.Sprintf("%d", toVertexInternal),
strconv.FormatFloat(cost, 'f', -1, 64),
fmt.Sprintf("%d", isContractExternal),
fmt.Sprintf("%d", isContractInternal),
})
if err != nil {
return err
}
} else {
err = writer.Write([]string{
fmt.Sprintf("%d", currentVertexExternal),
fmt.Sprintf("%d", toVertexExternal),
fmt.Sprintf("%d", currentVertexInternal),
fmt.Sprintf("%d", toVertexInternal),
strconv.FormatFloat(cost, 'f', -1, 64),
})
if err != nil {
return err
}
}
}
}
Expand Down
10 changes: 8 additions & 2 deletions export_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,14 @@ func TestExport(t *testing.T) {
t.Log("Please wait until contraction hierarchy is prepared")
g.PrepareContracts()
t.Log("TestExport is starting...")
// t.Log(len(g.contracts)) // 268420
// t.Log(len(g.Vertices)) // 588804
correctNumContractions := 91757
correctNumVertices := 187853
if len(g.contracts) != correctNumContractions {
t.Errorf("Number of contractions should be %d, but got %d", correctNumContractions, len(g.contracts))
}
if len(g.Vertices) != correctNumVertices {
t.Errorf("Number of vertices should be %d, but got %d", correctNumVertices, len(g.Vertices))
}
err := g.ExportToFile("data/export_pgrouting.csv")
if err != nil {
t.Error(err)
Expand Down
Loading

0 comments on commit e850b66

Please sign in to comment.