Fix bufferstream reading

include/pbf_blocks.h

@@ -17,13 +17,13 @@
    ------------------- */
 
 // Read and parse a protobuf message
-void readMessage(google::protobuf::Message *message, std::fstream *input, unsigned int size);
+void readMessage(google::protobuf::Message *message, std::istream &input, unsigned int size);
 
 // Read an osm.pbf sequence of header length -> BlobHeader -> Blob
 // and parse the unzipped contents into a message
-void readBlock(google::protobuf::Message *messagePtr, std::fstream *inputPtr);
+void readBlock(google::protobuf::Message *messagePtr, std::istream &input);
 
-void writeBlock(google::protobuf::Message *messagePtr, std::fstream *outputPtr, std::string headerType);
+void writeBlock(google::protobuf::Message *messagePtr, std::ostream &output, std::string headerType);
 /* -------------------
    Tag handling
    ------------------- */

include/read_pbf.h

@@ -51,7 +51,7 @@ class PbfReader
 	PbfReader();
 	virtual ~PbfReader();
 
-	int ReadPbfFile(const std::string &inputFile, std::unordered_set<std::string> &nodeKeys);
+	int ReadPbfFile(std::istream &inputFile, std::unordered_set<std::string> &nodeKeys);
 
 	///Pointer to output object. Loaded objects are sent here.
 	PbfReaderOutput * output;

src/pbf_blocks.cpp

@@ -8,35 +8,35 @@ using namespace std;
    ------------------- */
 
 // Read and parse a protobuf message
-void readMessage(google::protobuf::Message *message, fstream *input, unsigned int size) {
+void readMessage(google::protobuf::Message *message, istream &input, unsigned int size) {
 	vector<char> buffer(size);
-	input->read(&buffer.front(), size);
+	input.read(&buffer.front(), size);
 	message->ParseFromArray(&buffer.front(), size);
 }
 
 // Read an osm.pbf sequence of header length -> BlobHeader -> Blob
 // and parse the unzipped contents into a message
-void readBlock(google::protobuf::Message *messagePtr, fstream *inputPtr) {
+void readBlock(google::protobuf::Message *messagePtr, istream &input) {
 	// read the header length
 	unsigned int size;
-	inputPtr->read((char*)&size, sizeof(size));
-	if (inputPtr->eof()) { return; }
+	input.read((char*)&size, sizeof(size));
+	if (input.eof()) { return; }
 	endian_swap(size);
 
 	// get BlobHeader and parse
 	BlobHeader bh;
-	readMessage(&bh, inputPtr, size);
+	readMessage(&bh, input, size);
 
 	// get Blob and parse
 	Blob blob;
-	readMessage(&blob, inputPtr, bh.datasize());
+	readMessage(&blob, input, bh.datasize());
 
 	// Unzip the gzipped content
 	string contents = decompress_string(blob.zlib_data());
 	messagePtr->ParseFromString(contents);
 }
 
-void writeBlock(google::protobuf::Message *messagePtr, fstream *outputPtr, string headerType) {
+void writeBlock(google::protobuf::Message *messagePtr, ostream &output, string headerType) {
 	// encode the message
 	string serialised;
 	messagePtr->SerializeToString(&serialised);
@@ -59,9 +59,9 @@ void writeBlock(google::protobuf::Message *messagePtr, fstream *outputPtr, strin
 	// write out
 	uint bhLength=header_encoded.length();
 	endian_swap(bhLength);
-	outputPtr->write(reinterpret_cast<const char *>(&bhLength), 4);
-	outputPtr->write(header_encoded.c_str(), header_encoded.length() );
-	outputPtr->write(blob_encoded.c_str(), blob_encoded.length() );
+	output.write(reinterpret_cast<const char *>(&bhLength), 4);
+	output.write(header_encoded.c_str(), header_encoded.length() );
+	output.write(blob_encoded.c_str(), blob_encoded.length() );
 }
 
 /* -------------------

src/read_pbf.cpp

@@ -1,6 +1,9 @@
 #include <iostream>
 #include "read_pbf.h"
 #include "pbf_blocks.h"
+
+#include <boost/interprocess/streams/bufferstream.hpp>
+
 using namespace std;
 
 PbfReader::PbfReader()
@@ -141,20 +144,18 @@ bool PbfReader::ReadRelations(PrimitiveGroup &pg) {
 	return false;
 }
 
-int PbfReader::ReadPbfFile(const string &inputFile, unordered_set<string> &nodeKeys)
+int PbfReader::ReadPbfFile(std::istream &infile, unordered_set<string> &nodeKeys)
 {
 	// ----	Read PBF
 	// note that the order of reading and processing is:
 	//  1) output nodes -> (remember current position for rewinding to ways) (skip ways) -> (just remember all ways in any relation),
 	//  2) output ways, and also construct nodeId list for each way in relation -> output relations
 
-	fstream infile(inputFile, ios::in | ios::binary);
-	if (!infile) { cerr << "Couldn't open .pbf file " << inputFile << endl; return -1; }
 
 	if (output != nullptr) output->startOsmData();
 
 	HeaderBlock block;
-	readBlock(&block, &infile);
+	readBlock(&block, infile);
 
 	PrimitiveBlock pb;
 	PrimitiveGroup pg;
@@ -166,7 +167,7 @@ int PbfReader::ReadPbfFile(const string &inputFile, unordered_set<string> &nodeK
 
 	while (true) {
 		long long blockStart = infile.tellg();
-		readBlock(&pb, &infile);
+		readBlock(&pb, infile);
 		if (infile.eof()) {
 			if (!checkedRelations) {
 				checkedRelations = true;
@@ -239,7 +240,6 @@ int PbfReader::ReadPbfFile(const string &inputFile, unordered_set<string> &nodeK
 		ct++;
 	}
 	cout << endl;
-	infile.close();
 
 	if(output != nullptr)
 		output->endOsmData();
@@ -280,7 +280,7 @@ int ReadPbfBoundingBox(const std::string &inputFile, double &minLon, double &max
 	fstream infile(inputFile, ios::in | ios::binary);
 	if (!infile) { cerr << "Couldn't open .pbf file " << inputFile << endl; return -1; }
 	HeaderBlock block;
-	readBlock(&block, &infile);
+	readBlock(&block, infile);
 	if (block.has_bbox()) {
 		hasClippingBox = true;		
 		minLon = block.bbox().left()  /1000000000.0;

src/tilemaker.cpp

@@ -48,6 +48,7 @@ typedef unsigned uint;
 #include "shp_mem_tiles.h"
 
 #include <boost/asio/post.hpp>
+#include <boost/interprocess/streams/bufferstream.hpp>
 
 // Namespaces
 using namespace std;
@@ -212,7 +213,11 @@ int main(int argc, char* argv[]) {
 	if (!mapsplit) {
 		for (auto inputFile : inputFiles) {
 			cout << "Reading .pbf " << inputFile << endl;
-			int ret = pbfReader.ReadPbfFile(inputFile, nodeKeys);
+
+			ifstream infile(inputFile, ios::in | ios::binary);
+			if (!infile) { cerr << "Couldn't open .pbf file " << inputFile << endl; return -1; }
+
+			int ret = pbfReader.ReadPbfFile(infile, nodeKeys);
 			if (ret != 0) return ret;
 		}
 	}
@@ -267,6 +272,7 @@ int main(int argc, char* argv[]) {
 	for (unsigned run=0; run<runs; run++) {
 		// Read mapsplit tile and parse, if applicable
 		int srcZ = -1, srcX = -1, srcY = -1, tmsY = -1;
+
 		if (mapsplit) {
 			osmMemTiles.Clear();
 			attributeStore.clearOsmAttributes();
@@ -278,14 +284,13 @@ int main(int argc, char* argv[]) {
 			} else if (srcZ > config.startZoom) {
 				cout << "Mapsplit tiles (zoom " << srcZ << ") can't write data at zoom level " << config.startZoom << endl;
 			}
+
 			cout << "Reading tile " << srcZ << ": " << srcX << "," << srcY << " (" << (run+1) << "/" << runs << ")" << endl;
 			vector<char> pbf = mapsplitFile.readTile(srcZ,srcX,tmsY);
-			// Write to a temp file and read back in - can we do this better via a stringstream or similar?
-			ofstream tempfile;
-			tempfile.open("/tmp/temp.pbf");
-			copy(pbf.cbegin(), pbf.cend(), ostreambuf_iterator<char>(tempfile));
-			tempfile.close();
-			pbfReader.ReadPbfFile("/tmp/temp.pbf", nodeKeys);
+
+			boost::interprocess::bufferstream pbfstream(pbf.data(), pbf.size(),  ios::in | ios::binary);
+			pbfReader.ReadPbfFile(pbfstream, nodeKeys);
+
 			tileList.pop_back();
 			attributeStore.sortOsmAttributes();
 		}