Support including other ledger files

README.md

@@ -29,6 +29,9 @@ Example transaction:
 
 A ledger file is a list of transactions separated by a blank line.
 
+A ledger file may include other ledger files using `include <filepath>`. The
+`filepath` is relative to the including file.
+
 
 ## ledger
 

cmd/ledger/main.go

@@ -3,7 +3,6 @@ package main
 import (
 	"flag"
 	"fmt"
-	"os"
 	"strings"
 	"time"
 
@@ -68,16 +67,15 @@ func main() {
 		return
 	}
 
-	ledgerFileReader, err := os.Open(ledgerFileName)
+	ledgerFileReader, err := ledger.NewLedgerReader(ledgerFileName)
 	if err != nil {
 		fmt.Println(err)
 		return
 	}
-	defer ledgerFileReader.Close()
 
 	generalLedger, parseError := ledger.ParseLedger(ledgerFileReader)
 	if parseError != nil {
-		fmt.Printf("%s:%s\n", ledgerFileName, parseError.Error())
+		fmt.Printf("%s\n", parseError.Error())
 		return
 	}
 

cmd/limport/main.go

@@ -58,12 +58,11 @@ func main() {
 	}
 	defer csvFileReader.Close()
 
-	ledgerFileReader, err := os.Open(ledgerFileName)
+	ledgerFileReader, err := ledger.NewLedgerReader(ledgerFileName)
 	if err != nil {
 		fmt.Println("Ledger: ", err)
 		return
 	}
-	defer ledgerFileReader.Close()
 
 	generalLedger, parseError := ledger.ParseLedger(ledgerFileReader)
 	if parseError != nil {

cmd/llint/main.go

@@ -17,12 +17,11 @@ func main() {
 		usage(os.Args[0])
 	}
 	ledgerFileName := os.Args[1]
-	ledgerFileReader, err := os.Open(ledgerFileName)
+	ledgerFileReader, err := ledger.NewLedgerReader(ledgerFileName)
 	if err != nil {
 		fmt.Println("Ledger: ", err)
 		return
 	}
-	defer ledgerFileReader.Close()
 
 	c, e := ledger.ParseLedgerAsync(ledgerFileReader)
 	errorCount := 0

cmd/lweb/main.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"os"
 	"sync"
 	"time"
 
@@ -34,14 +33,13 @@ func getTransactions() ([]*ledger.Transaction, error) {
 	var buf bytes.Buffer
 	h := sha256.New()
 
-	ledgerFileReader, err := os.Open(ledgerFileName)
+	ledgerFileReader, err := ledger.NewLedgerReader(ledgerFileName)
 	if err != nil {
 		return nil, err
 
 	}
 	tr := io.TeeReader(ledgerFileReader, h)
 	io.Copy(&buf, tr)
-	ledgerFileReader.Close()
 
 	sum := h.Sum(nil)
 	if bytes.Equal(currentSum, sum) {
@@ -50,7 +48,7 @@ func getTransactions() ([]*ledger.Transaction, error) {
 
 	trans, terr := ledger.ParseLedger(&buf)
 	if terr != nil {
-		return nil, fmt.Errorf("%s:%s", ledgerFileName, terr.Error())
+		return nil, fmt.Errorf("%s", terr.Error())
 	}
 	currentSum = sum
 	currentTrans = trans

ledgerReader.go

@@ -0,0 +1,84 @@
+package ledger
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+)
+
+const (
+	markerPrefix = ";__ledger_file"
+)
+
+var includedFiles = make(map[string]bool)
+
+func NewLedgerReader(filename string) (*bytes.Buffer, error) {
+	var buf bytes.Buffer
+
+	err := includeFile(filename, &buf)
+	return &buf, err
+}
+
+// includeFile reads filename into buf, adding special marker comments
+// when there are step changes in file location due to 'include' directive.
+func includeFile(filename string, buf *bytes.Buffer) error {
+	filename = filepath.Clean(filename)
+	lineNum := 0
+
+	// check for include cyles
+	if includedFiles[filename] {
+		return fmt.Errorf("include cycle: '%s'", filename)
+	} else {
+		includedFiles[filename] = true
+	}
+	defer delete(includedFiles, filename)
+
+	f, err := os.Open(filename)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	s := bufio.NewScanner(f)
+
+	// mark the start of this file
+	fmt.Fprintln(buf, marker(filename, lineNum))
+
+	for s.Scan() {
+		line := s.Text()
+
+		if strings.HasPrefix(line, "include") {
+			pieces := strings.Split(line, " ")
+			if len(pieces) != 2 {
+				return fmt.Errorf("%s:%d: invalid include directive", filename, lineNum)
+			}
+
+			err := includeFile(filepath.Join(filename, "..", pieces[1]), buf)
+			if err != nil {
+				return fmt.Errorf("%s:%d: %s", filename, lineNum, err.Error())
+			}
+			lineNum++
+
+			// mark the resumption point for this file
+			fmt.Fprintln(buf, marker(filename, lineNum))
+		} else {
+			fmt.Fprintln(buf, s.Text())
+			lineNum++
+		}
+	}
+
+	return nil
+}
+
+func marker(filename string, lineNum int) string {
+	return fmt.Sprintf("%s*-*%s*-*%d", markerPrefix, filename, lineNum)
+}
+
+func parseMarker(s string) (string, int) {
+	v := strings.Split(s, "*-*")
+	lineNum, _ := strconv.Atoi(v[2])
+	return v[1], lineNum
+}

ledgerReader_test.go

@@ -0,0 +1,62 @@
+package ledger
+
+import (
+	"bytes"
+	"io/ioutil"
+	"path/filepath"
+	"testing"
+)
+
+var testDir string
+
+func TestLedgerScannerBasic(t *testing.T) {
+	r, err := NewLedgerReader("testdata/ledgerReader_input_0")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	parsed, err := ioutil.ReadAll(r)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected, _ := ioutil.ReadFile(filepath.Join("testdata", "ledgerReader_expected_0"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !bytes.Equal(parsed, expected) {
+		t.Fatalf("expected:\n%s\n\ngot:\n%s\n", expected, parsed)
+	}
+}
+
+func TestLedgerScannerSingleInclude(t *testing.T) {
+	r, err := NewLedgerReader("testdata/ledgerReader_input_1_root")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	parsed, err := ioutil.ReadAll(r)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected, _ := ioutil.ReadFile(filepath.Join("testdata", "ledgerReader_expected_1"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !bytes.Equal(parsed, expected) {
+		t.Fatalf("expected:\n%s\n\n got:\n%s", expected, parsed)
+	}
+}
+
+func TestMarkerSplit(t *testing.T) {
+	filename, lineNum := parseMarker(";__ledger_file*-*/somedir/somefile*-*45")
+	if filename != "/somedir/somefile" {
+		t.Fatalf("expected: %s got:%s", "/somedir/somefile", filename)
+	}
+	if lineNum != 45 {
+		t.Fatalf("expected: %d got:%d", 45, lineNum)
+	}
+}

parse.go

@@ -47,16 +47,28 @@ func ParseLedgerAsync(ledgerReader io.Reader) (c chan *Transaction, e chan error
 		var trans *Transaction
 		scanner := bufio.NewScanner(ledgerReader)
 		var line string
+		var filename string
 		var lineCount int
 
+		errorMsg := func(msg string) {
+			e <- fmt.Errorf("%s:%d: %s", filename, lineCount, msg)
+		}
+
 		for scanner.Scan() {
 			line = scanner.Text()
+
+			// update filename/line if sentinel comment is found
+			if strings.HasPrefix(line, markerPrefix) {
+				filename, lineCount = parseMarker(line)
+				continue
+			}
+
 			// remove heading and tailing space from the line
 			trimmedLine := strings.Trim(line, whitespace)
 			lineCount++
 
 			// handle comments
-			if commentIdx := strings.Index(trimmedLine, ";"); commentIdx >=0 {
+			if commentIdx := strings.Index(trimmedLine, ";"); commentIdx >= 0 {
 				trimmedLine = trimmedLine[:commentIdx]
 				if len(trimmedLine) == 0 {
 					continue
@@ -67,20 +79,21 @@ func ParseLedgerAsync(ledgerReader io.Reader) (c chan *Transaction, e chan error
 				if trans != nil {
 					transErr := balanceTransaction(trans)
 					if transErr != nil {
-						e <- fmt.Errorf("%d: Unable to balance transaction, %s", lineCount, transErr)
+						errorMsg("Unable to balance transaction, " + transErr.Error())
 					}
 					c <- trans
 					trans = nil
 				}
 			} else if trans == nil {
 				lineSplit := strings.SplitN(line, " ", 2)
 				if len(lineSplit) != 2 {
-					e <- fmt.Errorf("%d: Unable to parse payee line: %s", lineCount, line)
+					errorMsg("Unable to parse payee line: " + line)
+					continue
 				}
 				dateString := lineSplit[0]
 				transDate, dateErr := date.Parse(dateString)
 				if dateErr != nil {
-					e <- fmt.Errorf("%d: Unable to parse date: %s", lineCount, dateString)
+					errorMsg("Unable to parse date: " + dateString)
 				}
 				payeeString := lineSplit[1]
 				trans = &Transaction{Payee: payeeString, Date: transDate}
@@ -110,7 +123,7 @@ func ParseLedgerAsync(ledgerReader io.Reader) (c chan *Transaction, e chan error
 		if trans != nil {
 			transErr := balanceTransaction(trans)
 			if transErr != nil {
-				e <- fmt.Errorf("%d: Unable to balance transaction, %s", lineCount, transErr)
+				errorMsg("Unable to balance transaction, " + transErr.Error())
 			}
 			c <- trans
 			trans = nil

testdata/ledgerReader_expected_0

@@ -0,0 +1,7 @@
+;__ledger_file*-*testdata/ledgerReader_input_0*-*0
+aaa
+bbb
+ccc
+
+ddd
+eee

testdata/ledgerReader_expected_1

@@ -0,0 +1,14 @@
+;__ledger_file*-*testdata/ledgerReader_input_1_root*-*0
+aaa
+bbb
+;__ledger_file*-*testdata/ledgerReader_input_1_inc1*-*0
+xxx
+yyy
+;__ledger_file*-*testdata/ledgerReader_input_1_inc2*-*0
+111
+222
+333
+;__ledger_file*-*testdata/ledgerReader_input_1_inc1*-*3
+zzz
+;__ledger_file*-*testdata/ledgerReader_input_1_root*-*3
+ccc

testdata/ledgerReader_input_0

@@ -0,0 +1,6 @@
+aaa
+bbb
+ccc
+
+ddd
+eee

testdata/ledgerReader_input_1_inc1

@@ -0,0 +1,4 @@
+xxx
+yyy
+include ledgerReader_input_1_inc2
+zzz

testdata/ledgerReader_input_1_inc2

@@ -0,0 +1,3 @@
+111
+222
+333

testdata/ledgerReader_input_1_root

@@ -0,0 +1,4 @@
+aaa
+bbb
+include ledgerReader_input_1_inc1
+ccc