added regression test for issue #18

Signed-off-by: Tim Henderson <tadh@google.com>

lexer_test.go

@@ -292,3 +292,176 @@ func TestRegression(t *testing.T) {
 
 	}
 }
+
+func TestRegression2(t *testing.T) {
+
+	text := `# dhcpd.conf
+#
+# Sample configuration file for ISC dhcpd
+#
+
+# option definitions common to all supported networks...
+option domain-name "example.org";
+option domain-name-servers ns1.example.org, ns2.example.org;
+
+default-lease-time 600;
+max-lease-time 7200;
+
+# The ddns-updates-style parameter controls whether or not the server will
+# attempt to do a DNS update when a lease is confirmed. We default to the
+# behavior of the version 2 packages ('none', since DHCP v2 didn't
+# have support for DDNS.)
+ddns-update-style none;
+
+# If this DHCP server is the official DHCP server for the local
+# network, the authoritative directive should be uncommented.
+#authoritative;
+
+# Use this to send dhcp log messages to a different log file (you also
+# have to hack syslog.conf to complete the redirection).
+#log-facility local7;
+
+# No service will be given on this subnet, but declaring it helps the 
+# DHCP server to understand the network topology.
+
+#subnet 10.152.187.0 netmask 255.255.255.0 {
+#}
+
+# This is a very basic subnet declaration.
+
+#subnet 10.254.239.0 netmask 255.255.255.224 {
+#  range 10.254.239.10 10.254.239.20;
+#  option routers rtr-239-0-1.example.org, rtr-239-0-2.example.org;
+#}
+
+# This declaration allows BOOTP clients to get dynamic addresses,
+# which we don't really recommend.
+
+#subnet 10.254.239.32 netmask 255.255.255.224 {
+#  range dynamic-bootp 10.254.239.40 10.254.239.60;
+#  option broadcast-address 10.254.239.31;
+#  option routers rtr-239-32-1.example.org;
+#}
+
+# A slightly different configuration for an internal subnet.
+#subnet 10.5.5.0 netmask 255.255.255.224 {
+#  range 10.5.5.26 10.5.5.30;
+#  option domain-name-servers ns1.internal.example.org;
+#  option domain-name "internal.example.org";
+#  option routers 10.5.5.1;
+#  option broadcast-address 10.5.5.31;
+#  default-lease-time 600;
+#  max-lease-time 7200;
+#}
+
+# Hosts which require special configuration options can be listed in
+# host statements.   If no address is specified, the address will be
+# allocated dynamically (if possible), but the host-specific information
+# will still come from the host declaration.
+
+#host passacaglia {
+#  hardware ethernet 0:0:c0:5d:bd:95;
+#  filename "vmunix.passacaglia";
+#  server-name "toccata.example.com";
+#}
+
+# Fixed IP addresses can also be specified for hosts.   These addresses
+# should not also be listed as being available for dynamic assignment.
+# Hosts for which fixed IP addresses have been specified can boot using
+# BOOTP or DHCP.   Hosts for which no fixed address is specified can only
+# be booted with DHCP, unless there is an address range on the subnet
+# to which a BOOTP client is connected which has the dynamic-bootp flag
+# set.
+#host fantasia {
+#  hardware ethernet 08:00:07:26:c0:a5;
+#  fixed-address fantasia.example.com;
+#}
+
+# You can declare a class of clients and then do address allocation
+# based on that.   The example below shows a case where all clients
+# in a certain class get addresses on the 10.17.224/24 subnet, and all
+# other clients get addresses on the 10.0.29/24 subnet.
+
+#class "foo" {
+#  match if substring (option vendor-class-identifier, 0, 4) = "SUNW";
+#}
+
+#shared-network 224-29 {
+#  subnet 10.17.224.0 netmask 255.255.255.0 {
+#    option routers rtr-224.example.org;
+#  }
+#  subnet 10.0.29.0 netmask 255.255.255.0 {
+#    option routers rtr-29.example.org;
+#  }
+#  pool {
+#    allow members of "foo";
+#    range 10.17.224.10 10.17.224.250;
+#  }
+#  pool {
+#    deny members of "foo";
+#    range 10.0.29.10 10.0.29.230;
+#  }
+#}
+`
+
+	newLexer := func() *Lexer {
+		literals := []string{
+			"{",
+			"}",
+			";",
+			",",
+		}
+		tokens := []string{
+			"COMMENT",
+			"ID",
+		}
+		tokens = append(tokens, literals...)
+		tokenIds := map[string]int{}
+		for i, tok := range tokens {
+			tokenIds[tok] = i
+		}
+		lex := NewLexer()
+
+		skip := func(*Scanner, *machines.Match) (interface{}, error) {
+			return nil, nil
+		}
+		token := func(name string) Action {
+			return func(s *Scanner, m *machines.Match) (interface{}, error) {
+				return s.Token(tokenIds[name], string(m.Bytes), m), nil
+			}
+		}
+
+		lex.Add([]byte(`#[^\n]*\n?`), token("COMMENT"))
+		lex.Add([]byte(`([a-z]|[A-Z]|[0-9]|_|\-|\.)*`), token("ID"))
+		lex.Add([]byte(`"([^\\"]|(\\.))*"`), token("ID"))
+		lex.Add([]byte("[\n \t]"), skip)
+		for _, lit := range literals {
+			lex.Add([]byte(lit), token(lit))
+		}
+
+		err := lex.Compile()
+		if err != nil {
+			panic(err)
+		}
+
+		return lex
+	}
+
+	scanner, err := newLexer().Scanner([]byte(text))
+	if err != nil {
+		return
+	}
+	for tok, err, eof := scanner.Next(); !eof; tok, err, eof = scanner.Next() {
+		if err != nil {
+			t.Error(err)
+		}
+		token := tok.(*Token)
+		fmt.Printf("%-7v | %-10v | %v:%v-%v:%v\n",
+			token.Type,
+			string(token.Lexeme),
+			token.StartLine,
+			token.StartColumn,
+			token.EndLine,
+			token.EndColumn)
+	}
+}