Merge pull request #2452 from masatake/R-rewrite

R: rewrite the parser

.gitignore

@@ -97,3 +97,4 @@ Units/**/Makefile.am
 Units/**/Makefile.in
 Tmain/dist.m4
 Units/dist.m4
+VALGRIND.tmp.core.*

Tmain/list-fields-with-prefix.d/stdout-expected.txt

@@ -30,5 +30,6 @@ x       UCTAGSxpath          no      NONE             s--    no    xpath for the
 -       UCTAGShome           yes     Passwd           s--    no    home directory
 -       UCTAGSshell          yes     Passwd           s--    no    login shell
 -       UCTAGSdecorators     no      Python           s--    no    decorators on functions and classes
+-       UCTAGSassignmentop   no      R                s--    no    operator for assignment
 -       UCTAGSsectionMarker  no      ReStructuredText s--    no    character used for declaring section
 -       UCTAGSmixin          yes     Ruby             s--    no    how the class or module is mixed in (mixin:HOW:MODULE)

Tmain/list-fields.d/stdout-expected.txt

@@ -48,6 +48,7 @@ z	kind	no	NONE	s--	no	[tags output] prepend "kind:" to k/ (or K/) field output,
 -	home	yes	Passwd	s--	no	home directory
 -	shell	yes	Passwd	s--	no	login shell
 -	decorators	no	Python	s--	no	decorators on functions and classes
+-	assignmentop	no	R	s--	no	operator for assignment
 -	sectionMarker	no	ReStructuredText	s--	no	character used for declaring section
 -	mixin	yes	Ruby	s--	no	how the class or module is mixed in (mixin:HOW:MODULE)
 #

Tmain/list-roles.d/stdout-expected.txt

@@ -58,6 +58,9 @@ Python         i/module          indirectlyImported on      module imported in a
 Python         i/module          namespace          on      namespace from where classes/variables/functions are imported
 Python         x/unknown         imported           on      imported from the other module
 Python         x/unknown         indirectlyImported on      classes/variables/functions/modules imported in alternative name
+R              l/library         library            on      library attached by library fucntion
+R              l/library         require            on      library attached by require fucntion
+R              s/source          source             on      source loaded by source fucntion
 RpmSpec        m/macro           undef              on      undefined
 RpmSpec        p/patch           decl               on      declared for applying later
 Ruby           L/library         loaded             on      loaded by "load" method
@@ -137,6 +140,9 @@ Python         i/module          indirectlyImported on      module imported in a
 Python         i/module          namespace          on      namespace from where classes/variables/functions are imported
 Python         x/unknown         imported           on      imported from the other module
 Python         x/unknown         indirectlyImported on      classes/variables/functions/modules imported in alternative name
+R              l/library         library            on      library attached by library fucntion
+R              l/library         require            on      library attached by require fucntion
+R              s/source          source             on      source loaded by source fucntion
 RpmSpec        m/macro           undef              on      undefined
 RpmSpec        p/patch           decl               on      declared for applying later
 Ruby           L/library         loaded             on      loaded by "load" method

Units/parser-r.r/r-anonfunc.d/args.ctags

@@ -0,0 +1,4 @@
+--kinds-R=*
+--fields-R=*
+--fields=+E
+--sort=no

Units/parser-r.r/r-anonfunc.d/expected.tags

@@ -0,0 +1,5 @@
+anonFunccf813d470103	input.r	/^setGeneric("run", function(object) 1)$/;"	f	extras:anonymous
+object	input.r	/^setGeneric("run", function(object) 1)$/;"	z	function:anonFunccf813d470103
+anonFunccf813d470203	input.r	/^setMethod("run", c("C"), definition = function (object) {$/;"	f	extras:anonymous
+object	input.r	/^setMethod("run", c("C"), definition = function (object) {$/;"	z	function:anonFunccf813d470203
+c	input.r	/^c <- new ("C", x = 2)$/;"	g	assignmentop:<-

Units/parser-r.r/r-anonfunc.d/input.r

@@ -0,0 +1,14 @@
+# This can be a test case for S4 object system in the future.
+
+setClass("C", representation(x="numeric"))
+
+setGeneric("run", function(object) 1)
+
+setMethod("run", c("C"), definition = function (object) {
+		 object@x <- 4
+		 3 + object@x
+})
+
+c <- new ("C", x = 2)
+print(run(c))
+print(run('a'))

Units/parser-r.r/r-avoid-duplication.d/args.ctags

@@ -0,0 +1,3 @@
+--sort=no
+--fields=+n
+--kinds-R=+z

Units/parser-r.r/r-avoid-duplication.d/expected.tags

@@ -0,0 +1,15 @@
+x	input.r	/^x <- 1$/;"	g	line:1
+f	input.r	/^f <- function () {$/;"	f	line:4
+x	input.r	/^  x <- 3$/;"	v	line:5	function:f
+f	input.r	/^f <- function () {$/;"	f	line:10
+x	input.r	/^  x <- 4$/;"	v	line:11	function:f
+f	input.r	/^f <- function (x) {$/;"	f	line:14
+x	input.r	/^f <- function (x) {$/;"	z	line:14	function:f
+f	input.r	/^f <- function (x) {$/;"	f	line:18
+x	input.r	/^f <- function (x) {$/;"	z	line:18	function:f
+g	input.r	/^  g <- function () {$/;"	v	line:19	function:f
+x	input.r	/^    x <- 6$/;"	v	line:20	functionVar:f.g
+f	input.r	/^f <- function (x) {$/;"	f	line:24
+x	input.r	/^f <- function (x) {$/;"	z	line:24	function:f
+g	input.r	/^  g <- function () {$/;"	v	line:25	function:f
+x	input.r	/^    x <- 6$/;"	v	line:26	functionVar:f.g

Units/parser-r.r/r-avoid-duplication.d/input.r

@@ -0,0 +1,39 @@
+x <- 1
+x <- 2
+
+f <- function () {
+  x <- 3
+}
+
+# Next one `f' is captured because `f` is function
+# though `f' is already defined in the global scope.
+f <- function () {
+  x <- 4
+}
+
+f <- function (x) {
+  x <- 5
+}
+
+f <- function (x) {
+  g <- function () {
+    x <- 6
+  }
+}
+
+f <- function (x) {
+  g <- function () {
+    x <- 6
+  }
+  # Following `g' are not captured because `g' is functionVar
+  # and already defined.
+  g <- function (x) {
+    x <- 7
+  }
+  g <- function () {
+    x <- 8
+  }
+  g <- function () {
+    y <- 0
+  }
+}

Units/parser-r.r/r-broken-input.d/README

@@ -0,0 +1,2 @@
+Input files are broken. Therefore no expected.tags is here.
+

Units/parser-r.r/r-broken-input.d/input-0.r

@@ -0,0 +1 @@
+library(+)

Units/parser-r.r/r-broken-input.d/input-1.r

@@ -0,0 +1 @@
+source("abc"

Units/parser-r.r/r-broken-input.d/input-2.r

@@ -0,0 +1 @@
+library(+,

Units/parser-r.r/r-broken-input.d/input.r

@@ -0,0 +1 @@
+#

Units/parser-r.r/r-dots.d/args.ctags

@@ -0,0 +1,3 @@
+--sort=no
+--kinds-R=+z
+--fields=+{signature}

Units/parser-r.r/r-dots.d/expected.tags

@@ -0,0 +1,13 @@
+f0	input.r	/^f0 <- function (x0) {$/;"	f	signature:(x0)
+x0	input.r	/^f0 <- function (x0) {$/;"	z	function:f0
+f1	input.r	/^f1 <- function (x1 = c(1, 2, 3)) {$/;"	f	signature:(x1 = c (1, 2, 3))
+x1	input.r	/^f1 <- function (x1 = c(1, 2, 3)) {$/;"	z	function:f1
+f2	input.r	/^f2 <- function (x2, y2) {$/;"	f	signature:(x2, y2)
+x2	input.r	/^f2 <- function (x2, y2) {$/;"	z	function:f2
+y2	input.r	/^f2 <- function (x2, y2) {$/;"	z	function:f2
+f3	input.r	/^f3 <- function (x3 = c(1, 2, 3), y3 = c(c(1,2,3))) {$/;"	f	signature:(x3 = c (1, 2, 3), y3 = c (c (1, 2, 3)))
+x3	input.r	/^f3 <- function (x3 = c(1, 2, 3), y3 = c(c(1,2,3))) {$/;"	z	function:f3
+y3	input.r	/^f3 <- function (x3 = c(1, 2, 3), y3 = c(c(1,2,3))) {$/;"	z	function:f3
+f4	input.r	/^f4 <- function (x4 = c(1, 2, 3), y4 = c(c(1,2,3)), ...) {$/;"	f	signature:(x4 = c (1, 2, 3), y4 = c (c (1, 2, 3)), ...)
+x4	input.r	/^f4 <- function (x4 = c(1, 2, 3), y4 = c(c(1,2,3)), ...) {$/;"	z	function:f4
+y4	input.r	/^f4 <- function (x4 = c(1, 2, 3), y4 = c(c(1,2,3)), ...) {$/;"	z	function:f4

Units/parser-r.r/r-dots.d/input.r

@@ -0,0 +1,20 @@
+f0 <- function (x0) {
+   1
+}
+
+f1 <- function (x1 = c(1, 2, 3)) {
+   1
+}
+
+f2 <- function (x2, y2) {
+   1
+}
+
+f3 <- function (x3 = c(1, 2, 3), y3 = c(c(1,2,3))) {
+   1
+}
+
+f4 <- function (x4 = c(1, 2, 3), y4 = c(c(1,2,3)), ...) {
+   1
+}
+

Units/parser-r.r/r-extended.d/args.ctags

@@ -0,0 +1 @@
+--extras=+r

Units/parser-r.r/r-extended.d/expected.tags

@@ -1,3 +1,3 @@
+"mystuff.R"	input.r	/^  source("mystuff.R");$/;"	s
 .First	input.r	/^.First <- function() {$/;"	f
 MASS	input.r	/^  library(MASS)                         # attach a package$/;"	l
-file.path(Sys.getenv("HOME"	input.r	/^  source(file.path(Sys.getenv("HOME"), "R", "mystuff.R"))$/;"	s

Units/parser-r.r/r-extended.d/input.r

@@ -5,7 +5,12 @@
   options(digits=5, length=999)         # custom numbers and printout
   x11()                                 # for graphics
   par(pch = "+")                        # plotting character
-  source(file.path(Sys.getenv("HOME"), "R", "mystuff.R"))
-                                        # my personal functions
+  #
+  # In the following function call, "mystuff.R" should be
+  # captured. However, it is not easy.
+  #
+  # source(file.path(Sys.getenv("HOME"), "R", "mystuff.R"))
+  #                                       # my personal functions
   library(MASS)                         # attach a package
+  source("mystuff.R");
 }

Units/parser-r.r/r-external-entities.d/args.ctags

@@ -0,0 +1,4 @@
+--extras=+r
+--fields=+r
+--sort=no
+

Units/parser-r.r/r-external-entities.d/expected.tags

@@ -0,0 +1,4 @@
+"./input-0.r"	input.r	/^source(".\/input-0.r", echo = T)$/;"	s	roles:source
+splines	input.r	/^library(splines)            # attach package 'splines'$/;"	l	roles:library
+base	input.r	/^require(base)$/;"	l	roles:require
+x	input-0.r	/^x <- 1$/;"	g	roles:def

Units/parser-r.r/r-external-entities.d/input-0.r

@@ -0,0 +1 @@
+x <- 1

Units/parser-r.r/r-external-entities.d/input.r

@@ -0,0 +1,10 @@
+source("./input-0.r", echo = T)
+library()
+
+# Taken from
+# https://stat.ethz.ch/R-manual/R-devel/library/base/html/library.html
+library(lib.loc = .Library) # list all packages in the default library
+library(help = splines)     # documentation on package 'splines'
+library(splines)            # attach package 'splines'
+
+require(base)

Units/parser-r.r/r-list-indexing.d/args.ctags

@@ -0,0 +1 @@
+--sort=no

Units/parser-r.r/r-list-indexing.d/expected.tags

@@ -0,0 +1,9 @@
+x	input.r	/^x <- NULL$/;"	g
+y	input.r	/^x$x1 <- y <- 1:10$/;"	g
+z	input.r	/^x$x2 <- z <- 0:9 -> a$/;"	g
+a	input.r	/^x$x2 <- z <- 0:9 -> a$/;"	g
+f	input.r	/^f <- function () {$/;"	f
+X	input.r	/^	X <- NULL$/;"	v	function:f
+Y	input.r	/^	X$x1 <- Y <- 1:10$/;"	v	function:f
+Z	input.r	/^	X$x2 <- Z <- 0:9 -> A$/;"	v	function:f
+A	input.r	/^	X$x2 <- Z <- 0:9 -> A$/;"	v	function:f

Units/parser-r.r/r-list-indexing.d/input.r

@@ -0,0 +1,10 @@
+x <- NULL
+x$x1 <- y <- 1:10
+x$x2 <- z <- 0:9 -> a
+
+f <- function () {
+	X <- NULL
+	X$x1 <- Y <- 1:10
+	X$x2 <- Z <- 0:9 -> A
+}
+

Units/parser-r.r/r-loop-counters.d/args.ctags

@@ -0,0 +1,2 @@
+--kinds-R=+vz
+--sort=no

Units/parser-r.r/r-loop-counters.d/expected.tags

@@ -0,0 +1,6 @@
+`%*%`	input.r	/^`%*%` <- function (v, p) {$/;"	f
+v	input.r	/^`%*%` <- function (v, p) {$/;"	z	function:`%*%`
+p	input.r	/^`%*%` <- function (v, p) {$/;"	z	function:`%*%`
+r	input.r	/^    r <- 1$/;"	v	function:`%*%`
+i	input.r	/^        for (i in 1:p) {$/;"	v	function:`%*%`
+j	input.r	/^for (j in 1:3) {$/;"	g

Units/parser-r.r/r-loop-counters.d/input.r

@@ -0,0 +1,20 @@
+# defining an operator
+`%*%` <- function (v, p) {
+    r <- 1
+    if (p == 1) {
+        r <- v
+    } else {
+        for (i in 1:p) {
+            r <- r * v
+        }
+    }
+    r
+}
+
+print (c(2 %*% 10, 2%*%2, 2%*%3)[1:2])
+
+# j introduced as a global var.
+for (j in 1:3) {
+    print (j)
+}
+print (j)

Units/parser-r.r/r-scope.d/args.ctags

@@ -0,0 +1,4 @@
+--sort=no
+--fields-R=+{assignmentop}
+--fields=+eSn
+--kinds-R=+z

Units/parser-r.r/r-scope.d/expected.tags

@@ -0,0 +1,62 @@
+c00	input.r	/^c00 = 1$/;"	g	line:1	end:1	assignmentop:=
+c01	input.r	/^c01 <- 2$/;"	g	line:2	end:2	assignmentop:<-
+c02	input.r	/^c02 <<- 3$/;"	g	line:3	end:3	assignmentop:<<-
+f0	input.r	/^f0 <- function() {$/;"	f	line:5	signature:()	end:7	assignmentop:<-
+r0	input.r	/^   r0 <- c00 + c01 + c02$/;"	v	line:6	function:f0	end:6	assignmentop:<-
+f1	input.r	/^f1 <- function(a1) {$/;"	f	line:9	signature:(a1)	end:11	assignmentop:<-
+a1	input.r	/^f1 <- function(a1) {$/;"	z	line:9	function:f1
+r1	input.r	/^   r1 = c00 + c01 + c02 + a1$/;"	v	line:10	function:f1	end:10	assignmentop:=
+f2	input.r	/^f2 <- function(a20, a21) {$/;"	f	line:13	signature:(a20, a21)	end:15	assignmentop:<-
+a20	input.r	/^f2 <- function(a20, a21) {$/;"	z	line:13	function:f2
+a21	input.r	/^f2 <- function(a20, a21) {$/;"	z	line:13	function:f2
+r2	input.r	/^   r2 <<- c00 + c01 + c02 + a20 + a21$/;"	g	line:14	end:14	assignmentop:<<-
+f4	input.r	/^f4 <- function() c02 +$/;"	f	line:17	signature:()	end:19	assignmentop:<-
+f5	input.r	/^f5 <- function(a5) c(1,$/;"	f	line:21	signature:(a5)	end:24	assignmentop:<-
+a5	input.r	/^f5 <- function(a5) c(1,$/;"	z	line:21	function:f5
+r5	input.r	/^r5 <- 3,$/;"	v	line:23	function:f5	end:23	assignmentop:<-
+c50	input.r	/^f5((c50 = 4))$/;"	g	line:27	end:27	assignmentop:=
+c51	input.r	/^f5(c51 <- 4)$/;"	g	line:28	end:28	assignmentop:<-
+c52	input.r	/^f5(c52 <<- 4)$/;"	g	line:29	end:29	assignmentop:<<-
+c0	input-2.r	/^{c0 = 1$/;"	g	line:1	end:1	assignmentop:=
+d0	input-2.r	/^   d0 <- 2}$/;"	g	line:3	end:3	assignmentop:<-
+f0	input-2.r	/^{f0 = 1;$/;"	g	line:5	end:5	assignmentop:=
+g0	input-2.r	/^   g0 <- 2}$/;"	g	line:6	end:6	assignmentop:<-
+h0	input-2.r	/^{h0 = 1; j0 <- 2}$/;"	g	line:8	end:8	assignmentop:=
+j0	input-2.r	/^{h0 = 1; j0 <- 2}$/;"	g	line:8	end:8	assignmentop:<-
+f	input-2.r	/^f <- function () {$/;"	f	line:10	signature:()	end:32	assignmentop:<-
+c1	input-2.r	/^    {c1 = 1$/;"	v	line:11	function:f	end:11	assignmentop:=
+d1	input-2.r	/^       d1 <- 2}$/;"	v	line:13	function:f	end:13	assignmentop:<-
+f1	input-2.r	/^    {f1 = 1;$/;"	v	line:15	function:f	end:15	assignmentop:=
+g1	input-2.r	/^       g1 <- 2}$/;"	v	line:16	function:f	end:16	assignmentop:<-
+h1	input-2.r	/^    {h1 = 1; j1 <- 2}$/;"	v	line:18	function:f	end:18	assignmentop:=
+j1	input-2.r	/^    {h1 = 1; j1 <- 2}$/;"	v	line:18	function:f	end:18	assignmentop:<-
+g	input-2.r	/^    g = function () {$/;"	v	line:19	function:f	signature:()	end:28	assignmentop:=
+c2	input-2.r	/^	{c2 = 1$/;"	v	line:20	functionVar:f.g	end:20	assignmentop:=
+d2	input-2.r	/^	   d2 <- 2}$/;"	v	line:22	functionVar:f.g	end:22	assignmentop:<-
+f2	input-2.r	/^	{f2 = 1;$/;"	v	line:24	functionVar:f.g	end:24	assignmentop:=
+g2	input-2.r	/^	   g2 <- 2}$/;"	v	line:25	functionVar:f.g	end:25	assignmentop:<-
+h2	input-2.r	/^	{h2 = 1; j2 <- 2}$/;"	v	line:27	functionVar:f.g	end:27	assignmentop:=
+j2	input-2.r	/^	{h2 = 1; j2 <- 2}$/;"	v	line:27	functionVar:f.g	end:27	assignmentop:<-
+h0	input-2.r	/^    }; h0 = function () {$/;"	v	line:28	function:f	signature:()	end:31	assignmentop:=
+k	input-2.r	/^	   k <<- 3$/;"	g	line:29	end:29	assignmentop:<<-
+f	input-3.r	/^f <- function () {$/;"	f	line:1	signature:()	end:8	assignmentop:<-
+d	input-3.r	/^	d <- ($/;"	v	line:2	function:f	end:4	assignmentop:<-
+a	input-3.r	/^	<- 4) + 5)$/;"	v	line:4	function:f	end:4	assignmentop:<-
+f	input-4.r	/^f <- function () {$/;"	f	line:1	signature:()	end:3	assignmentop:<-
+c0	input-4.r	/^  {;c0 = 1;}$/;"	v	line:2	function:f	end:2	assignmentop:=
+g	input-4.r	/^}; g <- function () {$/;"	f	line:3	signature:()	end:5	assignmentop:<-
+c1	input-4.r	/^  {c1 = 2}$/;"	v	line:4	function:g	end:4	assignmentop:=
+`f<<-`	input-5.r	/^`f<<-` <- function() {$/;"	f	line:1	signature:()	end:3	assignmentop:<-
+f	input-6.r	/^f <- function (x, y) {$/;"	f	line:1	signature:(x, y)	end:3	assignmentop:<-
+x	input-6.r	/^f <- function (x, y) {$/;"	z	line:1	function:f
+y	input-6.r	/^f <- function (x, y) {$/;"	z	line:1	function:f
+z	input-6.r	/^} + y + (z <- 3) + 4$/;"	v	line:3	function:f	end:3	assignmentop:<-
+a	input-7.r	/^	a <- function ()$/;"	f	line:2	signature:()	end:5	assignmentop:<-
+b	input-7.r	/^	b <- function (x)$/;"	f	line:7	signature:(x)	end:8	assignmentop:<-
+x	input-7.r	/^	b <- function (x)$/;"	z	line:7	function:b
+c	input-7.r	/^	c <- function (x)$/;"	f	line:10	signature:(x)	end:11	assignmentop:<-
+x	input-7.r	/^	c <- function (x)$/;"	z	line:10	function:c
+f8	input-8.r	/^f8 <- function () {$/;"	f	line:2	signature:()	end:3	assignmentop:<-
+g8	input-8.r	/^} -> g8$/;"	v	line:3	function:f8	assignmentop:->
+x8	input-8.r	/^x8 <- 1 -> y8$/;"	g	line:6	end:6	assignmentop:<-
+y8	input-8.r	/^x8 <- 1 -> y8$/;"	g	line:6	assignmentop:->

Units/parser-r.r/r-scope.d/input-2.r

@@ -0,0 +1,32 @@
+{c0 = 1
+   ;
+   d0 <- 2}
+
+{f0 = 1;
+   g0 <- 2}
+
+{h0 = 1; j0 <- 2}
+
+f <- function () {
+    {c1 = 1
+       ;
+       d1 <- 2}
+
+    {f1 = 1;
+       g1 <- 2}
+
+    {h1 = 1; j1 <- 2}
+    g = function () {
+	{c2 = 1
+	   ;
+	   d2 <- 2}
+
+	{f2 = 1;
+	   g2 <- 2}
+
+	{h2 = 1; j2 <- 2}
+    }; h0 = function () {
+	   k <<- 3
+    } +
+    5
+}

Units/parser-r.r/r-scope.d/input-3.r

@@ -0,0 +1,8 @@
+f <- function () {
+	d <- (
+	  1 + 2 + 3 + (a
+	<- 4) + 5)
+
+
+
+}

Units/parser-r.r/r-scope.d/input-4.r

@@ -0,0 +1,5 @@
+f <- function () {
+  {;c0 = 1;}
+}; g <- function () {
+  {c1 = 2}
+}

Units/parser-r.r/r-scope.d/input-5.r

@@ -0,0 +1,3 @@
+`f<<-` <- function() {
+       1
+}