Merge branch 'master' into modular-canonicalizer

Jenkinsfile

@@ -6,6 +6,7 @@ def utils = new org.stan.Utils()
 def skipExpressionTests = false
 def skipRemainingStages = false
 def skipCompileTests = false
+def skipRebuildingBinaries = false
 def buildingAgentARM = "linux"
 
 /* Functions that runs a sh command and returns the stdout */
@@ -28,6 +29,12 @@ pipeline {
     agent none
     parameters {
         booleanParam(name:"compile_all", defaultValue: false, description:"Try compiling all models in test/integration/good")
+        string(defaultValue: '', name: 'cmdstan_pr',
+               description: "CmdStan PR to test against. Will check out this PR in the downstream Stan repo.")
+        string(defaultValue: '', name: 'stan_pr',
+               description: "Stan PR to test against. Will check out this PR in the downstream Stan repo.")
+        string(defaultValue: '', name: 'math_pr',
+               description: "Math PR to test against. Will check out this PR in the downstream Math repo.")
     }
     options {parallelsAlwaysFailFast()}
     stages {
@@ -49,12 +56,15 @@ pipeline {
                     def stanMathSigs = ['test/integration/signatures/stan_math_sigs.t'].join(" ")
                     skipExpressionTests = utils.verifyChanges(stanMathSigs)
 
-                    def sourceCodePaths = ['src'].join(" ")
-                    skipRemainingStages = utils.verifyChanges(sourceCodePaths)
+                    def runTestPaths = ['src', 'test/integration/good', 'test/stancjs'].join(" ")
+                    skipRemainingStages = utils.verifyChanges(runTestPaths)
 
                     def compileTests = ['test/integration/good'].join(" ")
                     skipCompileTests = utils.verifyChanges(compileTests)
 
+                    def sourceCodePaths = ['src'].join(" ")
+                    skipRebuildingBinaries = utils.verifyChanges(sourceCodePaths)
+
                     if (buildingTag()) {
                         buildingAgentARM = "arm-ec2"
                     }
@@ -180,6 +190,11 @@ pipeline {
 
                             writeFile(file:"performance-tests-cmdstan/cmdstan/make/local",
                                     text:"O=0\nCXX=${CXX}")
+
+                            utils.checkout_pr("cmdstan", "performance-tests-cmdstan/cmdstan", params.cmdstan_pr)
+                            utils.checkout_pr("stan", "performance-tests-cmdstan/cmdstan/stan", params.stan_pr)
+                            utils.checkout_pr("math", "performance-tests-cmdstan/cmdstan/stan/lib/stan_math", params.math_pr)
+
                             sh """
                                 cd performance-tests-cmdstan
                                 mkdir cmdstan/bin
@@ -209,23 +224,28 @@ pipeline {
                     }
                     agent { label 'linux' }
                     steps {
-                        unstash 'ubuntu-exe'
-                        sh """
-                            git clone --recursive --depth 50 https://github.com/stan-dev/performance-tests-cmdstan
-                        """
-                        sh """
-                            cd performance-tests-cmdstan
-                            git show HEAD --stat
-                            echo "example-models/regression_tests/mother.stan" > all.tests
-                            cat known_good_perf_all.tests >> all.tests
-                            echo "" >> all.tests
-                            cat shotgun_perf_all.tests >> all.tests
-                            cat all.tests
-                            echo "CXXFLAGS+=-march=core2" > cmdstan/make/local
-                            echo "PRECOMPILED_HEADERS=false" >> cmdstan/make/local
-                            cd cmdstan; make clean-all; git show HEAD --stat; cd ..
-                            CXX="${CXX}" ./compare-compilers.sh "--tests-file all.tests --num-samples=10" "\$(readlink -f ../bin/stanc)"
-                        """
+                        script {
+                            unstash 'ubuntu-exe'
+                            sh """
+                                git clone --recursive --depth 50 https://github.com/stan-dev/performance-tests-cmdstan
+                            """
+                            utils.checkout_pr("cmdstan", "performance-tests-cmdstan/cmdstan", params.cmdstan_pr)
+                            utils.checkout_pr("stan", "performance-tests-cmdstan/cmdstan/stan", params.stan_pr)
+                            utils.checkout_pr("math", "performance-tests-cmdstan/cmdstan/stan/lib/stan_math", params.math_pr)
+                            sh """
+                                cd performance-tests-cmdstan
+                                git show HEAD --stat
+                                echo "example-models/regression_tests/mother.stan" > all.tests
+                                cat known_good_perf_all.tests >> all.tests
+                                echo "" >> all.tests
+                                cat shotgun_perf_all.tests >> all.tests
+                                cat all.tests
+                                echo "CXXFLAGS+=-march=core2" > cmdstan/make/local
+                                echo "PRECOMPILED_HEADERS=false" >> cmdstan/make/local
+                                cd cmdstan; make clean-all; git show HEAD --stat; cd ..
+                                CXX="${CXX}" ./compare-compilers.sh "--tests-file all.tests --num-samples=10" "\$(readlink -f ../bin/stanc)"
+                            """
+                        }
 
                         xunit([GoogleTest(
                             deleteOutputFiles: false,
@@ -287,7 +307,7 @@ pipeline {
                     when {
                         beforeAgent true
                         expression {
-                            !skipRemainingStages
+                            !skipRebuildingBinaries
                         }
                     }
                     agent { label "osx && ocaml" }
@@ -311,7 +331,7 @@ pipeline {
                     when {
                         beforeAgent true
                         expression {
-                            !skipRemainingStages
+                            !skipRebuildingBinaries
                         }
                     }
                     agent {
@@ -338,7 +358,7 @@ pipeline {
                     when {
                         beforeAgent true
                         expression {
-                            !skipRemainingStages
+                            !skipRebuildingBinaries
                         }
                     }
                     agent {
@@ -366,7 +386,7 @@ pipeline {
                     when {
                         beforeAgent true
                         allOf {
-                            expression { !skipRemainingStages }
+                            expression { !skipRebuildingBinaries }
                             anyOf { buildingTag(); branch 'master' }
                         }
                     }
@@ -396,7 +416,7 @@ pipeline {
                     when {
                         beforeAgent true
                         allOf {
-                            expression { !skipRemainingStages }
+                            expression { !skipRebuildingBinaries }
                             anyOf { buildingTag(); branch 'master' }
                         }
                     }
@@ -426,7 +446,7 @@ pipeline {
                     when {
                         beforeAgent true
                         allOf {
-                            expression { !skipRemainingStages }
+                            expression { !skipRebuildingBinaries }
                             anyOf { buildingTag(); branch 'master' }
                         }
                     }
@@ -456,7 +476,7 @@ pipeline {
                     when {
                         beforeAgent true
                         allOf {
-                            expression { !skipRemainingStages }
+                            expression { !skipRebuildingBinaries }
                             anyOf { buildingTag(); branch 'master' }
                         }
                     }
@@ -487,7 +507,7 @@ pipeline {
                     when {
                         beforeAgent true
                         allOf {
-                            expression { !skipRemainingStages }
+                            expression { !skipRebuildingBinaries }
                             anyOf { buildingTag(); branch 'master' }
                         }
                     }
@@ -518,7 +538,7 @@ pipeline {
                     when {
                         beforeAgent true
                         allOf {
-                            expression { !skipRemainingStages }
+                            expression { !skipRebuildingBinaries }
                             anyOf { buildingTag(); branch 'master' }
                         }
                     }
@@ -550,7 +570,7 @@ pipeline {
                     when {
                         beforeAgent true
                         expression {
-                            !skipRemainingStages
+                            !skipRebuildingBinaries
                         }
                     }
                     agent {

src/frontend/Typechecker.ml

@@ -110,8 +110,10 @@ let verify_name_fresh_var loc tenv name =
   if Utils.is_unnormalized_distribution name then
     Semantic_error.ident_has_unnormalized_suffix loc name |> error
   else if
-    Env.mem tenv name
-    && not (Stan_math_signatures.is_stan_math_function_name name)
+    List.exists (Env.find tenv name) ~f:(function
+      | {kind= `StanMath; _} ->
+          false (* user variables can shadow library names *)
+      | _ -> true )
   then Semantic_error.ident_in_use loc name |> error
 
 (** verify that the variable being declared is previous unused.
@@ -960,7 +962,11 @@ let rec stmt_is_escape {stmt; _} =
 
 and list_until_escape xs =
   let rec aux accu = function
-    | next :: next' :: _ when stmt_is_escape next' ->
+    | [next; next'] when stmt_is_escape next' -> List.rev (next' :: next :: accu)
+    | next :: next' :: unreachable :: _ when stmt_is_escape next' ->
+        add_warning unreachable.smeta.loc
+          "Unreachable statement (following a reject, break, continue, or \
+           return) found, is this intended?" ;
         List.rev (next' :: next :: accu)
     | next :: rest -> aux (next :: accu) rest
     | [] -> List.rev accu in
@@ -1097,24 +1103,22 @@ and check_loop_body cf tenv loop_var loop_var_ty loop_body =
 and check_block loc cf tenv stmts =
   let _, checked_stmts =
     List.fold_map stmts ~init:tenv ~f:(check_statement cf) in
-  let stmts = list_until_escape checked_stmts in
   let return_type =
-    stmts
+    checked_stmts |> list_until_escape
     |> List.map ~f:(fun s -> s.smeta.return_type)
     |> List.fold ~init:NoReturnType
          ~f:(try_compute_block_statement_returntype loc) in
-  mk_typed_statement ~stmt:(Block stmts) ~return_type ~loc
+  mk_typed_statement ~stmt:(Block checked_stmts) ~return_type ~loc
 
 and check_profile loc cf tenv name stmts =
   let _, checked_stmts =
     List.fold_map stmts ~init:tenv ~f:(check_statement cf) in
-  let stmts = list_until_escape checked_stmts in
   let return_type =
-    stmts
+    checked_stmts |> list_until_escape
     |> List.map ~f:(fun s -> s.smeta.return_type)
     |> List.fold ~init:NoReturnType
          ~f:(try_compute_block_statement_returntype loc) in
-  mk_typed_statement ~stmt:(Profile (name, stmts)) ~return_type ~loc
+  mk_typed_statement ~stmt:(Profile (name, checked_stmts)) ~return_type ~loc
 
 (* variable declarations *)
 and verify_valid_transformation_for_type loc is_global sized_ty trans =

test/integration/bad/lang/reused_name.stan

@@ -0,0 +1,5 @@
+generated quantities {
+    vector[3] e;
+    // e is selected because it is also a Stan-math function 'e()'
+    vector[3] e;
+}

test/integration/bad/lang/stanc.expected

@@ -696,6 +696,17 @@ Expected an expression inside "[" and "]" but found a statement. Indexing should
   var[1 + i]
 not
   var[for (n in 1:N) ...]
+  $ ../../../../../install/default/bin/stanc reused_name.stan
+Semantic error in 'reused_name.stan', line 4, column 14 to column 15:
+   -------------------------------------------------
+     2:      vector[3] e;
+     3:      // e is selected because it is also a Stan-math function 'e()'
+     4:      vector[3] e;
+                       ^
+     5:  }
+   -------------------------------------------------
+
+Identifier 'e' is already in use.
   $ ../../../../../install/default/bin/stanc unterminated_comment.stan
 Syntax error in 'unterminated_comment.stan', line 4, column -1, lexing error:
    -------------------------------------------------

test/integration/good/lang/pretty.expected

@@ -609,3 +609,16 @@ generated quantities {
   xgq = 2713;
 }
 
+  $ ../../../../../install/default/bin/stanc --auto-format unreachable_statement.stan
+functions {
+  void foo(real x) {
+    if (x) 
+      ; // bug - misplaced ; makes the next statement unconditional
+    reject("Cannot be 0");
+    return;
+  }
+}
+
+Warning in 'unreachable_statement.stan', line 5, column 5: Unreachable
+    statement (following a reject, break, continue, or return) found, is this
+    intended?

test/integration/good/lang/unreachable_statement.stan

@@ -0,0 +1,7 @@
+functions {
+   void foo(real x){
+     if(x) ; // bug - misplaced ; makes the next statement unconditional
+       reject("Cannot be 0");
+     return;
+   }
+}

test/stancjs/data-generation.js

@@ -0,0 +1,21 @@
+var stanc = require('../../src/stancjs/stancjs.bc.js');
+var utils = require("./utils/utils.js");
+
+let datagen_model = `
+data {
+    int x[3, 4];
+    int y[5, 2, 4];
+    matrix[3, 4] z;
+    vector[3] w;
+    vector[3] p[4];
+}
+`
+
+let datagen = stanc.stanc("basic_model", datagen_model, ["debug-generate-data"]);
+const res = JSON.parse(datagen.result)
+console.log("dim(x) = (" + res.x.length + ", " + res.x[0].length + ")")
+console.log("dim(y) = (" + res.y.length + ", " + res.y[0].length + ", " + res.y[0][0].length + ")")
+console.log("dim(z) = (" + res.z.length + ", " + res.z[0].length + ")")
+console.log("dim(w) = (" + res.w.length + ")")
+console.log("dim(p) = (" + res.p.length + ", " + res.p[0].length + ")")
+console.log()

test/stancjs/stancjs.expected

@@ -45,6 +45,13 @@ model {
   y ~ normal(0, 1);
 }
 
+$ node data-generation.js
+dim(x) = (3, 4)
+dim(y) = (5, 2, 4)
+dim(z) = (3, 4)
+dim(w) = (3)
+dim(p) = (4, 3)
+
 $ node filename.js
 Semantic error in 'good_filename', line 6, column 4 to column 5: 
 Identifier 'z' not in scope.

test/stancjs/standalone-functions.js

@@ -50,5 +50,5 @@ let basic = stanc.stanc("basic_stanfuncs", basic_stanfuncs, ["standalone-functio
 utils.print_error(basic)
 var ind = basic.result.search("auto int_only_multiplication");
 if (ind == -1) {
-    console.log("ERROR: standalone-functions!")
+	console.log("ERROR: standalone-functions!")
 }