* updates tests to cover more cases of tokenizing

* uncomments tests that should now work

src/test/java/org/json/junit/CDLTest.java

@@ -30,7 +30,7 @@ public class CDLTest {
     );
 
     /**
-     * CDL.toJSONArray() adds all values asstrings, with no filtering or 
+     * CDL.toJSONArray() adds all values as strings, with no filtering or 
      * conversions. For testing, this means that the expected JSONObject 
      * values all must be quoted in the cases where the JSONObject parsing 
      * might normally convert the value into a non-string. 
@@ -264,8 +264,8 @@ public void checkSpecialChars() {
      */
     @Test
     public void textToJSONArray() {
-        JSONArray jsonArray = CDL.toJSONArray(lines);
-        JSONArray expectedJsonArray = new JSONArray(expectedLines);
+        JSONArray jsonArray = CDL.toJSONArray(this.lines);
+        JSONArray expectedJsonArray = new JSONArray(this.expectedLines);
         Util.compareActualVsExpectedJsonArrays(jsonArray, expectedJsonArray);
     }
 
@@ -289,10 +289,10 @@ public void jsonArrayToJSONArray() {
      */
     @Test
     public void textToJSONArrayAndBackToString() {
-        JSONArray jsonArray = CDL.toJSONArray(lines);
+        JSONArray jsonArray = CDL.toJSONArray(this.lines);
         String jsonStr = CDL.toString(jsonArray);
         JSONArray finalJsonArray = CDL.toJSONArray(jsonStr);
-        JSONArray expectedJsonArray = new JSONArray(expectedLines);
+        JSONArray expectedJsonArray = new JSONArray(this.expectedLines);
         Util.compareActualVsExpectedJsonArrays(finalJsonArray, expectedJsonArray);
     }
 

src/test/java/org/json/junit/JSONTokenerTest.java

@@ -12,7 +12,9 @@
 import java.io.Reader;
 import java.io.StringReader;
 
+import org.json.JSONArray;
 import org.json.JSONException;
+import org.json.JSONObject;
 import org.json.JSONTokener;
 import org.junit.Test;
 
@@ -64,7 +66,99 @@ public void verifyBackFailureDoubleBack() throws IOException {
             }
        }
     }
+
+    @Test
+    public void testValid() {
+        checkValid("0",Number.class);
+        checkValid(" 0  ",Number.class);
+        checkValid("23",Number.class);
+        checkValid("23.5",Number.class);
+        checkValid(" 23.5  ",Number.class);
+        checkValid("null",null);
+        checkValid(" null  ",null);
+        checkValid("true",Boolean.class);
+        checkValid(" true\n",Boolean.class);
+        checkValid("false",Boolean.class);
+        checkValid("\nfalse  ",Boolean.class);
+        checkValid("{}",JSONObject.class);
+        checkValid(" {}  ",JSONObject.class);
+        checkValid("{\"a\":1}",JSONObject.class);
+        checkValid(" {\"a\":1}  ",JSONObject.class);
+        checkValid("[]",JSONArray.class);
+        checkValid(" []  ",JSONArray.class);
+        checkValid("[1,2]",JSONArray.class);
+        checkValid("\n\n[1,2]\n\n",JSONArray.class);
+        checkValid("1 2", String.class);
+    }
+
+    @Test
+    public void testErrors() {
+        // Check that stream can detect that a value is found after
+        // the first one
+        checkError(" { \"a\":1 }  4 ");
+        checkError("null \"a\"");
+        checkError("{} true");
+    }
+
+    private Object checkValid(String testStr, Class<?> aClass)  {
+        Object result = nextValue(testStr);
 
+        // Check class of object returned
+        if( null == aClass ) {
+            if(JSONObject.NULL.equals(result)) {
+                // OK
+            } else {
+                throw new JSONException("Unexpected class: "+result.getClass().getSimpleName());
+            }
+        } else {
+            if( null == result ) {
+                throw new JSONException("Unexpected null result");
+            } else if(!aClass.isAssignableFrom(result.getClass()) ) {
+                throw new JSONException("Unexpected class: "+result.getClass().getSimpleName());
+            }
+        }
+
+        return result;
+    }
+
+    private void checkError(String testStr) {
+        try {
+            nextValue(testStr);
+
+            fail("Error should be triggered: (\""+testStr+"\")");
+        } catch (JSONException e) {
+            // OK
+        }
+    }
+
+    /**
+     * Verifies that JSONTokener can read a stream that contains a value. After
+     * the reading is done, check that the stream is left in the correct state
+     * by reading the characters after. All valid cases should reach end of stream.
+     * @param testStr
+     * @return
+     * @throws Exception
+     */
+    private Object nextValue(String testStr) throws JSONException {
+        try(StringReader sr = new StringReader(testStr);){
+            JSONTokener tokener = new JSONTokener(sr);
+
+            Object result = tokener.nextValue();
+
+            if( result == null ) {
+                throw new JSONException("Unable to find value token in JSON stream: ("+tokener+"): "+testStr);
+            }
+
+            char c = tokener.nextClean();
+            if( 0 != c ) {
+                throw new JSONException("Unexpected character found at end of JSON stream: "+c+ " ("+tokener+"): "+testStr);
+            }
+
+            return result;
+        }
+
+    }
+
     /**
      * Tests the failure of the skipTo method with a buffered reader. Preferably
      * we'd like this not to fail but at this time we don't have a good recovery.

src/test/java/org/json/junit/XMLTest.java

@@ -6,6 +6,13 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+
 import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
@@ -743,14 +750,10 @@ private void compareStringToJSONObject(String xmlStr, String expectedStr) {
      * @param expectedStr the expected JSON string
      */
     private void compareReaderToJSONObject(String xmlStr, String expectedStr) {
-        /*
-         * Commenting out this method until the JSON-java code is updated
-         * to support XML.toJSONObject(reader)
         JSONObject expectedJsonObject = new JSONObject(expectedStr);
         Reader reader = new StringReader(xmlStr);
         JSONObject jsonObject = XML.toJSONObject(reader);
         Util.compareActualVsExpectedJsonObjects(jsonObject,expectedJsonObject);
-        */
     }
 
     /**
@@ -764,22 +767,19 @@ private void compareReaderToJSONObject(String xmlStr, String expectedStr) {
      * @throws IOException
      */
     private void compareFileToJSONObject(String xmlStr, String expectedStr) {
-        /*
-         * Commenting out this method until the JSON-java code is updated
-         * to support XML.toJSONObject(reader)
         try {
             JSONObject expectedJsonObject = new JSONObject(expectedStr);
-            File tempFile = testFolder.newFile("fileToJSONObject.xml");
-            FileWriter fileWriter = new FileWriter(tempFile);
-            fileWriter.write(xmlStr);
-            fileWriter.close();
-            Reader reader = new FileReader(tempFile);
-            JSONObject jsonObject = XML.toJSONObject(reader);
-            Util.compareActualVsExpectedJsonObjects(jsonObject,expectedJsonObject);
+            File tempFile = this.testFolder.newFile("fileToJSONObject.xml");
+            try(FileWriter fileWriter = new FileWriter(tempFile);){
+                fileWriter.write(xmlStr);
+            }
+            try(Reader reader = new FileReader(tempFile);){
+                JSONObject jsonObject = XML.toJSONObject(reader);
+                Util.compareActualVsExpectedJsonObjects(jsonObject,expectedJsonObject);
+            }
         } catch (IOException e) {
-            assertTrue("file writer error: " +e.getMessage(), false);
+            fail("file writer error: " +e.getMessage());
         }
-        */
     }
 
     /**