New testcases for token_get_all() function

ext/tokenizer/tests/token_get_all_basic.phpt

@@ -0,0 +1,97 @@
+--TEST--
+Test token_get_all() function : basic functionality 
+--FILE--
+<?php
+/* Prototype  : array token_get_all(string $source)
+ * Description : splits the given source into an array of PHP languange tokens
+ * Source code: ext/tokenizer/tokenizer.c
+*/
+
+echo "*** Testing token_get_all() : basic functionality ***\n";
+
+// with php open/close tags
+$source = '<?php echo "Hello World"; ?>';
+echo "-- source string with PHP open and close tags --\n";
+var_dump( token_get_all($source) );
+
+// without php open/close tags testing for T_INLINE_HTML
+$source = "echo 'Hello World';";
+echo "-- source string without PHP open and close tags --\n";
+var_dump( token_get_all($source) );
+
+echo "Done"
+?>
+--EXPECTF--
+*** Testing token_get_all() : basic functionality ***
+-- source string with PHP open and close tags --
+array(7) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [3]=>
+  array(3) {
+    [0]=>
+    int(315)
+    [1]=>
+    string(13) ""Hello World""
+    [2]=>
+    int(1)
+  }
+  [4]=>
+  string(1) ";"
+  [5]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [6]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(1)
+  }
+}
+-- source string without PHP open and close tags --
+array(1) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(311)
+    [1]=>
+    string(19) "echo 'Hello World';"
+    [2]=>
+    int(1)
+  }
+}
+Done

ext/tokenizer/tests/token_get_all_error.phpt

@@ -0,0 +1,35 @@
+--TEST--
+Test token_get_all() function : error conditions 
+--FILE--
+<?php
+/* Prototype  : array token_get_all(string $source)
+ * Description: splits the given source into an array of PHP languange tokens 
+ * Source code: ext/tokenizer/tokenizer.c
+*/
+
+echo "*** Testing token_get_all() : error conditions ***\n";
+
+// with zero arguments
+echo "\n-- Testing token_get_all() function with zero arguments --\n";
+var_dump( token_get_all());
+
+// with one more than the expected number of arguments
+echo "-- Testing token_get_all() function with more than expected no. of arguments --\n";
+$source = '<?php ?>';
+$extra_arg = 10;
+var_dump( token_get_all($source, $extra_arg));
+
+echo "Done"
+?>
+--EXPECTF--
+*** Testing token_get_all() : error conditions ***
+
+-- Testing token_get_all() function with zero arguments --
+
+Warning: token_get_all() expects exactly 1 parameter, 0 given in %s on line %d
+NULL
+-- Testing token_get_all() function with more than expected no. of arguments --
+
+Warning: token_get_all() expects exactly 1 parameter, 2 given in %s on line %d
+NULL
+Done