From 9974db50010fb9b379905f29aa33c5cd4e06921c Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Tue, 6 Jan 2015 21:18:38 +0100
Subject: [PATCH 01/95] Added support for sorting contents of the JSON markup,
 for printing out "normalized JSON markup" (the top element only, without the
 bracketed header), for concatenating input string values with embedded
 newlines into single-line strings with embedded "\n"

---
 JSON.sh | 126 ++++++++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 119 insertions(+), 7 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index ac0148b..1b42430 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -1,5 +1,9 @@
 #!/usr/bin/env bash
 
+# https://github.com/dominictarr/JSON.sh/blob/master/JSON.sh
+# MIT / Apache 2 licenses (C) 2014 by "dominictarr" checked out 2015-01-04
+# further development (C) 2015 Jim Klimov <EvgenyKlimov@eaton.com>
+
 throw () {
   echo "$*" >&2
   exit 1
@@ -8,15 +12,31 @@ throw () {
 BRIEF=0
 LEAFONLY=0
 PRUNE=0
+SORTDATA=""
+NORMALIZE=0
 
 usage() {
   echo
-  echo "Usage: JSON.sh [-b] [-l] [-p] [-h]"
+  echo "Usage: JSON.sh [-b] [-l] [-p] [-N] [-S|-S='args']"
+  echo "       JSON.sh [-N|-N='args'] < markup.json"
+  echo "       JSON.sh [-h]"
+  echo "-h - This help text."
   echo
   echo "-p - Prune empty. Exclude fields with empty values."
   echo "-l - Leaf only. Only show leaf nodes, which stops data duplication."
   echo "-b - Brief. Combines 'Leaf only' and 'Prune empty' options."
-  echo "-h - This help text."
+  echo
+  echo "Sorting is also available, although limited to single-line strings in"
+  echo "the markup (multilines are automatically escaped into backslash+n):"
+  echo "-S - Sort the contents of items in JSON markup and leaf-list markup:"
+  echo "     'sort' objects by key names and then values, and arrays by values"
+  echo "-S='args' - use 'sort \$args' for content sorting, e.g. use -S='-n -r'"
+  echo "     for reverse numeric sort"
+  echo "-N - Normalize the input JSON markup into a single-line JSON output;"
+  echo "     in this mode syntax and spacing are normalized, data order remains"
+  echo "-N='args' - Normalize the input JSON markup into a single-line JSON"
+  echo "     output with contents sorted like for -S='args', e.g. use -N='-n'"
+  echo "     This is equivalent to -N -S='args', just more compact to write"
   echo
 }
 
@@ -25,7 +45,7 @@ parse_options() {
   local ARGN=$#
   while [ $ARGN -ne 0 ]
   do
-    case $1 in
+    case "$1" in
       -h) usage
           exit 0
       ;;
@@ -37,6 +57,15 @@ parse_options() {
       ;;
       -p) PRUNE=1
       ;;
+      -N) NORMALIZE=1
+      ;;
+      -N=*) SORTDATA="sort `echo "$1" | sed 's,^-N=,,'`"
+          NORMALIZE=1
+      ;;
+      -S) SORTDATA="sort"
+      ;;
+      -S=*) SORTDATA="sort `echo "$1" | sed 's,^-S=,,'`"
+      ;;
       ?*) echo "ERROR: Unknown option."
           usage
           exit 0
@@ -45,6 +74,9 @@ parse_options() {
     shift 1
     ARGN=$((ARGN-1))
   done
+
+  # For normalized data, we do the whole job and just return the top object
+  [ "$NORMALIZE" -eq 1 ] && BRIEF=0 && LEAFONLY=0 && PRUNE=0
 }
 
 awk_egrep () {
@@ -60,6 +92,49 @@ awk_egrep () {
   }' pattern=$pattern_string
 }
 
+strip_newlines() {
+  # replace line returns inside strings in input with \n string
+  local ILINE
+  local LINESTRIP
+  local NUMQ
+  local ODD
+  local INSTRING=0
+
+  # the first "grep" should ensure that input has a trailing newline
+  grep '' | while IFS="" read -r ILINE; do
+    # Remove escaped quotes:
+    LINESTRIP="${ILINE//\\\"}"
+    # Remove all chars but remaining quotes:
+    LINESTRIP="${LINESTRIP//[^\"]}"
+    # Count unescaped quotes:
+    NUMQ="${#LINESTRIP}"
+    ODD="$(($NUMQ%2))"
+
+    if [ "$ODD" -eq 1 -a "$INSTRING" -eq 0 ]; then
+      printf '%s\\n' "$ILINE"
+      INSTRING=1
+      continue
+    fi
+
+    if [ "$ODD" -eq 1 -a "$INSTRING" -eq 1 ]; then
+      printf '%s\n' "$ILINE"
+      INSTRING=0
+      continue
+    fi
+
+    if [ "$ODD" -eq 0 -a "$INSTRING" -eq 1 ]; then
+      printf '%s\\n' "$ILINE"
+      continue
+    fi
+
+    if [ "$ODD" -eq 0 -a "$INSTRING" -eq 0 ]; then
+      printf '%s\n' "$ILINE"
+      continue
+    fi
+  done
+  :
+}
+
 tokenize () {
   local GREP
   local ESCAPE
@@ -82,17 +157,20 @@ tokenize () {
     CHAR='[^[:cntrl:]"\\\\]'
   fi
 
-  local STRING="\"$CHAR*($ESCAPE$CHAR*)*\""
+  local STRINGVAL="$CHAR*($ESCAPE$CHAR*)*"
+  local STRING="(\"$STRINGVAL\")"
   local NUMBER='-?(0|[1-9][0-9]*)([.][0-9]*)?([eE][+-]?[0-9]*)?'
   local KEYWORD='null|false|true'
   local SPACE='[[:space:]]+'
 
+  strip_newlines | \
   $GREP "$STRING|$NUMBER|$KEYWORD|$SPACE|." | egrep -v "^$SPACE$"
 }
 
 parse_array () {
   local index=0
   local ary=''
+  local aryml=''
   read -r token
   case "$token" in
     ']') ;;
@@ -101,7 +179,11 @@ parse_array () {
       do
         parse_value "$1" "$index"
         index=$((index+1))
-        ary="$ary""$value" 
+        ary="$ary""$value"
+        if [ -n "$SORTDATA" ]; then
+            [ -z "$aryml" ] && aryml="$value" || aryml="$aryml
+$value"
+        fi
         read -r token
         case "$token" in
           ']') break ;;
@@ -112,6 +194,9 @@ parse_array () {
       done
       ;;
   esac
+  if [ -n "$SORTDATA" ]; then
+    ary="`echo "$aryml" | $SORTDATA | tr '\n' ',' | sed 's|,*$||' | sed 's|^,*||'`"
+  fi
   [ "$BRIEF" -eq 0 ] && value=`printf '[%s]' "$ary"` || value=
   :
 }
@@ -119,6 +204,7 @@ parse_array () {
 parse_object () {
   local key
   local obj=''
+  local objml=''
   read -r token
   case "$token" in
     '}') ;;
@@ -136,7 +222,11 @@ parse_object () {
         esac
         read -r token
         parse_value "$1" "$key"
-        obj="$obj$key:$value"        
+        obj="$obj$key:$value"
+        if [ -n "$SORTDATA" ]; then
+            [ -z "$objml" ] && objml="$key:$value" || objml="$objml
+$key:$value"
+        fi
         read -r token
         case "$token" in
           '}') break ;;
@@ -147,6 +237,9 @@ parse_object () {
       done
     ;;
   esac
+  if [ -n "$SORTDATA" ]; then
+    obj="`echo "$objml" | $SORTDATA | tr '\n' ',' | sed 's|,*$||' | sed 's|^,*||'`"
+  fi
   [ "$BRIEF" -eq 0 ] && value=`printf '{%s}' "$obj"` || value=
   :
 }
@@ -163,12 +256,22 @@ parse_value () {
        [ "$value" = '""' ] && isempty=1
        ;;
   esac
+
+  if [ "$NORMALIZE" -eq 1 ]; then
+    # Ensure a "true" output from the "if" for "return"
+    [ "$jpath" != '' ] || printf "%s\n" "$value"
+    return
+  fi
+
+  ### Skip printing larger objects in brief mode
   [ "$value" = '' ] && return
+
   [ "$LEAFONLY" -eq 0 ] && [ "$PRUNE" -eq 0 ] && print=1
   [ "$LEAFONLY" -eq 1 ] && [ "$isleaf" -eq 1 ] && [ $PRUNE -eq 0 ] && print=1
   [ "$LEAFONLY" -eq 0 ] && [ "$PRUNE" -eq 1 ] && [ "$isempty" -eq 0 ] && print=1
   [ "$LEAFONLY" -eq 1 ] && [ "$isleaf" -eq 1 ] && \
     [ $PRUNE -eq 1 ] && [ $isempty -eq 0 ] && print=1
+
   [ "$print" -eq 1 ] && printf "[%s]\t%s\n" "$jpath" "$value"
   :
 }
@@ -183,8 +286,17 @@ parse () {
   esac
 }
 
+smart_parse() {
+  tokenize | if [ -n "$SORTDATA" ] ; then
+      ( NORMALIZE=1 LEAFONLY=0 BRIEF=0 parse ) \
+      | tokenize | parse
+    else
+      parse
+    fi
+}
+
 if ([ "$0" = "$BASH_SOURCE" ] || ! [ -n "$BASH_SOURCE" ]);
 then
   parse_options "$@"
-  tokenize | parse
+  smart_parse
 fi

From e04de1a8467539d1bff007c78ac66c9ea687761c Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 7 Jan 2015 01:45:39 +0100
Subject: [PATCH 02/95] Added tests for sorting (with multiple parameters) and
 normalization

---
 test/valid-test.sh                            | 31 +++++++++++++------
 test/valid/array.normalized                   |  1 +
 test/valid/array.sorted                       |  5 +++
 test/valid/embedded.normalized                |  1 +
 test/valid/embedded.sorted                    |  2 ++
 test/valid/empty_array.normalized             |  1 +
 test/valid/empty_array.sorted                 |  1 +
 test/valid/empty_object.normalized            |  1 +
 test/valid/empty_object.sorted                |  1 +
 test/valid/many_object.normalized             |  1 +
 test/valid/many_object.sorted                 |  3 ++
 test/valid/multiline_escapedquotes.json       |  5 +++
 test/valid/multiline_escapedquotes.normalized |  1 +
 test/valid/multiline_escapedquotes.parsed     |  5 +++
 test/valid/multiline_escapedquotes.sorted     |  5 +++
 .../multiline_escapedquotes_indented.json     |  5 +++
 ...ultiline_escapedquotes_indented.normalized |  1 +
 .../multiline_escapedquotes_indented.parsed   |  5 +++
 .../multiline_escapedquotes_indented.sorted   |  5 +++
 test/valid/multiline_simple_key.json          |  2 ++
 test/valid/multiline_simple_key.normalized    |  1 +
 test/valid/multiline_simple_key.parsed        |  2 ++
 test/valid/multiline_simple_key.sorted        |  2 ++
 test/valid/multiline_simple_value.json        |  2 ++
 test/valid/multiline_simple_value.normalized  |  1 +
 test/valid/multiline_simple_value.parsed      |  2 ++
 test/valid/multiline_simple_value.sorted      |  2 ++
 test/valid/nested_array.normalized            |  1 +
 test/valid/nested_array.sorted                |  9 ++++++
 test/valid/nested_object.normalized           |  1 +
 test/valid/nested_object.sorted               |  5 +++
 test/valid/number.normalized                  |  1 +
 test/valid/number.sorted                      |  1 +
 test/valid/object.normalized                  |  1 +
 test/valid/object.sorted                      |  2 ++
 test/valid/singleline_escapedquotes_key.json  |  1 +
 .../singleline_escapedquotes_key.normalized   |  1 +
 .../valid/singleline_escapedquotes_key.parsed |  2 ++
 .../valid/singleline_escapedquotes_key.sorted |  2 ++
 .../valid/singleline_escapedquotes_value.json |  1 +
 .../singleline_escapedquotes_value.normalized |  1 +
 .../singleline_escapedquotes_value.parsed     |  2 ++
 .../singleline_escapedquotes_value.sorted     |  2 ++
 test/valid/string.normalized                  |  1 +
 test/valid/string.sorted                      |  1 +
 test/valid/string_in_array.normalized         |  1 +
 test/valid/string_in_array.sorted             |  2 ++
 test/valid/string_in_object.normalized        |  1 +
 test/valid/string_in_object.sorted            |  2 ++
 test/valid/tab_escape.normalized              |  1 +
 test/valid/tab_escape.sorted                  |  1 +
 51 files changed, 128 insertions(+), 9 deletions(-)
 create mode 100644 test/valid/array.normalized
 create mode 100644 test/valid/array.sorted
 create mode 100644 test/valid/embedded.normalized
 create mode 100644 test/valid/embedded.sorted
 create mode 100644 test/valid/empty_array.normalized
 create mode 100644 test/valid/empty_array.sorted
 create mode 100644 test/valid/empty_object.normalized
 create mode 100644 test/valid/empty_object.sorted
 create mode 100644 test/valid/many_object.normalized
 create mode 100644 test/valid/many_object.sorted
 create mode 100644 test/valid/multiline_escapedquotes.json
 create mode 100644 test/valid/multiline_escapedquotes.normalized
 create mode 100644 test/valid/multiline_escapedquotes.parsed
 create mode 100644 test/valid/multiline_escapedquotes.sorted
 create mode 100644 test/valid/multiline_escapedquotes_indented.json
 create mode 100644 test/valid/multiline_escapedquotes_indented.normalized
 create mode 100644 test/valid/multiline_escapedquotes_indented.parsed
 create mode 100644 test/valid/multiline_escapedquotes_indented.sorted
 create mode 100644 test/valid/multiline_simple_key.json
 create mode 100644 test/valid/multiline_simple_key.normalized
 create mode 100644 test/valid/multiline_simple_key.parsed
 create mode 100644 test/valid/multiline_simple_key.sorted
 create mode 100644 test/valid/multiline_simple_value.json
 create mode 100644 test/valid/multiline_simple_value.normalized
 create mode 100644 test/valid/multiline_simple_value.parsed
 create mode 100644 test/valid/multiline_simple_value.sorted
 create mode 100644 test/valid/nested_array.normalized
 create mode 100644 test/valid/nested_array.sorted
 create mode 100644 test/valid/nested_object.normalized
 create mode 100644 test/valid/nested_object.sorted
 create mode 100644 test/valid/number.normalized
 create mode 100644 test/valid/number.sorted
 create mode 100644 test/valid/object.normalized
 create mode 100644 test/valid/object.sorted
 create mode 100644 test/valid/singleline_escapedquotes_key.json
 create mode 100644 test/valid/singleline_escapedquotes_key.normalized
 create mode 100644 test/valid/singleline_escapedquotes_key.parsed
 create mode 100644 test/valid/singleline_escapedquotes_key.sorted
 create mode 100644 test/valid/singleline_escapedquotes_value.json
 create mode 100644 test/valid/singleline_escapedquotes_value.normalized
 create mode 100644 test/valid/singleline_escapedquotes_value.parsed
 create mode 100644 test/valid/singleline_escapedquotes_value.sorted
 create mode 100644 test/valid/string.normalized
 create mode 100644 test/valid/string.sorted
 create mode 100644 test/valid/string_in_array.normalized
 create mode 100644 test/valid/string_in_array.sorted
 create mode 100644 test/valid/string_in_object.normalized
 create mode 100644 test/valid/string_in_object.sorted
 create mode 100644 test/valid/tab_escape.normalized
 create mode 100644 test/valid/tab_escape.sorted

diff --git a/test/valid-test.sh b/test/valid-test.sh
index cad183b..fdc7db2 100755
--- a/test/valid-test.sh
+++ b/test/valid-test.sh
@@ -1,21 +1,34 @@
 #! /usr/bin/env bash
 
+# To disambiguate tests on sorting, use one locale
+LANG=C
+LC_ALL=C
+export LANG LC_ALL
+
 cd ${0%/*}
 fails=0
 i=0
 tests=`ls valid/*.json -1l | wc -l`
+tests=$(($tests*3))
 echo "1..$tests"
 for input in valid/*.json
 do
-  expected="${input%.json}.parsed"
-  i=$((i+1))
-  if ! ../JSON.sh < "$input" | diff -u - "$expected" 
-  then
-    echo "not ok $i - $input"
-    fails=$((fails+1))
-  else
-    echo "ok $i - $input"    
-  fi
+  for EXT in parsed sorted normalized; do
+    expected="${input%.json}.$EXT"
+    i=$((i+1))
+    case "$EXT" in
+      sorted) OPTIONS="-S='-n -r'" ;;
+      normalized) OPTIONS="-N" ;;
+      parsed|*) OPTIONS="" ;;
+    esac
+    if ! eval ../JSON.sh $OPTIONS < "$input" | diff -u - "$expected" 
+    then
+      echo "not ok $i - $input $EXT"
+      fails=$((fails+1))
+    else
+      echo "ok $i - $input $EXT"
+    fi
+  done
 done
 echo "$fails test(s) failed"
 exit $fails
diff --git a/test/valid/array.normalized b/test/valid/array.normalized
new file mode 100644
index 0000000..ee44631
--- /dev/null
+++ b/test/valid/array.normalized
@@ -0,0 +1 @@
+[1,2,3,"hello"]
diff --git a/test/valid/array.sorted b/test/valid/array.sorted
new file mode 100644
index 0000000..0bcd687
--- /dev/null
+++ b/test/valid/array.sorted
@@ -0,0 +1,5 @@
+[0]	3
+[1]	2
+[2]	1
+[3]	"hello"
+[]	[3,2,1,"hello"]
diff --git a/test/valid/embedded.normalized b/test/valid/embedded.normalized
new file mode 100644
index 0000000..f327913
--- /dev/null
+++ b/test/valid/embedded.normalized
@@ -0,0 +1 @@
+{"foo":"{\"foo\":\"bar\"}"}
diff --git a/test/valid/embedded.sorted b/test/valid/embedded.sorted
new file mode 100644
index 0000000..041eaf9
--- /dev/null
+++ b/test/valid/embedded.sorted
@@ -0,0 +1,2 @@
+["foo"]	"{\"foo\":\"bar\"}"
+[]	{"foo":"{\"foo\":\"bar\"}"}
diff --git a/test/valid/empty_array.normalized b/test/valid/empty_array.normalized
new file mode 100644
index 0000000..fe51488
--- /dev/null
+++ b/test/valid/empty_array.normalized
@@ -0,0 +1 @@
+[]
diff --git a/test/valid/empty_array.sorted b/test/valid/empty_array.sorted
new file mode 100644
index 0000000..d24d150
--- /dev/null
+++ b/test/valid/empty_array.sorted
@@ -0,0 +1 @@
+[]	[]
diff --git a/test/valid/empty_object.normalized b/test/valid/empty_object.normalized
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/test/valid/empty_object.normalized
@@ -0,0 +1 @@
+{}
diff --git a/test/valid/empty_object.sorted b/test/valid/empty_object.sorted
new file mode 100644
index 0000000..4cdea2a
--- /dev/null
+++ b/test/valid/empty_object.sorted
@@ -0,0 +1 @@
+[]	{}
diff --git a/test/valid/many_object.normalized b/test/valid/many_object.normalized
new file mode 100644
index 0000000..abfdad2
--- /dev/null
+++ b/test/valid/many_object.normalized
@@ -0,0 +1 @@
+{"key1":"string","key2":3573}
diff --git a/test/valid/many_object.sorted b/test/valid/many_object.sorted
new file mode 100644
index 0000000..7ea3be5
--- /dev/null
+++ b/test/valid/many_object.sorted
@@ -0,0 +1,3 @@
+["key2"]	3573
+["key1"]	"string"
+[]	{"key2":3573,"key1":"string"}
diff --git a/test/valid/multiline_escapedquotes.json b/test/valid/multiline_escapedquotes.json
new file mode 100644
index 0000000..4315c38
--- /dev/null
+++ b/test/valid/multiline_escapedquotes.json
@@ -0,0 +1,5 @@
+{"s1
+s2 \" s3 ": "abs" ,
+"s4": "qwe", "d2": "qwer
+t
+yu","d2":123}
diff --git a/test/valid/multiline_escapedquotes.normalized b/test/valid/multiline_escapedquotes.normalized
new file mode 100644
index 0000000..6eea35d
--- /dev/null
+++ b/test/valid/multiline_escapedquotes.normalized
@@ -0,0 +1 @@
+{"s1\ns2 \" s3 ":"abs","s4":"qwe","d2":"qwer\nt\nyu","d2":123}
diff --git a/test/valid/multiline_escapedquotes.parsed b/test/valid/multiline_escapedquotes.parsed
new file mode 100644
index 0000000..7a8aee6
--- /dev/null
+++ b/test/valid/multiline_escapedquotes.parsed
@@ -0,0 +1,5 @@
+["s1\ns2 \" s3 "]	"abs"
+["s4"]	"qwe"
+["d2"]	"qwer\nt\nyu"
+["d2"]	123
+[]	{"s1\ns2 \" s3 ":"abs","s4":"qwe","d2":"qwer\nt\nyu","d2":123}
diff --git a/test/valid/multiline_escapedquotes.sorted b/test/valid/multiline_escapedquotes.sorted
new file mode 100644
index 0000000..ecbd9e9
--- /dev/null
+++ b/test/valid/multiline_escapedquotes.sorted
@@ -0,0 +1,5 @@
+["s4"]	"qwe"
+["s1\ns2 \" s3 "]	"abs"
+["d2"]	123
+["d2"]	"qwer\nt\nyu"
+[]	{"s4":"qwe","s1\ns2 \" s3 ":"abs","d2":123,"d2":"qwer\nt\nyu"}
diff --git a/test/valid/multiline_escapedquotes_indented.json b/test/valid/multiline_escapedquotes_indented.json
new file mode 100644
index 0000000..74dacab
--- /dev/null
+++ b/test/valid/multiline_escapedquotes_indented.json
@@ -0,0 +1,5 @@
+{"s1
+s2 \" s3 ": "abs" ,
+"s4": "qwe", "d2": "qwer  
+t
+   yu","d2":123}
diff --git a/test/valid/multiline_escapedquotes_indented.normalized b/test/valid/multiline_escapedquotes_indented.normalized
new file mode 100644
index 0000000..2b65429
--- /dev/null
+++ b/test/valid/multiline_escapedquotes_indented.normalized
@@ -0,0 +1 @@
+{"s1\ns2 \" s3 ":"abs","s4":"qwe","d2":"qwer  \nt\n   yu","d2":123}
diff --git a/test/valid/multiline_escapedquotes_indented.parsed b/test/valid/multiline_escapedquotes_indented.parsed
new file mode 100644
index 0000000..cb7fbcb
--- /dev/null
+++ b/test/valid/multiline_escapedquotes_indented.parsed
@@ -0,0 +1,5 @@
+["s1\ns2 \" s3 "]	"abs"
+["s4"]	"qwe"
+["d2"]	"qwer  \nt\n   yu"
+["d2"]	123
+[]	{"s1\ns2 \" s3 ":"abs","s4":"qwe","d2":"qwer  \nt\n   yu","d2":123}
diff --git a/test/valid/multiline_escapedquotes_indented.sorted b/test/valid/multiline_escapedquotes_indented.sorted
new file mode 100644
index 0000000..bae5acb
--- /dev/null
+++ b/test/valid/multiline_escapedquotes_indented.sorted
@@ -0,0 +1,5 @@
+["s4"]	"qwe"
+["s1\ns2 \" s3 "]	"abs"
+["d2"]	123
+["d2"]	"qwer  \nt\n   yu"
+[]	{"s4":"qwe","s1\ns2 \" s3 ":"abs","d2":123,"d2":"qwer  \nt\n   yu"}
diff --git a/test/valid/multiline_simple_key.json b/test/valid/multiline_simple_key.json
new file mode 100644
index 0000000..38d71d1
--- /dev/null
+++ b/test/valid/multiline_simple_key.json
@@ -0,0 +1,2 @@
+{"s1
+s2": "abs"}
\ No newline at end of file
diff --git a/test/valid/multiline_simple_key.normalized b/test/valid/multiline_simple_key.normalized
new file mode 100644
index 0000000..39302df
--- /dev/null
+++ b/test/valid/multiline_simple_key.normalized
@@ -0,0 +1 @@
+{"s1\ns2":"abs"}
diff --git a/test/valid/multiline_simple_key.parsed b/test/valid/multiline_simple_key.parsed
new file mode 100644
index 0000000..af9dee7
--- /dev/null
+++ b/test/valid/multiline_simple_key.parsed
@@ -0,0 +1,2 @@
+["s1\ns2"]	"abs"
+[]	{"s1\ns2":"abs"}
diff --git a/test/valid/multiline_simple_key.sorted b/test/valid/multiline_simple_key.sorted
new file mode 100644
index 0000000..af9dee7
--- /dev/null
+++ b/test/valid/multiline_simple_key.sorted
@@ -0,0 +1,2 @@
+["s1\ns2"]	"abs"
+[]	{"s1\ns2":"abs"}
diff --git a/test/valid/multiline_simple_value.json b/test/valid/multiline_simple_value.json
new file mode 100644
index 0000000..022ac26
--- /dev/null
+++ b/test/valid/multiline_simple_value.json
@@ -0,0 +1,2 @@
+{"s":"ab c
+d e"}
\ No newline at end of file
diff --git a/test/valid/multiline_simple_value.normalized b/test/valid/multiline_simple_value.normalized
new file mode 100644
index 0000000..3ee6927
--- /dev/null
+++ b/test/valid/multiline_simple_value.normalized
@@ -0,0 +1 @@
+{"s":"ab c\nd e"}
diff --git a/test/valid/multiline_simple_value.parsed b/test/valid/multiline_simple_value.parsed
new file mode 100644
index 0000000..00b40af
--- /dev/null
+++ b/test/valid/multiline_simple_value.parsed
@@ -0,0 +1,2 @@
+["s"]	"ab c\nd e"
+[]	{"s":"ab c\nd e"}
diff --git a/test/valid/multiline_simple_value.sorted b/test/valid/multiline_simple_value.sorted
new file mode 100644
index 0000000..00b40af
--- /dev/null
+++ b/test/valid/multiline_simple_value.sorted
@@ -0,0 +1,2 @@
+["s"]	"ab c\nd e"
+[]	{"s":"ab c\nd e"}
diff --git a/test/valid/nested_array.normalized b/test/valid/nested_array.normalized
new file mode 100644
index 0000000..97aade5
--- /dev/null
+++ b/test/valid/nested_array.normalized
@@ -0,0 +1 @@
+[1,[],[4,"hello",{}],{"array":[]}]
diff --git a/test/valid/nested_array.sorted b/test/valid/nested_array.sorted
new file mode 100644
index 0000000..f323363
--- /dev/null
+++ b/test/valid/nested_array.sorted
@@ -0,0 +1,9 @@
+[0]	1
+[1,"array"]	[]
+[1]	{"array":[]}
+[2]	[]
+[3,0]	4
+[3,1]	{}
+[3,2]	"hello"
+[3]	[4,{},"hello"]
+[]	[1,{"array":[]},[],[4,{},"hello"]]
diff --git a/test/valid/nested_object.normalized b/test/valid/nested_object.normalized
new file mode 100644
index 0000000..68c057f
--- /dev/null
+++ b/test/valid/nested_object.normalized
@@ -0,0 +1 @@
+{"object":{"key":"value","empty":{}},"number":5}
diff --git a/test/valid/nested_object.sorted b/test/valid/nested_object.sorted
new file mode 100644
index 0000000..8609e30
--- /dev/null
+++ b/test/valid/nested_object.sorted
@@ -0,0 +1,5 @@
+["object","key"]	"value"
+["object","empty"]	{}
+["object"]	{"key":"value","empty":{}}
+["number"]	5
+[]	{"object":{"key":"value","empty":{}},"number":5}
diff --git a/test/valid/number.normalized b/test/valid/number.normalized
new file mode 100644
index 0000000..00750ed
--- /dev/null
+++ b/test/valid/number.normalized
@@ -0,0 +1 @@
+3
diff --git a/test/valid/number.sorted b/test/valid/number.sorted
new file mode 100644
index 0000000..2a1fecb
--- /dev/null
+++ b/test/valid/number.sorted
@@ -0,0 +1 @@
+[]	3
diff --git a/test/valid/object.normalized b/test/valid/object.normalized
new file mode 100644
index 0000000..f523ccf
--- /dev/null
+++ b/test/valid/object.normalized
@@ -0,0 +1 @@
+{"key":"Value"}
diff --git a/test/valid/object.sorted b/test/valid/object.sorted
new file mode 100644
index 0000000..9f50711
--- /dev/null
+++ b/test/valid/object.sorted
@@ -0,0 +1,2 @@
+["key"]	"Value"
+[]	{"key":"Value"}
diff --git a/test/valid/singleline_escapedquotes_key.json b/test/valid/singleline_escapedquotes_key.json
new file mode 100644
index 0000000..1aaa032
--- /dev/null
+++ b/test/valid/singleline_escapedquotes_key.json
@@ -0,0 +1 @@
+{"s1 \" s2": "abs"}
\ No newline at end of file
diff --git a/test/valid/singleline_escapedquotes_key.normalized b/test/valid/singleline_escapedquotes_key.normalized
new file mode 100644
index 0000000..938af32
--- /dev/null
+++ b/test/valid/singleline_escapedquotes_key.normalized
@@ -0,0 +1 @@
+{"s1 \" s2":"abs"}
diff --git a/test/valid/singleline_escapedquotes_key.parsed b/test/valid/singleline_escapedquotes_key.parsed
new file mode 100644
index 0000000..66e89aa
--- /dev/null
+++ b/test/valid/singleline_escapedquotes_key.parsed
@@ -0,0 +1,2 @@
+["s1 \" s2"]	"abs"
+[]	{"s1 \" s2":"abs"}
diff --git a/test/valid/singleline_escapedquotes_key.sorted b/test/valid/singleline_escapedquotes_key.sorted
new file mode 100644
index 0000000..66e89aa
--- /dev/null
+++ b/test/valid/singleline_escapedquotes_key.sorted
@@ -0,0 +1,2 @@
+["s1 \" s2"]	"abs"
+[]	{"s1 \" s2":"abs"}
diff --git a/test/valid/singleline_escapedquotes_value.json b/test/valid/singleline_escapedquotes_value.json
new file mode 100644
index 0000000..78dc4f2
--- /dev/null
+++ b/test/valid/singleline_escapedquotes_value.json
@@ -0,0 +1 @@
+{"s1 s2": "quoted \"substring\" value"}
\ No newline at end of file
diff --git a/test/valid/singleline_escapedquotes_value.normalized b/test/valid/singleline_escapedquotes_value.normalized
new file mode 100644
index 0000000..075a978
--- /dev/null
+++ b/test/valid/singleline_escapedquotes_value.normalized
@@ -0,0 +1 @@
+{"s1 s2":"quoted \"substring\" value"}
diff --git a/test/valid/singleline_escapedquotes_value.parsed b/test/valid/singleline_escapedquotes_value.parsed
new file mode 100644
index 0000000..1339ac3
--- /dev/null
+++ b/test/valid/singleline_escapedquotes_value.parsed
@@ -0,0 +1,2 @@
+["s1 s2"]	"quoted \"substring\" value"
+[]	{"s1 s2":"quoted \"substring\" value"}
diff --git a/test/valid/singleline_escapedquotes_value.sorted b/test/valid/singleline_escapedquotes_value.sorted
new file mode 100644
index 0000000..1339ac3
--- /dev/null
+++ b/test/valid/singleline_escapedquotes_value.sorted
@@ -0,0 +1,2 @@
+["s1 s2"]	"quoted \"substring\" value"
+[]	{"s1 s2":"quoted \"substring\" value"}
diff --git a/test/valid/string.normalized b/test/valid/string.normalized
new file mode 100644
index 0000000..31f592f
--- /dev/null
+++ b/test/valid/string.normalized
@@ -0,0 +1 @@
+"hello this is a string"
diff --git a/test/valid/string.sorted b/test/valid/string.sorted
new file mode 100644
index 0000000..b1fb986
--- /dev/null
+++ b/test/valid/string.sorted
@@ -0,0 +1 @@
+[]	"hello this is a string"
diff --git a/test/valid/string_in_array.normalized b/test/valid/string_in_array.normalized
new file mode 100644
index 0000000..89179f6
--- /dev/null
+++ b/test/valid/string_in_array.normalized
@@ -0,0 +1 @@
+["hello this is a string"]
diff --git a/test/valid/string_in_array.sorted b/test/valid/string_in_array.sorted
new file mode 100644
index 0000000..a49e6d9
--- /dev/null
+++ b/test/valid/string_in_array.sorted
@@ -0,0 +1,2 @@
+[0]	"hello this is a string"
+[]	["hello this is a string"]
diff --git a/test/valid/string_in_object.normalized b/test/valid/string_in_object.normalized
new file mode 100644
index 0000000..357dfdc
--- /dev/null
+++ b/test/valid/string_in_object.normalized
@@ -0,0 +1 @@
+{"key":"hello this is a string"}
diff --git a/test/valid/string_in_object.sorted b/test/valid/string_in_object.sorted
new file mode 100644
index 0000000..e266552
--- /dev/null
+++ b/test/valid/string_in_object.sorted
@@ -0,0 +1,2 @@
+["key"]	"hello this is a string"
+[]	{"key":"hello this is a string"}
diff --git a/test/valid/tab_escape.normalized b/test/valid/tab_escape.normalized
new file mode 100644
index 0000000..b7e42b8
--- /dev/null
+++ b/test/valid/tab_escape.normalized
@@ -0,0 +1 @@
+"hello\tworld"
diff --git a/test/valid/tab_escape.sorted b/test/valid/tab_escape.sorted
new file mode 100644
index 0000000..ee69dd9
--- /dev/null
+++ b/test/valid/tab_escape.sorted
@@ -0,0 +1 @@
+[]	"hello\tworld"

From 2e6a44b257c72418473e3fa3569704484a517a36 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 7 Jan 2015 01:46:24 +0100
Subject: [PATCH 03/95] Tweaks to ensure better portability (now tested in
 Linux and Solaris)

---
 JSON.sh | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index 1b42430..e35d816 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -40,6 +40,11 @@ usage() {
   echo
 }
 
+unquote() {
+    # Remove single or double quotes surrounding the token
+    sed "s,^'\(.*\)'\$,\1," | sed 's,^\"\(.*\)\"$,\1,'
+}
+
 parse_options() {
   set -- "$@"
   local ARGN=$#
@@ -59,14 +64,14 @@ parse_options() {
       ;;
       -N) NORMALIZE=1
       ;;
-      -N=*) SORTDATA="sort `echo "$1" | sed 's,^-N=,,'`"
+      -N=*) SORTDATA="sort `echo "$1" | sed 's,^-N=,,' | unquote `"
           NORMALIZE=1
       ;;
       -S) SORTDATA="sort"
       ;;
-      -S=*) SORTDATA="sort `echo "$1" | sed 's,^-S=,,'`"
+      -S=*) SORTDATA="sort `echo "$1" | sed 's,^-S=,,' | unquote `"
       ;;
-      ?*) echo "ERROR: Unknown option."
+      ?*) echo "ERROR: Unknown option '$1'."
           usage
           exit 0
       ;;
@@ -195,7 +200,7 @@ $value"
       ;;
   esac
   if [ -n "$SORTDATA" ]; then
-    ary="`echo "$aryml" | $SORTDATA | tr '\n' ',' | sed 's|,*$||' | sed 's|^,*||'`"
+    ary="`echo -E "$aryml" | $SORTDATA | tr '\n' ',' | sed 's|,*$||' | sed 's|^,*||'`"
   fi
   [ "$BRIEF" -eq 0 ] && value=`printf '[%s]' "$ary"` || value=
   :
@@ -238,7 +243,7 @@ $key:$value"
     ;;
   esac
   if [ -n "$SORTDATA" ]; then
-    obj="`echo "$objml" | $SORTDATA | tr '\n' ',' | sed 's|,*$||' | sed 's|^,*||'`"
+    obj="`echo -E "$objml" | $SORTDATA | tr '\n' ',' | sed 's|,*$||' | sed 's|^,*||'`"
   fi
   [ "$BRIEF" -eq 0 ] && value=`printf '{%s}' "$obj"` || value=
   :

From 9297df3bda0762c731e0c7d0863ef43646dfb5c3 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 7 Jan 2015 02:09:08 +0100
Subject: [PATCH 04/95] Added tests for combination of normalized+sorted
 content

---
 test/valid-test.sh                                           | 5 +++--
 test/valid/array.normalized_sorted                           | 1 +
 test/valid/embedded.normalized_sorted                        | 1 +
 test/valid/empty_array.normalized_sorted                     | 1 +
 test/valid/empty_object.normalized_sorted                    | 1 +
 test/valid/many_object.normalized_sorted                     | 1 +
 test/valid/multiline_escapedquotes.normalized_sorted         | 1 +
 .../valid/multiline_escapedquotes_indented.normalized_sorted | 1 +
 test/valid/multiline_simple_key.normalized_sorted            | 1 +
 test/valid/multiline_simple_value.normalized_sorted          | 1 +
 test/valid/nested_array.normalized_sorted                    | 1 +
 test/valid/nested_object.normalized_sorted                   | 1 +
 test/valid/number.normalized_sorted                          | 1 +
 test/valid/object.normalized_sorted                          | 1 +
 test/valid/singleline_escapedquotes_key.normalized_sorted    | 1 +
 test/valid/singleline_escapedquotes_value.normalized_sorted  | 1 +
 test/valid/string.normalized_sorted                          | 1 +
 test/valid/string_in_array.normalized_sorted                 | 1 +
 test/valid/string_in_object.normalized_sorted                | 1 +
 test/valid/tab_escape.normalized_sorted                      | 1 +
 20 files changed, 22 insertions(+), 2 deletions(-)
 create mode 100644 test/valid/array.normalized_sorted
 create mode 100644 test/valid/embedded.normalized_sorted
 create mode 100644 test/valid/empty_array.normalized_sorted
 create mode 100644 test/valid/empty_object.normalized_sorted
 create mode 100644 test/valid/many_object.normalized_sorted
 create mode 100644 test/valid/multiline_escapedquotes.normalized_sorted
 create mode 100644 test/valid/multiline_escapedquotes_indented.normalized_sorted
 create mode 100644 test/valid/multiline_simple_key.normalized_sorted
 create mode 100644 test/valid/multiline_simple_value.normalized_sorted
 create mode 100644 test/valid/nested_array.normalized_sorted
 create mode 100644 test/valid/nested_object.normalized_sorted
 create mode 100644 test/valid/number.normalized_sorted
 create mode 100644 test/valid/object.normalized_sorted
 create mode 100644 test/valid/singleline_escapedquotes_key.normalized_sorted
 create mode 100644 test/valid/singleline_escapedquotes_value.normalized_sorted
 create mode 100644 test/valid/string.normalized_sorted
 create mode 100644 test/valid/string_in_array.normalized_sorted
 create mode 100644 test/valid/string_in_object.normalized_sorted
 create mode 100644 test/valid/tab_escape.normalized_sorted

diff --git a/test/valid-test.sh b/test/valid-test.sh
index fdc7db2..e662323 100755
--- a/test/valid-test.sh
+++ b/test/valid-test.sh
@@ -9,16 +9,17 @@ cd ${0%/*}
 fails=0
 i=0
 tests=`ls valid/*.json -1l | wc -l`
-tests=$(($tests*3))
+tests=$(($tests*4))
 echo "1..$tests"
 for input in valid/*.json
 do
-  for EXT in parsed sorted normalized; do
+  for EXT in parsed sorted normalized normalized_sorted; do
     expected="${input%.json}.$EXT"
     i=$((i+1))
     case "$EXT" in
       sorted) OPTIONS="-S='-n -r'" ;;
       normalized) OPTIONS="-N" ;;
+      normalized_sorted) OPTIONS="-N=-n" ;;
       parsed|*) OPTIONS="" ;;
     esac
     if ! eval ../JSON.sh $OPTIONS < "$input" | diff -u - "$expected" 
diff --git a/test/valid/array.normalized_sorted b/test/valid/array.normalized_sorted
new file mode 100644
index 0000000..b5a9be5
--- /dev/null
+++ b/test/valid/array.normalized_sorted
@@ -0,0 +1 @@
+["hello",1,2,3]
diff --git a/test/valid/embedded.normalized_sorted b/test/valid/embedded.normalized_sorted
new file mode 100644
index 0000000..f327913
--- /dev/null
+++ b/test/valid/embedded.normalized_sorted
@@ -0,0 +1 @@
+{"foo":"{\"foo\":\"bar\"}"}
diff --git a/test/valid/empty_array.normalized_sorted b/test/valid/empty_array.normalized_sorted
new file mode 100644
index 0000000..fe51488
--- /dev/null
+++ b/test/valid/empty_array.normalized_sorted
@@ -0,0 +1 @@
+[]
diff --git a/test/valid/empty_object.normalized_sorted b/test/valid/empty_object.normalized_sorted
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/test/valid/empty_object.normalized_sorted
@@ -0,0 +1 @@
+{}
diff --git a/test/valid/many_object.normalized_sorted b/test/valid/many_object.normalized_sorted
new file mode 100644
index 0000000..abfdad2
--- /dev/null
+++ b/test/valid/many_object.normalized_sorted
@@ -0,0 +1 @@
+{"key1":"string","key2":3573}
diff --git a/test/valid/multiline_escapedquotes.normalized_sorted b/test/valid/multiline_escapedquotes.normalized_sorted
new file mode 100644
index 0000000..fe90225
--- /dev/null
+++ b/test/valid/multiline_escapedquotes.normalized_sorted
@@ -0,0 +1 @@
+{"d2":"qwer\nt\nyu","d2":123,"s1\ns2 \" s3 ":"abs","s4":"qwe"}
diff --git a/test/valid/multiline_escapedquotes_indented.normalized_sorted b/test/valid/multiline_escapedquotes_indented.normalized_sorted
new file mode 100644
index 0000000..4521493
--- /dev/null
+++ b/test/valid/multiline_escapedquotes_indented.normalized_sorted
@@ -0,0 +1 @@
+{"d2":"qwer  \nt\n   yu","d2":123,"s1\ns2 \" s3 ":"abs","s4":"qwe"}
diff --git a/test/valid/multiline_simple_key.normalized_sorted b/test/valid/multiline_simple_key.normalized_sorted
new file mode 100644
index 0000000..39302df
--- /dev/null
+++ b/test/valid/multiline_simple_key.normalized_sorted
@@ -0,0 +1 @@
+{"s1\ns2":"abs"}
diff --git a/test/valid/multiline_simple_value.normalized_sorted b/test/valid/multiline_simple_value.normalized_sorted
new file mode 100644
index 0000000..3ee6927
--- /dev/null
+++ b/test/valid/multiline_simple_value.normalized_sorted
@@ -0,0 +1 @@
+{"s":"ab c\nd e"}
diff --git a/test/valid/nested_array.normalized_sorted b/test/valid/nested_array.normalized_sorted
new file mode 100644
index 0000000..3cb16fb
--- /dev/null
+++ b/test/valid/nested_array.normalized_sorted
@@ -0,0 +1 @@
+[["hello",{},4],[],{"array":[]},1]
diff --git a/test/valid/nested_object.normalized_sorted b/test/valid/nested_object.normalized_sorted
new file mode 100644
index 0000000..efe1ce2
--- /dev/null
+++ b/test/valid/nested_object.normalized_sorted
@@ -0,0 +1 @@
+{"number":5,"object":{"empty":{},"key":"value"}}
diff --git a/test/valid/number.normalized_sorted b/test/valid/number.normalized_sorted
new file mode 100644
index 0000000..00750ed
--- /dev/null
+++ b/test/valid/number.normalized_sorted
@@ -0,0 +1 @@
+3
diff --git a/test/valid/object.normalized_sorted b/test/valid/object.normalized_sorted
new file mode 100644
index 0000000..f523ccf
--- /dev/null
+++ b/test/valid/object.normalized_sorted
@@ -0,0 +1 @@
+{"key":"Value"}
diff --git a/test/valid/singleline_escapedquotes_key.normalized_sorted b/test/valid/singleline_escapedquotes_key.normalized_sorted
new file mode 100644
index 0000000..938af32
--- /dev/null
+++ b/test/valid/singleline_escapedquotes_key.normalized_sorted
@@ -0,0 +1 @@
+{"s1 \" s2":"abs"}
diff --git a/test/valid/singleline_escapedquotes_value.normalized_sorted b/test/valid/singleline_escapedquotes_value.normalized_sorted
new file mode 100644
index 0000000..075a978
--- /dev/null
+++ b/test/valid/singleline_escapedquotes_value.normalized_sorted
@@ -0,0 +1 @@
+{"s1 s2":"quoted \"substring\" value"}
diff --git a/test/valid/string.normalized_sorted b/test/valid/string.normalized_sorted
new file mode 100644
index 0000000..31f592f
--- /dev/null
+++ b/test/valid/string.normalized_sorted
@@ -0,0 +1 @@
+"hello this is a string"
diff --git a/test/valid/string_in_array.normalized_sorted b/test/valid/string_in_array.normalized_sorted
new file mode 100644
index 0000000..89179f6
--- /dev/null
+++ b/test/valid/string_in_array.normalized_sorted
@@ -0,0 +1 @@
+["hello this is a string"]
diff --git a/test/valid/string_in_object.normalized_sorted b/test/valid/string_in_object.normalized_sorted
new file mode 100644
index 0000000..357dfdc
--- /dev/null
+++ b/test/valid/string_in_object.normalized_sorted
@@ -0,0 +1 @@
+{"key":"hello this is a string"}
diff --git a/test/valid/tab_escape.normalized_sorted b/test/valid/tab_escape.normalized_sorted
new file mode 100644
index 0000000..b7e42b8
--- /dev/null
+++ b/test/valid/tab_escape.normalized_sorted
@@ -0,0 +1 @@
+"hello\tworld"

From c98786e42282c6e9c33a99113c4d117646aa0f8c Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 7 Jan 2015 03:01:27 +0100
Subject: [PATCH 05/95] Added tests for an array with several empty strings
 (should remain in place) and meaningless whitespace in markup between
 elements (should be ignored)

---
 test/valid/array_with_empty_elements.json        |  6 ++++++
 test/valid/array_with_empty_elements.normalized  |  1 +
 .../array_with_empty_elements.normalized_sorted  |  1 +
 test/valid/array_with_empty_elements.sorted      | 16 ++++++++++++++++
 4 files changed, 24 insertions(+)
 create mode 100644 test/valid/array_with_empty_elements.json
 create mode 100644 test/valid/array_with_empty_elements.normalized
 create mode 100644 test/valid/array_with_empty_elements.normalized_sorted
 create mode 100644 test/valid/array_with_empty_elements.sorted

diff --git a/test/valid/array_with_empty_elements.json b/test/valid/array_with_empty_elements.json
new file mode 100644
index 0000000..3bde6fa
--- /dev/null
+++ b/test/valid/array_with_empty_elements.json
@@ -0,0 +1,6 @@
+["", {"k1":"v1",
+	"k2":"","k3":1e15,
+"k0":0,"arr":[3,1,2,4]},
+ "",
+ "av1",
+"v1",0]
diff --git a/test/valid/array_with_empty_elements.normalized b/test/valid/array_with_empty_elements.normalized
new file mode 100644
index 0000000..76bdb17
--- /dev/null
+++ b/test/valid/array_with_empty_elements.normalized
@@ -0,0 +1 @@
+["",{"k1":"v1","k2":"","k3":1e15,"k0":0,"arr":[3,1,2,4]},"","av1","v1",0]
diff --git a/test/valid/array_with_empty_elements.normalized_sorted b/test/valid/array_with_empty_elements.normalized_sorted
new file mode 100644
index 0000000..4d70dca
--- /dev/null
+++ b/test/valid/array_with_empty_elements.normalized_sorted
@@ -0,0 +1 @@
+["","","av1","v1",0,{"arr":[1,2,3,4],"k0":0,"k1":"v1","k2":"","k3":1e15}]
diff --git a/test/valid/array_with_empty_elements.sorted b/test/valid/array_with_empty_elements.sorted
new file mode 100644
index 0000000..2afd91b
--- /dev/null
+++ b/test/valid/array_with_empty_elements.sorted
@@ -0,0 +1,16 @@
+[0]	""
+[1]	""
+[2]	"av1"
+[3]	"v1"
+[4]	0
+[5,"arr",0]	1
+[5,"arr",1]	2
+[5,"arr",2]	3
+[5,"arr",3]	4
+[5,"arr"]	[1,2,3,4]
+[5,"k0"]	0
+[5,"k1"]	"v1"
+[5,"k2"]	""
+[5,"k3"]	1e15
+[5]	{"arr":[1,2,3,4],"k0":0,"k1":"v1","k2":"","k3":1e15}
+[]	["","","av1","v1",0,{"arr":[1,2,3,4],"k0":0,"k1":"v1","k2":"","k3":1e15}]

From 966a2ee56aa730e60ead561bf5b2a86779c09a2b Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 7 Jan 2015 05:00:30 +0100
Subject: [PATCH 06/95] Added an option to abort at strings with embedded
 newlines rather than trying to rectify them

---
 JSON.sh | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/JSON.sh b/JSON.sh
index e35d816..74bb823 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -14,10 +14,11 @@ LEAFONLY=0
 PRUNE=0
 SORTDATA=""
 NORMALIZE=0
+TOXIC_NEWLINE=0
 
 usage() {
   echo
-  echo "Usage: JSON.sh [-b] [-l] [-p] [-N] [-S|-S='args']"
+  echo "Usage: JSON.sh [-b] [-l] [-p] [-N] [-S|-S='args'] [--no-newline]"
   echo "       JSON.sh [-N|-N='args'] < markup.json"
   echo "       JSON.sh [-h]"
   echo "-h - This help text."
@@ -25,6 +26,8 @@ usage() {
   echo "-p - Prune empty. Exclude fields with empty values."
   echo "-l - Leaf only. Only show leaf nodes, which stops data duplication."
   echo "-b - Brief. Combines 'Leaf only' and 'Prune empty' options."
+  echo "--no-newline - rather than concatenating detected line breaks in markup,"
+  echo "     return with error when this is seen in input"
   echo
   echo "Sorting is also available, although limited to single-line strings in"
   echo "the markup (multilines are automatically escaped into backslash+n):"
@@ -71,6 +74,9 @@ parse_options() {
       ;;
       -S=*) SORTDATA="sort `echo "$1" | sed 's,^-S=,,' | unquote `"
       ;;
+      --no-newline)
+          TOXIC_NEWLINE=1
+      ;;
       ?*) echo "ERROR: Unknown option '$1'."
           usage
           exit 0
@@ -104,6 +110,7 @@ strip_newlines() {
   local NUMQ
   local ODD
   local INSTRING=0
+  local LINENUM=0
 
   # the first "grep" should ensure that input has a trailing newline
   grep '' | while IFS="" read -r ILINE; do
@@ -114,8 +121,12 @@ strip_newlines() {
     # Count unescaped quotes:
     NUMQ="${#LINESTRIP}"
     ODD="$(($NUMQ%2))"
+    LINENUM="$(($LINENUM+1))"
 
     if [ "$ODD" -eq 1 -a "$INSTRING" -eq 0 ]; then
+      [ "$TOXIC_NEWLINE" = 1 ] && \
+        echo "Invalid JSON markup detected: newline in a string value: at line #$LINENUM" >&2 && \
+        exit 121
       printf '%s\\n' "$ILINE"
       INSTRING=1
       continue

From 67b63690a7a03f90a21a2d1e32bfc9118bf690fc Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 7 Jan 2015 11:05:08 +0100
Subject: [PATCH 07/95] Add a way to easily generate expected-valid-result
 files in a standard manner

---
 test/valid/generate-results.sh | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100755 test/valid/generate-results.sh

diff --git a/test/valid/generate-results.sh b/test/valid/generate-results.sh
new file mode 100755
index 0000000..fb142b9
--- /dev/null
+++ b/test/valid/generate-results.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+
+# Generate the reference results in a standardized manner
+# (script options, extensions, locale)
+LANG=C
+LC_ALL=C
+export LANG
+export C
+
+JSONSH=../../JSON.sh
+
+for F in *.json ; do
+	B="`basename "$F" .json`"
+	echo "=== Generating results for '$F'..."
+
+	EXT=parsed
+	$JSONSH < "$F" > "$B.$EXT" || echo "ERROR with $EXT"
+
+	EXT=sorted
+	$JSONSH -S="-n -r" < "$F" > "$B.$EXT" || echo "ERROR with $EXT"
+
+	EXT=normalized
+	$JSONSH -N < "$F" > "$B.$EXT" || echo "ERROR with $EXT"
+
+	EXT=normalized_sorted
+	$JSONSH -N='-n' < "$F" > "$B.$EXT" || echo "ERROR with $EXT"
+done
+

From 29603a3d9904fca05ba99bed7b06170ad6f6ff7e Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 7 Jan 2015 11:06:02 +0100
Subject: [PATCH 08/95] Fixed tests for array_with_empty_elements: the "sorted"
 variant was made in a wrong locale initially, and "parsed" was missing

---
 test/valid/array_with_empty_elements.parsed | 16 +++++++++++
 test/valid/array_with_empty_elements.sorted | 32 ++++++++++-----------
 2 files changed, 32 insertions(+), 16 deletions(-)
 create mode 100644 test/valid/array_with_empty_elements.parsed

diff --git a/test/valid/array_with_empty_elements.parsed b/test/valid/array_with_empty_elements.parsed
new file mode 100644
index 0000000..3ec2517
--- /dev/null
+++ b/test/valid/array_with_empty_elements.parsed
@@ -0,0 +1,16 @@
+[0]	""
+[1,"k1"]	"v1"
+[1,"k2"]	""
+[1,"k3"]	1e15
+[1,"k0"]	0
+[1,"arr",0]	3
+[1,"arr",1]	1
+[1,"arr",2]	2
+[1,"arr",3]	4
+[1,"arr"]	[3,1,2,4]
+[1]	{"k1":"v1","k2":"","k3":1e15,"k0":0,"arr":[3,1,2,4]}
+[2]	""
+[3]	"av1"
+[4]	"v1"
+[5]	0
+[]	["",{"k1":"v1","k2":"","k3":1e15,"k0":0,"arr":[3,1,2,4]},"","av1","v1",0]
diff --git a/test/valid/array_with_empty_elements.sorted b/test/valid/array_with_empty_elements.sorted
index 2afd91b..10c308c 100644
--- a/test/valid/array_with_empty_elements.sorted
+++ b/test/valid/array_with_empty_elements.sorted
@@ -1,16 +1,16 @@
-[0]	""
-[1]	""
-[2]	"av1"
-[3]	"v1"
-[4]	0
-[5,"arr",0]	1
-[5,"arr",1]	2
-[5,"arr",2]	3
-[5,"arr",3]	4
-[5,"arr"]	[1,2,3,4]
-[5,"k0"]	0
-[5,"k1"]	"v1"
-[5,"k2"]	""
-[5,"k3"]	1e15
-[5]	{"arr":[1,2,3,4],"k0":0,"k1":"v1","k2":"","k3":1e15}
-[]	["","","av1","v1",0,{"arr":[1,2,3,4],"k0":0,"k1":"v1","k2":"","k3":1e15}]
+[0,"k3"]	1e15
+[0,"k2"]	""
+[0,"k1"]	"v1"
+[0,"k0"]	0
+[0,"arr",0]	4
+[0,"arr",1]	3
+[0,"arr",2]	2
+[0,"arr",3]	1
+[0,"arr"]	[4,3,2,1]
+[0]	{"k3":1e15,"k2":"","k1":"v1","k0":0,"arr":[4,3,2,1]}
+[1]	0
+[2]	"v1"
+[3]	"av1"
+[4]	""
+[5]	""
+[]	[{"k3":1e15,"k2":"","k1":"v1","k0":0,"arr":[4,3,2,1]},0,"v1","av1","",""]

From fd1bb4c1badc0b38f9a4a2fa00165d9524741ea4 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 7 Jan 2015 11:07:29 +0100
Subject: [PATCH 09/95] JSON.sh optimization: relocated call of
 strip_newlines() into start of processing so it is only done once (not twice
 for SORTDATA enabled mode)

---
 JSON.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/JSON.sh b/JSON.sh
index 74bb823..769eb01 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -179,7 +179,6 @@ tokenize () {
   local KEYWORD='null|false|true'
   local SPACE='[[:space:]]+'
 
-  strip_newlines | \
   $GREP "$STRING|$NUMBER|$KEYWORD|$SPACE|." | egrep -v "^$SPACE$"
 }
 
@@ -303,6 +302,7 @@ parse () {
 }
 
 smart_parse() {
+  strip_newlines | \
   tokenize | if [ -n "$SORTDATA" ] ; then
       ( NORMALIZE=1 LEAFONLY=0 BRIEF=0 parse ) \
       | tokenize | parse

From def8dc65590669f879c927d4de957810f4ca2bbe Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 7 Jan 2015 11:55:19 +0100
Subject: [PATCH 10/95] Allow to (re)generate expected results just for named
 files rather than everythig (which remains the default)

---
 test/valid/generate-results.sh | 33 ++++++++++++++++++++++++++-------
 1 file changed, 26 insertions(+), 7 deletions(-)

diff --git a/test/valid/generate-results.sh b/test/valid/generate-results.sh
index fb142b9..7bdf4a7 100755
--- a/test/valid/generate-results.sh
+++ b/test/valid/generate-results.sh
@@ -9,20 +9,39 @@ export C
 
 JSONSH=../../JSON.sh
 
-for F in *.json ; do
+generate() {
+	F="$1"
+	[ -s "$F" ] || return
+
 	B="`basename "$F" .json`"
 	echo "=== Generating results for '$F'..."
+	RES=0
 
 	EXT=parsed
-	$JSONSH < "$F" > "$B.$EXT" || echo "ERROR with $EXT"
+	$JSONSH < "$F" > "$B.$EXT" || \
+	    { RES=$?; echo "ERROR with $EXT"; }
 
 	EXT=sorted
-	$JSONSH -S="-n -r" < "$F" > "$B.$EXT" || echo "ERROR with $EXT"
+	$JSONSH -S="-n -r" < "$F" > "$B.$EXT" || \
+	    { RES=$?; echo "ERROR with $EXT"; }
 
 	EXT=normalized
-	$JSONSH -N < "$F" > "$B.$EXT" || echo "ERROR with $EXT"
+	$JSONSH -N < "$F" > "$B.$EXT" || \
+	    { RES=$?; echo "ERROR with $EXT"; }
 
 	EXT=normalized_sorted
-	$JSONSH -N='-n' < "$F" > "$B.$EXT" || echo "ERROR with $EXT"
-done
-
+	$JSONSH -N='-n' < "$F" > "$B.$EXT" || \
+	    { RES=$?; echo "ERROR with $EXT"; }
+
+	return $RES
+}
+
+if [ $# -gt 0 ]; then
+    for F in "$@" ; do
+	generate "$F"
+    done
+else
+    for F in *.json ; do
+	generate "$F"
+    done
+fi

From 0e8dcf17ad00fa0251c15fbbb53a587e3cdfadf5 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 7 Jan 2015 16:23:16 +0100
Subject: [PATCH 11/95] JSON.sh can now "extract" and display only those items
 whose jpath matches a specified regex

---
 JSON.sh | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/JSON.sh b/JSON.sh
index 769eb01..4573560 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -14,11 +14,12 @@ LEAFONLY=0
 PRUNE=0
 SORTDATA=""
 NORMALIZE=0
+EXTRACT_JPATH=""
 TOXIC_NEWLINE=0
 
 usage() {
   echo
-  echo "Usage: JSON.sh [-b] [-l] [-p] [-N] [-S|-S='args'] [--no-newline]"
+  echo "Usage: JSON.sh [-b] [-l] [-p] [-x 'regex'] [-S|-S='args'] [--no-newline]"
   echo "       JSON.sh [-N|-N='args'] < markup.json"
   echo "       JSON.sh [-h]"
   echo "-h - This help text."
@@ -26,6 +27,10 @@ usage() {
   echo "-p - Prune empty. Exclude fields with empty values."
   echo "-l - Leaf only. Only show leaf nodes, which stops data duplication."
   echo "-b - Brief. Combines 'Leaf only' and 'Prune empty' options."
+  echo "-x 'regex' - rather than showing all document from the root element,"
+  echo "     extract the items rooted at path(s) matching the regex (see the"
+  echo "     comma-separated list of nested hierarchy names in general output,"
+  echo "     brackets not included) e.g. regex='^\"level1\",\"level2arr\",0'"
   echo "--no-newline - rather than concatenating detected line breaks in markup,"
   echo "     return with error when this is seen in input"
   echo
@@ -35,6 +40,8 @@ usage() {
   echo "     'sort' objects by key names and then values, and arrays by values"
   echo "-S='args' - use 'sort \$args' for content sorting, e.g. use -S='-n -r'"
   echo "     for reverse numeric sort"
+  echo
+  echo "An input JSON markup can be normalized into single-line no-whitespace:"
   echo "-N - Normalize the input JSON markup into a single-line JSON output;"
   echo "     in this mode syntax and spacing are normalized, data order remains"
   echo "-N='args' - Normalize the input JSON markup into a single-line JSON"
@@ -74,6 +81,9 @@ parse_options() {
       ;;
       -S=*) SORTDATA="sort `echo "$1" | sed 's,^-S=,,' | unquote `"
       ;;
+      -x) EXTRACT_JPATH="$2"
+          shift
+      ;;
       --no-newline)
           TOXIC_NEWLINE=1
       ;;
@@ -287,6 +297,11 @@ parse_value () {
   [ "$LEAFONLY" -eq 1 ] && [ "$isleaf" -eq 1 ] && \
     [ $PRUNE -eq 1 ] && [ $isempty -eq 0 ] && print=1
 
+  if [ "$print" -eq 1 -a -n "$EXTRACT_JPATH" ] ; then
+    ### BASH regex matching:
+    [[ ${jpath} =~ ${EXTRACT_JPATH} ]] || print=0
+  fi
+
   [ "$print" -eq 1 ] && printf "[%s]\t%s\n" "$jpath" "$value"
   :
 }

From 8096cb25a86ebbe349ea96d4678416d86794993c Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 7 Jan 2015 16:48:44 +0100
Subject: [PATCH 12/95] JSON.sh now has debugging option (to trace why some
 lines are printed or not) and supports listing/pruning of empty arrays and
 objects as leaves

---
 JSON.sh | 35 ++++++++++++++++++++++++++---------
 1 file changed, 26 insertions(+), 9 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index 4573560..84b6853 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -16,10 +16,11 @@ SORTDATA=""
 NORMALIZE=0
 EXTRACT_JPATH=""
 TOXIC_NEWLINE=0
+DEBUG=0
 
 usage() {
   echo
-  echo "Usage: JSON.sh [-b] [-l] [-p] [-x 'regex'] [-S|-S='args'] [--no-newline]"
+  echo "Usage: JSON.sh [-b] [-l] [-p] [-x 'regex'] [-S|-S='args'] [--no-newline] [-d]"
   echo "       JSON.sh [-N|-N='args'] < markup.json"
   echo "       JSON.sh [-h]"
   echo "-h - This help text."
@@ -33,6 +34,7 @@ usage() {
   echo "     brackets not included) e.g. regex='^\"level1\",\"level2arr\",0'"
   echo "--no-newline - rather than concatenating detected line breaks in markup,"
   echo "     return with error when this is seen in input"
+  echo "-d - Enable debugging traces to stderr"
   echo
   echo "Sorting is also available, although limited to single-line strings in"
   echo "the markup (multilines are automatically escaped into backslash+n):"
@@ -87,6 +89,8 @@ parse_options() {
       --no-newline)
           TOXIC_NEWLINE=1
       ;;
+      -d) DEBUG=$(($DEBUG+1))
+      ;;
       ?*) echo "ERROR: Unknown option '$1'."
           usage
           exit 0
@@ -272,8 +276,12 @@ $key:$value"
 parse_value () {
   local jpath="${1:+$1,}$2" isleaf=0 isempty=0 print=0
   case "$token" in
-    '{') parse_object "$jpath" ;;
-    '[') parse_array  "$jpath" ;;
+    '{') parse_object "$jpath"
+       [ "$value" = '{}' ] && isempty=1
+       ;;
+    '[') parse_array  "$jpath"
+       [ "$value" = '[]' ] && isempty=1
+       ;;
     # At this point, the only valid single-character tokens are digits.
     ''|[!0-9]) throw "EXPECTED value GOT ${token:-EOF}" ;;
     *) value=$token
@@ -292,17 +300,26 @@ parse_value () {
   [ "$value" = '' ] && return
 
   [ "$LEAFONLY" -eq 0 ] && [ "$PRUNE" -eq 0 ] && print=1
-  [ "$LEAFONLY" -eq 1 ] && [ "$isleaf" -eq 1 ] && [ $PRUNE -eq 0 ] && print=1
-  [ "$LEAFONLY" -eq 0 ] && [ "$PRUNE" -eq 1 ] && [ "$isempty" -eq 0 ] && print=1
+  [ "$LEAFONLY" -eq 1 ] && [ "$isleaf" -eq 1 ] && [ $PRUNE -eq 0 ] && print=2
+  [ "$LEAFONLY" -eq 0 ] && [ "$PRUNE" -eq 1 ] && [ "$isempty" -eq 0 ] && print=3
   [ "$LEAFONLY" -eq 1 ] && [ "$isleaf" -eq 1 ] && \
-    [ $PRUNE -eq 1 ] && [ $isempty -eq 0 ] && print=1
+    [ $PRUNE -eq 1 ] && [ $isempty -eq 0 ] && print=4
+  ### A special case of an empty array or object - for leaf printing
+  ### without pruning, we are interested in these:
+  [ "$LEAFONLY" -eq 1 ] && [ "$isleaf" -eq 0 ] && [ "$isempty" -eq 1 ] && \
+    [ $PRUNE -eq 0 ] && print=5
 
-  if [ "$print" -eq 1 -a -n "$EXTRACT_JPATH" ] ; then
+  if [ "$print" -ne 0 -a -n "$EXTRACT_JPATH" ] ; then
     ### BASH regex matching:
-    [[ ${jpath} =~ ${EXTRACT_JPATH} ]] || print=0
+    [[ ${jpath} =~ ${EXTRACT_JPATH} ]] || print=-1
   fi
 
-  [ "$print" -eq 1 ] && printf "[%s]\t%s\n" "$jpath" "$value"
+  [ "$DEBUG" -gt 0 ] && \
+    echo "=== KEY='$jpath' VALUE='$value' B='$BRIEF'" \
+	"isleaf='$isleaf'/L='$LEAFONLY' isempty='$isempty'/P='$PRUNE':" \
+	"print='$print'" >&2
+
+  [ "$print" -gt 0 ] && printf "[%s]\t%s\n" "$jpath" "$value"
   :
 }
 

From d64ca2ef8520844bd8e52ecc750297fa6ad2a099 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Thu, 8 Jan 2015 11:43:50 +0100
Subject: [PATCH 13/95] JSON.sh: added a mode to help "cook" input text into
 escaped strings valid as JSON content

---
 JSON.sh | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/JSON.sh b/JSON.sh
index 84b6853..a608718 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -17,6 +17,7 @@ NORMALIZE=0
 EXTRACT_JPATH=""
 TOXIC_NEWLINE=0
 DEBUG=0
+COOKASTRING=0
 
 usage() {
   echo
@@ -50,6 +51,12 @@ usage() {
   echo "     output with contents sorted like for -S='args', e.g. use -N='-n'"
   echo "     This is equivalent to -N -S='args', just more compact to write"
   echo
+  echo "To help JSON-related scripting, with '-Q' an input plaintext can be cooked"
+  echo "into a string valid for JSON (backslashes, quotes and newlines escaped,"
+  echo "with no trailing newline); after cooking, the script exits:"
+  echo '       COOKEDSTRING="`somecommand 2>&1 | JSON.sh -Q`"'
+  echo "This can also be used to pack JSON in JSON."
+  echo
 }
 
 unquote() {
@@ -91,6 +98,8 @@ parse_options() {
       ;;
       -d) DEBUG=$(($DEBUG+1))
       ;;
+      -Q) COOKASTRING=1
+      ;;
       ?*) echo "ERROR: Unknown option '$1'."
           usage
           exit 0
@@ -165,6 +174,16 @@ strip_newlines() {
   :
 }
 
+cook_a_string() {
+    ### Escape backslashes, double-quotes and newlines, in this order
+    grep '' | sed -e 's,\\,\\\\,g' -e 's,\",\\",g' | \
+    { FIRST=''; while IFS="" read -r ILINE; do
+      printf '%s%s' "$FIRST" "$ILINE"
+      [ -z "$FIRST" ] && FIRST='\n'
+      done; }
+    :
+}
+
 tokenize () {
   local GREP
   local ESCAPE
@@ -346,5 +365,9 @@ smart_parse() {
 if ([ "$0" = "$BASH_SOURCE" ] || ! [ -n "$BASH_SOURCE" ]);
 then
   parse_options "$@"
-  smart_parse
+  if [ "$COOKASTRING" -eq 1 ]; then
+    cook_a_string
+  else
+    smart_parse
+  fi
 fi

From 9c0441253110c85b8ca9605f80462dafd98c7027 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Thu, 8 Jan 2015 15:51:48 +0100
Subject: [PATCH 14/95] JSON.sh debugging greatly enhanced; content-cooking
 mode now also escapes tabs

---
 JSON.sh | 115 +++++++++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 105 insertions(+), 10 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index a608718..1c38023 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -16,9 +16,12 @@ SORTDATA=""
 NORMALIZE=0
 EXTRACT_JPATH=""
 TOXIC_NEWLINE=0
-DEBUG=0
 COOKASTRING=0
 
+### Beside command-line, debugging can be enabled by envvars from the caller
+[ x"$DEBUG" = xy -o x"$DEBUG" = xyes ] && DEBUG=1
+[ -n "$DEBUG" -a "$DEBUG" -ge 0 ] 2>/dev/null || DEBUG=0
+
 usage() {
   echo
   echo "Usage: JSON.sh [-b] [-l] [-p] [-x 'regex'] [-S|-S='args'] [--no-newline] [-d]"
@@ -35,7 +38,7 @@ usage() {
   echo "     brackets not included) e.g. regex='^\"level1\",\"level2arr\",0'"
   echo "--no-newline - rather than concatenating detected line breaks in markup,"
   echo "     return with error when this is seen in input"
-  echo "-d - Enable debugging traces to stderr"
+  echo "-d - Enable debugging traces to stderr (repeat or use -d=NUM to bump)"
   echo
   echo "Sorting is also available, although limited to single-line strings in"
   echo "the markup (multilines are automatically escaped into backslash+n):"
@@ -64,6 +67,41 @@ unquote() {
     sed "s,^'\(.*\)'\$,\1," | sed 's,^\"\(.*\)\"$,\1,'
 }
 
+### Empty and non-numeric and non-positive values should be filtered out here
+is_positive() {
+    [ -n "$1" -a "$1" -gt 0 ] 2>/dev/null
+}
+default_posval() {
+    eval is_positive "\$$1" || eval "$1"="$2"
+}
+
+print_debug() {
+    # Required params:
+    #   $1	Debug level of the message
+    #   $2..	The message to print to stderr (if $DEBUG>=$1)
+    local DL="$1"
+    shift
+    [ "$DEBUG" -ge "$DL" ] 2>/dev/null && \
+        echo -E "[$$]DEBUG($DL): $@" >&2
+    :
+}
+
+tee_stderr() {
+    TEE_TAG="TEE_STDERR: "
+    [ -n "$1" ] && TEE_TAG="$1:"
+    [ -n "$2" -a "$2" -ge 0 ] 2>/dev/null && \
+        TEE_DEBUG="$2" || \
+        TEE_DEBUG=$DEBUGLEVEL_PRINTTOKEN_PIPELINE
+
+    ### If debug is not enabled, skip tee'ing quickly with little impact
+    [ "$DEBUG" -lt "$TEE_DEBUG" ] 2>/dev/null && cat || \
+    while IFS= read -r LINE; do
+        echo -E "$LINE"
+        print_debug "$TEE_DEBUG" "$TEE_TAG" "$LINE"
+    done
+    :
+}
+
 parse_options() {
   set -- "$@"
   local ARGN=$#
@@ -98,6 +136,8 @@ parse_options() {
       ;;
       -d) DEBUG=$(($DEBUG+1))
       ;;
+      -d=*) DEBUG="`echo "$1" | sed 's,^-d=,,'`"
+      ;;
       -Q) COOKASTRING=1
       ;;
       ?*) echo "ERROR: Unknown option '$1'."
@@ -135,8 +175,10 @@ strip_newlines() {
   local INSTRING=0
   local LINENUM=0
 
-  # the first "grep" should ensure that input has a trailing newline
-  grep '' | while IFS="" read -r ILINE; do
+  # The first "grep" should ensure that input has a trailing newline
+  grep '' | \
+  tee_stderr BEFORE_STRIP $DEBUGLEVEL_PRINTTOKEN_PIPELINE | \
+  while IFS="" read -r ILINE; do
     # Remove escaped quotes:
     LINESTRIP="${ILINE//\\\"}"
     # Remove all chars but remaining quotes:
@@ -148,7 +190,7 @@ strip_newlines() {
 
     if [ "$ODD" -eq 1 -a "$INSTRING" -eq 0 ]; then
       [ "$TOXIC_NEWLINE" = 1 ] && \
-        echo "Invalid JSON markup detected: newline in a string value: at line #$LINENUM" >&2 && \
+        echo "ERROR: Invalid JSON markup detected: newline in a string value: at line #$LINENUM" >&2 && \
         exit 121
       printf '%s\\n' "$ILINE"
       INSTRING=1
@@ -175,8 +217,8 @@ strip_newlines() {
 }
 
 cook_a_string() {
-    ### Escape backslashes, double-quotes and newlines, in this order
-    grep '' | sed -e 's,\\,\\\\,g' -e 's,\",\\",g' | \
+    ### Escape backslashes, double-quotes, tabs and newlines, in this order
+    grep '' | sed -e 's,\\,\\\\,g' -e 's,\",\\",g' -e 's,\t,\\t,g' | \
     { FIRST=''; while IFS="" read -r ILINE; do
       printf '%s%s' "$FIRST" "$ILINE"
       [ -z "$FIRST" ] && FIRST='\n'
@@ -212,6 +254,7 @@ tokenize () {
   local KEYWORD='null|false|true'
   local SPACE='[[:space:]]+'
 
+  tee_stderr BEFORE_TOKENIZER $DEBUGLEVEL_PRINTTOKEN_PIPELINE | \
   $GREP "$STRING|$NUMBER|$KEYWORD|$SPACE|." | egrep -v "^$SPACE$"
 }
 
@@ -220,6 +263,7 @@ parse_array () {
   local ary=''
   local aryml=''
   read -r token
+  print_debug $DEBUGLEVEL_PRINTTOKEN "parse_array(1):" "token=$token"
   case "$token" in
     ']') ;;
     *)
@@ -233,12 +277,14 @@ parse_array () {
 $value"
         fi
         read -r token
+        print_debug $DEBUGLEVEL_PRINTTOKEN "parse_array(2):" "token=$token"
         case "$token" in
           ']') break ;;
           ',') ary="$ary," ;;
           *) throw "EXPECTED , or ] GOT ${token:-EOF}" ;;
         esac
         read -r token
+        print_debug $DEBUGLEVEL_PRINTTOKEN "parse_array(3):" "token=$token"
       done
       ;;
   esac
@@ -254,6 +300,7 @@ parse_object () {
   local obj=''
   local objml=''
   read -r token
+  print_debug $DEBUGLEVEL_PRINTTOKEN "parse_object(1):" "token=$token"
   case "$token" in
     '}') ;;
     *)
@@ -264,11 +311,13 @@ parse_object () {
           *) throw "EXPECTED string GOT ${token:-EOF}" ;;
         esac
         read -r token
+        print_debug $DEBUGLEVEL_PRINTTOKEN "parse_object(2):" "token=$token"
         case "$token" in
           ':') ;;
           *) throw "EXPECTED : GOT ${token:-EOF}" ;;
         esac
         read -r token
+        print_debug $DEBUGLEVEL_PRINTTOKEN "parse_object(3):" "token=$token"
         parse_value "$1" "$key"
         obj="$obj$key:$value"
         if [ -n "$SORTDATA" ]; then
@@ -276,12 +325,14 @@ parse_object () {
 $key:$value"
         fi
         read -r token
+        print_debug $DEBUGLEVEL_PRINTTOKEN "parse_object(4):" "token=$token"
         case "$token" in
           '}') break ;;
           ',') obj="$obj," ;;
           *) throw "EXPECTED , or } GOT ${token:-EOF}" ;;
         esac
         read -r token
+        print_debug $DEBUGLEVEL_PRINTTOKEN "parse_object(5):" "token=$token"
       done
     ;;
   esac
@@ -311,7 +362,11 @@ parse_value () {
 
   if [ "$NORMALIZE" -eq 1 ]; then
     # Ensure a "true" output from the "if" for "return"
-    [ "$jpath" != '' ] || printf "%s\n" "$value"
+    if [ "$jpath" != '' ]; then : ; else
+        print_debug $DEBUGLEVEL_PRINTPATHVAL \
+            "Non-root keys were skipped due to normalization mode"
+	printf "%s\n" "$value"
+    fi
     return
   fi
 
@@ -333,8 +388,8 @@ parse_value () {
     [[ ${jpath} =~ ${EXTRACT_JPATH} ]] || print=-1
   fi
 
-  [ "$DEBUG" -gt 0 ] && \
-    echo "=== KEY='$jpath' VALUE='$value' B='$BRIEF'" \
+  print_debug $DEBUGLEVEL_PRINTPATHVAL \
+	"JPATH='$jpath' VALUE='$value' B='$BRIEF'" \
 	"isleaf='$isleaf'/L='$LEAFONLY' isempty='$isempty'/P='$PRUNE':" \
 	"print='$print'" >&2
 
@@ -344,8 +399,10 @@ parse_value () {
 
 parse () {
   read -r token
+  print_debug $DEBUGLEVEL_PRINTTOKEN "parse(1):" "token=$token"
   parse_value
   read -r token
+  print_debug $DEBUGLEVEL_PRINTTOKEN "parse(2):" "token=$token"
   case "$token" in
     '') ;;
     *) throw "EXPECTED EOF GOT $token" ;;
@@ -362,9 +419,47 @@ smart_parse() {
     fi
 }
 
+###########################################################
+### Active logic
+
+### Caller can disable specific debuggers by setting their level too high
+default_posval DEBUGLEVEL_PRINTPATHVAL		1
+default_posval DEBUGLEVEL_PRINTTOKEN		2
+default_posval DEBUGLEVEL_PRINTTOKEN_PIPELINE	3
+default_posval DEBUGLEVEL_TRACE_X		4
+default_posval DEBUGLEVEL_TRACE_V		5
+default_posval DEBUGLEVEL_MERGE_ERROUT		4
+
 if ([ "$0" = "$BASH_SOURCE" ] || ! [ -n "$BASH_SOURCE" ]);
 then
   parse_options "$@"
+  # Note that the options enable some debug level
+
+  [ "$DEBUG" -ge "$DEBUGLEVEL_MERGE_ERROUT" ] && \
+    exec 2>&1 && \
+    echo "[$$]DEBUG: Merge stderr and stdout for easier tracing with less" \
+	"(DEBUGLEVEL_MERGE_ERROUT=$DEBUGLEVEL_MERGE_ERROUT)" >&2
+  [ "$DEBUG" -gt 0 ] && \
+    echo "[$$]DEBUG: Enabled (debugging level $DEBUG)" >&2
+  [ "$DEBUG" -ge "$DEBUGLEVEL_PRINTPATHVAL" ] && \
+    echo "[$$]DEBUG: Enabled tracing of path:value printing decisions" \
+	"(DEBUGLEVEL_PRINTPATHVAL=$DEBUGLEVEL_PRINTPATHVAL)" >&2
+  [ "$DEBUG" -ge "$DEBUGLEVEL_PRINTTOKEN" ] && \
+    echo "[$$]DEBUG: Enabled printing of each processed token" \
+	"(DEBUGLEVEL_PRINTTOKEN=$DEBUGLEVEL_PRINTTOKEN)" >&2
+  [ "$DEBUG" -ge "$DEBUGLEVEL_PRINTTOKEN_PIPELINE" ] && \
+    echo "[$$]DEBUG: Enabled tracing of read-in token conversions" \
+	"(DEBUGLEVEL_PRINTTOKEN_PIPELINE=$DEBUGLEVEL_PRINTTOKEN_PIPELINE)" >&2
+  [ "$DEBUG" -ge "$DEBUGLEVEL_TRACE_V" ] && \
+    echo "[$$]DEBUG: Enable execution tracing (-v)" \
+	"(DEBUGLEVEL_TRACE_V=$DEBUGLEVEL_TRACE_V)" >&2 && \
+    set +v
+  [ "$DEBUG" -ge "$DEBUGLEVEL_TRACE_X" ] && \
+    echo "[$$]DEBUG: Enable execution tracing (-x)" \
+	"(DEBUGLEVEL_TRACE_X=$DEBUGLEVEL_TRACE_X)" >&2 && \
+    set -x
+
+  tee_stderr RAW_INPUT $DEBUGLEVEL_PRINTTOKEN_PIPELINE | \
   if [ "$COOKASTRING" -eq 1 ]; then
     cook_a_string
   else

From 7e5e3f6d487358d7d4264e68a24731965ac32eff Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Thu, 8 Jan 2015 16:10:59 +0100
Subject: [PATCH 15/95] JSON.sh now tolerant of tab characters inside strings
 in JSON contents

---
 JSON.sh | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index 1c38023..3022fe2 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -248,14 +248,17 @@ tokenize () {
     CHAR='[^[:cntrl:]"\\\\]'
   fi
 
-  local STRINGVAL="$CHAR*($ESCAPE$CHAR*)*"
+  # Allow tabs inside strings
+  local CHART="($CHAR|[[:blank:]])"
+  local STRINGVAL="$CHART*($ESCAPE$CHART*)*"
   local STRING="(\"$STRINGVAL\")"
   local NUMBER='-?(0|[1-9][0-9]*)([.][0-9]*)?([eE][+-]?[0-9]*)?'
   local KEYWORD='null|false|true'
   local SPACE='[[:space:]]+'
 
   tee_stderr BEFORE_TOKENIZER $DEBUGLEVEL_PRINTTOKEN_PIPELINE | \
-  $GREP "$STRING|$NUMBER|$KEYWORD|$SPACE|." | egrep -v "^$SPACE$"
+  $GREP "$STRING|$NUMBER|$KEYWORD|$SPACE|." | egrep -v "^$SPACE$" | \
+  tee_stderr AFTER_TOKENIZER $DEBUGLEVEL_PRINTTOKEN_PIPELINE
 }
 
 parse_array () {

From 6b9e724ba7fe93bbb4dd5092902e469ed0c92709 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Fri, 9 Jan 2015 01:35:59 +0100
Subject: [PATCH 16/95] JSON.sh: for feature parity, support -x="regex" as well
 as -x "regex"

---
 JSON.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/JSON.sh b/JSON.sh
index 3022fe2..b8ae29d 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -131,6 +131,8 @@ parse_options() {
       -x) EXTRACT_JPATH="$2"
           shift
       ;;
+      -x=*) EXTRACT_JPATH="`echo "$1" | sed 's,^-x=,,'`"
+      ;;
       --no-newline)
           TOXIC_NEWLINE=1
       ;;

From 2e34b777477aa5a10591c7aa0586b9a53865680b Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Sun, 11 Jan 2015 14:11:51 +0100
Subject: [PATCH 17/95] Moved the examples from the pull-request into the
 project README file

---
 README.md                                     | 431 +++++++++++++++++-
 test/valid/documented_example.json            |  13 +
 test/valid/documented_example.normalized      |   1 +
 .../documented_example.normalized_sorted      |   1 +
 test/valid/documented_example.parsed          |  47 ++
 test/valid/documented_example.sorted          |  47 ++
 6 files changed, 536 insertions(+), 4 deletions(-)
 create mode 100644 test/valid/documented_example.json
 create mode 100644 test/valid/documented_example.normalized
 create mode 100644 test/valid/documented_example.normalized_sorted
 create mode 100644 test/valid/documented_example.parsed
 create mode 100644 test/valid/documented_example.sorted

diff --git a/README.md b/README.md
index 63f4c97..bcf1ee4 100755
--- a/README.md
+++ b/README.md
@@ -1,12 +1,25 @@
 # JSON.sh
 
-yo, so it's a json parser written in bash
+Yo, so it's a JSON parser written in bash!
 
-pipe json to it, and it traverses the json objects and prints out the 
-path to the current object (as a JSON array) and then the object, without whitespace.
+Pipe JSON to it, and it traverses the json objects and prints out the 
+path to the current object (as a JSON array) and then the object,
+without whitespace between syntactic constructs (whitespace inside
+content strings is preserved).
+
+Further features include the ability to "extract" the JSON paths which
+match a specified regex, ability to sort the content (array and object
+items), and ability to "cook" raw text into escaped strings acceptable
+for passing in JSON markup. Finally, there is a mode to "normalize" an
+input JSON markup into a single-line no-extra-whitespace string, with
+optional sorting applied, so that the script can be used as a filter to
+normalize two JSON documents so they can be compared for differences.
+
+A simple example follows here, and more complex ones are presented
+after the break for command-line options:
 
 ``` bash
-$ json_parse < package.json
+$ ./JSON.sh < package.json
 ["name"]  "JSON.sh"
 ["version"]  "0.0.0"
 ["description"]  ""
@@ -20,6 +33,24 @@ $ json_parse < package.json
 #  ... etc
 ```
 
+* Pruning of empty arrays/structs (from line-by-line output) is supported
+as well as for strings-only before:
+```bash
+:; ELINE='{"emptyarr":[],"emptyobj":{},"emptystr":""}'
+
+# Here you see them...
+:; echo -E "$ELINE" | ./JSON.sh
+["emptyarr"]    []
+["emptyobj"]    {}
+["emptystr"]    ""
+[]      {"emptyarr":[],"emptyobj":{},"emptystr":""}
+
+# Here you don't ;)
+:; echo -E "$ELINE" | ./JSON.sh -p
+[]      {"emptyarr":[],"emptyobj":{},"emptystr":""}
+```
+
+
 a more complex example:
 
 ``` bash
@@ -41,6 +72,398 @@ curl registry.npmjs.org/express | ./JSON.sh | egrep '\["versions","[^"]*"\]'
 -h
 > Show help text.
 
+## Complex usage examples
+
+A picture shows more than a thousand words, heh?
+So here is a few thousand words for you, to display the new features ;)
+
+* First, define a complicated value contrived just for show-off. This example
+can be found in the source checkout as `tests/valid/documented_example.json`.
+It may be arguable that newlines in the markup are invalid... but for some
+practical example, if the actual data is generated by some shell-script
+(storing a copy of a multiline file or command output, etc.) - then the new
+features in `JSON.sh` precisely allow to turn that into (more) valid markup ;)
+```bash
+:; LINE='{"var1":"val1","split
+key":"value","var0":"escaped \" quote","splitValue":"there
+  are a newline and three spaces (one after \"there\" and two before \"are\")",
+"array":["z","a","b",3,20,0,"","","\""
+,"escaping\"several\"\"
+quote\"s and
+newlines"],"aNumber":1,"var8":"string\nwith\nproper\\\nnewlines",
+"var38":"","emptyarr":[],"emptyobj":{},
+"arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"},
+  {"var":"val2","str":"z"},{"var":"val2","str":"x"},
+  {"var":"val1","str":"S"},{"var":"val1","str":"\""},
+{"var":"val1","str":5},{"var":"val1","str":"5"}]}'
+
+### Examples related to sorting will also need this to be reproducible:
+:; LANG=C; LC_ALL=C; export LANG; export LC_ALL
+```
+
+* Note also the use of `echo -E` to avoid shell's processing of escaped
+characters (such as `\\\n` in `var8`):
+```bash
+:; echo -E "$LINE"
+{"var1":"val1","split
+key":"value","var0":"escaped \" quote","splitValue":"there
+  are a newline and three spaces (one after \"there\" and two before \"are\")",
+"array":["z","a","b",3,20,0,"","","\""
+,"escaping\"several\"\"
+quote\"s and
+newlines"],"aNumber":1,"var8":"string\nwith\nproper\\\nnewlines",
+"var38":"","emptyarr":[],"emptyobj":{},
+"arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"},
+  {"var":"val2","str":"z"},{"var":"val2","str":"x"},
+  {"var":"val1","str":"S"},{"var":"val1","str":"\""},
+{"var":"val1","str":5},{"var":"val1","str":"5"}]}
+```
+
+* There is a mode to detect invalid input (due to newlines in strings):
+```bash
+:; echo "$LINE" | ./JSON.sh --no-newline
+Invalid JSON markup detected: newline in a string value: at line #1
+EXPECTED value GOT EOF
+```
+
+* Otherwise automatic conversion of these takes place:
+```bash
+:; echo -E "$LINE" | ./JSON.sh
+["var1"]        "val1"
+["split\nkey"]  "value"
+["var0"]        "escaped \" quote"
+["splitValue"]  "there\n  are a newline and three spaces (one after \"there\" and two before \"are\")"
+["array",0]     "z"
+["array",1]     "a"
+["array",2]     "b"
+["array",3]     3
+["array",4]     20
+["array",5]     0
+["array",6]     ""
+["array",7]     ""
+["array",8]     "\""
+["array",9]     "escaping\"several\"\"\nquote\"s and\nnewlines"
+["array"]       ["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"]
+["aNumber"]     1
+["var8"]        "string\nwith\nproper\\\nnewlines"
+["var38"]       ""
+["emptyarr"]    []
+["emptyobj"]    {}
+["arrOfObjs",0,"var"]   "val1"
+["arrOfObjs",0,"str"]   "s"
+["arrOfObjs",0] {"var":"val1","str":"s"}
+["arrOfObjs",1,"var"]   "val30"
+["arrOfObjs",1,"str"]   "s"
+["arrOfObjs",1] {"var":"val30","str":"s"}
+["arrOfObjs",2,"var"]   "val2"
+["arrOfObjs",2,"str"]   "z"
+["arrOfObjs",2] {"var":"val2","str":"z"}
+["arrOfObjs",3,"var"]   "val2"
+["arrOfObjs",3,"str"]   "x"
+["arrOfObjs",3] {"var":"val2","str":"x"}
+["arrOfObjs",4,"var"]   "val1"
+["arrOfObjs",4,"str"]   "S"
+["arrOfObjs",4] {"var":"val1","str":"S"}
+["arrOfObjs",5,"var"]   "val1"
+["arrOfObjs",5,"str"]   "\""
+["arrOfObjs",5] {"var":"val1","str":"\""}
+["arrOfObjs",6,"var"]   "val1"
+["arrOfObjs",6,"str"]   5
+["arrOfObjs",6] {"var":"val1","str":5}
+["arrOfObjs",7,"var"]   "val1"
+["arrOfObjs",7,"str"]   "5"
+["arrOfObjs",7] {"var":"val1","str":"5"}
+["arrOfObjs"]   [{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}]
+[]      {"var1":"val1","split\nkey":"value","var0":"escaped \" quote","splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","array":["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"],"aNumber":1,"var8":"string\nwith\nproper\\\nnewlines","var38":"","emptyarr":[],"emptyobj":{},"arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}]}
+```
+
+* Returning a valid JSON markup string without the JSON path (i.e. use of
+`JSON.sh` as a filter to convert scripted output into more valid JSON:
+```bash
+:; echo -E "$LINE" | ./JSON.sh -N
+{"var1":"val1","split\nkey":"value","var0":"escaped \" quote","splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","array":["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"],"aNumber":1,"var8":"string\nwith\nproper\\\nnewlines","var38":"","emptyarr":[],"emptyobj":{},"arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}]}
+```
+
+* Sorted output with defaults taken by `sort` program in your OS, i.e.
+alphabetic order (where `3` is greater than `20`), etc. and according to
+currently exported locale/collation (influencing order of numbers over
+punctuation over letters, sorting of letters with diacritics, etc.):
+```bash
+:; $ echo -E "$LINE" | ./JSON.sh -S
+["aNumber"]     1
+["arrOfObjs",0,"str"]   "5"
+["arrOfObjs",0,"var"]   "val1"
+["arrOfObjs",0] {"str":"5","var":"val1"}
+["arrOfObjs",1,"str"]   "S"
+["arrOfObjs",1,"var"]   "val1"
+["arrOfObjs",1] {"str":"S","var":"val1"}
+["arrOfObjs",2,"str"]   "\""
+["arrOfObjs",2,"var"]   "val1"
+["arrOfObjs",2] {"str":"\"","var":"val1"}
+["arrOfObjs",3,"str"]   "s"
+["arrOfObjs",3,"var"]   "val1"
+["arrOfObjs",3] {"str":"s","var":"val1"}
+["arrOfObjs",4,"str"]   "s"
+["arrOfObjs",4,"var"]   "val30"
+["arrOfObjs",4] {"str":"s","var":"val30"}
+["arrOfObjs",5,"str"]   "x"
+["arrOfObjs",5,"var"]   "val2"
+["arrOfObjs",5] {"str":"x","var":"val2"}
+["arrOfObjs",6,"str"]   "z"
+["arrOfObjs",6,"var"]   "val2"
+["arrOfObjs",6] {"str":"z","var":"val2"}
+["arrOfObjs",7,"str"]   5
+["arrOfObjs",7,"var"]   "val1"
+["arrOfObjs",7] {"str":5,"var":"val1"}
+["arrOfObjs"]   [{"str":"5","var":"val1"},{"str":"S","var":"val1"},{"str":"\"","var":"val1"},{"str":"s","var":"val1"},{"str":"s","var":"val30"},{"str":"x","var":"val2"},{"str":"z","var":"val2"},{"str":5,"var":"val1"}]
+["array",0]     ""
+["array",1]     ""
+["array",2]     "\""
+["array",3]     "a"
+["array",4]     "b"
+["array",5]     "escaping\"several\"\"\nquote\"s and\nnewlines"
+["array",6]     "z"
+["array",7]     0
+["array",8]     20
+["array",9]     3
+["array"]       ["","","\"","a","b","escaping\"several\"\"\nquote\"s and\nnewlines","z",0,20,3]
+["emptyarr"]    []
+["emptyobj"]    {}
+["splitValue"]  "there\n  are a newline and three spaces (one after \"there\" and two before \"are\")"
+["split\nkey"]  "value"
+["var0"]        "escaped \" quote"
+["var1"]        "val1"
+["var38"]       ""
+["var8"]        "string\nwith\nproper\\\nnewlines"
+[]      {"aNumber":1,"arrOfObjs":[{"str":"5","var":"val1"},{"str":"S","var":"val1"},{"str":"\"","var":"val1"},{"str":"s","var":"val1"},{"str":"s","var":"val30"},{"str":"x","var":"val2"},{"str":"z","var":"val2"},{"str":5,"var":"val1"}],"array":["","","\"","a","b","escaping\"several\"\"\nquote\"s and\nnewlines","z",0,20,3],"emptyarr":[],"emptyobj":{},"splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","split\nkey":"value","var0":"escaped \" quote","var1":"val1","var38":"","var8":"string\nwith\nproper\\\nnewlines"}
+```
+
+* Sorting with parameters, several can be passed as a single quoted string -
+for example we request numeric (`20` is greater than `3` - though only for
+standalone number tokens) and reversed (`a` is after `z`) sorting:
+```bash
+:; echo -E "$LINE" | ./JSON.sh -S='-r -n'
+["var8"]        "string\nwith\nproper\\\nnewlines"
+["var38"]       ""
+["var1"]        "val1"
+["var0"]        "escaped \" quote"
+["split\nkey"]  "value"
+["splitValue"]  "there\n  are a newline and three spaces (one after \"there\" and two before \"are\")"
+["emptyobj"]    {}
+["emptyarr"]    []
+["array",0]     20
+["array",1]     3
+["array",2]     0
+["array",3]     "z"
+["array",4]     "escaping\"several\"\"\nquote\"s and\nnewlines"
+["array",5]     "b"
+["array",6]     "a"
+["array",7]     "\""
+["array",8]     ""
+["array",9]     ""
+["array"]       [20,3,0,"z","escaping\"several\"\"\nquote\"s and\nnewlines","b","a","\"","",""]
+["arrOfObjs",0,"var"]   "val30"
+["arrOfObjs",0,"str"]   "s"
+["arrOfObjs",0] {"var":"val30","str":"s"}
+["arrOfObjs",1,"var"]   "val2"
+["arrOfObjs",1,"str"]   "z"
+["arrOfObjs",1] {"var":"val2","str":"z"}
+["arrOfObjs",2,"var"]   "val2"
+["arrOfObjs",2,"str"]   "x"
+["arrOfObjs",2] {"var":"val2","str":"x"}
+["arrOfObjs",3,"var"]   "val1"
+["arrOfObjs",3,"str"]   5
+["arrOfObjs",3] {"var":"val1","str":5}
+["arrOfObjs",4,"var"]   "val1"
+["arrOfObjs",4,"str"]   "s"
+["arrOfObjs",4] {"var":"val1","str":"s"}
+["arrOfObjs",5,"var"]   "val1"
+["arrOfObjs",5,"str"]   "\""
+["arrOfObjs",5] {"var":"val1","str":"\""}
+["arrOfObjs",6,"var"]   "val1"
+["arrOfObjs",6,"str"]   "S"
+["arrOfObjs",6] {"var":"val1","str":"S"}
+["arrOfObjs",7,"var"]   "val1"
+["arrOfObjs",7,"str"]   "5"
+["arrOfObjs",7] {"var":"val1","str":"5"}
+["arrOfObjs"]   [{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":5},{"var":"val1","str":"s"},{"var":"val1","str":"\""},{"var":"val1","str":"S"},{"var":"val1","str":"5"}]
+["aNumber"]     1
+[]      {"var8":"string\nwith\nproper\\\nnewlines","var38":"","var1":"val1","var0":"escaped \" quote","split\nkey":"value","splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","emptyobj":{},"emptyarr":[],"array":[20,3,0,"z","escaping\"several\"\"\nquote\"s and\nnewlines","b","a","\"","",""],"arrOfObjs":[{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":5},{"var":"val1","str":"s"},{"var":"val1","str":"\""},{"var":"val1","str":"S"},{"var":"val1","str":"5"}],"aNumber":1}
+```
+
+* Normalized output can also be sorted, upon request - although *NOTE* that if
+your document schema has arrays whose order of items has syntactic meaning for
+your application (aka "tuples"), such ordering will likely make the document
+invalid for your application's use-case; this warning *should* be irrelevant
+for objects (`key:value` pairs) though:
+```bash
+:; echo -E "$LINE" | ./JSON.sh -N='-n'
+{"aNumber":1,"arrOfObjs":[{"str":"5","var":"val1"},{"str":"S","var":"val1"},{"str":"\"","var":"val1"},{"str":"s","var":"val1"},{"str":"s","var":"val30"},{"str":"x","var":"val2"},{"str":"z","var":"val2"},{"str":5,"var":"val1"}],"array":["","","\"","a","b","escaping\"several\"\"\nquote\"s and\nnewlines","z",0,3,20],"emptyarr":[],"emptyobj":{},"splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","split\nkey":"value","var0":"escaped \" quote","var1":"val1","var38":"","var8":"string\nwith\nproper\\\nnewlines"}
+
+:; echo -E "$LINE" | ./JSON.sh -N=-r
+{"var8":"string\nwith\nproper\\\nnewlines","var38":"","var1":"val1","var0":"escaped \" quote","split\nkey":"value","splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","emptyobj":{},"emptyarr":[],"array":[3,20,0,"z","escaping\"several\"\"\nquote\"s and\nnewlines","b","a","\"","",""],"arrOfObjs":[{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":5},{"var":"val1","str":"s"},{"var":"val1","str":"\""},{"var":"val1","str":"S"},{"var":"val1","str":"5"}],"aNumber":1}
+
+:; echo -E "$LINE" | ./JSON.sh -N="-r -n"
+{"var8":"string\nwith\nproper\\\nnewlines","var38":"","var1":"val1","var0":"escaped \" quote","split\nkey":"value","splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","emptyobj":{},"emptyarr":[],"array":[20,3,0,"z","escaping\"several\"\"\nquote\"s and\nnewlines","b","a","\"","",""],"arrOfObjs":[{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":5},{"var":"val1","str":"s"},{"var":"val1","str":"\""},{"var":"val1","str":"S"},{"var":"val1","str":"5"}],"aNumber":1}
+```
+
+* And note that the normalized output returns (maybe sorted) JSON markup of
+the top-level item without whitespaces between syntactic elements, and other
+`JSON.sh` modifiers are essentially ignored (`-x` option is detailed below):
+```bash
+:; echo -E "$LINE" | ./JSON.sh -x 'empty' -N
+{"var1":"val1","split\nkey":"value","var0":"escaped \" quote","splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","array":["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"],"aNumber":1,"var8":"string\nwith\nproper\\\nnewlines","var38":"","emptyarr":[],"emptyobj":{},"arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}]}
+
+### Normalization mode can still be used for validation of input markup though:
+:; echo -E "$LINE" | ./JSON.sh --no-newline -N
+Invalid JSON markup detected: newline in a string value: at line #1
+EXPECTED value GOT EOF
+```
+
+* As developers of the script itself, we can debug why something is or is not
+printed and thanks to which logical block (interesting excerpts copypasted);
+several keys have been defined to pring different debug values if the debug
+level is big enough, and can be easily redefined by `export` from the caller
+(see `JSON.sh` source):
+```bash
+:; echo -E "$LINE" | ./JSON.sh -p -l -d
+# Leaf value printed
+=== KEY='"var1"' VALUE='"val1"' B='1' isleaf='1'/L='1' isempty='0'/P='1': print='4'
+["var1"]        "val1"
+# Empty value pruned from line-by-line output (not from JSON markup, not from index numbering):
+=== KEY='"array",6' VALUE='""' B='1' isleaf='1'/L='1' isempty='1'/P='1': print='0'
+=== KEY='"array",7' VALUE='""' B='1' isleaf='1'/L='1' isempty='1'/P='1': print='0'
+=== KEY='"var38"' VALUE='""' B='1' isleaf='1'/L='1' isempty='1'/P='1': print='0'
+# Empty arrays and objects are NOW also pruned on request (this is different from brief mode which just does not output objects/arrays at all):
+=== KEY='"emptyarr"' VALUE='[]' B='0' isleaf='0'/L='1' isempty='1'/P='1': print='0'
+=== KEY='"emptyobj"' VALUE='{}' B='0' isleaf='0'/L='1' isempty='1'/P='1': print='0'
+# Non-leaf items skipped from line-by-line printing:
+=== KEY='"arrOfObjs",7' VALUE='{"var":"val1","str":"5"}' B='0' isleaf='0'/L='1' isempty='0'/P='1': print='0'
+=== KEY='"arrOfObjs"' VALUE='[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}]' B='0' isleaf='0'/L='1' isempty='0'/P='1': print='0'
+=== KEY='' VALUE='...' B='0' isleaf='0'/L='1' isempty='0'/P='1': print='0'
+```
+
+* Last but not least, we now have an "extractor" to simplify scripted requests
+to particular entries by their jpaths, which helps scripted interaction with
+the JSON markup:
+```bash
+:; echo -E "$LINE" | ./JSON.sh -x 'empty'
+["emptyarr"]    []
+["emptyobj"]    {}
+
+:; echo -E "$LINE" | ./JSON.sh -x 'var'
+["var1"]        "val1"
+["var0"]        "escaped \" quote"
+["var8"]        "string\nwith\nproper\\\nnewlines"
+["var38"]       ""
+["arrOfObjs",0,"var"]   "val1"
+["arrOfObjs",1,"var"]   "val30"
+["arrOfObjs",2,"var"]   "val2"
+["arrOfObjs",3,"var"]   "val2"
+["arrOfObjs",4,"var"]   "val1"
+["arrOfObjs",5,"var"]   "val1"
+["arrOfObjs",6,"var"]   "val1"
+["arrOfObjs",7,"var"]   "val1"
+
+# Regex can be used:
+:; echo -E "$LINE" | ./JSON.sh -x '^\"var'
+["var1"]        "val1"
+["var0"]        "escaped \" quote"
+["var8"]        "string\nwith\nproper\\\nnewlines"
+["var38"]       ""
+
+:; echo -E "$LINE" | ./JSON.sh -x 'var\"$'
+["arrOfObjs",0,"var"]   "val1"
+["arrOfObjs",1,"var"]   "val30"
+["arrOfObjs",2,"var"]   "val2"
+["arrOfObjs",3,"var"]   "val2"
+["arrOfObjs",4,"var"]   "val1"
+["arrOfObjs",5,"var"]   "val1"
+["arrOfObjs",6,"var"]   "val1"
+["arrOfObjs",7,"var"]   "val1"
+
+# You can also pick array elements...
+:; echo -E "$LINE" | ./JSON.sh -x 'arrOfObjs\",[0-9]*$'
+["arrOfObjs",0] {"var":"val1","str":"s"}
+["arrOfObjs",1] {"var":"val30","str":"s"}
+["arrOfObjs",2] {"var":"val2","str":"z"}
+["arrOfObjs",3] {"var":"val2","str":"x"}
+["arrOfObjs",4] {"var":"val1","str":"S"}
+["arrOfObjs",5] {"var":"val1","str":"\""}
+["arrOfObjs",6] {"var":"val1","str":5}
+["arrOfObjs",7] {"var":"val1","str":"5"}
+
+#... unless of course you use leaf-only mode:
+:; echo -E "$LINE" | ./JSON.sh -x 'arrOfObjs\",[0-9]*$' -l
+
+#...or you can pick just the contents of the arrays:
+:; echo -E "$LINE" | ./JSON.sh -x 'arrOfObjs\",[0-9]+,.+$'
+["arrOfObjs",0,"var"]   "val1"
+["arrOfObjs",0,"str"]   "s"
+["arrOfObjs",1,"var"]   "val30"
+["arrOfObjs",1,"str"]   "s"
+["arrOfObjs",2,"var"]   "val2"
+["arrOfObjs",2,"str"]   "z"
+["arrOfObjs",3,"var"]   "val2"
+["arrOfObjs",3,"str"]   "x"
+["arrOfObjs",4,"var"]   "val1"
+["arrOfObjs",4,"str"]   "S"
+["arrOfObjs",5,"var"]   "val1"
+["arrOfObjs",5,"str"]   "\""
+["arrOfObjs",6,"var"]   "val1"
+["arrOfObjs",6,"str"]   5
+["arrOfObjs",7,"var"]   "val1"
+["arrOfObjs",7,"str"]   "5"
+
+#...or perhaps just the items in these arrays starting with an "s":
+:; echo -E "$LINE" | ./JSON.sh -x 'arrOfObjs\",[0-9]+,\"s.+$'
+["arrOfObjs",0,"str"]   "s"
+["arrOfObjs",1,"str"]   "s"
+["arrOfObjs",2,"str"]   "z"
+["arrOfObjs",3,"str"]   "x"
+["arrOfObjs",4,"str"]   "S"
+["arrOfObjs",5,"str"]   "\""
+["arrOfObjs",6,"str"]   5
+["arrOfObjs",7,"str"]   "5"
+
+# Note that only jpaths (not contents themselves) are matched:
+:; echo -E "$LINE" | ./JSON.sh -x '\n' -l
+["split\nkey"]  "value"
+```
+
+* Another new feature to help scripting is "cooking" of input strings into
+escaped JSON that should be valid markup (with no trailing newline as well);
+this currently allows to escape newlines, backslashes and TAB characters which
+otherwise made the `JSON.sh` parser sad:
+```bash
+:; RAWLINE='[ This is text
+It has
+Several "lines"
+maybe \n escaped \"
+}'
+
+:; ESCAPED="`echo -E "$RAWLINE" | ./JSON.sh -Q`"; echo -E "'$ESCAPED'"
+'[ This is text\nIt has\nSeveral \"lines\"\nmaybe \\n escaped \\\"\n}'
+```
+
+For a more practical example, let's turn some text-file dumps into
+JSON markup with escaped newlines:
+
+```bash
+:; ( echo '['; for F in /etc/motd /etc/release ; do \
+     printf '{"filename":"'"$F"'","contents":"%s"},\n' \
+       "`cat "$F"`"; done; echo '{}]' ) | ./JSON.sh -N
+[{"filename":"/etc/motd","contents":"The Illumos Project        SunOS 5.11      illumos-ad69a33 January 2015"},{"filename":"/etc/release","contents":"             OpenIndiana Development oi_151.1.8 X86 (powered by illumos)\n        Copyright 2011 Oracle and/or its affiliates. All rights reserved.\n                        Use is subject to license terms.\n                           Assembled 19 February 2013"},{}]
+```
+
+Escaping for TAB characters in string contents during "cooking", as well as
+toleration during processing, can be seen in `/etc/motd` of this example both
+above and below:
+```
+:; cat /etc/motd | ./JSON.sh -Q ; echo ""
+The Illumos Project\tSunOS 5.11\tillumos-ad69a33\tJanuary 2015
+```
+
 ## Cool Links
 
 * [step-/JSON.awk](https://github.com/step-/JSON.awk) JSON.sh ported to awk
diff --git a/test/valid/documented_example.json b/test/valid/documented_example.json
new file mode 100644
index 0000000..4622cb4
--- /dev/null
+++ b/test/valid/documented_example.json
@@ -0,0 +1,13 @@
+{"var1":"val1","split
+key":"value","var0":"escaped \" quote","splitValue":"there
+  are a newline and three spaces (one after \"there\" and two before \"are\")",
+"array":["z","a","b",3,20,0,"","","\""
+,"escaping\"several\"\"
+quote\"s and
+newlines"],"aNumber":1,"var8":"string\nwith\nproper\\\nnewlines",
+"var38":"","emptyarr":[],"emptyobj":{},"smptystr":"",
+"arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"},
+  {"var":"val2","str":"z"},{"var":"val2","str":"x"},
+  {"var":"val1","str":"S"},{"var":"val1","str":"\""},
+{"var":"val1","str":5},{"var":"val1","str":"5"}]}
+
diff --git a/test/valid/documented_example.normalized b/test/valid/documented_example.normalized
new file mode 100644
index 0000000..3b10c89
--- /dev/null
+++ b/test/valid/documented_example.normalized
@@ -0,0 +1 @@
+{"var1":"val1","split\nkey":"value","var0":"escaped \" quote","splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","array":["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"],"aNumber":1,"var8":"string\nwith\nproper\\\nnewlines","var38":"","emptyarr":[],"emptyobj":{},"smptystr":"","arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}]}
diff --git a/test/valid/documented_example.normalized_sorted b/test/valid/documented_example.normalized_sorted
new file mode 100644
index 0000000..2a51ec4
--- /dev/null
+++ b/test/valid/documented_example.normalized_sorted
@@ -0,0 +1 @@
+{"aNumber":1,"arrOfObjs":[{"str":"5","var":"val1"},{"str":"S","var":"val1"},{"str":"\"","var":"val1"},{"str":"s","var":"val1"},{"str":"s","var":"val30"},{"str":"x","var":"val2"},{"str":"z","var":"val2"},{"str":5,"var":"val1"}],"array":["","","\"","a","b","escaping\"several\"\"\nquote\"s and\nnewlines","z",0,3,20],"emptyarr":[],"emptyobj":{},"smptystr":"","splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","split\nkey":"value","var0":"escaped \" quote","var1":"val1","var38":"","var8":"string\nwith\nproper\\\nnewlines"}
diff --git a/test/valid/documented_example.parsed b/test/valid/documented_example.parsed
new file mode 100644
index 0000000..bda2d64
--- /dev/null
+++ b/test/valid/documented_example.parsed
@@ -0,0 +1,47 @@
+["var1"]	"val1"
+["split\nkey"]	"value"
+["var0"]	"escaped \" quote"
+["splitValue"]	"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")"
+["array",0]	"z"
+["array",1]	"a"
+["array",2]	"b"
+["array",3]	3
+["array",4]	20
+["array",5]	0
+["array",6]	""
+["array",7]	""
+["array",8]	"\""
+["array",9]	"escaping\"several\"\"\nquote\"s and\nnewlines"
+["array"]	["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"]
+["aNumber"]	1
+["var8"]	"string\nwith\nproper\\\nnewlines"
+["var38"]	""
+["emptyarr"]	[]
+["emptyobj"]	{}
+["smptystr"]	""
+["arrOfObjs",0,"var"]	"val1"
+["arrOfObjs",0,"str"]	"s"
+["arrOfObjs",0]	{"var":"val1","str":"s"}
+["arrOfObjs",1,"var"]	"val30"
+["arrOfObjs",1,"str"]	"s"
+["arrOfObjs",1]	{"var":"val30","str":"s"}
+["arrOfObjs",2,"var"]	"val2"
+["arrOfObjs",2,"str"]	"z"
+["arrOfObjs",2]	{"var":"val2","str":"z"}
+["arrOfObjs",3,"var"]	"val2"
+["arrOfObjs",3,"str"]	"x"
+["arrOfObjs",3]	{"var":"val2","str":"x"}
+["arrOfObjs",4,"var"]	"val1"
+["arrOfObjs",4,"str"]	"S"
+["arrOfObjs",4]	{"var":"val1","str":"S"}
+["arrOfObjs",5,"var"]	"val1"
+["arrOfObjs",5,"str"]	"\""
+["arrOfObjs",5]	{"var":"val1","str":"\""}
+["arrOfObjs",6,"var"]	"val1"
+["arrOfObjs",6,"str"]	5
+["arrOfObjs",6]	{"var":"val1","str":5}
+["arrOfObjs",7,"var"]	"val1"
+["arrOfObjs",7,"str"]	"5"
+["arrOfObjs",7]	{"var":"val1","str":"5"}
+["arrOfObjs"]	[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}]
+[]	{"var1":"val1","split\nkey":"value","var0":"escaped \" quote","splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","array":["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"],"aNumber":1,"var8":"string\nwith\nproper\\\nnewlines","var38":"","emptyarr":[],"emptyobj":{},"smptystr":"","arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}]}
diff --git a/test/valid/documented_example.sorted b/test/valid/documented_example.sorted
new file mode 100644
index 0000000..3cf1663
--- /dev/null
+++ b/test/valid/documented_example.sorted
@@ -0,0 +1,47 @@
+["var8"]	"string\nwith\nproper\\\nnewlines"
+["var38"]	""
+["var1"]	"val1"
+["var0"]	"escaped \" quote"
+["split\nkey"]	"value"
+["splitValue"]	"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")"
+["smptystr"]	""
+["emptyobj"]	{}
+["emptyarr"]	[]
+["array",0]	20
+["array",1]	3
+["array",2]	0
+["array",3]	"z"
+["array",4]	"escaping\"several\"\"\nquote\"s and\nnewlines"
+["array",5]	"b"
+["array",6]	"a"
+["array",7]	"\""
+["array",8]	""
+["array",9]	""
+["array"]	[20,3,0,"z","escaping\"several\"\"\nquote\"s and\nnewlines","b","a","\"","",""]
+["arrOfObjs",0,"var"]	"val30"
+["arrOfObjs",0,"str"]	"s"
+["arrOfObjs",0]	{"var":"val30","str":"s"}
+["arrOfObjs",1,"var"]	"val2"
+["arrOfObjs",1,"str"]	"z"
+["arrOfObjs",1]	{"var":"val2","str":"z"}
+["arrOfObjs",2,"var"]	"val2"
+["arrOfObjs",2,"str"]	"x"
+["arrOfObjs",2]	{"var":"val2","str":"x"}
+["arrOfObjs",3,"var"]	"val1"
+["arrOfObjs",3,"str"]	5
+["arrOfObjs",3]	{"var":"val1","str":5}
+["arrOfObjs",4,"var"]	"val1"
+["arrOfObjs",4,"str"]	"s"
+["arrOfObjs",4]	{"var":"val1","str":"s"}
+["arrOfObjs",5,"var"]	"val1"
+["arrOfObjs",5,"str"]	"\""
+["arrOfObjs",5]	{"var":"val1","str":"\""}
+["arrOfObjs",6,"var"]	"val1"
+["arrOfObjs",6,"str"]	"S"
+["arrOfObjs",6]	{"var":"val1","str":"S"}
+["arrOfObjs",7,"var"]	"val1"
+["arrOfObjs",7,"str"]	"5"
+["arrOfObjs",7]	{"var":"val1","str":"5"}
+["arrOfObjs"]	[{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":5},{"var":"val1","str":"s"},{"var":"val1","str":"\""},{"var":"val1","str":"S"},{"var":"val1","str":"5"}]
+["aNumber"]	1
+[]	{"var8":"string\nwith\nproper\\\nnewlines","var38":"","var1":"val1","var0":"escaped \" quote","split\nkey":"value","splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","smptystr":"","emptyobj":{},"emptyarr":[],"array":[20,3,0,"z","escaping\"several\"\"\nquote\"s and\nnewlines","b","a","\"","",""],"arrOfObjs":[{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":5},{"var":"val1","str":"s"},{"var":"val1","str":"\""},{"var":"val1","str":"S"},{"var":"val1","str":"5"}],"aNumber":1}

From c104dd102c7cf4327acc293887045c4e9db7de91 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Sun, 11 Jan 2015 14:29:07 +0100
Subject: [PATCH 18/95] README updated for newly added usage() modes

---
 README.md | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 68 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index bcf1ee4..a8fb7b0 100755
--- a/README.md
+++ b/README.md
@@ -60,18 +60,82 @@ curl registry.npmjs.org/express | ./JSON.sh | egrep '\["versions","[^"]*"\]'
 
 ## Options
 
--b
+### Usual queries, full and filtered
+``` bash
+Usage: JSON.sh [-b] [-l] [-p] [-x 'regex'] [-S|-S='args'] [--no-newline] [-d]
+```
+
+* -b
 > Brief output. Combines 'Leaf only' and 'Prune empty' options.
 
--l
+* -l
 > Leaf only. Only show leaf nodes, which stops data duplication.
 
--p
-> Prune empty. Exclude fields with empty values.
+* -p
+> Prune empty. Exclude fields with empty values (strings, arrays, objects).
+
+* -x 'regex'
+* -x='regex'
+> "Extract" - rather than showing all document from the root element,
+extract the items rooted at path(s) matching the regex (see the
+comma-separated list of nested hierarchy names in general output,
+brackets not included) e.g. `-x='^"level1","level2arr",0'`
+
+* --no-newline
+> rather than concatenating detected line breaks in markup, return
+with error when this is seen in input
+
+Sorting is also available, although limited to single-line strings in
+the markup (multilines are automatically escaped into backslash+n):
+* -S
+> Sort the contents of items in JSON markup and leaf-list markup:
+`sort` objects by key names and then values, and arrays by values
+
+> -S='args'
+> use `sort $args` for content sorting, e.g. use `-S='-n -r'` for
+reverse numeric sort
+
+* -d
+> Enable debugging traces to stderr (repeat or use `-d=NUM` to bump)
+
+
+### Normalization (with optional sorting)
+``` bash
+Usage: JSON.sh [-N|-N='args'] [-d] < markup.json
+```
+
+An input JSON markup can be normalized into single-line no-whitespace:
+* -N
+> Normalize the input JSON markup into a single-line JSON output;
+in this mode syntax and spacing are normalized, data order remains
+
+* -N='args'
+> Normalize the input JSON markup into a single-line JSON output with
+contents sorted like for `-S='args'`, e.g. use `-N='-n'`.
+This is equivalent to `-N -S='args'`, just more compact to write.
+
+### Cook raw data
+
+``` bash
+Usage: COOKEDSTRING="`somecommand 2>&1 | JSON.sh -Q`"
+```
+
+To help JSON-related scripting, with `-Q` an input plaintext can be "cooked"
+into a string valid for JSON (backslashes, quotes and newlines escaped, with
+no trailing newline); after cooking, the script exits.
+
+This mode can also be used to pack JSON in JSON.
+
+
+### Ask for help
+``` bash
+Usage: JSON.sh [-h]
+```
 
 -h
 > Show help text.
 
+
 ## Complex usage examples
 
 A picture shows more than a thousand words, heh?

From 8a56928dd36c0d418722f74dd1b091538d4db7ff Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Sun, 11 Jan 2015 14:35:41 +0100
Subject: [PATCH 19/95] README markup updated for prettiness and a few bugs
 fixed too ;)

---
 README.md | 45 +++++++++++++++++++++++++--------------------
 1 file changed, 25 insertions(+), 20 deletions(-)

diff --git a/README.md b/README.md
index a8fb7b0..3613402 100755
--- a/README.md
+++ b/README.md
@@ -65,38 +65,42 @@ curl registry.npmjs.org/express | ./JSON.sh | egrep '\["versions","[^"]*"\]'
 Usage: JSON.sh [-b] [-l] [-p] [-x 'regex'] [-S|-S='args'] [--no-newline] [-d]
 ```
 
-* -b
+* `-b`
 > Brief output. Combines 'Leaf only' and 'Prune empty' options.
 
-* -l
+* `-l`
 > Leaf only. Only show leaf nodes, which stops data duplication.
 
-* -p
+* `-p`
 > Prune empty. Exclude fields with empty values (strings, arrays, objects).
 
-* -x 'regex'
-* -x='regex'
+* `-x 'regex'` or `-x='regex'`
 > "Extract" - rather than showing all document from the root element,
 extract the items rooted at path(s) matching the regex (see the
 comma-separated list of nested hierarchy names in general output,
 brackets not included) e.g. `-x='^"level1","level2arr",0'`
 
-* --no-newline
-> rather than concatenating detected line breaks in markup, return
-with error when this is seen in input
-
 Sorting is also available, although limited to single-line strings in
 the markup (multilines are automatically escaped into backslash+n):
-* -S
+
+* `-S`
 > Sort the contents of items in JSON markup and leaf-list markup:
 `sort` objects by key names and then values, and arrays by values
 
-> -S='args'
-> use `sort $args` for content sorting, e.g. use `-S='-n -r'` for
+* `-S='args'`
+> Use `sort $args` for content sorting, e.g. use `-S='-n -r'` for
 reverse numeric sort
 
-* -d
-> Enable debugging traces to stderr (repeat or use `-d=NUM` to bump)
+Other options:
+
+* `--no-newline`
+> rather than concatenating detected line breaks in markup, return
+with error when this is seen in input
+
+* `-d [-d...]` or `-d=NUM`
+> Enable debugging traces to `stderr` (repeat or use `-d=NUM` to bump,
+see the script source for details on what can be debugged and how to
+select what you want)
 
 
 ### Normalization (with optional sorting)
@@ -105,11 +109,11 @@ Usage: JSON.sh [-N|-N='args'] [-d] < markup.json
 ```
 
 An input JSON markup can be normalized into single-line no-whitespace:
-* -N
+* `-N`
 > Normalize the input JSON markup into a single-line JSON output;
 in this mode syntax and spacing are normalized, data order remains
 
-* -N='args'
+* `-N='args'`
 > Normalize the input JSON markup into a single-line JSON output with
 contents sorted like for `-S='args'`, e.g. use `-N='-n'`.
 This is equivalent to `-N -S='args'`, just more compact to write.
@@ -117,14 +121,15 @@ This is equivalent to `-N -S='args'`, just more compact to write.
 ### Cook raw data
 
 ``` bash
-Usage: COOKEDSTRING="`somecommand 2>&1 | JSON.sh -Q`"
+Usage: COOKEDSTRING="`somecommand 2>&1 | ./JSON.sh -Q`"
 ```
 
-To help JSON-related scripting, with `-Q` an input plaintext can be "cooked"
+* `-Q`
+> To help JSON-related scripting, with `-Q` an input plaintext can be "cooked"
 into a string valid for JSON (backslashes, quotes and newlines escaped, with
 no trailing newline); after cooking, the script exits.
 
-This mode can also be used to pack JSON in JSON.
+This mode can also be used to pack JSON into JSON.
 
 
 ### Ask for help
@@ -132,7 +137,7 @@ This mode can also be used to pack JSON in JSON.
 Usage: JSON.sh [-h]
 ```
 
--h
+* `-h`
 > Show help text.
 
 

From 8a53cb7ca976b3b731110a735a7c70d6eecb6205 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Sun, 11 Jan 2015 14:38:56 +0100
Subject: [PATCH 20/95] Usage() for -x updated to stress object vs. array in
 jpath

---
 JSON.sh   | 2 +-
 README.md | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index b8ae29d..2723f75 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -35,7 +35,7 @@ usage() {
   echo "-x 'regex' - rather than showing all document from the root element,"
   echo "     extract the items rooted at path(s) matching the regex (see the"
   echo "     comma-separated list of nested hierarchy names in general output,"
-  echo "     brackets not included) e.g. regex='^\"level1\",\"level2arr\",0'"
+  echo "     brackets not included) e.g. regex='^\"level1obj\",\"level2arr\",0'"
   echo "--no-newline - rather than concatenating detected line breaks in markup,"
   echo "     return with error when this is seen in input"
   echo "-d - Enable debugging traces to stderr (repeat or use -d=NUM to bump)"
diff --git a/README.md b/README.md
index 3613402..ae3bc19 100755
--- a/README.md
+++ b/README.md
@@ -78,7 +78,7 @@ Usage: JSON.sh [-b] [-l] [-p] [-x 'regex'] [-S|-S='args'] [--no-newline] [-d]
 > "Extract" - rather than showing all document from the root element,
 extract the items rooted at path(s) matching the regex (see the
 comma-separated list of nested hierarchy names in general output,
-brackets not included) e.g. `-x='^"level1","level2arr",0'`
+brackets not included) e.g. `-x='^"level1obj","level2arr",0'`
 
 Sorting is also available, although limited to single-line strings in
 the markup (multilines are automatically escaped into backslash+n):
@@ -128,7 +128,6 @@ Usage: COOKEDSTRING="`somecommand 2>&1 | ./JSON.sh -Q`"
 > To help JSON-related scripting, with `-Q` an input plaintext can be "cooked"
 into a string valid for JSON (backslashes, quotes and newlines escaped, with
 no trailing newline); after cooking, the script exits.
-
 This mode can also be used to pack JSON into JSON.
 
 

From 70e1af2762c3bc09eba5fb88e4baac505b4666d2 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Sun, 11 Jan 2015 14:40:52 +0100
Subject: [PATCH 21/95] README markup updated for prettiness

---
 README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index ae3bc19..07ad0b2 100755
--- a/README.md
+++ b/README.md
@@ -60,7 +60,7 @@ curl registry.npmjs.org/express | ./JSON.sh | egrep '\["versions","[^"]*"\]'
 
 ## Options
 
-### Usual queries, full and filtered
+### Usual queries, full or filtered
 ``` bash
 Usage: JSON.sh [-b] [-l] [-p] [-x 'regex'] [-S|-S='args'] [--no-newline] [-d]
 ```
@@ -119,11 +119,11 @@ contents sorted like for `-S='args'`, e.g. use `-N='-n'`.
 This is equivalent to `-N -S='args'`, just more compact to write.
 
 ### Cook raw data
-
 ``` bash
 Usage: COOKEDSTRING="`somecommand 2>&1 | ./JSON.sh -Q`"
 ```
 
+Cooking:
 * `-Q`
 > To help JSON-related scripting, with `-Q` an input plaintext can be "cooked"
 into a string valid for JSON (backslashes, quotes and newlines escaped, with
@@ -136,6 +136,7 @@ This mode can also be used to pack JSON into JSON.
 Usage: JSON.sh [-h]
 ```
 
+Helping:
 * `-h`
 > Show help text.
 

From 8021e55aeb5b20cffa3ea723cdfac3ca2237edd7 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Sat, 10 Jan 2015 06:10:23 +0100
Subject: [PATCH 22/95] Typo fix in documented_example.*: smptystr => emptystr

---
 test/valid/documented_example.json              | 2 +-
 test/valid/documented_example.normalized        | 2 +-
 test/valid/documented_example.normalized_sorted | 2 +-
 test/valid/documented_example.parsed            | 4 ++--
 test/valid/documented_example.sorted            | 4 ++--
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/test/valid/documented_example.json b/test/valid/documented_example.json
index 4622cb4..11b8cbf 100644
--- a/test/valid/documented_example.json
+++ b/test/valid/documented_example.json
@@ -5,7 +5,7 @@ key":"value","var0":"escaped \" quote","splitValue":"there
 ,"escaping\"several\"\"
 quote\"s and
 newlines"],"aNumber":1,"var8":"string\nwith\nproper\\\nnewlines",
-"var38":"","emptyarr":[],"emptyobj":{},"smptystr":"",
+"var38":"","emptyarr":[],"emptyobj":{},"emptystr":"",
 "arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"},
   {"var":"val2","str":"z"},{"var":"val2","str":"x"},
   {"var":"val1","str":"S"},{"var":"val1","str":"\""},
diff --git a/test/valid/documented_example.normalized b/test/valid/documented_example.normalized
index 3b10c89..7f5de8f 100644
--- a/test/valid/documented_example.normalized
+++ b/test/valid/documented_example.normalized
@@ -1 +1 @@
-{"var1":"val1","split\nkey":"value","var0":"escaped \" quote","splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","array":["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"],"aNumber":1,"var8":"string\nwith\nproper\\\nnewlines","var38":"","emptyarr":[],"emptyobj":{},"smptystr":"","arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}]}
+{"var1":"val1","split\nkey":"value","var0":"escaped \" quote","splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","array":["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"],"aNumber":1,"var8":"string\nwith\nproper\\\nnewlines","var38":"","emptyarr":[],"emptyobj":{},"emptystr":"","arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}]}
diff --git a/test/valid/documented_example.normalized_sorted b/test/valid/documented_example.normalized_sorted
index 2a51ec4..6aed423 100644
--- a/test/valid/documented_example.normalized_sorted
+++ b/test/valid/documented_example.normalized_sorted
@@ -1 +1 @@
-{"aNumber":1,"arrOfObjs":[{"str":"5","var":"val1"},{"str":"S","var":"val1"},{"str":"\"","var":"val1"},{"str":"s","var":"val1"},{"str":"s","var":"val30"},{"str":"x","var":"val2"},{"str":"z","var":"val2"},{"str":5,"var":"val1"}],"array":["","","\"","a","b","escaping\"several\"\"\nquote\"s and\nnewlines","z",0,3,20],"emptyarr":[],"emptyobj":{},"smptystr":"","splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","split\nkey":"value","var0":"escaped \" quote","var1":"val1","var38":"","var8":"string\nwith\nproper\\\nnewlines"}
+{"aNumber":1,"arrOfObjs":[{"str":"5","var":"val1"},{"str":"S","var":"val1"},{"str":"\"","var":"val1"},{"str":"s","var":"val1"},{"str":"s","var":"val30"},{"str":"x","var":"val2"},{"str":"z","var":"val2"},{"str":5,"var":"val1"}],"array":["","","\"","a","b","escaping\"several\"\"\nquote\"s and\nnewlines","z",0,3,20],"emptyarr":[],"emptyobj":{},"emptystr":"","splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","split\nkey":"value","var0":"escaped \" quote","var1":"val1","var38":"","var8":"string\nwith\nproper\\\nnewlines"}
diff --git a/test/valid/documented_example.parsed b/test/valid/documented_example.parsed
index bda2d64..78ee6dd 100644
--- a/test/valid/documented_example.parsed
+++ b/test/valid/documented_example.parsed
@@ -18,7 +18,7 @@
 ["var38"]	""
 ["emptyarr"]	[]
 ["emptyobj"]	{}
-["smptystr"]	""
+["emptystr"]	""
 ["arrOfObjs",0,"var"]	"val1"
 ["arrOfObjs",0,"str"]	"s"
 ["arrOfObjs",0]	{"var":"val1","str":"s"}
@@ -44,4 +44,4 @@
 ["arrOfObjs",7,"str"]	"5"
 ["arrOfObjs",7]	{"var":"val1","str":"5"}
 ["arrOfObjs"]	[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}]
-[]	{"var1":"val1","split\nkey":"value","var0":"escaped \" quote","splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","array":["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"],"aNumber":1,"var8":"string\nwith\nproper\\\nnewlines","var38":"","emptyarr":[],"emptyobj":{},"smptystr":"","arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}]}
+[]	{"var1":"val1","split\nkey":"value","var0":"escaped \" quote","splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","array":["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"],"aNumber":1,"var8":"string\nwith\nproper\\\nnewlines","var38":"","emptyarr":[],"emptyobj":{},"emptystr":"","arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}]}
diff --git a/test/valid/documented_example.sorted b/test/valid/documented_example.sorted
index 3cf1663..fc2bb14 100644
--- a/test/valid/documented_example.sorted
+++ b/test/valid/documented_example.sorted
@@ -4,7 +4,7 @@
 ["var0"]	"escaped \" quote"
 ["split\nkey"]	"value"
 ["splitValue"]	"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")"
-["smptystr"]	""
+["emptystr"]	""
 ["emptyobj"]	{}
 ["emptyarr"]	[]
 ["array",0]	20
@@ -44,4 +44,4 @@
 ["arrOfObjs",7]	{"var":"val1","str":"5"}
 ["arrOfObjs"]	[{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":5},{"var":"val1","str":"s"},{"var":"val1","str":"\""},{"var":"val1","str":"S"},{"var":"val1","str":"5"}]
 ["aNumber"]	1
-[]	{"var8":"string\nwith\nproper\\\nnewlines","var38":"","var1":"val1","var0":"escaped \" quote","split\nkey":"value","splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","smptystr":"","emptyobj":{},"emptyarr":[],"array":[20,3,0,"z","escaping\"several\"\"\nquote\"s and\nnewlines","b","a","\"","",""],"arrOfObjs":[{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":5},{"var":"val1","str":"s"},{"var":"val1","str":"\""},{"var":"val1","str":"S"},{"var":"val1","str":"5"}],"aNumber":1}
+[]	{"var8":"string\nwith\nproper\\\nnewlines","var38":"","var1":"val1","var0":"escaped \" quote","split\nkey":"value","splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","emptystr":"","emptyobj":{},"emptyarr":[],"array":[20,3,0,"z","escaping\"several\"\"\nquote\"s and\nnewlines","b","a","\"","",""],"arrOfObjs":[{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":5},{"var":"val1","str":"s"},{"var":"val1","str":"\""},{"var":"val1","str":"S"},{"var":"val1","str":"5"}],"aNumber":1}

From c501592f6e068b9f4844f136026872f4aadcefb2 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Mon, 12 Jan 2015 11:28:52 +0100
Subject: [PATCH 23/95] Added and documented new options to
 sorting/normalization to only set up sorting of objects or arrays, not both
 at once (to keep tuples intact)

---
 JSON.sh   | 63 +++++++++++++++++++++++++++++++++++++++++--------------
 README.md | 34 +++++++++++++++++++++++-------
 2 files changed, 74 insertions(+), 23 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index 2723f75..e7cd3d7 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -12,16 +12,13 @@ throw () {
 BRIEF=0
 LEAFONLY=0
 PRUNE=0
-SORTDATA=""
+SORTDATA_OBJ=""
+SORTDATA_ARR=""
 NORMALIZE=0
 EXTRACT_JPATH=""
 TOXIC_NEWLINE=0
 COOKASTRING=0
 
-### Beside command-line, debugging can be enabled by envvars from the caller
-[ x"$DEBUG" = xy -o x"$DEBUG" = xyes ] && DEBUG=1
-[ -n "$DEBUG" -a "$DEBUG" -ge 0 ] 2>/dev/null || DEBUG=0
-
 usage() {
   echo
   echo "Usage: JSON.sh [-b] [-l] [-p] [-x 'regex'] [-S|-S='args'] [--no-newline] [-d]"
@@ -46,6 +43,8 @@ usage() {
   echo "     'sort' objects by key names and then values, and arrays by values"
   echo "-S='args' - use 'sort \$args' for content sorting, e.g. use -S='-n -r'"
   echo "     for reverse numeric sort"
+  echo "-So|-So='args' - enable sorting (with given arguments) only for objects"
+  echo "-Sa|-Sa='args' - enable sorting (with given arguments) only for arrays"
   echo
   echo "An input JSON markup can be normalized into single-line no-whitespace:"
   echo "-N - Normalize the input JSON markup into a single-line JSON output;"
@@ -53,6 +52,8 @@ usage() {
   echo "-N='args' - Normalize the input JSON markup into a single-line JSON"
   echo "     output with contents sorted like for -S='args', e.g. use -N='-n'"
   echo "     This is equivalent to -N -S='args', just more compact to write"
+  echo "-No='args' - enable sorting (with given arguments) only for objects"
+  echo "-Na='args' - enable sorting (with given arguments) only for arrays"
   echo
   echo "To help JSON-related scripting, with '-Q' an input plaintext can be cooked"
   echo "into a string valid for JSON (backslashes, quotes and newlines escaped,"
@@ -62,6 +63,12 @@ usage() {
   echo
 }
 
+validate_debuglevel() {
+    ### Beside command-line, debugging can be enabled by envvars from the caller
+    [ x"$DEBUG" = xy -o x"$DEBUG" = xyes ] && DEBUG=1
+    [ -n "$DEBUG" -a "$DEBUG" -ge 0 ] 2>/dev/null || DEBUG=0
+}
+
 unquote() {
     # Remove single or double quotes surrounding the token
     sed "s,^'\(.*\)'\$,\1," | sed 's,^\"\(.*\)\"$,\1,'
@@ -121,12 +128,32 @@ parse_options() {
       ;;
       -N) NORMALIZE=1
       ;;
-      -N=*) SORTDATA="sort `echo "$1" | sed 's,^-N=,,' | unquote `"
-          NORMALIZE=1
+      -N=*) NORMALIZE=1
+          SORTDATA_OBJ="sort `echo "$1" | sed 's,^-N=,,' | unquote `"
+          SORTDATA_ARR="sort `echo "$1" | sed 's,^-N=,,' | unquote `"
+      ;;
+      -No=*) NORMALIZE=1
+          SORTDATA_OBJ="sort `echo "$1" | sed 's,^-No=,,' | unquote `"
+      ;;
+      -Na=*) NORMALIZE=1
+          SORTDATA_ARR="sort `echo "$1" | sed 's,^-Na=,,' | unquote `"
       ;;
-      -S) SORTDATA="sort"
+      -S) SORTDATA_OBJ="sort"
+          SORTDATA_ARR="sort"
       ;;
-      -S=*) SORTDATA="sort `echo "$1" | sed 's,^-S=,,' | unquote `"
+      -So) SORTDATA_OBJ="sort"
+      ;;
+      -Sa) SORTDATA_ARR="sort"
+      ;;
+      -S=*)
+          SORTDATA_OBJ="sort `echo "$1" | sed 's,^-S=,,' | unquote `"
+          SORTDATA_ARR="sort `echo "$1" | sed 's,^-S=,,' | unquote `"
+      ;;
+      -So=*)
+          SORTDATA_OBJ="sort `echo "$1" | sed 's,^-So=,,' | unquote `"
+      ;;
+      -Sa=*)
+          SORTDATA_ARR="sort `echo "$1" | sed 's,^-Sa=,,' | unquote `"
       ;;
       -x) EXTRACT_JPATH="$2"
           shift
@@ -151,6 +178,8 @@ parse_options() {
     ARGN=$((ARGN-1))
   done
 
+  validate_debuglevel
+
   # For normalized data, we do the whole job and just return the top object
   [ "$NORMALIZE" -eq 1 ] && BRIEF=0 && LEAFONLY=0 && PRUNE=0
 }
@@ -277,7 +306,7 @@ parse_array () {
         parse_value "$1" "$index"
         index=$((index+1))
         ary="$ary""$value"
-        if [ -n "$SORTDATA" ]; then
+        if [ -n "$SORTDATA_ARR" ]; then
             [ -z "$aryml" ] && aryml="$value" || aryml="$aryml
 $value"
         fi
@@ -293,8 +322,8 @@ $value"
       done
       ;;
   esac
-  if [ -n "$SORTDATA" ]; then
-    ary="`echo -E "$aryml" | $SORTDATA | tr '\n' ',' | sed 's|,*$||' | sed 's|^,*||'`"
+  if [ -n "$SORTDATA_ARR" ]; then
+    ary="`echo -E "$aryml" | $SORTDATA_ARR | tr '\n' ',' | sed 's|,*$||' | sed 's|^,*||'`"
   fi
   [ "$BRIEF" -eq 0 ] && value=`printf '[%s]' "$ary"` || value=
   :
@@ -325,7 +354,7 @@ parse_object () {
         print_debug $DEBUGLEVEL_PRINTTOKEN "parse_object(3):" "token=$token"
         parse_value "$1" "$key"
         obj="$obj$key:$value"
-        if [ -n "$SORTDATA" ]; then
+        if [ -n "$SORTDATA_OBJ" ]; then
             [ -z "$objml" ] && objml="$key:$value" || objml="$objml
 $key:$value"
         fi
@@ -341,8 +370,8 @@ $key:$value"
       done
     ;;
   esac
-  if [ -n "$SORTDATA" ]; then
-    obj="`echo -E "$objml" | $SORTDATA | tr '\n' ',' | sed 's|,*$||' | sed 's|^,*||'`"
+  if [ -n "$SORTDATA_OBJ" ]; then
+    obj="`echo -E "$objml" | $SORTDATA_OBJ | tr '\n' ',' | sed 's|,*$||' | sed 's|^,*||'`"
   fi
   [ "$BRIEF" -eq 0 ] && value=`printf '{%s}' "$obj"` || value=
   :
@@ -416,7 +445,8 @@ parse () {
 
 smart_parse() {
   strip_newlines | \
-  tokenize | if [ -n "$SORTDATA" ] ; then
+  tokenize | if [ -n "$SORTDATA_OBJ$SORTDATA_ARR" ] ; then
+      ### Any type of sort was enabled
       ( NORMALIZE=1 LEAFONLY=0 BRIEF=0 parse ) \
       | tokenize | parse
     else
@@ -428,6 +458,7 @@ smart_parse() {
 ### Active logic
 
 ### Caller can disable specific debuggers by setting their level too high
+validate_debuglevel
 default_posval DEBUGLEVEL_PRINTPATHVAL		1
 default_posval DEBUGLEVEL_PRINTTOKEN		2
 default_posval DEBUGLEVEL_PRINTTOKEN_PIPELINE	3
diff --git a/README.md b/README.md
index 07ad0b2..2266061 100755
--- a/README.md
+++ b/README.md
@@ -91,6 +91,14 @@ the markup (multilines are automatically escaped into backslash+n):
 > Use `sort $args` for content sorting, e.g. use `-S='-n -r'` for
 reverse numeric sort
 
+* `-So='args'` and/or `-Sa='args'`, or `-So` or `-Sa`
+> Only enable `sort` and set the arguments for either objects (`-So`)
+or arrays/tuples (`-Sa`). This way sorting of tuples can be avoided
+to keep data in valid order (as defined by the programmatic users of
+the markup) and/or different rules can be used for arrays vs. objects.
+Essentially, the singular `-S{='args'}` option just enables both the
+`-Sa` and `-So` options with the same values.
+
 Other options:
 
 * `--no-newline`
@@ -118,6 +126,12 @@ in this mode syntax and spacing are normalized, data order remains
 contents sorted like for `-S='args'`, e.g. use `-N='-n'`.
 This is equivalent to `-N -S='args'`, just more compact to write.
 
+* `-No='args'` and/or `-Na='args'`
+> Normalize with sorting like above, but only enable and set the `sort`
+arguments for either objects (`-No`) or arrays/tuples (`-Na`). This way
+sorting of tuples can be avoided to keep data in valid order (as defined
+by the programmatic users of the markup).
+
 ### Cook raw data
 ``` bash
 Usage: COOKEDSTRING="`somecommand 2>&1 | ./JSON.sh -Q`"
@@ -125,12 +139,11 @@ Usage: COOKEDSTRING="`somecommand 2>&1 | ./JSON.sh -Q`"
 
 Cooking:
 * `-Q`
-> To help JSON-related scripting, with `-Q` an input plaintext can be "cooked"
-into a string valid for JSON (backslashes, quotes and newlines escaped, with
-no trailing newline); after cooking, the script exits.
+> To help JSON-related scripting, a block of input plaintext can be
+"cooked" into a string valid for JSON (backslashes, quotes and newlines
+escaped, with no trailing newline); after cooking, the script exits.
 This mode can also be used to pack JSON into JSON.
 
-
 ### Ask for help
 ``` bash
 Usage: JSON.sh [-h]
@@ -140,7 +153,6 @@ Helping:
 * `-h`
 > Show help text.
 
-
 ## Complex usage examples
 
 A picture shows more than a thousand words, heh?
@@ -363,8 +375,9 @@ standalone number tokens) and reversed (`a` is after `z`) sorting:
 * Normalized output can also be sorted, upon request - although *NOTE* that if
 your document schema has arrays whose order of items has syntactic meaning for
 your application (aka "tuples"), such ordering will likely make the document
-invalid for your application's use-case; this warning *should* be irrelevant
-for objects (`key:value` pairs) though:
+invalid for your application's use-case - and in such case you might want to
+use `-No{='args'}` to only sort objects; this warning *should* be irrelevant
+for objects (the `{"key":value}` pairs) though:
 ```bash
 :; echo -E "$LINE" | ./JSON.sh -N='-n'
 {"aNumber":1,"arrOfObjs":[{"str":"5","var":"val1"},{"str":"S","var":"val1"},{"str":"\"","var":"val1"},{"str":"s","var":"val1"},{"str":"s","var":"val30"},{"str":"x","var":"val2"},{"str":"z","var":"val2"},{"str":5,"var":"val1"}],"array":["","","\"","a","b","escaping\"several\"\"\nquote\"s and\nnewlines","z",0,3,20],"emptyarr":[],"emptyobj":{},"splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","split\nkey":"value","var0":"escaped \" quote","var1":"val1","var38":"","var8":"string\nwith\nproper\\\nnewlines"}
@@ -374,6 +387,13 @@ for objects (`key:value` pairs) though:
 
 :; echo -E "$LINE" | ./JSON.sh -N="-r -n"
 {"var8":"string\nwith\nproper\\\nnewlines","var38":"","var1":"val1","var0":"escaped \" quote","split\nkey":"value","splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","emptyobj":{},"emptyarr":[],"array":[20,3,0,"z","escaping\"several\"\"\nquote\"s and\nnewlines","b","a","\"","",""],"arrOfObjs":[{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":5},{"var":"val1","str":"s"},{"var":"val1","str":"\""},{"var":"val1","str":"S"},{"var":"val1","str":"5"}],"aNumber":1}
+
+### Normalize sorting only objects (arrays/tuples remain in original order):
+:; echo -E "$LINE" | ./JSON.sh -No='-r -n'
+{"var8":"string\nwith\nproper\\\nnewlines","var38":"","var1":"val1","var0":"escaped \" quote","splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","split\nkey":"value","emptystr":"","emptyobj":{},"emptyarr":[],"arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}],"array":["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"],"aNumber":1}
+
+:; echo -E "$LINE" | ./JSON.sh -No='-n'
+{"aNumber":1,"array":["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"],"arrOfObjs":[{"str":"s","var":"val1"},{"str":"s","var":"val30"},{"str":"z","var":"val2"},{"str":"x","var":"val2"},{"str":"S","var":"val1"},{"str":"\"","var":"val1"},{"str":5,"var":"val1"},{"str":"5","var":"val1"}],"emptyarr":[],"emptyobj":{},"emptystr":"","split\nkey":"value","splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","var0":"escaped \" quote","var1":"val1","var38":"","var8":"string\nwith\nproper\\\nnewlines"}
 ```
 
 * And note that the normalized output returns (maybe sorted) JSON markup of

From 9bcc85082ee9aa1e9e1013e555a5049f8c6cafcd Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 1 Apr 2015 13:11:14 +0200
Subject: [PATCH 24/95] JSON.sh: added ability to normalize numbers into a
 common format for easier string-to-string comparison of normalized JSON
 markup

---
 JSON.sh | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/JSON.sh b/JSON.sh
index e7cd3d7..b3f23fe 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -15,6 +15,9 @@ PRUNE=0
 SORTDATA_OBJ=""
 SORTDATA_ARR=""
 NORMALIZE=0
+NORMALIZE_NUMBERS=0
+NORMALIZE_NUMBERS_FORMAT='%.6f'
+NORMALIZE_NUMBERS_STRIP=0
 EXTRACT_JPATH=""
 TOXIC_NEWLINE=0
 COOKASTRING=0
@@ -23,6 +26,7 @@ usage() {
   echo
   echo "Usage: JSON.sh [-b] [-l] [-p] [-x 'regex'] [-S|-S='args'] [--no-newline] [-d]"
   echo "       JSON.sh [-N|-N='args'] < markup.json"
+  echo "       JSON.sh [...] [-Nnx|-Nnx='fmtstr'|-Nn|-Nn='fmtstr'] < markup.json"
   echo "       JSON.sh [-h]"
   echo "-h - This help text."
   echo
@@ -55,6 +59,11 @@ usage() {
   echo "-No='args' - enable sorting (with given arguments) only for objects"
   echo "-Na='args' - enable sorting (with given arguments) only for arrays"
   echo
+  echo "Numeric values can be normalized (e.g. convert engineering into layman)"
+  echo "-Nn='fmtstr' - printf the detected numeric values with the fmtstr conversion"
+  echo "-Nn  - assume 'fmtstr'='%.6f' (with 6 precision digits after period)"
+  echo "-Nnx - -Nn + strip trailing zeroes and trailing period (for whole numbers)"
+  echo
   echo "To help JSON-related scripting, with '-Q' an input plaintext can be cooked"
   echo "into a string valid for JSON (backslashes, quotes and newlines escaped,"
   echo "with no trailing newline); after cooking, the script exits:"
@@ -138,6 +147,18 @@ parse_options() {
       -Na=*) NORMALIZE=1
           SORTDATA_ARR="sort `echo "$1" | sed 's,^-Na=,,' | unquote `"
       ;;
+      -Nnx) NORMALIZE_NUMBERS_STRIP=1
+            NORMALIZE_NUMBERS=1
+      ;;
+      -Nnx=*) NORMALIZE_NUMBERS_STRIP=1
+            NORMALIZE_NUMBERS=1
+            NORMALIZE_NUMBERS_FORMAT="`echo "$1" | sed 's,^-Nnx=,,' | unquote `"
+      ;;
+      -Nn) NORMALIZE_NUMBERS=1
+      ;;
+      -Nn=*) NORMALIZE_NUMBERS=1
+          NORMALIZE_NUMBERS_FORMAT="`echo "$1" | sed 's,^-Nn=,,' | unquote `"
+      ;;
       -S) SORTDATA_OBJ="sort"
           SORTDATA_ARR="sort"
       ;;
@@ -377,6 +398,7 @@ $key:$value"
   :
 }
 
+REGEX_NUMBER='^-?(0|[1-9][0-9]*)([.][0-9]*)?([eE][+-]?[0-9]*)?$'
 parse_value () {
   local jpath="${1:+$1,}$2" isleaf=0 isempty=0 print=0
   case "$token" in
@@ -388,6 +410,30 @@ parse_value () {
        ;;
     # At this point, the only valid single-character tokens are digits.
     ''|[!0-9]) throw "EXPECTED value GOT ${token:-EOF}" ;;
+    -*|[0-9]*|.*)  # Potential number - separate hit in case for efficiency
+       print_debug $DEBUGLEVEL_PRINTPATHVAL \
+            "token '$token' is a suspected number" >&2
+       if [ "$NORMALIZE_NUMBERS" = 1 ] && \
+         [[ "$token" =~ ${REGEX_NUMBER} ]] \
+       ; then
+            value="`printf "$NORMALIZE_NUMBERS_FORMAT" "$token"`" || \
+            value=$token
+            print_debug $DEBUGLEVEL_PRINTPATHVAL "normalized numeric token" \
+                "'$token' into '$value'" >&2
+            if [ "$NORMALIZE_NUMBERS_STRIP" = 1 ]; then
+                local valuetmp="`echo "$value" | sed -e 's,0*$,,g' -e 's,\.$,,'`" && \
+                value="$valuetmp"
+                unset valuetmp
+                print_debug $DEBUGLEVEL_PRINTPATHVAL "stripped numeric token" \
+                    "'$token' into '$value'" >&2
+            fi
+       else
+        # Not a number or no normalization - process like default
+            value=$token
+       fi
+       isleaf=1
+       [ "$value" = '""' -o "$value" = '' ] && isempty=1
+       ;;
     *) value=$token
        isleaf=1
        [ "$value" = '""' ] && isempty=1

From a4e52bc7cdea45b5bbdab69c3c82b1f7363a8242 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 1 Apr 2015 13:28:41 +0200
Subject: [PATCH 25/95] JSON.sh: a leading plus is technically valid character
 in a number as well

---
 JSON.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index b3f23fe..15396eb 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -304,7 +304,7 @@ tokenize () {
   local CHART="($CHAR|[[:blank:]])"
   local STRINGVAL="$CHART*($ESCAPE$CHART*)*"
   local STRING="(\"$STRINGVAL\")"
-  local NUMBER='-?(0|[1-9][0-9]*)([.][0-9]*)?([eE][+-]?[0-9]*)?'
+  local NUMBER='[+-]?(0|[1-9][0-9]*)([.][0-9]*)?([eE][+-]?[0-9]*)?'
   local KEYWORD='null|false|true'
   local SPACE='[[:space:]]+'
 
@@ -398,7 +398,7 @@ $key:$value"
   :
 }
 
-REGEX_NUMBER='^-?(0|[1-9][0-9]*)([.][0-9]*)?([eE][+-]?[0-9]*)?$'
+REGEX_NUMBER='^[+-]?(0|[1-9][0-9]*)([.][0-9]*)?([eE][+-]?[0-9]*)?$'
 parse_value () {
   local jpath="${1:+$1,}$2" isleaf=0 isempty=0 print=0
   case "$token" in
@@ -410,7 +410,7 @@ parse_value () {
        ;;
     # At this point, the only valid single-character tokens are digits.
     ''|[!0-9]) throw "EXPECTED value GOT ${token:-EOF}" ;;
-    -*|[0-9]*|.*)  # Potential number - separate hit in case for efficiency
+    +*|-*|[0-9]*|.*)  # Potential number - separate hit in case for efficiency
        print_debug $DEBUGLEVEL_PRINTPATHVAL \
             "token '$token' is a suspected number" >&2
        if [ "$NORMALIZE_NUMBERS" = 1 ] && \

From 2e7a322b2a1b9f9854cd2f9567dec9abc0ca8d43 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 1 Apr 2015 14:20:48 +0200
Subject: [PATCH 26/95] JSON.sh: support numbers starting with a decimal point
 (no leading zeroes)

---
 JSON.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index 15396eb..62ee03a 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -304,7 +304,7 @@ tokenize () {
   local CHART="($CHAR|[[:blank:]])"
   local STRINGVAL="$CHART*($ESCAPE$CHART*)*"
   local STRING="(\"$STRINGVAL\")"
-  local NUMBER='[+-]?(0|[1-9][0-9]*)([.][0-9]*)?([eE][+-]?[0-9]*)?'
+  local NUMBER='[+-]?([.][0-9]+|(0+|[1-9][0-9]*)([.][0-9]*)?)([eE][+-]?[0-9]*)?'
   local KEYWORD='null|false|true'
   local SPACE='[[:space:]]+'
 
@@ -398,7 +398,7 @@ $key:$value"
   :
 }
 
-REGEX_NUMBER='^[+-]?(0|[1-9][0-9]*)([.][0-9]*)?([eE][+-]?[0-9]*)?$'
+REGEX_NUMBER='^[+-]?([.][0-9]+|(0+|[1-9][0-9]*)([.][0-9]*)?)([eE][+-]?[0-9]*)?$'
 parse_value () {
   local jpath="${1:+$1,}$2" isleaf=0 isempty=0 print=0
   case "$token" in

From b8f03a7993835f785b4139c0dbd195219f86cda6 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 1 Apr 2015 14:22:59 +0200
Subject: [PATCH 27/95] Added tests and expected results for number
 normalization

---
 test/invalid/bad_numbers.json                     |  3 +++
 test/valid/number_representations.json            | 12 ++++++++++++
 test/valid/number_representations.normalized      |  1 +
 ...umber_representations.normalized_numnormalized |  1 +
 ...resentations.normalized_numnormalized_stripped |  1 +
 .../number_representations.normalized_sorted      |  1 +
 test/valid/number_representations.numnormalized   | 15 +++++++++++++++
 .../number_representations.numnormalized_stripped | 15 +++++++++++++++
 test/valid/number_representations.parsed          | 15 +++++++++++++++
 test/valid/number_representations.sorted          | 15 +++++++++++++++
 10 files changed, 79 insertions(+)
 create mode 100644 test/invalid/bad_numbers.json
 create mode 100644 test/valid/number_representations.json
 create mode 100644 test/valid/number_representations.normalized
 create mode 100644 test/valid/number_representations.normalized_numnormalized
 create mode 100644 test/valid/number_representations.normalized_numnormalized_stripped
 create mode 100644 test/valid/number_representations.normalized_sorted
 create mode 100644 test/valid/number_representations.numnormalized
 create mode 100644 test/valid/number_representations.numnormalized_stripped
 create mode 100644 test/valid/number_representations.parsed
 create mode 100644 test/valid/number_representations.sorted

diff --git a/test/invalid/bad_numbers.json b/test/invalid/bad_numbers.json
new file mode 100644
index 0000000..894b15e
--- /dev/null
+++ b/test/invalid/bad_numbers.json
@@ -0,0 +1,3 @@
+[++2,--1,-+3,+-4,-f,+x,
+1e++4]
+
diff --git a/test/valid/number_representations.json b/test/valid/number_representations.json
new file mode 100644
index 0000000..7c528b1
--- /dev/null
+++ b/test/valid/number_representations.json
@@ -0,0 +1,12 @@
+[1,0,-1,
+00,-00.40,
+2e3,
+2.1e2,
+2.2e-4,
+-1.3e+2,
+1e14,
+-.3,
+.5,
+0.4,
++3.25]
+
diff --git a/test/valid/number_representations.normalized b/test/valid/number_representations.normalized
new file mode 100644
index 0000000..76f6b87
--- /dev/null
+++ b/test/valid/number_representations.normalized
@@ -0,0 +1 @@
+[1,0,-1,00,-00.40,2e3,2.1e2,2.2e-4,-1.3e+2,1e14,-.3,.5,0.4,+3.25]
diff --git a/test/valid/number_representations.normalized_numnormalized b/test/valid/number_representations.normalized_numnormalized
new file mode 100644
index 0000000..ff0e4c7
--- /dev/null
+++ b/test/valid/number_representations.normalized_numnormalized
@@ -0,0 +1 @@
+[-130.000000000000,-1.000000000000,-0.400000000000,-0.300000000000,0.000000000000,0.000000000000,0.000220000000,0.400000000000,0.500000000000,1.000000000000,3.250000000000,210.000000000000,2000.000000000000,100000000000000.000000000000]
diff --git a/test/valid/number_representations.normalized_numnormalized_stripped b/test/valid/number_representations.normalized_numnormalized_stripped
new file mode 100644
index 0000000..e77be75
--- /dev/null
+++ b/test/valid/number_representations.normalized_numnormalized_stripped
@@ -0,0 +1 @@
+[-130,-1,-0.4,-0.3,0,0,0.00022,0.4,0.5,1,3.25,210,2000,100000000000000]
diff --git a/test/valid/number_representations.normalized_sorted b/test/valid/number_representations.normalized_sorted
new file mode 100644
index 0000000..305b358
--- /dev/null
+++ b/test/valid/number_representations.normalized_sorted
@@ -0,0 +1 @@
+[-1.3e+2,-1,-00.40,-.3,+3.25,0,00,0.4,.5,1,1e14,2e3,2.1e2,2.2e-4]
diff --git a/test/valid/number_representations.numnormalized b/test/valid/number_representations.numnormalized
new file mode 100644
index 0000000..da48718
--- /dev/null
+++ b/test/valid/number_representations.numnormalized
@@ -0,0 +1,15 @@
+[0]	1.000000000000
+[1]	0.000000000000
+[2]	-1.000000000000
+[3]	0.000000000000
+[4]	-0.400000000000
+[5]	2000.000000000000
+[6]	210.000000000000
+[7]	0.000220000000
+[8]	-130.000000000000
+[9]	100000000000000.000000000000
+[10]	-0.300000000000
+[11]	0.500000000000
+[12]	0.400000000000
+[13]	3.250000000000
+[]	[1.000000000000,0.000000000000,-1.000000000000,0.000000000000,-0.400000000000,2000.000000000000,210.000000000000,0.000220000000,-130.000000000000,100000000000000.000000000000,-0.300000000000,0.500000000000,0.400000000000,3.250000000000]
diff --git a/test/valid/number_representations.numnormalized_stripped b/test/valid/number_representations.numnormalized_stripped
new file mode 100644
index 0000000..13e9b8c
--- /dev/null
+++ b/test/valid/number_representations.numnormalized_stripped
@@ -0,0 +1,15 @@
+[0]	1
+[1]	0
+[2]	-1
+[3]	0
+[4]	-0.4
+[5]	2000
+[6]	210
+[7]	0.00022
+[8]	-130
+[9]	100000000000000
+[10]	-0.3
+[11]	0.5
+[12]	0.4
+[13]	3.25
+[]	[1,0,-1,0,-0.4,2000,210,0.00022,-130,100000000000000,-0.3,0.5,0.4,3.25]
diff --git a/test/valid/number_representations.parsed b/test/valid/number_representations.parsed
new file mode 100644
index 0000000..9435ad5
--- /dev/null
+++ b/test/valid/number_representations.parsed
@@ -0,0 +1,15 @@
+[0]	1
+[1]	0
+[2]	-1
+[3]	00
+[4]	-00.40
+[5]	2e3
+[6]	2.1e2
+[7]	2.2e-4
+[8]	-1.3e+2
+[9]	1e14
+[10]	-.3
+[11]	.5
+[12]	0.4
+[13]	+3.25
+[]	[1,0,-1,00,-00.40,2e3,2.1e2,2.2e-4,-1.3e+2,1e14,-.3,.5,0.4,+3.25]
diff --git a/test/valid/number_representations.sorted b/test/valid/number_representations.sorted
new file mode 100644
index 0000000..d1982f9
--- /dev/null
+++ b/test/valid/number_representations.sorted
@@ -0,0 +1,15 @@
+[0]	2.2e-4
+[1]	2.1e2
+[2]	2e3
+[3]	1e14
+[4]	1
+[5]	.5
+[6]	0.4
+[7]	00
+[8]	0
+[9]	+3.25
+[10]	-.3
+[11]	-00.40
+[12]	-1
+[13]	-1.3e+2
+[]	[2.2e-4,2.1e2,2e3,1e14,1,.5,0.4,00,0,+3.25,-.3,-00.40,-1,-1.3e+2]

From f451815bf7cf3d00e320c9d922c6950eec180dd1 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 1 Apr 2015 14:24:07 +0200
Subject: [PATCH 28/95] valid-test.sh : support listing of tests to run on
 command line (default to all test files as before); mode to (re)generate
 expected test result files

---
 test/valid-test.sh | 87 +++++++++++++++++++++++++++++++++++++---------
 1 file changed, 71 insertions(+), 16 deletions(-)

diff --git a/test/valid-test.sh b/test/valid-test.sh
index e662323..c31dcdb 100755
--- a/test/valid-test.sh
+++ b/test/valid-test.sh
@@ -1,5 +1,9 @@
 #! /usr/bin/env bash
 
+# NOTE: Developer of a new feature can pre-create expected result files:
+#   JSON_TEST_GENERATE=auto     conditionally (don't replace nonempty files)
+#   JSON_TEST_GENERATE=yes      recreate (replace existing results if any)
+
 # To disambiguate tests on sorting, use one locale
 LANG=C
 LC_ALL=C
@@ -7,29 +11,80 @@ export LANG LC_ALL
 
 cd ${0%/*}
 fails=0
+passes=0
+skips=0
+generated=0
 i=0
-tests=`ls valid/*.json -1l | wc -l`
-tests=$(($tests*4))
+
+CHOMPEXT='\.\(parsed\|sorted\|numnormalized\|normalized\|json\).*$'
+[ $# -gt 0 ] && \
+    FILES="$(for F in "$@"; do echo valid/"`basename "$F" | sed "s,${CHOMPEXT},,"`".json ; done | sort | uniq)" || \
+    FILES="`ls valid/*.json -1`"
+
+[ -z "$FILES" ] && echo "error - no files found to test!" >&2 && exit 1
+
+tests="`echo "$FILES" | wc -l`"
+### We currently have up to 8 extensions to consider per test
+tests=$(($tests*8))
 echo "1..$tests"
-for input in valid/*.json
+
+for input in $FILES
 do
-  for EXT in parsed sorted normalized normalized_sorted; do
+  for EXT in parsed sorted normalized normalized_sorted \
+        numnormalized numnormalized_stripped \
+        normalized_numnormalized normalized_numnormalized_stripped \
+  ; do
+    if [ ! -f "$input" ]; then
+      echo "error - missing input file '$input', assuming all its tests failed"
+      fails=$(($fails+8))
+      break
+    fi
+
     expected="${input%.json}.$EXT"
-    i=$((i+1))
-    case "$EXT" in
-      sorted) OPTIONS="-S='-n -r'" ;;
-      normalized) OPTIONS="-N" ;;
-      normalized_sorted) OPTIONS="-N=-n" ;;
-      parsed|*) OPTIONS="" ;;
-    esac
-    if ! eval ../JSON.sh $OPTIONS < "$input" | diff -u - "$expected" 
-    then
-      echo "not ok $i - $input $EXT"
-      fails=$((fails+1))
+    if [ -f "$expected" -o -n "$JSON_TEST_GENERATE" ]; then
+      i=$((i+1))
+      case "$EXT" in
+        sorted) OPTIONS="-S='-n -r'" ;;
+        normalized) OPTIONS="-N" ;;
+        normalized_sorted) OPTIONS="-N=-n" ;;
+        numnormalized) OPTIONS="-Nn=%.12f" ;;
+        numnormalized_stripped) OPTIONS="-Nnx" ;;
+        normalized_numnormalized) OPTIONS="-N=-n -Nn=%.12f" ;;
+        normalized_numnormalized_stripped) OPTIONS="-N=-n -Nnx" ;;
+        parsed|*) OPTIONS="" ;;
+      esac
+      if [ "$JSON_TEST_GENERATE" = yes ] || \
+         [ "$JSON_TEST_GENERATE" = auto -a ! -s "$expected" ]
+      then
+        if ! eval ../JSON.sh $OPTIONS < "$input" > "$expected"
+        then
+          echo "generation not ok $i - $input $EXT"
+          fails=$((fails+1))
+          mv -f "$expected" "$expected.failed"
+        else
+          echo "generation ok $i - $input $EXT"
+          passes=$(($passes+1))
+          generated=$(($generated+1))
+        fi
+        continue
+      fi
+
+      if ! eval ../JSON.sh $OPTIONS < "$input" | diff -u - "$expected" 
+      then
+        echo "not ok $i - $input $EXT"
+        fails=$((fails+1))
+      else
+        echo "ok $i - $input $EXT"
+        passes=$(($passes+1))
+      fi
     else
-      echo "ok $i - $input $EXT"
+      # echo "skip (missing result file) - $input $EXT"
+      skips=$(($skips+1))
     fi
   done
 done
+[ -n "$JSON_TEST_GENERATE" ] && echo "$generated expected results generated"
+[ -n "$skips" ] && echo "$skips test(s) skipped (missing expected results file)"
+echo "$passes test(s) succeeded"
 echo "$fails test(s) failed"
 exit $fails

From b81eb00490deeb6d310ea540476bce404af70609 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 1 Apr 2015 14:25:14 +0200
Subject: [PATCH 29/95] Generated numnormalized results for older existing
 tests

---
 test/valid/array.normalized_numnormalized     |  1 +
 .../array.normalized_numnormalized_stripped   |  1 +
 test/valid/array.numnormalized                |  5 ++
 test/valid/array.numnormalized_stripped       |  5 ++
 ...th_empty_elements.normalized_numnormalized |  1 +
 ...elements.normalized_numnormalized_stripped |  1 +
 .../array_with_empty_elements.numnormalized   | 16 +++++++
 ...with_empty_elements.numnormalized_stripped | 16 +++++++
 ...ocumented_example.normalized_numnormalized |  1 +
 ..._example.normalized_numnormalized_stripped |  1 +
 test/valid/documented_example.numnormalized   | 47 +++++++++++++++++++
 .../documented_example.numnormalized_stripped | 47 +++++++++++++++++++
 test/valid/embedded.normalized_numnormalized  |  1 +
 ...embedded.normalized_numnormalized_stripped |  1 +
 test/valid/embedded.numnormalized             |  2 +
 test/valid/embedded.numnormalized_stripped    |  2 +
 .../empty_array.normalized_numnormalized      |  1 +
 ...ty_array.normalized_numnormalized_stripped |  1 +
 test/valid/empty_array.numnormalized          |  1 +
 test/valid/empty_array.numnormalized_stripped |  1 +
 .../empty_object.normalized_numnormalized     |  1 +
 ...y_object.normalized_numnormalized_stripped |  1 +
 test/valid/empty_object.numnormalized         |  1 +
 .../valid/empty_object.numnormalized_stripped |  1 +
 .../many_object.normalized_numnormalized      |  1 +
 ...y_object.normalized_numnormalized_stripped |  1 +
 test/valid/many_object.numnormalized          |  3 ++
 test/valid/many_object.numnormalized_stripped |  3 ++
 ...ine_escapedquotes.normalized_numnormalized |  1 +
 ...edquotes.normalized_numnormalized_stripped |  1 +
 .../multiline_escapedquotes.numnormalized     |  5 ++
 ...iline_escapedquotes.numnormalized_stripped |  5 ++
 ...edquotes_indented.normalized_numnormalized |  1 +
 ...indented.normalized_numnormalized_stripped |  1 +
 ...iline_escapedquotes_indented.numnormalized |  5 ++
 ...apedquotes_indented.numnormalized_stripped |  5 ++
 ...tiline_simple_key.normalized_numnormalized |  1 +
 ...mple_key.normalized_numnormalized_stripped |  1 +
 test/valid/multiline_simple_key.numnormalized |  2 +
 ...ultiline_simple_key.numnormalized_stripped |  2 +
 ...line_simple_value.normalized_numnormalized |  1 +
 ...le_value.normalized_numnormalized_stripped |  1 +
 .../multiline_simple_value.numnormalized      |  2 +
 ...tiline_simple_value.numnormalized_stripped |  2 +
 .../nested_array.normalized_numnormalized     |  1 +
 ...ed_array.normalized_numnormalized_stripped |  1 +
 test/valid/nested_array.numnormalized         |  9 ++++
 .../valid/nested_array.numnormalized_stripped |  9 ++++
 .../nested_object.normalized_numnormalized    |  1 +
 ...d_object.normalized_numnormalized_stripped |  1 +
 test/valid/nested_object.numnormalized        |  5 ++
 .../nested_object.numnormalized_stripped      |  5 ++
 test/valid/number.normalized_numnormalized    |  1 +
 .../number.normalized_numnormalized_stripped  |  1 +
 test/valid/number.numnormalized               |  1 +
 test/valid/number.numnormalized_stripped      |  1 +
 test/valid/object.normalized_numnormalized    |  1 +
 .../object.normalized_numnormalized_stripped  |  1 +
 test/valid/object.numnormalized               |  2 +
 test/valid/object.numnormalized_stripped      |  2 +
 ...escapedquotes_key.normalized_numnormalized |  1 +
 ...otes_key.normalized_numnormalized_stripped |  1 +
 ...singleline_escapedquotes_key.numnormalized |  2 +
 ...e_escapedquotes_key.numnormalized_stripped |  2 +
 ...capedquotes_value.normalized_numnormalized |  1 +
 ...es_value.normalized_numnormalized_stripped |  1 +
 ...ngleline_escapedquotes_value.numnormalized |  2 +
 ...escapedquotes_value.numnormalized_stripped |  2 +
 test/valid/string.normalized_numnormalized    |  1 +
 .../string.normalized_numnormalized_stripped  |  1 +
 test/valid/string.numnormalized               |  1 +
 test/valid/string.numnormalized_stripped      |  1 +
 .../string_in_array.normalized_numnormalized  |  1 +
 ...in_array.normalized_numnormalized_stripped |  1 +
 test/valid/string_in_array.numnormalized      |  2 +
 .../string_in_array.numnormalized_stripped    |  2 +
 .../string_in_object.normalized_numnormalized |  1 +
 ...n_object.normalized_numnormalized_stripped |  1 +
 test/valid/string_in_object.numnormalized     |  2 +
 .../string_in_object.numnormalized_stripped   |  2 +
 .../valid/tab_escape.normalized_numnormalized |  1 +
 ...b_escape.normalized_numnormalized_stripped |  1 +
 test/valid/tab_escape.numnormalized           |  1 +
 test/valid/tab_escape.numnormalized_stripped  |  1 +
 84 files changed, 274 insertions(+)
 create mode 100644 test/valid/array.normalized_numnormalized
 create mode 100644 test/valid/array.normalized_numnormalized_stripped
 create mode 100644 test/valid/array.numnormalized
 create mode 100644 test/valid/array.numnormalized_stripped
 create mode 100644 test/valid/array_with_empty_elements.normalized_numnormalized
 create mode 100644 test/valid/array_with_empty_elements.normalized_numnormalized_stripped
 create mode 100644 test/valid/array_with_empty_elements.numnormalized
 create mode 100644 test/valid/array_with_empty_elements.numnormalized_stripped
 create mode 100644 test/valid/documented_example.normalized_numnormalized
 create mode 100644 test/valid/documented_example.normalized_numnormalized_stripped
 create mode 100644 test/valid/documented_example.numnormalized
 create mode 100644 test/valid/documented_example.numnormalized_stripped
 create mode 100644 test/valid/embedded.normalized_numnormalized
 create mode 100644 test/valid/embedded.normalized_numnormalized_stripped
 create mode 100644 test/valid/embedded.numnormalized
 create mode 100644 test/valid/embedded.numnormalized_stripped
 create mode 100644 test/valid/empty_array.normalized_numnormalized
 create mode 100644 test/valid/empty_array.normalized_numnormalized_stripped
 create mode 100644 test/valid/empty_array.numnormalized
 create mode 100644 test/valid/empty_array.numnormalized_stripped
 create mode 100644 test/valid/empty_object.normalized_numnormalized
 create mode 100644 test/valid/empty_object.normalized_numnormalized_stripped
 create mode 100644 test/valid/empty_object.numnormalized
 create mode 100644 test/valid/empty_object.numnormalized_stripped
 create mode 100644 test/valid/many_object.normalized_numnormalized
 create mode 100644 test/valid/many_object.normalized_numnormalized_stripped
 create mode 100644 test/valid/many_object.numnormalized
 create mode 100644 test/valid/many_object.numnormalized_stripped
 create mode 100644 test/valid/multiline_escapedquotes.normalized_numnormalized
 create mode 100644 test/valid/multiline_escapedquotes.normalized_numnormalized_stripped
 create mode 100644 test/valid/multiline_escapedquotes.numnormalized
 create mode 100644 test/valid/multiline_escapedquotes.numnormalized_stripped
 create mode 100644 test/valid/multiline_escapedquotes_indented.normalized_numnormalized
 create mode 100644 test/valid/multiline_escapedquotes_indented.normalized_numnormalized_stripped
 create mode 100644 test/valid/multiline_escapedquotes_indented.numnormalized
 create mode 100644 test/valid/multiline_escapedquotes_indented.numnormalized_stripped
 create mode 100644 test/valid/multiline_simple_key.normalized_numnormalized
 create mode 100644 test/valid/multiline_simple_key.normalized_numnormalized_stripped
 create mode 100644 test/valid/multiline_simple_key.numnormalized
 create mode 100644 test/valid/multiline_simple_key.numnormalized_stripped
 create mode 100644 test/valid/multiline_simple_value.normalized_numnormalized
 create mode 100644 test/valid/multiline_simple_value.normalized_numnormalized_stripped
 create mode 100644 test/valid/multiline_simple_value.numnormalized
 create mode 100644 test/valid/multiline_simple_value.numnormalized_stripped
 create mode 100644 test/valid/nested_array.normalized_numnormalized
 create mode 100644 test/valid/nested_array.normalized_numnormalized_stripped
 create mode 100644 test/valid/nested_array.numnormalized
 create mode 100644 test/valid/nested_array.numnormalized_stripped
 create mode 100644 test/valid/nested_object.normalized_numnormalized
 create mode 100644 test/valid/nested_object.normalized_numnormalized_stripped
 create mode 100644 test/valid/nested_object.numnormalized
 create mode 100644 test/valid/nested_object.numnormalized_stripped
 create mode 100644 test/valid/number.normalized_numnormalized
 create mode 100644 test/valid/number.normalized_numnormalized_stripped
 create mode 100644 test/valid/number.numnormalized
 create mode 100644 test/valid/number.numnormalized_stripped
 create mode 100644 test/valid/object.normalized_numnormalized
 create mode 100644 test/valid/object.normalized_numnormalized_stripped
 create mode 100644 test/valid/object.numnormalized
 create mode 100644 test/valid/object.numnormalized_stripped
 create mode 100644 test/valid/singleline_escapedquotes_key.normalized_numnormalized
 create mode 100644 test/valid/singleline_escapedquotes_key.normalized_numnormalized_stripped
 create mode 100644 test/valid/singleline_escapedquotes_key.numnormalized
 create mode 100644 test/valid/singleline_escapedquotes_key.numnormalized_stripped
 create mode 100644 test/valid/singleline_escapedquotes_value.normalized_numnormalized
 create mode 100644 test/valid/singleline_escapedquotes_value.normalized_numnormalized_stripped
 create mode 100644 test/valid/singleline_escapedquotes_value.numnormalized
 create mode 100644 test/valid/singleline_escapedquotes_value.numnormalized_stripped
 create mode 100644 test/valid/string.normalized_numnormalized
 create mode 100644 test/valid/string.normalized_numnormalized_stripped
 create mode 100644 test/valid/string.numnormalized
 create mode 100644 test/valid/string.numnormalized_stripped
 create mode 100644 test/valid/string_in_array.normalized_numnormalized
 create mode 100644 test/valid/string_in_array.normalized_numnormalized_stripped
 create mode 100644 test/valid/string_in_array.numnormalized
 create mode 100644 test/valid/string_in_array.numnormalized_stripped
 create mode 100644 test/valid/string_in_object.normalized_numnormalized
 create mode 100644 test/valid/string_in_object.normalized_numnormalized_stripped
 create mode 100644 test/valid/string_in_object.numnormalized
 create mode 100644 test/valid/string_in_object.numnormalized_stripped
 create mode 100644 test/valid/tab_escape.normalized_numnormalized
 create mode 100644 test/valid/tab_escape.normalized_numnormalized_stripped
 create mode 100644 test/valid/tab_escape.numnormalized
 create mode 100644 test/valid/tab_escape.numnormalized_stripped

diff --git a/test/valid/array.normalized_numnormalized b/test/valid/array.normalized_numnormalized
new file mode 100644
index 0000000..c70d6ea
--- /dev/null
+++ b/test/valid/array.normalized_numnormalized
@@ -0,0 +1 @@
+["hello",1.000000000000,2.000000000000,3.000000000000]
diff --git a/test/valid/array.normalized_numnormalized_stripped b/test/valid/array.normalized_numnormalized_stripped
new file mode 100644
index 0000000..b5a9be5
--- /dev/null
+++ b/test/valid/array.normalized_numnormalized_stripped
@@ -0,0 +1 @@
+["hello",1,2,3]
diff --git a/test/valid/array.numnormalized b/test/valid/array.numnormalized
new file mode 100644
index 0000000..02487eb
--- /dev/null
+++ b/test/valid/array.numnormalized
@@ -0,0 +1,5 @@
+[0]	1.000000000000
+[1]	2.000000000000
+[2]	3.000000000000
+[3]	"hello"
+[]	[1.000000000000,2.000000000000,3.000000000000,"hello"]
diff --git a/test/valid/array.numnormalized_stripped b/test/valid/array.numnormalized_stripped
new file mode 100644
index 0000000..d564cd9
--- /dev/null
+++ b/test/valid/array.numnormalized_stripped
@@ -0,0 +1,5 @@
+[0]	1
+[1]	2
+[2]	3
+[3]	"hello"
+[]	[1,2,3,"hello"]
diff --git a/test/valid/array_with_empty_elements.normalized_numnormalized b/test/valid/array_with_empty_elements.normalized_numnormalized
new file mode 100644
index 0000000..b86bcb5
--- /dev/null
+++ b/test/valid/array_with_empty_elements.normalized_numnormalized
@@ -0,0 +1 @@
+["","","av1","v1",0.000000000000,{"arr":[1.000000000000,2.000000000000,3.000000000000,4.000000000000],"k0":0.000000000000,"k1":"v1","k2":"","k3":1000000000000000.000000000000}]
diff --git a/test/valid/array_with_empty_elements.normalized_numnormalized_stripped b/test/valid/array_with_empty_elements.normalized_numnormalized_stripped
new file mode 100644
index 0000000..9894eb5
--- /dev/null
+++ b/test/valid/array_with_empty_elements.normalized_numnormalized_stripped
@@ -0,0 +1 @@
+["","","av1","v1",0,{"arr":[1,2,3,4],"k0":0,"k1":"v1","k2":"","k3":1000000000000000}]
diff --git a/test/valid/array_with_empty_elements.numnormalized b/test/valid/array_with_empty_elements.numnormalized
new file mode 100644
index 0000000..5a37807
--- /dev/null
+++ b/test/valid/array_with_empty_elements.numnormalized
@@ -0,0 +1,16 @@
+[0]	""
+[1,"k1"]	"v1"
+[1,"k2"]	""
+[1,"k3"]	1000000000000000.000000000000
+[1,"k0"]	0.000000000000
+[1,"arr",0]	3.000000000000
+[1,"arr",1]	1.000000000000
+[1,"arr",2]	2.000000000000
+[1,"arr",3]	4.000000000000
+[1,"arr"]	[3.000000000000,1.000000000000,2.000000000000,4.000000000000]
+[1]	{"k1":"v1","k2":"","k3":1000000000000000.000000000000,"k0":0.000000000000,"arr":[3.000000000000,1.000000000000,2.000000000000,4.000000000000]}
+[2]	""
+[3]	"av1"
+[4]	"v1"
+[5]	0.000000000000
+[]	["",{"k1":"v1","k2":"","k3":1000000000000000.000000000000,"k0":0.000000000000,"arr":[3.000000000000,1.000000000000,2.000000000000,4.000000000000]},"","av1","v1",0.000000000000]
diff --git a/test/valid/array_with_empty_elements.numnormalized_stripped b/test/valid/array_with_empty_elements.numnormalized_stripped
new file mode 100644
index 0000000..1405a83
--- /dev/null
+++ b/test/valid/array_with_empty_elements.numnormalized_stripped
@@ -0,0 +1,16 @@
+[0]	""
+[1,"k1"]	"v1"
+[1,"k2"]	""
+[1,"k3"]	1000000000000000
+[1,"k0"]	0
+[1,"arr",0]	3
+[1,"arr",1]	1
+[1,"arr",2]	2
+[1,"arr",3]	4
+[1,"arr"]	[3,1,2,4]
+[1]	{"k1":"v1","k2":"","k3":1000000000000000,"k0":0,"arr":[3,1,2,4]}
+[2]	""
+[3]	"av1"
+[4]	"v1"
+[5]	0
+[]	["",{"k1":"v1","k2":"","k3":1000000000000000,"k0":0,"arr":[3,1,2,4]},"","av1","v1",0]
diff --git a/test/valid/documented_example.normalized_numnormalized b/test/valid/documented_example.normalized_numnormalized
new file mode 100644
index 0000000..a927426
--- /dev/null
+++ b/test/valid/documented_example.normalized_numnormalized
@@ -0,0 +1 @@
+{"aNumber":1.000000000000,"arrOfObjs":[{"str":"5","var":"val1"},{"str":"S","var":"val1"},{"str":"\"","var":"val1"},{"str":"s","var":"val1"},{"str":"s","var":"val30"},{"str":"x","var":"val2"},{"str":"z","var":"val2"},{"str":5.000000000000,"var":"val1"}],"array":["","","\"","a","b","escaping\"several\"\"\nquote\"s and\nnewlines","z",0.000000000000,3.000000000000,20.000000000000],"emptyarr":[],"emptyobj":{},"emptystr":"","splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","split\nkey":"value","var0":"escaped \" quote","var1":"val1","var38":"","var8":"string\nwith\nproper\\\nnewlines"}
diff --git a/test/valid/documented_example.normalized_numnormalized_stripped b/test/valid/documented_example.normalized_numnormalized_stripped
new file mode 100644
index 0000000..6aed423
--- /dev/null
+++ b/test/valid/documented_example.normalized_numnormalized_stripped
@@ -0,0 +1 @@
+{"aNumber":1,"arrOfObjs":[{"str":"5","var":"val1"},{"str":"S","var":"val1"},{"str":"\"","var":"val1"},{"str":"s","var":"val1"},{"str":"s","var":"val30"},{"str":"x","var":"val2"},{"str":"z","var":"val2"},{"str":5,"var":"val1"}],"array":["","","\"","a","b","escaping\"several\"\"\nquote\"s and\nnewlines","z",0,3,20],"emptyarr":[],"emptyobj":{},"emptystr":"","splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","split\nkey":"value","var0":"escaped \" quote","var1":"val1","var38":"","var8":"string\nwith\nproper\\\nnewlines"}
diff --git a/test/valid/documented_example.numnormalized b/test/valid/documented_example.numnormalized
new file mode 100644
index 0000000..d7b020b
--- /dev/null
+++ b/test/valid/documented_example.numnormalized
@@ -0,0 +1,47 @@
+["var1"]	"val1"
+["split\nkey"]	"value"
+["var0"]	"escaped \" quote"
+["splitValue"]	"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")"
+["array",0]	"z"
+["array",1]	"a"
+["array",2]	"b"
+["array",3]	3.000000000000
+["array",4]	20.000000000000
+["array",5]	0.000000000000
+["array",6]	""
+["array",7]	""
+["array",8]	"\""
+["array",9]	"escaping\"several\"\"\nquote\"s and\nnewlines"
+["array"]	["z","a","b",3.000000000000,20.000000000000,0.000000000000,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"]
+["aNumber"]	1.000000000000
+["var8"]	"string\nwith\nproper\\\nnewlines"
+["var38"]	""
+["emptyarr"]	[]
+["emptyobj"]	{}
+["emptystr"]	""
+["arrOfObjs",0,"var"]	"val1"
+["arrOfObjs",0,"str"]	"s"
+["arrOfObjs",0]	{"var":"val1","str":"s"}
+["arrOfObjs",1,"var"]	"val30"
+["arrOfObjs",1,"str"]	"s"
+["arrOfObjs",1]	{"var":"val30","str":"s"}
+["arrOfObjs",2,"var"]	"val2"
+["arrOfObjs",2,"str"]	"z"
+["arrOfObjs",2]	{"var":"val2","str":"z"}
+["arrOfObjs",3,"var"]	"val2"
+["arrOfObjs",3,"str"]	"x"
+["arrOfObjs",3]	{"var":"val2","str":"x"}
+["arrOfObjs",4,"var"]	"val1"
+["arrOfObjs",4,"str"]	"S"
+["arrOfObjs",4]	{"var":"val1","str":"S"}
+["arrOfObjs",5,"var"]	"val1"
+["arrOfObjs",5,"str"]	"\""
+["arrOfObjs",5]	{"var":"val1","str":"\""}
+["arrOfObjs",6,"var"]	"val1"
+["arrOfObjs",6,"str"]	5.000000000000
+["arrOfObjs",6]	{"var":"val1","str":5.000000000000}
+["arrOfObjs",7,"var"]	"val1"
+["arrOfObjs",7,"str"]	"5"
+["arrOfObjs",7]	{"var":"val1","str":"5"}
+["arrOfObjs"]	[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5.000000000000},{"var":"val1","str":"5"}]
+[]	{"var1":"val1","split\nkey":"value","var0":"escaped \" quote","splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","array":["z","a","b",3.000000000000,20.000000000000,0.000000000000,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"],"aNumber":1.000000000000,"var8":"string\nwith\nproper\\\nnewlines","var38":"","emptyarr":[],"emptyobj":{},"emptystr":"","arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5.000000000000},{"var":"val1","str":"5"}]}
diff --git a/test/valid/documented_example.numnormalized_stripped b/test/valid/documented_example.numnormalized_stripped
new file mode 100644
index 0000000..78ee6dd
--- /dev/null
+++ b/test/valid/documented_example.numnormalized_stripped
@@ -0,0 +1,47 @@
+["var1"]	"val1"
+["split\nkey"]	"value"
+["var0"]	"escaped \" quote"
+["splitValue"]	"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")"
+["array",0]	"z"
+["array",1]	"a"
+["array",2]	"b"
+["array",3]	3
+["array",4]	20
+["array",5]	0
+["array",6]	""
+["array",7]	""
+["array",8]	"\""
+["array",9]	"escaping\"several\"\"\nquote\"s and\nnewlines"
+["array"]	["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"]
+["aNumber"]	1
+["var8"]	"string\nwith\nproper\\\nnewlines"
+["var38"]	""
+["emptyarr"]	[]
+["emptyobj"]	{}
+["emptystr"]	""
+["arrOfObjs",0,"var"]	"val1"
+["arrOfObjs",0,"str"]	"s"
+["arrOfObjs",0]	{"var":"val1","str":"s"}
+["arrOfObjs",1,"var"]	"val30"
+["arrOfObjs",1,"str"]	"s"
+["arrOfObjs",1]	{"var":"val30","str":"s"}
+["arrOfObjs",2,"var"]	"val2"
+["arrOfObjs",2,"str"]	"z"
+["arrOfObjs",2]	{"var":"val2","str":"z"}
+["arrOfObjs",3,"var"]	"val2"
+["arrOfObjs",3,"str"]	"x"
+["arrOfObjs",3]	{"var":"val2","str":"x"}
+["arrOfObjs",4,"var"]	"val1"
+["arrOfObjs",4,"str"]	"S"
+["arrOfObjs",4]	{"var":"val1","str":"S"}
+["arrOfObjs",5,"var"]	"val1"
+["arrOfObjs",5,"str"]	"\""
+["arrOfObjs",5]	{"var":"val1","str":"\""}
+["arrOfObjs",6,"var"]	"val1"
+["arrOfObjs",6,"str"]	5
+["arrOfObjs",6]	{"var":"val1","str":5}
+["arrOfObjs",7,"var"]	"val1"
+["arrOfObjs",7,"str"]	"5"
+["arrOfObjs",7]	{"var":"val1","str":"5"}
+["arrOfObjs"]	[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}]
+[]	{"var1":"val1","split\nkey":"value","var0":"escaped \" quote","splitValue":"there\n  are a newline and three spaces (one after \"there\" and two before \"are\")","array":["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"],"aNumber":1,"var8":"string\nwith\nproper\\\nnewlines","var38":"","emptyarr":[],"emptyobj":{},"emptystr":"","arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}]}
diff --git a/test/valid/embedded.normalized_numnormalized b/test/valid/embedded.normalized_numnormalized
new file mode 100644
index 0000000..f327913
--- /dev/null
+++ b/test/valid/embedded.normalized_numnormalized
@@ -0,0 +1 @@
+{"foo":"{\"foo\":\"bar\"}"}
diff --git a/test/valid/embedded.normalized_numnormalized_stripped b/test/valid/embedded.normalized_numnormalized_stripped
new file mode 100644
index 0000000..f327913
--- /dev/null
+++ b/test/valid/embedded.normalized_numnormalized_stripped
@@ -0,0 +1 @@
+{"foo":"{\"foo\":\"bar\"}"}
diff --git a/test/valid/embedded.numnormalized b/test/valid/embedded.numnormalized
new file mode 100644
index 0000000..041eaf9
--- /dev/null
+++ b/test/valid/embedded.numnormalized
@@ -0,0 +1,2 @@
+["foo"]	"{\"foo\":\"bar\"}"
+[]	{"foo":"{\"foo\":\"bar\"}"}
diff --git a/test/valid/embedded.numnormalized_stripped b/test/valid/embedded.numnormalized_stripped
new file mode 100644
index 0000000..041eaf9
--- /dev/null
+++ b/test/valid/embedded.numnormalized_stripped
@@ -0,0 +1,2 @@
+["foo"]	"{\"foo\":\"bar\"}"
+[]	{"foo":"{\"foo\":\"bar\"}"}
diff --git a/test/valid/empty_array.normalized_numnormalized b/test/valid/empty_array.normalized_numnormalized
new file mode 100644
index 0000000..fe51488
--- /dev/null
+++ b/test/valid/empty_array.normalized_numnormalized
@@ -0,0 +1 @@
+[]
diff --git a/test/valid/empty_array.normalized_numnormalized_stripped b/test/valid/empty_array.normalized_numnormalized_stripped
new file mode 100644
index 0000000..fe51488
--- /dev/null
+++ b/test/valid/empty_array.normalized_numnormalized_stripped
@@ -0,0 +1 @@
+[]
diff --git a/test/valid/empty_array.numnormalized b/test/valid/empty_array.numnormalized
new file mode 100644
index 0000000..d24d150
--- /dev/null
+++ b/test/valid/empty_array.numnormalized
@@ -0,0 +1 @@
+[]	[]
diff --git a/test/valid/empty_array.numnormalized_stripped b/test/valid/empty_array.numnormalized_stripped
new file mode 100644
index 0000000..d24d150
--- /dev/null
+++ b/test/valid/empty_array.numnormalized_stripped
@@ -0,0 +1 @@
+[]	[]
diff --git a/test/valid/empty_object.normalized_numnormalized b/test/valid/empty_object.normalized_numnormalized
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/test/valid/empty_object.normalized_numnormalized
@@ -0,0 +1 @@
+{}
diff --git a/test/valid/empty_object.normalized_numnormalized_stripped b/test/valid/empty_object.normalized_numnormalized_stripped
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/test/valid/empty_object.normalized_numnormalized_stripped
@@ -0,0 +1 @@
+{}
diff --git a/test/valid/empty_object.numnormalized b/test/valid/empty_object.numnormalized
new file mode 100644
index 0000000..4cdea2a
--- /dev/null
+++ b/test/valid/empty_object.numnormalized
@@ -0,0 +1 @@
+[]	{}
diff --git a/test/valid/empty_object.numnormalized_stripped b/test/valid/empty_object.numnormalized_stripped
new file mode 100644
index 0000000..4cdea2a
--- /dev/null
+++ b/test/valid/empty_object.numnormalized_stripped
@@ -0,0 +1 @@
+[]	{}
diff --git a/test/valid/many_object.normalized_numnormalized b/test/valid/many_object.normalized_numnormalized
new file mode 100644
index 0000000..376ddac
--- /dev/null
+++ b/test/valid/many_object.normalized_numnormalized
@@ -0,0 +1 @@
+{"key1":"string","key2":3573.000000000000}
diff --git a/test/valid/many_object.normalized_numnormalized_stripped b/test/valid/many_object.normalized_numnormalized_stripped
new file mode 100644
index 0000000..abfdad2
--- /dev/null
+++ b/test/valid/many_object.normalized_numnormalized_stripped
@@ -0,0 +1 @@
+{"key1":"string","key2":3573}
diff --git a/test/valid/many_object.numnormalized b/test/valid/many_object.numnormalized
new file mode 100644
index 0000000..b04a65b
--- /dev/null
+++ b/test/valid/many_object.numnormalized
@@ -0,0 +1,3 @@
+["key1"]	"string"
+["key2"]	3573.000000000000
+[]	{"key1":"string","key2":3573.000000000000}
diff --git a/test/valid/many_object.numnormalized_stripped b/test/valid/many_object.numnormalized_stripped
new file mode 100644
index 0000000..fd1fccb
--- /dev/null
+++ b/test/valid/many_object.numnormalized_stripped
@@ -0,0 +1,3 @@
+["key1"]	"string"
+["key2"]	3573
+[]	{"key1":"string","key2":3573}
diff --git a/test/valid/multiline_escapedquotes.normalized_numnormalized b/test/valid/multiline_escapedquotes.normalized_numnormalized
new file mode 100644
index 0000000..d049f7e
--- /dev/null
+++ b/test/valid/multiline_escapedquotes.normalized_numnormalized
@@ -0,0 +1 @@
+{"d2":"qwer\nt\nyu","d2":123.000000000000,"s1\ns2 \" s3 ":"abs","s4":"qwe"}
diff --git a/test/valid/multiline_escapedquotes.normalized_numnormalized_stripped b/test/valid/multiline_escapedquotes.normalized_numnormalized_stripped
new file mode 100644
index 0000000..fe90225
--- /dev/null
+++ b/test/valid/multiline_escapedquotes.normalized_numnormalized_stripped
@@ -0,0 +1 @@
+{"d2":"qwer\nt\nyu","d2":123,"s1\ns2 \" s3 ":"abs","s4":"qwe"}
diff --git a/test/valid/multiline_escapedquotes.numnormalized b/test/valid/multiline_escapedquotes.numnormalized
new file mode 100644
index 0000000..9b72344
--- /dev/null
+++ b/test/valid/multiline_escapedquotes.numnormalized
@@ -0,0 +1,5 @@
+["s1\ns2 \" s3 "]	"abs"
+["s4"]	"qwe"
+["d2"]	"qwer\nt\nyu"
+["d2"]	123.000000000000
+[]	{"s1\ns2 \" s3 ":"abs","s4":"qwe","d2":"qwer\nt\nyu","d2":123.000000000000}
diff --git a/test/valid/multiline_escapedquotes.numnormalized_stripped b/test/valid/multiline_escapedquotes.numnormalized_stripped
new file mode 100644
index 0000000..7a8aee6
--- /dev/null
+++ b/test/valid/multiline_escapedquotes.numnormalized_stripped
@@ -0,0 +1,5 @@
+["s1\ns2 \" s3 "]	"abs"
+["s4"]	"qwe"
+["d2"]	"qwer\nt\nyu"
+["d2"]	123
+[]	{"s1\ns2 \" s3 ":"abs","s4":"qwe","d2":"qwer\nt\nyu","d2":123}
diff --git a/test/valid/multiline_escapedquotes_indented.normalized_numnormalized b/test/valid/multiline_escapedquotes_indented.normalized_numnormalized
new file mode 100644
index 0000000..64a0716
--- /dev/null
+++ b/test/valid/multiline_escapedquotes_indented.normalized_numnormalized
@@ -0,0 +1 @@
+{"d2":"qwer  \nt\n   yu","d2":123.000000000000,"s1\ns2 \" s3 ":"abs","s4":"qwe"}
diff --git a/test/valid/multiline_escapedquotes_indented.normalized_numnormalized_stripped b/test/valid/multiline_escapedquotes_indented.normalized_numnormalized_stripped
new file mode 100644
index 0000000..4521493
--- /dev/null
+++ b/test/valid/multiline_escapedquotes_indented.normalized_numnormalized_stripped
@@ -0,0 +1 @@
+{"d2":"qwer  \nt\n   yu","d2":123,"s1\ns2 \" s3 ":"abs","s4":"qwe"}
diff --git a/test/valid/multiline_escapedquotes_indented.numnormalized b/test/valid/multiline_escapedquotes_indented.numnormalized
new file mode 100644
index 0000000..bc66cee
--- /dev/null
+++ b/test/valid/multiline_escapedquotes_indented.numnormalized
@@ -0,0 +1,5 @@
+["s1\ns2 \" s3 "]	"abs"
+["s4"]	"qwe"
+["d2"]	"qwer  \nt\n   yu"
+["d2"]	123.000000000000
+[]	{"s1\ns2 \" s3 ":"abs","s4":"qwe","d2":"qwer  \nt\n   yu","d2":123.000000000000}
diff --git a/test/valid/multiline_escapedquotes_indented.numnormalized_stripped b/test/valid/multiline_escapedquotes_indented.numnormalized_stripped
new file mode 100644
index 0000000..cb7fbcb
--- /dev/null
+++ b/test/valid/multiline_escapedquotes_indented.numnormalized_stripped
@@ -0,0 +1,5 @@
+["s1\ns2 \" s3 "]	"abs"
+["s4"]	"qwe"
+["d2"]	"qwer  \nt\n   yu"
+["d2"]	123
+[]	{"s1\ns2 \" s3 ":"abs","s4":"qwe","d2":"qwer  \nt\n   yu","d2":123}
diff --git a/test/valid/multiline_simple_key.normalized_numnormalized b/test/valid/multiline_simple_key.normalized_numnormalized
new file mode 100644
index 0000000..39302df
--- /dev/null
+++ b/test/valid/multiline_simple_key.normalized_numnormalized
@@ -0,0 +1 @@
+{"s1\ns2":"abs"}
diff --git a/test/valid/multiline_simple_key.normalized_numnormalized_stripped b/test/valid/multiline_simple_key.normalized_numnormalized_stripped
new file mode 100644
index 0000000..39302df
--- /dev/null
+++ b/test/valid/multiline_simple_key.normalized_numnormalized_stripped
@@ -0,0 +1 @@
+{"s1\ns2":"abs"}
diff --git a/test/valid/multiline_simple_key.numnormalized b/test/valid/multiline_simple_key.numnormalized
new file mode 100644
index 0000000..af9dee7
--- /dev/null
+++ b/test/valid/multiline_simple_key.numnormalized
@@ -0,0 +1,2 @@
+["s1\ns2"]	"abs"
+[]	{"s1\ns2":"abs"}
diff --git a/test/valid/multiline_simple_key.numnormalized_stripped b/test/valid/multiline_simple_key.numnormalized_stripped
new file mode 100644
index 0000000..af9dee7
--- /dev/null
+++ b/test/valid/multiline_simple_key.numnormalized_stripped
@@ -0,0 +1,2 @@
+["s1\ns2"]	"abs"
+[]	{"s1\ns2":"abs"}
diff --git a/test/valid/multiline_simple_value.normalized_numnormalized b/test/valid/multiline_simple_value.normalized_numnormalized
new file mode 100644
index 0000000..3ee6927
--- /dev/null
+++ b/test/valid/multiline_simple_value.normalized_numnormalized
@@ -0,0 +1 @@
+{"s":"ab c\nd e"}
diff --git a/test/valid/multiline_simple_value.normalized_numnormalized_stripped b/test/valid/multiline_simple_value.normalized_numnormalized_stripped
new file mode 100644
index 0000000..3ee6927
--- /dev/null
+++ b/test/valid/multiline_simple_value.normalized_numnormalized_stripped
@@ -0,0 +1 @@
+{"s":"ab c\nd e"}
diff --git a/test/valid/multiline_simple_value.numnormalized b/test/valid/multiline_simple_value.numnormalized
new file mode 100644
index 0000000..00b40af
--- /dev/null
+++ b/test/valid/multiline_simple_value.numnormalized
@@ -0,0 +1,2 @@
+["s"]	"ab c\nd e"
+[]	{"s":"ab c\nd e"}
diff --git a/test/valid/multiline_simple_value.numnormalized_stripped b/test/valid/multiline_simple_value.numnormalized_stripped
new file mode 100644
index 0000000..00b40af
--- /dev/null
+++ b/test/valid/multiline_simple_value.numnormalized_stripped
@@ -0,0 +1,2 @@
+["s"]	"ab c\nd e"
+[]	{"s":"ab c\nd e"}
diff --git a/test/valid/nested_array.normalized_numnormalized b/test/valid/nested_array.normalized_numnormalized
new file mode 100644
index 0000000..874f2f0
--- /dev/null
+++ b/test/valid/nested_array.normalized_numnormalized
@@ -0,0 +1 @@
+[["hello",{},4.000000000000],[],{"array":[]},1.000000000000]
diff --git a/test/valid/nested_array.normalized_numnormalized_stripped b/test/valid/nested_array.normalized_numnormalized_stripped
new file mode 100644
index 0000000..3cb16fb
--- /dev/null
+++ b/test/valid/nested_array.normalized_numnormalized_stripped
@@ -0,0 +1 @@
+[["hello",{},4],[],{"array":[]},1]
diff --git a/test/valid/nested_array.numnormalized b/test/valid/nested_array.numnormalized
new file mode 100644
index 0000000..6e33067
--- /dev/null
+++ b/test/valid/nested_array.numnormalized
@@ -0,0 +1,9 @@
+[0]	1.000000000000
+[1]	[]
+[2,0]	4.000000000000
+[2,1]	"hello"
+[2,2]	{}
+[2]	[4.000000000000,"hello",{}]
+[3,"array"]	[]
+[3]	{"array":[]}
+[]	[1.000000000000,[],[4.000000000000,"hello",{}],{"array":[]}]
diff --git a/test/valid/nested_array.numnormalized_stripped b/test/valid/nested_array.numnormalized_stripped
new file mode 100644
index 0000000..4638c15
--- /dev/null
+++ b/test/valid/nested_array.numnormalized_stripped
@@ -0,0 +1,9 @@
+[0]	1
+[1]	[]
+[2,0]	4
+[2,1]	"hello"
+[2,2]	{}
+[2]	[4,"hello",{}]
+[3,"array"]	[]
+[3]	{"array":[]}
+[]	[1,[],[4,"hello",{}],{"array":[]}]
diff --git a/test/valid/nested_object.normalized_numnormalized b/test/valid/nested_object.normalized_numnormalized
new file mode 100644
index 0000000..220de56
--- /dev/null
+++ b/test/valid/nested_object.normalized_numnormalized
@@ -0,0 +1 @@
+{"number":5.000000000000,"object":{"empty":{},"key":"value"}}
diff --git a/test/valid/nested_object.normalized_numnormalized_stripped b/test/valid/nested_object.normalized_numnormalized_stripped
new file mode 100644
index 0000000..efe1ce2
--- /dev/null
+++ b/test/valid/nested_object.normalized_numnormalized_stripped
@@ -0,0 +1 @@
+{"number":5,"object":{"empty":{},"key":"value"}}
diff --git a/test/valid/nested_object.numnormalized b/test/valid/nested_object.numnormalized
new file mode 100644
index 0000000..61ab077
--- /dev/null
+++ b/test/valid/nested_object.numnormalized
@@ -0,0 +1,5 @@
+["object","key"]	"value"
+["object","empty"]	{}
+["object"]	{"key":"value","empty":{}}
+["number"]	5.000000000000
+[]	{"object":{"key":"value","empty":{}},"number":5.000000000000}
diff --git a/test/valid/nested_object.numnormalized_stripped b/test/valid/nested_object.numnormalized_stripped
new file mode 100644
index 0000000..8609e30
--- /dev/null
+++ b/test/valid/nested_object.numnormalized_stripped
@@ -0,0 +1,5 @@
+["object","key"]	"value"
+["object","empty"]	{}
+["object"]	{"key":"value","empty":{}}
+["number"]	5
+[]	{"object":{"key":"value","empty":{}},"number":5}
diff --git a/test/valid/number.normalized_numnormalized b/test/valid/number.normalized_numnormalized
new file mode 100644
index 0000000..81a5cdb
--- /dev/null
+++ b/test/valid/number.normalized_numnormalized
@@ -0,0 +1 @@
+3.000000000000
diff --git a/test/valid/number.normalized_numnormalized_stripped b/test/valid/number.normalized_numnormalized_stripped
new file mode 100644
index 0000000..00750ed
--- /dev/null
+++ b/test/valid/number.normalized_numnormalized_stripped
@@ -0,0 +1 @@
+3
diff --git a/test/valid/number.numnormalized b/test/valid/number.numnormalized
new file mode 100644
index 0000000..4c0c749
--- /dev/null
+++ b/test/valid/number.numnormalized
@@ -0,0 +1 @@
+[]	3.000000000000
diff --git a/test/valid/number.numnormalized_stripped b/test/valid/number.numnormalized_stripped
new file mode 100644
index 0000000..2a1fecb
--- /dev/null
+++ b/test/valid/number.numnormalized_stripped
@@ -0,0 +1 @@
+[]	3
diff --git a/test/valid/object.normalized_numnormalized b/test/valid/object.normalized_numnormalized
new file mode 100644
index 0000000..f523ccf
--- /dev/null
+++ b/test/valid/object.normalized_numnormalized
@@ -0,0 +1 @@
+{"key":"Value"}
diff --git a/test/valid/object.normalized_numnormalized_stripped b/test/valid/object.normalized_numnormalized_stripped
new file mode 100644
index 0000000..f523ccf
--- /dev/null
+++ b/test/valid/object.normalized_numnormalized_stripped
@@ -0,0 +1 @@
+{"key":"Value"}
diff --git a/test/valid/object.numnormalized b/test/valid/object.numnormalized
new file mode 100644
index 0000000..9f50711
--- /dev/null
+++ b/test/valid/object.numnormalized
@@ -0,0 +1,2 @@
+["key"]	"Value"
+[]	{"key":"Value"}
diff --git a/test/valid/object.numnormalized_stripped b/test/valid/object.numnormalized_stripped
new file mode 100644
index 0000000..9f50711
--- /dev/null
+++ b/test/valid/object.numnormalized_stripped
@@ -0,0 +1,2 @@
+["key"]	"Value"
+[]	{"key":"Value"}
diff --git a/test/valid/singleline_escapedquotes_key.normalized_numnormalized b/test/valid/singleline_escapedquotes_key.normalized_numnormalized
new file mode 100644
index 0000000..938af32
--- /dev/null
+++ b/test/valid/singleline_escapedquotes_key.normalized_numnormalized
@@ -0,0 +1 @@
+{"s1 \" s2":"abs"}
diff --git a/test/valid/singleline_escapedquotes_key.normalized_numnormalized_stripped b/test/valid/singleline_escapedquotes_key.normalized_numnormalized_stripped
new file mode 100644
index 0000000..938af32
--- /dev/null
+++ b/test/valid/singleline_escapedquotes_key.normalized_numnormalized_stripped
@@ -0,0 +1 @@
+{"s1 \" s2":"abs"}
diff --git a/test/valid/singleline_escapedquotes_key.numnormalized b/test/valid/singleline_escapedquotes_key.numnormalized
new file mode 100644
index 0000000..66e89aa
--- /dev/null
+++ b/test/valid/singleline_escapedquotes_key.numnormalized
@@ -0,0 +1,2 @@
+["s1 \" s2"]	"abs"
+[]	{"s1 \" s2":"abs"}
diff --git a/test/valid/singleline_escapedquotes_key.numnormalized_stripped b/test/valid/singleline_escapedquotes_key.numnormalized_stripped
new file mode 100644
index 0000000..66e89aa
--- /dev/null
+++ b/test/valid/singleline_escapedquotes_key.numnormalized_stripped
@@ -0,0 +1,2 @@
+["s1 \" s2"]	"abs"
+[]	{"s1 \" s2":"abs"}
diff --git a/test/valid/singleline_escapedquotes_value.normalized_numnormalized b/test/valid/singleline_escapedquotes_value.normalized_numnormalized
new file mode 100644
index 0000000..075a978
--- /dev/null
+++ b/test/valid/singleline_escapedquotes_value.normalized_numnormalized
@@ -0,0 +1 @@
+{"s1 s2":"quoted \"substring\" value"}
diff --git a/test/valid/singleline_escapedquotes_value.normalized_numnormalized_stripped b/test/valid/singleline_escapedquotes_value.normalized_numnormalized_stripped
new file mode 100644
index 0000000..075a978
--- /dev/null
+++ b/test/valid/singleline_escapedquotes_value.normalized_numnormalized_stripped
@@ -0,0 +1 @@
+{"s1 s2":"quoted \"substring\" value"}
diff --git a/test/valid/singleline_escapedquotes_value.numnormalized b/test/valid/singleline_escapedquotes_value.numnormalized
new file mode 100644
index 0000000..1339ac3
--- /dev/null
+++ b/test/valid/singleline_escapedquotes_value.numnormalized
@@ -0,0 +1,2 @@
+["s1 s2"]	"quoted \"substring\" value"
+[]	{"s1 s2":"quoted \"substring\" value"}
diff --git a/test/valid/singleline_escapedquotes_value.numnormalized_stripped b/test/valid/singleline_escapedquotes_value.numnormalized_stripped
new file mode 100644
index 0000000..1339ac3
--- /dev/null
+++ b/test/valid/singleline_escapedquotes_value.numnormalized_stripped
@@ -0,0 +1,2 @@
+["s1 s2"]	"quoted \"substring\" value"
+[]	{"s1 s2":"quoted \"substring\" value"}
diff --git a/test/valid/string.normalized_numnormalized b/test/valid/string.normalized_numnormalized
new file mode 100644
index 0000000..31f592f
--- /dev/null
+++ b/test/valid/string.normalized_numnormalized
@@ -0,0 +1 @@
+"hello this is a string"
diff --git a/test/valid/string.normalized_numnormalized_stripped b/test/valid/string.normalized_numnormalized_stripped
new file mode 100644
index 0000000..31f592f
--- /dev/null
+++ b/test/valid/string.normalized_numnormalized_stripped
@@ -0,0 +1 @@
+"hello this is a string"
diff --git a/test/valid/string.numnormalized b/test/valid/string.numnormalized
new file mode 100644
index 0000000..b1fb986
--- /dev/null
+++ b/test/valid/string.numnormalized
@@ -0,0 +1 @@
+[]	"hello this is a string"
diff --git a/test/valid/string.numnormalized_stripped b/test/valid/string.numnormalized_stripped
new file mode 100644
index 0000000..b1fb986
--- /dev/null
+++ b/test/valid/string.numnormalized_stripped
@@ -0,0 +1 @@
+[]	"hello this is a string"
diff --git a/test/valid/string_in_array.normalized_numnormalized b/test/valid/string_in_array.normalized_numnormalized
new file mode 100644
index 0000000..89179f6
--- /dev/null
+++ b/test/valid/string_in_array.normalized_numnormalized
@@ -0,0 +1 @@
+["hello this is a string"]
diff --git a/test/valid/string_in_array.normalized_numnormalized_stripped b/test/valid/string_in_array.normalized_numnormalized_stripped
new file mode 100644
index 0000000..89179f6
--- /dev/null
+++ b/test/valid/string_in_array.normalized_numnormalized_stripped
@@ -0,0 +1 @@
+["hello this is a string"]
diff --git a/test/valid/string_in_array.numnormalized b/test/valid/string_in_array.numnormalized
new file mode 100644
index 0000000..a49e6d9
--- /dev/null
+++ b/test/valid/string_in_array.numnormalized
@@ -0,0 +1,2 @@
+[0]	"hello this is a string"
+[]	["hello this is a string"]
diff --git a/test/valid/string_in_array.numnormalized_stripped b/test/valid/string_in_array.numnormalized_stripped
new file mode 100644
index 0000000..a49e6d9
--- /dev/null
+++ b/test/valid/string_in_array.numnormalized_stripped
@@ -0,0 +1,2 @@
+[0]	"hello this is a string"
+[]	["hello this is a string"]
diff --git a/test/valid/string_in_object.normalized_numnormalized b/test/valid/string_in_object.normalized_numnormalized
new file mode 100644
index 0000000..357dfdc
--- /dev/null
+++ b/test/valid/string_in_object.normalized_numnormalized
@@ -0,0 +1 @@
+{"key":"hello this is a string"}
diff --git a/test/valid/string_in_object.normalized_numnormalized_stripped b/test/valid/string_in_object.normalized_numnormalized_stripped
new file mode 100644
index 0000000..357dfdc
--- /dev/null
+++ b/test/valid/string_in_object.normalized_numnormalized_stripped
@@ -0,0 +1 @@
+{"key":"hello this is a string"}
diff --git a/test/valid/string_in_object.numnormalized b/test/valid/string_in_object.numnormalized
new file mode 100644
index 0000000..e266552
--- /dev/null
+++ b/test/valid/string_in_object.numnormalized
@@ -0,0 +1,2 @@
+["key"]	"hello this is a string"
+[]	{"key":"hello this is a string"}
diff --git a/test/valid/string_in_object.numnormalized_stripped b/test/valid/string_in_object.numnormalized_stripped
new file mode 100644
index 0000000..e266552
--- /dev/null
+++ b/test/valid/string_in_object.numnormalized_stripped
@@ -0,0 +1,2 @@
+["key"]	"hello this is a string"
+[]	{"key":"hello this is a string"}
diff --git a/test/valid/tab_escape.normalized_numnormalized b/test/valid/tab_escape.normalized_numnormalized
new file mode 100644
index 0000000..b7e42b8
--- /dev/null
+++ b/test/valid/tab_escape.normalized_numnormalized
@@ -0,0 +1 @@
+"hello\tworld"
diff --git a/test/valid/tab_escape.normalized_numnormalized_stripped b/test/valid/tab_escape.normalized_numnormalized_stripped
new file mode 100644
index 0000000..b7e42b8
--- /dev/null
+++ b/test/valid/tab_escape.normalized_numnormalized_stripped
@@ -0,0 +1 @@
+"hello\tworld"
diff --git a/test/valid/tab_escape.numnormalized b/test/valid/tab_escape.numnormalized
new file mode 100644
index 0000000..ee69dd9
--- /dev/null
+++ b/test/valid/tab_escape.numnormalized
@@ -0,0 +1 @@
+[]	"hello\tworld"
diff --git a/test/valid/tab_escape.numnormalized_stripped b/test/valid/tab_escape.numnormalized_stripped
new file mode 100644
index 0000000..ee69dd9
--- /dev/null
+++ b/test/valid/tab_escape.numnormalized_stripped
@@ -0,0 +1 @@
+[]	"hello\tworld"

From b2cbb39c13b5d0790f6588aa446bf8ba5fee7f5c Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Tue, 14 Apr 2015 14:48:23 +0200
Subject: [PATCH 30/95] JSON.sh: support empty documents (no token in input,
 empty jpath when we hit EOF) as an empty object

---
 JSON.sh                                            | 14 ++++++++++++--
 test/valid/empty_document.json                     |  0
 test/valid/empty_document.normalized               |  1 +
 test/valid/empty_document.normalized_numnormalized |  1 +
 ...mpty_document.normalized_numnormalized_stripped |  1 +
 test/valid/empty_document.normalized_sorted        |  1 +
 test/valid/empty_document.numnormalized            |  1 +
 test/valid/empty_document.numnormalized_stripped   |  1 +
 test/valid/empty_document.parsed                   |  1 +
 test/valid/empty_document.sorted                   |  1 +
 10 files changed, 20 insertions(+), 2 deletions(-)
 create mode 100644 test/valid/empty_document.json
 create mode 100644 test/valid/empty_document.normalized
 create mode 100644 test/valid/empty_document.normalized_numnormalized
 create mode 100644 test/valid/empty_document.normalized_numnormalized_stripped
 create mode 100644 test/valid/empty_document.normalized_sorted
 create mode 100644 test/valid/empty_document.numnormalized
 create mode 100644 test/valid/empty_document.numnormalized_stripped
 create mode 100644 test/valid/empty_document.parsed
 create mode 100644 test/valid/empty_document.sorted

diff --git a/JSON.sh b/JSON.sh
index d1027ab..f159609 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -13,6 +13,7 @@ throw () {
 BRIEF=0
 LEAFONLY=0
 PRUNE=0
+ALLOWEMPTYINPUT=1
 NORMALIZE_SOLIDUS=0
 SORTDATA_OBJ=""
 SORTDATA_ARR=""
@@ -26,13 +27,14 @@ COOKASTRING=0
 
 usage() {
   echo
-  echo "Usage: JSON.sh [-b] [-l] [-p] [-s] [--no-newline] [-d] \ "
+  echo "Usage: JSON.sh [-b] [-l] [-p] [ -P] [-s] [--no-newline] [-d] \ "
   echo "               [-x 'regex'] [-S|-S='args'] [-N|-N='args'] \ "
   echo "               [-Nnx|-Nnx='fmtstr'|-Nn|-Nn='fmtstr'] < markup.json"
   echo "       JSON.sh [-h]"
   echo "-h - This help text."
   echo
   echo "-p - Prune empty. Exclude fields with empty values."
+  echo "-P - Pedantic mode, forbids acception of empty input documents."
   echo "-l - Leaf only. Only show leaf nodes, which stops data duplication."
   echo "-b - Brief. Combines 'Leaf only' and 'Prune empty' options."
   echo "-s - Remove escaping of the solidus symbol (stright slash)."
@@ -138,6 +140,8 @@ parse_options() {
       ;;
       -p) PRUNE=1
       ;;
+      -P) ALLOWEMPTYINPUT=0
+      ;;
       -s) NORMALIZE_SOLIDUS=1
       ;;
       -N) NORMALIZE=1
@@ -414,7 +418,13 @@ parse_value () {
        [ "$value" = '[]' ] && isempty=1
        ;;
     # At this point, the only valid single-character tokens are digits.
-    ''|[!0-9]) throw "EXPECTED value GOT ${token:-EOF}" ;;
+    ''|[!0-9]) if [ -z "$token" -a -z "$jpath" ] && [ "$ALLOWEMPTYINPUT" = 1 ]; then
+            print_debug $DEBUGLEVEL_PRINTPATHVAL \
+                'Got a NULL document as input (no jpath, no token)' >&2
+            value='{}'
+        else
+            throw "EXPECTED value GOT ${token:-EOF}"
+        fi ;;
     +*|-*|[0-9]*|.*)  # Potential number - separate hit in case for efficiency
        print_debug $DEBUGLEVEL_PRINTPATHVAL \
             "token '$token' is a suspected number" >&2
diff --git a/test/valid/empty_document.json b/test/valid/empty_document.json
new file mode 100644
index 0000000..e69de29
diff --git a/test/valid/empty_document.normalized b/test/valid/empty_document.normalized
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/test/valid/empty_document.normalized
@@ -0,0 +1 @@
+{}
diff --git a/test/valid/empty_document.normalized_numnormalized b/test/valid/empty_document.normalized_numnormalized
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/test/valid/empty_document.normalized_numnormalized
@@ -0,0 +1 @@
+{}
diff --git a/test/valid/empty_document.normalized_numnormalized_stripped b/test/valid/empty_document.normalized_numnormalized_stripped
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/test/valid/empty_document.normalized_numnormalized_stripped
@@ -0,0 +1 @@
+{}
diff --git a/test/valid/empty_document.normalized_sorted b/test/valid/empty_document.normalized_sorted
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/test/valid/empty_document.normalized_sorted
@@ -0,0 +1 @@
+{}
diff --git a/test/valid/empty_document.numnormalized b/test/valid/empty_document.numnormalized
new file mode 100644
index 0000000..4cdea2a
--- /dev/null
+++ b/test/valid/empty_document.numnormalized
@@ -0,0 +1 @@
+[]	{}
diff --git a/test/valid/empty_document.numnormalized_stripped b/test/valid/empty_document.numnormalized_stripped
new file mode 100644
index 0000000..4cdea2a
--- /dev/null
+++ b/test/valid/empty_document.numnormalized_stripped
@@ -0,0 +1 @@
+[]	{}
diff --git a/test/valid/empty_document.parsed b/test/valid/empty_document.parsed
new file mode 100644
index 0000000..4cdea2a
--- /dev/null
+++ b/test/valid/empty_document.parsed
@@ -0,0 +1 @@
+[]	{}
diff --git a/test/valid/empty_document.sorted b/test/valid/empty_document.sorted
new file mode 100644
index 0000000..4cdea2a
--- /dev/null
+++ b/test/valid/empty_document.sorted
@@ -0,0 +1 @@
+[]	{}

From 38746ff074c5bd9e0ca9c4359e50d78c8fb5e223 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Tue, 21 Apr 2015 17:03:24 +0200
Subject: [PATCH 31/95] JSON.sh: null-string comparison prefers separate
 clauses

---
 JSON.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/JSON.sh b/JSON.sh
index f159609..10945e4 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -418,7 +418,7 @@ parse_value () {
        [ "$value" = '[]' ] && isempty=1
        ;;
     # At this point, the only valid single-character tokens are digits.
-    ''|[!0-9]) if [ -z "$token" -a -z "$jpath" ] && [ "$ALLOWEMPTYINPUT" = 1 ]; then
+    ''|[!0-9]) if [ "$ALLOWEMPTYINPUT" = 1 -a -z "$jpath" ] && [ -z "$token" ]; then
             print_debug $DEBUGLEVEL_PRINTPATHVAL \
                 'Got a NULL document as input (no jpath, no token)' >&2
             value='{}'

From b44764082f44a47f0adfc4504b0adf3789bc379d Mon Sep 17 00:00:00 2001
From: Jim Klimov <jimklimov@cos.ru>
Date: Tue, 7 Jul 2015 20:45:07 +0300
Subject: [PATCH 32/95] invalid-test.sh valid-test.sh : reordered counters into
 "ls -1 pattern"

---
 test/invalid-test.sh | 2 +-
 test/valid-test.sh   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/invalid-test.sh b/test/invalid-test.sh
index b2350ec..c29fa89 100755
--- a/test/invalid-test.sh
+++ b/test/invalid-test.sh
@@ -6,7 +6,7 @@ cd ${0%/*}
 # http://en.wikipedia.org/wiki/Test_Anything_Protocol
 
 fails=0
-tests=`ls invalid/* -1 | wc -l`
+tests=`ls -1 invalid/* | wc -l`
 
 echo "1..${tests##* }"
 for input in invalid/*
diff --git a/test/valid-test.sh b/test/valid-test.sh
index c31dcdb..d46ffb4 100755
--- a/test/valid-test.sh
+++ b/test/valid-test.sh
@@ -19,7 +19,7 @@ i=0
 CHOMPEXT='\.\(parsed\|sorted\|numnormalized\|normalized\|json\).*$'
 [ $# -gt 0 ] && \
     FILES="$(for F in "$@"; do echo valid/"`basename "$F" | sed "s,${CHOMPEXT},,"`".json ; done | sort | uniq)" || \
-    FILES="`ls valid/*.json -1`"
+    FILES="`ls -1 valid/*.json`"
 
 [ -z "$FILES" ] && echo "error - no files found to test!" >&2 && exit 1
 

From 297d4a3e358be1fa0ff8ceeb318d4be343eb5040 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jimklimov@cos.ru>
Date: Tue, 7 Jul 2015 20:46:06 +0300
Subject: [PATCH 33/95] JSON.sh : added link to my repo

---
 JSON.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/JSON.sh b/JSON.sh
index 10945e4..4346a31 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -4,6 +4,7 @@
 # MIT / Apache 2 licenses (C) 2014 by "dominictarr" checked out 2015-01-04
 # MIT / Apache 2 licenses (C) 2015 by "dominictarr" merged 0.2.0 2015-04-01
 # further development (C) 2015 Jim Klimov <EvgenyKlimov@eaton.com>
+# at fork https://github.com/jimklimov/JSON.sh
 
 throw () {
   echo "$*" >&2

From 88041925f0f64b6604a54b60518238b14e08b58e Mon Sep 17 00:00:00 2001
From: Jim Klimov <jimklimov@cos.ru>
Date: Tue, 7 Jul 2015 21:32:40 +0300
Subject: [PATCH 34/95] JSON.sh quote token values in EXPECTED and in debug
 outputs

---
 JSON.sh | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index 4346a31..ac5d497 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -328,7 +328,7 @@ parse_array () {
   local ary=''
   local aryml=''
   read -r token
-  print_debug $DEBUGLEVEL_PRINTTOKEN "parse_array(1):" "token=$token"
+  print_debug $DEBUGLEVEL_PRINTTOKEN "parse_array(1):" "token='$token'"
   case "$token" in
     ']') ;;
     *)
@@ -342,14 +342,14 @@ parse_array () {
 $value"
         fi
         read -r token
-        print_debug $DEBUGLEVEL_PRINTTOKEN "parse_array(2):" "token=$token"
+        print_debug $DEBUGLEVEL_PRINTTOKEN "parse_array(2):" "token='$token'"
         case "$token" in
           ']') break ;;
           ',') ary="$ary," ;;
-          *) throw "EXPECTED , or ] GOT ${token:-EOF}" ;;
+          *) throw "EXPECTED ',' or ']' GOT '${token:-EOF}'" ;;
         esac
         read -r token
-        print_debug $DEBUGLEVEL_PRINTTOKEN "parse_array(3):" "token=$token"
+        print_debug $DEBUGLEVEL_PRINTTOKEN "parse_array(3):" "token='$token'"
       done
       ;;
   esac
@@ -365,7 +365,7 @@ parse_object () {
   local obj=''
   local objml=''
   read -r token
-  print_debug $DEBUGLEVEL_PRINTTOKEN "parse_object(1):" "token=$token"
+  print_debug $DEBUGLEVEL_PRINTTOKEN "parse_object(1):" "token='$token'"
   case "$token" in
     '}') ;;
     *)
@@ -373,16 +373,16 @@ parse_object () {
       do
         case "$token" in
           '"'*'"') key=$token ;;
-          *) throw "EXPECTED string GOT ${token:-EOF}" ;;
+          *) throw "EXPECTED string GOT '${token:-EOF}'" ;;
         esac
         read -r token
-        print_debug $DEBUGLEVEL_PRINTTOKEN "parse_object(2):" "token=$token"
+        print_debug $DEBUGLEVEL_PRINTTOKEN "parse_object(2):" "token='$token'"
         case "$token" in
           ':') ;;
-          *) throw "EXPECTED : GOT ${token:-EOF}" ;;
+          *) throw "EXPECTED : GOT '${token:-EOF}'" ;;
         esac
         read -r token
-        print_debug $DEBUGLEVEL_PRINTTOKEN "parse_object(3):" "token=$token"
+        print_debug $DEBUGLEVEL_PRINTTOKEN "parse_object(3):" "token='$token'"
         parse_value "$1" "$key"
         obj="$obj$key:$value"
         if [ -n "$SORTDATA_OBJ" ]; then
@@ -390,14 +390,14 @@ parse_object () {
 $key:$value"
         fi
         read -r token
-        print_debug $DEBUGLEVEL_PRINTTOKEN "parse_object(4):" "token=$token"
+        print_debug $DEBUGLEVEL_PRINTTOKEN "parse_object(4):" "token='$token'"
         case "$token" in
           '}') break ;;
           ',') obj="$obj," ;;
-          *) throw "EXPECTED , or } GOT ${token:-EOF}" ;;
+          *) throw "EXPECTED ',' or '}' GOT '${token:-EOF}'" ;;
         esac
         read -r token
-        print_debug $DEBUGLEVEL_PRINTTOKEN "parse_object(5):" "token=$token"
+        print_debug $DEBUGLEVEL_PRINTTOKEN "parse_object(5):" "token='$token'"
       done
     ;;
   esac
@@ -424,7 +424,7 @@ parse_value () {
                 'Got a NULL document as input (no jpath, no token)' >&2
             value='{}'
         else
-            throw "EXPECTED value GOT ${token:-EOF}"
+            throw "EXPECTED value GOT '${token:-EOF}'"
         fi ;;
     +*|-*|[0-9]*|.*)  # Potential number - separate hit in case for efficiency
        print_debug $DEBUGLEVEL_PRINTPATHVAL \
@@ -498,13 +498,13 @@ parse_value () {
 
 parse () {
   read -r token
-  print_debug $DEBUGLEVEL_PRINTTOKEN "parse(1):" "token=$token"
+  print_debug $DEBUGLEVEL_PRINTTOKEN "parse(1):" "token='$token'"
   parse_value
   read -r token
-  print_debug $DEBUGLEVEL_PRINTTOKEN "parse(2):" "token=$token"
+  print_debug $DEBUGLEVEL_PRINTTOKEN "parse(2):" "token='$token'"
   case "$token" in
     '') ;;
-    *) throw "EXPECTED EOF GOT $token" ;;
+    *) throw "EXPECTED EOF GOT '$token'" ;;
   esac
 }
 

From d274e81dec98c7c71122567473614babda745743 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jimklimov@cos.ru>
Date: Tue, 7 Jul 2015 22:25:19 +0300
Subject: [PATCH 35/95] JSON.sh : adapting to Solaris - better guess grep,
 egrep and awk

---
 JSON.sh | 53 +++++++++++++++++++++++++++++++++++++++++------------
 1 file changed, 41 insertions(+), 12 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index ac5d497..227c62a 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -26,6 +26,28 @@ EXTRACT_JPATH=""
 TOXIC_NEWLINE=0
 COOKASTRING=0
 
+# May be passed by caller; also may pass AWK_OPTS *for it* then
+[ -z "$AWK" ] && AWK_OPTS="" && \
+for P in gawk /usr/xpg4/bin/awk nawk oawk awk ; do case "$P" in
+    /*) [ -x "$P" ] && AWK="$P"; break;;
+    *) AWK="`which "$P" 2>/dev/null`" && [ -n "$AWK" ] && break;;
+esac; done
+
+# Different OSes have different greps... we like a GNU one
+[ -z "$GGREP" ] && \
+for P in ggrep /usr/xpg4/bin/grep grep ; do case "$P" in
+    /*) [ -x "$P" ] && GGREP="$P"; break;;
+    *) GGREP="`which "$P" 2>/dev/null`" && [ -n "$GGREP" ] && break;;
+esac; done
+[ -n "$GGREP" ] && [ -x "$GGREP" ] || throw "No GNU GREP was found!"
+
+[ -z "$GEGREP" ] && \
+for P in gegrep /usr/xpg4/bin/egrep egrep ; do case "$P" in
+    /*) [ -x "$P" ] && GEGREP="$P"; break;;
+    *) GEGREP="`which "$P" 2>/dev/null`" && [ -n "$GEGREP" ] && break;;
+esac; done
+[ -n "$GEGREP" ] && [ -x "$GEGREP" ] || throw "No GNU EGREP was found!"
+
 usage() {
   echo
   echo "Usage: JSON.sh [-b] [-l] [-p] [ -P] [-s] [--no-newline] [-d] \ "
@@ -218,7 +240,10 @@ parse_options() {
 awk_egrep () {
   local pattern_string=$1
 
-  gawk '{
+  [ -z "$AWK" ] && throw "No AWK found!"
+  [ ! -x "$AWK" ] && throw "Not executable AWK='$AWK'!"
+
+  ${AWK} $AWK_OPTS '{
     while ($0) {
       start=match($0, pattern);
       token=substr($0, start, RLENGTH);
@@ -237,8 +262,8 @@ strip_newlines() {
   local INSTRING=0
   local LINENUM=0
 
-  # The first "grep" should ensure that input has a trailing newline
-  grep '' | \
+  # The first "grep" should ensure that input for "while" has a trailing newline
+  $GGREP '' | \
   tee_stderr BEFORE_STRIP $DEBUGLEVEL_PRINTTOKEN_PIPELINE | \
   while IFS="" read -r ILINE; do
     # Remove escaped quotes:
@@ -280,7 +305,7 @@ strip_newlines() {
 
 cook_a_string() {
     ### Escape backslashes, double-quotes, tabs and newlines, in this order
-    grep '' | sed -e 's,\\,\\\\,g' -e 's,\",\\",g' -e 's,\t,\\t,g' | \
+    $GGREP '' | sed -e 's,\\,\\\\,g' -e 's,\",\\",g' -e 's,\t,\\t,g' | \
     { FIRST=''; while IFS="" read -r ILINE; do
       printf '%s%s' "$FIRST" "$ILINE"
       [ -z "$FIRST" ] && FIRST='\n'
@@ -289,23 +314,27 @@ cook_a_string() {
 }
 
 tokenize () {
-  local GREP
+  local GREP_O
   local ESCAPE
   local CHAR
 
-  if echo "test string" | egrep -ao --color=never "test" &>/dev/null
+  if echo "test string" | $GEGREP -ao --color=never "test" >/dev/null 2>/dev/null
   then
-    GREP='egrep -ao --color=never'
-  else
-    GREP='egrep -ao'
+    GREP_O="$GEGREP -ao --color=never"
+  elif echo "test string" | $GEGREP -ao "test" >/dev/null 2>/dev/null
+  then
+    GREP_O="$GEGREP -ao"
+  elif echo "test string" | $GEGREP -o "test" >/dev/null 2>/dev/null
+  then
+    GREP_O="$GEGREP -o"
   fi
 
-  if echo "test string" | egrep -o "test" &>/dev/null
+  if [ -n "$GREP_O" ] && echo "test string" | $GREP_O "test" >/dev/null
   then
     ESCAPE='(\\[^u[:cntrl:]]|\\u[0-9a-fA-F]{4})'
     CHAR='[^[:cntrl:]"\\]'
   else
-    GREP=awk_egrep
+    GREP_O=awk_egrep
     ESCAPE='(\\\\[^u[:cntrl:]]|\\u[0-9a-fA-F]{4})'
     CHAR='[^[:cntrl:]"\\\\]'
   fi
@@ -319,7 +348,7 @@ tokenize () {
   local SPACE='[[:space:]]+'
 
   tee_stderr BEFORE_TOKENIZER $DEBUGLEVEL_PRINTTOKEN_PIPELINE | \
-  $GREP "$STRING|$NUMBER|$KEYWORD|$SPACE|." | egrep -v "^$SPACE$" | \
+  $GREP_O "$STRING|$NUMBER|$KEYWORD|$SPACE|." | $GEGREP -v "^$SPACE$" | \
   tee_stderr AFTER_TOKENIZER $DEBUGLEVEL_PRINTTOKEN_PIPELINE
 }
 

From ea4c9b6850d65101cd0061ab99a8f4fb403c4c0b Mon Sep 17 00:00:00 2001
From: Jim Klimov <jimklimov@cos.ru>
Date: Tue, 7 Jul 2015 22:26:51 +0300
Subject: [PATCH 36/95] Removed  invalid/empty.json: this is not in fact
 invalid JSON, and conflicts with valid/empty_document.json

---
 test/invalid/empty.json | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 test/invalid/empty.json

diff --git a/test/invalid/empty.json b/test/invalid/empty.json
deleted file mode 100644
index e69de29..0000000

From 0394bc626d62955bb281b369db0c98b66bcf14a2 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jimklimov@cos.ru>
Date: Tue, 7 Jul 2015 22:34:35 +0300
Subject: [PATCH 37/95] JSON.sh : typo fix in recent changes

---
 JSON.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index 227c62a..9271e19 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -29,21 +29,21 @@ COOKASTRING=0
 # May be passed by caller; also may pass AWK_OPTS *for it* then
 [ -z "$AWK" ] && AWK_OPTS="" && \
 for P in gawk /usr/xpg4/bin/awk nawk oawk awk ; do case "$P" in
-    /*) [ -x "$P" ] && AWK="$P"; break;;
+    /*) [ -x "$P" ] && AWK="$P" && break;;
     *) AWK="`which "$P" 2>/dev/null`" && [ -n "$AWK" ] && break;;
 esac; done
 
 # Different OSes have different greps... we like a GNU one
 [ -z "$GGREP" ] && \
 for P in ggrep /usr/xpg4/bin/grep grep ; do case "$P" in
-    /*) [ -x "$P" ] && GGREP="$P"; break;;
+    /*) [ -x "$P" ] && GGREP="$P" && break;;
     *) GGREP="`which "$P" 2>/dev/null`" && [ -n "$GGREP" ] && break;;
 esac; done
 [ -n "$GGREP" ] && [ -x "$GGREP" ] || throw "No GNU GREP was found!"
 
 [ -z "$GEGREP" ] && \
 for P in gegrep /usr/xpg4/bin/egrep egrep ; do case "$P" in
-    /*) [ -x "$P" ] && GEGREP="$P"; break;;
+    /*) [ -x "$P" ] && GEGREP="$P" && break;;
     *) GEGREP="`which "$P" 2>/dev/null`" && [ -n "$GEGREP" ] && break;;
 esac; done
 [ -n "$GEGREP" ] && [ -x "$GEGREP" ] || throw "No GNU EGREP was found!"

From f2b13761bd6e284352b314d6bd65323d6affacc6 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jimklimov@cos.ru>
Date: Wed, 8 Jul 2015 00:23:24 +0300
Subject: [PATCH 38/95] JSON.sh : refactored binfile lookup into a function;
 script now mostly works in older Solaris 10 as well

---
 JSON.sh | 78 +++++++++++++++++++++++++++++++++------------------------
 1 file changed, 45 insertions(+), 33 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index 9271e19..b132e0d 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -26,28 +26,39 @@ EXTRACT_JPATH=""
 TOXIC_NEWLINE=0
 COOKASTRING=0
 
+findbin() {
+    # Locates a named binary or one from path, prints to stdout
+    local BIN
+    for P in "$@" ; do case "$P" in
+	/*) [ -x "$P" ] && BIN="$P" && break;;
+        *) BIN="`which "$P" 2>/dev/null | tail -1`" && [ -n "$BIN" ] && [ -x "$BIN" ] && break || BIN="";;
+    esac; done
+    [ -n "$BIN" ] && [ -x "$BIN" ] && echo "$BIN" && return 0
+    return 1
+}
+
 # May be passed by caller; also may pass AWK_OPTS *for it* then
 [ -z "$AWK" ] && AWK_OPTS="" && \
-for P in gawk /usr/xpg4/bin/awk nawk oawk awk ; do case "$P" in
-    /*) [ -x "$P" ] && AWK="$P" && break;;
-    *) AWK="`which "$P" 2>/dev/null`" && [ -n "$AWK" ] && break;;
-esac; done
+    AWK="`findbin /usr/xpg4/bin/awk gawk nawk oawk awk`"
+# Error-checked in one optional place it may be needed
 
 # Different OSes have different greps... we like a GNU one
 [ -z "$GGREP" ] && \
-for P in ggrep /usr/xpg4/bin/grep grep ; do case "$P" in
-    /*) [ -x "$P" ] && GGREP="$P" && break;;
-    *) GGREP="`which "$P" 2>/dev/null`" && [ -n "$GGREP" ] && break;;
-esac; done
+    GGREP="`findbin ggrep /usr/xpg4/bin/grep grep`"
 [ -n "$GGREP" ] && [ -x "$GGREP" ] || throw "No GNU GREP was found!"
 
 [ -z "$GEGREP" ] && \
-for P in gegrep /usr/xpg4/bin/egrep egrep ; do case "$P" in
-    /*) [ -x "$P" ] && GEGREP="$P" && break;;
-    *) GEGREP="`which "$P" 2>/dev/null`" && [ -n "$GEGREP" ] && break;;
-esac; done
+    GEGREP="`findbin gegrep /usr/xpg4/bin/egrep egrep`"
 [ -n "$GEGREP" ] && [ -x "$GEGREP" ] || throw "No GNU EGREP was found!"
 
+[ -z "$GSORT" ] && \
+    GSORT="`findbin gsort sort /usr/xpg4/bin/sort`"
+[ -n "$GSORT" ] && [ -x "$GSORT" ] || throw "No GNU SORT was found!"
+
+[ -z "$GSED" ] && \
+    GSED="`findbin /usr/xpg4/bin/sed gsed sed`"
+[ -n "$GSED" ] && [ -x "$GSED" ] || throw "No GNU SED was found!"
+
 usage() {
   echo
   echo "Usage: JSON.sh [-b] [-l] [-p] [ -P] [-s] [--no-newline] [-d] \ "
@@ -108,7 +119,8 @@ validate_debuglevel() {
 
 unquote() {
     # Remove single or double quotes surrounding the token
-    sed "s,^'\(.*\)'\$,\1," | sed 's,^\"\(.*\)\"$,\1,'
+    $GSED "s,^'\(.*\)'\$,\1," 2>/dev/null | \
+    $GSED 's,^\"\(.*\)\"$,\1,' 2>/dev/null
 }
 
 ### Empty and non-numeric and non-positive values should be filtered out here
@@ -170,55 +182,55 @@ parse_options() {
       -N) NORMALIZE=1
       ;;
       -N=*) NORMALIZE=1
-          SORTDATA_OBJ="sort `echo "$1" | sed 's,^-N=,,' | unquote `"
-          SORTDATA_ARR="sort `echo "$1" | sed 's,^-N=,,' | unquote `"
+          SORTDATA_OBJ="$GSORT `echo "$1" | $GSED 's,^-N=,,' 2>/dev/null | unquote `"
+          SORTDATA_ARR="$GSORT `echo "$1" | $GSED 's,^-N=,,' 2>/dev/null | unquote `"
       ;;
       -No=*) NORMALIZE=1
-          SORTDATA_OBJ="sort `echo "$1" | sed 's,^-No=,,' | unquote `"
+          SORTDATA_OBJ="$GSORT `echo "$1" | $GSED 's,^-No=,,' 2>/dev/null | unquote `"
       ;;
       -Na=*) NORMALIZE=1
-          SORTDATA_ARR="sort `echo "$1" | sed 's,^-Na=,,' | unquote `"
+          SORTDATA_ARR="$GSORT `echo "$1" | $GSED 's,^-Na=,,' 2>/dev/null | unquote `"
       ;;
       -Nnx) NORMALIZE_NUMBERS_STRIP=1
             NORMALIZE_NUMBERS=1
       ;;
       -Nnx=*) NORMALIZE_NUMBERS_STRIP=1
             NORMALIZE_NUMBERS=1
-            NORMALIZE_NUMBERS_FORMAT="`echo "$1" | sed 's,^-Nnx=,,' | unquote `"
+            NORMALIZE_NUMBERS_FORMAT="`echo "$1" | $GSED 's,^-Nnx=,,' 2>/dev/null | unquote `"
       ;;
       -Nn) NORMALIZE_NUMBERS=1
       ;;
       -Nn=*) NORMALIZE_NUMBERS=1
-          NORMALIZE_NUMBERS_FORMAT="`echo "$1" | sed 's,^-Nn=,,' | unquote `"
+          NORMALIZE_NUMBERS_FORMAT="`echo "$1" | $GSED 's,^-Nn=,,' 2>/dev/null | unquote `"
       ;;
-      -S) SORTDATA_OBJ="sort"
-          SORTDATA_ARR="sort"
+      -S) SORTDATA_OBJ="$GSORT"
+          SORTDATA_ARR="$GSORT"
       ;;
-      -So) SORTDATA_OBJ="sort"
+      -So) SORTDATA_OBJ="$GSORT"
       ;;
-      -Sa) SORTDATA_ARR="sort"
+      -Sa) SORTDATA_ARR="$GSORT"
       ;;
       -S=*)
-          SORTDATA_OBJ="sort `echo "$1" | sed 's,^-S=,,' | unquote `"
-          SORTDATA_ARR="sort `echo "$1" | sed 's,^-S=,,' | unquote `"
+          SORTDATA_OBJ="$GSORT `echo "$1" | $GSED 's,^-S=,,' 2>/dev/null | unquote `"
+          SORTDATA_ARR="$GSORT `echo "$1" | $GSED 's,^-S=,,' 2>/dev/null | unquote `"
       ;;
       -So=*)
-          SORTDATA_OBJ="sort `echo "$1" | sed 's,^-So=,,' | unquote `"
+          SORTDATA_OBJ="$GSORT `echo "$1" | $GSED 's,^-So=,,' 2>/dev/null | unquote `"
       ;;
       -Sa=*)
-          SORTDATA_ARR="sort `echo "$1" | sed 's,^-Sa=,,' | unquote `"
+          SORTDATA_ARR="$GSORT `echo "$1" | $GSED 's,^-Sa=,,' 2>/dev/null | unquote `"
       ;;
       -x) EXTRACT_JPATH="$2"
           shift
       ;;
-      -x=*) EXTRACT_JPATH="`echo "$1" | sed 's,^-x=,,'`"
+      -x=*) EXTRACT_JPATH="`echo "$1" | $GSED 's,^-x=,,' 2>/dev/null`"
       ;;
       --no-newline)
           TOXIC_NEWLINE=1
       ;;
       -d) DEBUG=$(($DEBUG+1))
       ;;
-      -d=*) DEBUG="`echo "$1" | sed 's,^-d=,,'`"
+      -d=*) DEBUG="`echo "$1" | $GSED 's,^-d=,,' 2>/dev/null`"
       ;;
       -Q) COOKASTRING=1
       ;;
@@ -305,7 +317,7 @@ strip_newlines() {
 
 cook_a_string() {
     ### Escape backslashes, double-quotes, tabs and newlines, in this order
-    $GGREP '' | sed -e 's,\\,\\\\,g' -e 's,\",\\",g' -e 's,\t,\\t,g' | \
+    $GGREP '' | $GSED -e 's,\\,\\\\,g' -e 's,\",\\",g' -e 's,\t,\\t,g' 2>/dev/null | \
     { FIRST=''; while IFS="" read -r ILINE; do
       printf '%s%s' "$FIRST" "$ILINE"
       [ -z "$FIRST" ] && FIRST='\n'
@@ -383,7 +395,7 @@ $value"
       ;;
   esac
   if [ -n "$SORTDATA_ARR" ]; then
-    ary="`echo -E "$aryml" | $SORTDATA_ARR | tr '\n' ',' | sed 's|,*$||' | sed 's|^,*||'`"
+    ary="`echo -E "$aryml" | $SORTDATA_ARR | tr '\n' ',' | $GSED 's|,*$||' 2>/dev/null | $GSED 's|^,*||' 2>/dev/null`"
   fi
   [ "$BRIEF" -eq 0 ] && value=`printf '[%s]' "$ary"` || value=
   :
@@ -431,7 +443,7 @@ $key:$value"
     ;;
   esac
   if [ -n "$SORTDATA_OBJ" ]; then
-    obj="`echo -E "$objml" | $SORTDATA_OBJ | tr '\n' ',' | sed 's|,*$||' | sed 's|^,*||'`"
+    obj="`echo -E "$objml" | $SORTDATA_OBJ | tr '\n' ',' | $GSED 's|,*$||' 2>/dev/null | $GSED 's|^,*||' 2>/dev/null`"
   fi
   [ "$BRIEF" -eq 0 ] && value=`printf '{%s}' "$obj"` || value=
   :
@@ -466,7 +478,7 @@ parse_value () {
             print_debug $DEBUGLEVEL_PRINTPATHVAL "normalized numeric token" \
                 "'$token' into '$value'" >&2
             if [ "$NORMALIZE_NUMBERS_STRIP" = 1 ]; then
-                local valuetmp="`echo "$value" | sed -e 's,0*$,,g' -e 's,\.$,,'`" && \
+                local valuetmp="`echo "$value" | $GSED -e 's,0*$,,g' -e 's,\.$,,' 2>/dev/null`" && \
                 value="$valuetmp"
                 unset valuetmp
                 print_debug $DEBUGLEVEL_PRINTPATHVAL "stripped numeric token" \

From a062ada7754f80278bf9944f2be131a46201477c Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Tue, 28 Jul 2015 17:38:13 +0200
Subject: [PATCH 39/95] pedantic-empty-test.sh : initial commit (test to
 validate empty documents with different pedantic settings)

---
 test/pedantic-empty-test.sh | 39 +++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100755 test/pedantic-empty-test.sh

diff --git a/test/pedantic-empty-test.sh b/test/pedantic-empty-test.sh
new file mode 100755
index 0000000..a222758
--- /dev/null
+++ b/test/pedantic-empty-test.sh
@@ -0,0 +1,39 @@
+#! /usr/bin/env bash
+
+cd ${0%/*}
+
+# make test output TAP compatible
+# http://en.wikipedia.org/wiki/Test_Anything_Protocol
+
+fails=0
+tests=8
+i=0
+
+echo "1..${tests##* }"
+
+for DOC in "" " " "
+" "
+  
+ "; do
+  i=$(($i+1))
+  if echo "$DOC" | ../JSON.sh
+  then
+    echo "ok $i - empty input '$DOC' is okay in non-pedantic mode"
+  else
+    echo "not ok $i - empty input '$DOC' was rejected in non-pedantic mode"
+    fails=$((fails+1))
+  fi
+
+  i=$(($i+1))
+  if echo "$DOC" | ../JSON.sh -P
+  then
+    echo "not ok $i - empty input '$DOC' should be rejected in pedantic mode"
+    fails=$((fails+1))
+  else
+    echo "ok $i - empty input '$DOC' was rejected in pedantic mode"
+  fi
+done
+
+echo "$i test(s) executed"
+echo "$fails test(s) failed"
+exit $fails

From fb6b97c319dbd9234f574076029de1c6fde9a0e4 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 19 Aug 2015 15:59:33 +0200
Subject: [PATCH 40/95] JSON.sh : licensing header reformatted to fit internal
 project for which the tool was adapted

---
 JSON.sh | 51 +++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 45 insertions(+), 6 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index b132e0d..510751b 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -1,10 +1,49 @@
 #!/usr/bin/env bash
-
-# https://github.com/dominictarr/JSON.sh/blob/master/JSON.sh
-# MIT / Apache 2 licenses (C) 2014 by "dominictarr" checked out 2015-01-04
-# MIT / Apache 2 licenses (C) 2015 by "dominictarr" merged 0.2.0 2015-04-01
-# further development (C) 2015 Jim Klimov <EvgenyKlimov@eaton.com>
-# at fork https://github.com/jimklimov/JSON.sh
+#
+# Copyright (C) 2014-2015 Dominic Tarr
+# Copyright (C) 2015 Eaton
+#
+#! \file    JSON.sh
+#  \brief   A json parser written in bash
+#  \author  Dominic Tarr <https://github.com/dominictarr>
+#  \author  Jim Klimov <EvgenyKlimov@Eaton.com>
+#  \details Based on Dominic Tarr JSON.sh
+#           https://github.com/dominictarr/JSON.sh/blob/master/JSON.sh
+#           Forked and further modified by Eaton / Jim Klimov
+#           https://github.com/jimklimov/JSON.sh
+#
+# The MIT License (MIT)
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 throw () {
   echo "$*" >&2

From 3f86a4db1ae5c76feb675d268131aaa205396e54 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Mon, 11 Jan 2016 11:34:05 +0100
Subject: [PATCH 41/95] JSON.sh : added paths to GNU tools that are expected on
 Solaris/OpenIndiana

---
 JSON.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index 510751b..70fdaf0 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -69,7 +69,7 @@ findbin() {
     # Locates a named binary or one from path, prints to stdout
     local BIN
     for P in "$@" ; do case "$P" in
-	/*) [ -x "$P" ] && BIN="$P" && break;;
+    /*) [ -x "$P" ] && BIN="$P" && break;;
         *) BIN="`which "$P" 2>/dev/null | tail -1`" && [ -n "$BIN" ] && [ -x "$BIN" ] && break || BIN="";;
     esac; done
     [ -n "$BIN" ] && [ -x "$BIN" ] && echo "$BIN" && return 0
@@ -83,19 +83,19 @@ findbin() {
 
 # Different OSes have different greps... we like a GNU one
 [ -z "$GGREP" ] && \
-    GGREP="`findbin ggrep /usr/xpg4/bin/grep grep`"
+    GGREP="`findbin /{usr,opt}/{gnu,sfw}/bin/grep ggrep /usr/xpg4/bin/grep grep`"
 [ -n "$GGREP" ] && [ -x "$GGREP" ] || throw "No GNU GREP was found!"
 
 [ -z "$GEGREP" ] && \
-    GEGREP="`findbin gegrep /usr/xpg4/bin/egrep egrep`"
+    GEGREP="`findbin /{usr,opt}/{gnu,sfw}/bin/egrep gegrep /usr/xpg4/bin/egrep egrep`"
 [ -n "$GEGREP" ] && [ -x "$GEGREP" ] || throw "No GNU EGREP was found!"
 
 [ -z "$GSORT" ] && \
-    GSORT="`findbin gsort sort /usr/xpg4/bin/sort`"
+    GSORT="`findbin /{usr,opt}/{gnu,sfw}/bin/sort gsort sort /usr/xpg4/bin/sort`"
 [ -n "$GSORT" ] && [ -x "$GSORT" ] || throw "No GNU SORT was found!"
 
 [ -z "$GSED" ] && \
-    GSED="`findbin /usr/xpg4/bin/sed gsed sed`"
+    GSED="`findbin /{usr,opt}/{gnu,sfw}/bin/sed /usr/xpg4/bin/sed gsed sed`"
 [ -n "$GSED" ] && [ -x "$GSED" ] || throw "No GNU SED was found!"
 
 usage() {

From cf12fc9bb682e4ad1e6c86f4c3a9a2bac54ede18 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Mon, 11 Jan 2016 15:02:56 +0100
Subject: [PATCH 42/95] JSON.sh : refactored the CLI-driven logic into
 jsonsh_cli() and separated away setting and reporting of DEBUG variables with
 flag-locks so they are only reported once

---
 JSON.sh | 117 +++++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 78 insertions(+), 39 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index 70fdaf0..d13b27d 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -69,7 +69,7 @@ findbin() {
     # Locates a named binary or one from path, prints to stdout
     local BIN
     for P in "$@" ; do case "$P" in
-    /*) [ -x "$P" ] && BIN="$P" && break;;
+        /*) [ -x "$P" ] && BIN="$P" && break;;
         *) BIN="`which "$P" 2>/dev/null | tail -1`" && [ -n "$BIN" ] && [ -x "$BIN" ] && break || BIN="";;
     esac; done
     [ -n "$BIN" ] && [ -x "$BIN" ] && echo "$BIN" && return 0
@@ -268,8 +268,12 @@ parse_options() {
           TOXIC_NEWLINE=1
       ;;
       -d) DEBUG=$(($DEBUG+1))
+          JSONSH_DEBUGGING_SETUP=notdone
+          JSONSH_DEBUGGING_REPORT=notdone
       ;;
       -d=*) DEBUG="`echo "$1" | $GSED 's,^-d=,,' 2>/dev/null`"
+          JSONSH_DEBUGGING_SETUP=notdone
+          JSONSH_DEBUGGING_REPORT=notdone
       ;;
       -Q) COOKASTRING=1
       ;;
@@ -599,51 +603,86 @@ smart_parse() {
     fi
 }
 
-###########################################################
-### Active logic
+JSONSH_DEBUGGING_SETUP=notdone
+JSONSH_DEBUGGING_REPORT=notdone
+JSONSH_DEBUGGING_DEFAULTS=notdone
+jsonsh_debugging_defaults() {
+    [ x"$JSONSH_DEBUGGING_DEFAULTS" = xdone ] && return 0
+    ### Caller can disable specific debuggers by setting their level too high
+    validate_debuglevel
+    default_posval DEBUGLEVEL_PRINTPATHVAL		1
+    default_posval DEBUGLEVEL_PRINTTOKEN		2
+    default_posval DEBUGLEVEL_PRINTTOKEN_PIPELINE	3
+    default_posval DEBUGLEVEL_TRACE_X		4
+    default_posval DEBUGLEVEL_TRACE_V		5
+    default_posval DEBUGLEVEL_MERGE_ERROUT		4
+    JSONSH_DEBUGGING_DEFAULTS="done"
+}
 
-### Caller can disable specific debuggers by setting their level too high
-validate_debuglevel
-default_posval DEBUGLEVEL_PRINTPATHVAL		1
-default_posval DEBUGLEVEL_PRINTTOKEN		2
-default_posval DEBUGLEVEL_PRINTTOKEN_PIPELINE	3
-default_posval DEBUGLEVEL_TRACE_X		4
-default_posval DEBUGLEVEL_TRACE_V		5
-default_posval DEBUGLEVEL_MERGE_ERROUT		4
+jsonsh_debugging_setup() {
+    [ x"$JSONSH_DEBUGGING_SETUP" = xdone ] && return 0
+    # Note that the CLI options enable some debug level
 
-if ([ "$0" = "$BASH_SOURCE" ] || ! [ -n "$BASH_SOURCE" ]);
-then
-  parse_options "$@"
-  # Note that the options enable some debug level
-
-  [ "$DEBUG" -ge "$DEBUGLEVEL_MERGE_ERROUT" ] && \
-    exec 2>&1 && \
-    echo "[$$]DEBUG: Merge stderr and stdout for easier tracing with less" \
-	"(DEBUGLEVEL_MERGE_ERROUT=$DEBUGLEVEL_MERGE_ERROUT)" >&2
-  [ "$DEBUG" -gt 0 ] && \
-    echo "[$$]DEBUG: Enabled (debugging level $DEBUG)" >&2
-  [ "$DEBUG" -ge "$DEBUGLEVEL_PRINTPATHVAL" ] && \
-    echo "[$$]DEBUG: Enabled tracing of path:value printing decisions" \
-	"(DEBUGLEVEL_PRINTPATHVAL=$DEBUGLEVEL_PRINTPATHVAL)" >&2
-  [ "$DEBUG" -ge "$DEBUGLEVEL_PRINTTOKEN" ] && \
-    echo "[$$]DEBUG: Enabled printing of each processed token" \
-	"(DEBUGLEVEL_PRINTTOKEN=$DEBUGLEVEL_PRINTTOKEN)" >&2
-  [ "$DEBUG" -ge "$DEBUGLEVEL_PRINTTOKEN_PIPELINE" ] && \
-    echo "[$$]DEBUG: Enabled tracing of read-in token conversions" \
-	"(DEBUGLEVEL_PRINTTOKEN_PIPELINE=$DEBUGLEVEL_PRINTTOKEN_PIPELINE)" >&2
-  [ "$DEBUG" -ge "$DEBUGLEVEL_TRACE_V" ] && \
-    echo "[$$]DEBUG: Enable execution tracing (-v)" \
-	"(DEBUGLEVEL_TRACE_V=$DEBUGLEVEL_TRACE_V)" >&2 && \
-    set +v
-  [ "$DEBUG" -ge "$DEBUGLEVEL_TRACE_X" ] && \
-    echo "[$$]DEBUG: Enable execution tracing (-x)" \
-	"(DEBUGLEVEL_TRACE_X=$DEBUGLEVEL_TRACE_X)" >&2 && \
-    set -x
+    [ "$DEBUG" -ge "$DEBUGLEVEL_MERGE_ERROUT" ] && \
+        exec 2>&1
+    [ "$DEBUG" -ge "$DEBUGLEVEL_TRACE_V" ] && \
+        set +v
+    [ "$DEBUG" -ge "$DEBUGLEVEL_TRACE_X" ] && \
+        set -x
+
+    JSONSH_DEBUGGING_SETUP="done"
+}
+
+jsonsh_debugging_report() {
+    [ x"$JSONSH_DEBUGGING_REPORT" = xdone ] && return 0
+    # Note that the CLI options enable some debug level
+
+    [ "$DEBUG" -ge "$DEBUGLEVEL_MERGE_ERROUT" ] && \
+        echo "[$$]DEBUG: Merge stderr and stdout for easier tracing with less" \
+        "(DEBUGLEVEL_MERGE_ERROUT=$DEBUGLEVEL_MERGE_ERROUT)" >&2
+    [ "$DEBUG" -gt 0 ] && \
+        echo "[$$]DEBUG: Enabled (debugging level $DEBUG)" >&2
+    [ "$DEBUG" -ge "$DEBUGLEVEL_PRINTPATHVAL" ] && \
+        echo "[$$]DEBUG: Enabled tracing of path:value printing decisions" \
+        "(DEBUGLEVEL_PRINTPATHVAL=$DEBUGLEVEL_PRINTPATHVAL)" >&2
+    [ "$DEBUG" -ge "$DEBUGLEVEL_PRINTTOKEN" ] && \
+        echo "[$$]DEBUG: Enabled printing of each processed token" \
+        "(DEBUGLEVEL_PRINTTOKEN=$DEBUGLEVEL_PRINTTOKEN)" >&2
+    [ "$DEBUG" -ge "$DEBUGLEVEL_PRINTTOKEN_PIPELINE" ] && \
+        echo "[$$]DEBUG: Enabled tracing of read-in token conversions" \
+        "(DEBUGLEVEL_PRINTTOKEN_PIPELINE=$DEBUGLEVEL_PRINTTOKEN_PIPELINE)" >&2
+    [ "$DEBUG" -ge "$DEBUGLEVEL_TRACE_V" ] && \
+        echo "[$$]DEBUG: Enable execution tracing (-v)" \
+        "(DEBUGLEVEL_TRACE_V=$DEBUGLEVEL_TRACE_V)" >&2
+    [ "$DEBUG" -ge "$DEBUGLEVEL_TRACE_X" ] && \
+        echo "[$$]DEBUG: Enable execution tracing (-x)" \
+        "(DEBUGLEVEL_TRACE_X=$DEBUGLEVEL_TRACE_X)" >&2
+
+    JSONSH_DEBUGGING_REPORT="done"
+}
 
+jsonsh_cli() {
+  # All the logic needed to parse the CLI options and the JSON stdin
+  # for a common case "cat file.json | tokenize | parse" can suffice
+  # NOTE: If the caller sets up some specific different debugging envvars
+  # then consider changing JSONSH_DEBUGGING_SETUP and JSONSH_DEBUGGING_REPORT
+  # to e.g. "notdone" as well
+  parse_options "$@"
+  jsonsh_debugging_setup
+  jsonsh_debugging_report
   tee_stderr RAW_INPUT $DEBUGLEVEL_PRINTTOKEN_PIPELINE | \
   if [ "$COOKASTRING" -eq 1 ]; then
     cook_a_string
   else
     smart_parse
   fi
+}
+
+###########################################################
+### Active logic
+jsonsh_debugging_defaults
+
+if ([ "$0" = "$BASH_SOURCE" ] || ! [ -n "$BASH_SOURCE" ]);
+then
+  jsonsh_cli "$@"
 fi

From 26ecf00db9a1f67944330617f5bba3ed2e70cdf9 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Thu, 14 Jan 2016 15:18:38 +0100
Subject: [PATCH 43/95] JSON.sh : fixing detection of bash-sourcing

---
 JSON.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/JSON.sh b/JSON.sh
index d13b27d..b995c4b 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -682,7 +682,9 @@ jsonsh_cli() {
 ### Active logic
 jsonsh_debugging_defaults
 
-if ([ "$0" = "$BASH_SOURCE" ] || ! [ -n "$BASH_SOURCE" ]);
+# If not sourced into a bash script, parse stdin and quit
+if ([ "$0" = "$BASH_SOURCE[0]" ] || [ "$0" = "$BASH_SOURCE" ] || [ -z "${BASH-}" ]);
 then
   jsonsh_cli "$@"
+  exit $?
 fi

From 0833dbef7ffda54b03fb71e06773cdfca2f997ec Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Thu, 14 Jan 2016 15:42:36 +0100
Subject: [PATCH 44/95] JSON.sh : added cook_a_string_arg() routine (-QQ flag)

---
 JSON.sh | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index b995c4b..782e44e 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -64,6 +64,7 @@ NORMALIZE_NUMBERS_STRIP=0
 EXTRACT_JPATH=""
 TOXIC_NEWLINE=0
 COOKASTRING=0
+COOKASTRING_INPUT=""
 
 findbin() {
     # Locates a named binary or one from path, prints to stdout
@@ -146,6 +147,8 @@ usage() {
   echo "into a string valid for JSON (backslashes, quotes and newlines escaped,"
   echo "with no trailing newline); after cooking, the script exits:"
   echo '       COOKEDSTRING="`somecommand 2>&1 | JSON.sh -Q`"'
+  echo "A '-QQ' mode also exists to cook a (single) command-line argument:"
+  echo '       COOKEDSTRING="`JSON.sh -QQ "$SAVED_INPUT"`"'
   echo "This can also be used to pack JSON in JSON."
   echo
 }
@@ -277,6 +280,10 @@ parse_options() {
       ;;
       -Q) COOKASTRING=1
       ;;
+      -QQ) COOKASTRING=2
+          COOKASTRING_INPUT="$2"
+          shift
+      ;;
       ?*) echo "ERROR: Unknown option '$1'."
           usage
           exit 0
@@ -368,6 +375,16 @@ cook_a_string() {
     :
 }
 
+cook_a_string_arg() {
+    # Use routine above to cook a string passed as "$1" unless it is trivial
+    [[ -z "$1" ]] && return 0
+    [[ "$1" =~ ^[A-Za-z0-9\ \-\.\+\\\/\:\;\(\)\{\}]*$ ]] >/dev/null && \
+        echo "$1" && \
+        return 0
+
+    echo "$1" | cook_a_string
+}
+
 tokenize () {
   local GREP_O
   local ESCAPE
@@ -671,11 +688,11 @@ jsonsh_cli() {
   jsonsh_debugging_setup
   jsonsh_debugging_report
   tee_stderr RAW_INPUT $DEBUGLEVEL_PRINTTOKEN_PIPELINE | \
-  if [ "$COOKASTRING" -eq 1 ]; then
-    cook_a_string
-  else
-    smart_parse
-  fi
+  case "$COOKASTRING" in
+  1) cook_a_string ;;
+  2) cook_a_string_arg "$COOKASTRING_INPUT" ;;
+  *) smart_parse ;;
+  esac
 }
 
 ###########################################################

From 50c6372dc15eb0406e051e9b2bd997ff0afe17ed Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Thu, 14 Jan 2016 15:45:31 +0100
Subject: [PATCH 45/95] JSON.sh : added jsonsh_cli_subshell()

---
 JSON.sh | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/JSON.sh b/JSON.sh
index 782e44e..c623b72 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -695,6 +695,13 @@ jsonsh_cli() {
   esac
 }
 
+jsonsh_cli_subshell() (
+  # Same as above, but isolated in a subshell (no variables come back)
+  jsonsh_cli "$@"
+  exit $?
+)
+
+
 ###########################################################
 ### Active logic
 jsonsh_debugging_defaults

From 638c7114525fdf6415b08e9db9ce68ceb623e475 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Thu, 14 Jan 2016 16:06:25 +0100
Subject: [PATCH 46/95] JSON.sh : when using -QQ flag to call
 cook_a_string_arg() routine, ignore stdin

---
 JSON.sh | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index c623b72..3293398 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -149,7 +149,7 @@ usage() {
   echo '       COOKEDSTRING="`somecommand 2>&1 | JSON.sh -Q`"'
   echo "A '-QQ' mode also exists to cook a (single) command-line argument:"
   echo '       COOKEDSTRING="`JSON.sh -QQ "$SAVED_INPUT"`"'
-  echo "This can also be used to pack JSON in JSON."
+  echo "This can also be used to pack JSON in JSON. Note that '-QQ' ignores stdin."
   echo
 }
 
@@ -687,12 +687,19 @@ jsonsh_cli() {
   parse_options "$@"
   jsonsh_debugging_setup
   jsonsh_debugging_report
-  tee_stderr RAW_INPUT $DEBUGLEVEL_PRINTTOKEN_PIPELINE | \
-  case "$COOKASTRING" in
-  1) cook_a_string ;;
-  2) cook_a_string_arg "$COOKASTRING_INPUT" ;;
-  *) smart_parse ;;
-  esac
+  if [[ "$COOKASTRING" -eq 2 ]]; then
+    if [[ "$DEBUG" -ge "$DEBUGLEVEL_PRINTTOKEN" ]] || \
+       [[ "$DEBUG" -ge "$DEBUGLEVEL_PRINTTOKEN_PIPELINE" ]] ; then
+        echo "[$$]DEBUG: Cooking an argument into JSON string and exiting:" "$1" >&2
+    fi
+    cook_a_string_arg "$COOKASTRING_INPUT"
+  else
+    tee_stderr RAW_INPUT $DEBUGLEVEL_PRINTTOKEN_PIPELINE | \
+    case "$COOKASTRING" in
+      1) cook_a_string ;;
+      *) smart_parse ;;
+    esac
+  fi
 }
 
 jsonsh_cli_subshell() (

From 61b83060b93c9f1db83611995f83121b9d8c5aa0 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Thu, 14 Jan 2016 17:08:39 +0100
Subject: [PATCH 47/95] JSON.sh : make sure we run in BASH

---
 JSON.sh | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/JSON.sh b/JSON.sh
index 3293398..86edb35 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -45,6 +45,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+if [ -z "${BASH-}" ] ; then
+    # NOTE: This can break scripts which source this file and are not in bash
+    echo "ERROR: JSON.sh requires to be run with BASH interpreter! Subshelling..." >&2
+    /usr/bin/bash "$0" "$@"
+    exit $?
+fi
+
 throw () {
   echo "$*" >&2
   exit 1

From aeda42a3c01a0f08d8013c11b0ea1dc45282b60c Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Thu, 14 Jan 2016 17:09:31 +0100
Subject: [PATCH 48/95] JSON.sh : make sure strings with quotes and backslashes
 are cooked slowly (Solaris BASH needed this :\ go figure)

---
 JSON.sh | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index 86edb35..9d28609 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -385,10 +385,17 @@ cook_a_string() {
 cook_a_string_arg() {
     # Use routine above to cook a string passed as "$1" unless it is trivial
     [[ -z "$1" ]] && return 0
-    [[ "$1" =~ ^[A-Za-z0-9\ \-\.\+\\\/\:\;\(\)\{\}]*$ ]] >/dev/null && \
-        echo "$1" && \
+    # Strangely, for some OSes it does not suffice that all chars must be from
+    # the first pattern - should explicitly test that some are not forbidden
+    if ! [[ "$1" =~ [\\\"] ]] >/dev/null && \
+         [[ "$1" =~ ^[A-Za-z0-9\ \-\.\+\/\@\:\;\!\%\,\&\(\)\{\}]*$ ]] >/dev/null \
+    ; then
+        print_debug $DEBUGLEVEL_PRINTTOKEN_PIPELINE "cook_a_string_arg(): input trivial, not cooking: '$1'"
+        echo "$1"
         return 0
+    fi
 
+    print_debug $DEBUGLEVEL_PRINTTOKEN_PIPELINE "cook_a_string_arg(): input not trivial, cooking: '$1'"
     echo "$1" | cook_a_string
 }
 

From e26d26b7f627f4b4f65eb9184c640057587e3173 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Thu, 14 Jan 2016 17:19:40 +0100
Subject: [PATCH 49/95] cook-test.sh : adding a simple test for
 cook_a_string_arg()

---
 test/cook-test.sh | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100755 test/cook-test.sh

diff --git a/test/cook-test.sh b/test/cook-test.sh
new file mode 100755
index 0000000..2b8b999
--- /dev/null
+++ b/test/cook-test.sh
@@ -0,0 +1,30 @@
+#! /usr/bin/env bash
+
+cd ${0%/*}
+
+. ../JSON.sh
+
+cooktest() {
+    INPUT="$1"
+    EXPECT="$2"
+    i=$((i+1))
+    OUT="$(cook_a_string_arg "$INPUT")"
+    if [ $? = 0 -a x"$OUT" = x"$EXPECT" ]; then
+        echo "ok $i - '$INPUT' => '$OUT'"
+    else
+        echo "not ok $i - '$INPUT' => '$OUT' (expected '$EXPECT')"
+        fails=$((fails+1))
+    fi
+}
+
+fails=0
+i=0
+
+echo "1..4"
+cooktest 'a@b' 'a@b'
+cooktest 'a"b' 'a\"b'
+cooktest 'a\"b' 'a\\\"b'
+cooktest 'a b' 'a b'
+
+echo "$fails test(s) failed"
+exit $fails

From 28df951ed9e07379285a8f8be49b0ce0fb9f9cf8 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Thu, 14 Jan 2016 17:39:56 +0100
Subject: [PATCH 50/95] JSON.sh : converted all reasonable bracket-tests to
 builtin double-brackets

---
 JSON.sh | 148 ++++++++++++++++++++++++++++----------------------------
 1 file changed, 74 insertions(+), 74 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index 9d28609..ba13f8a 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -77,34 +77,34 @@ findbin() {
     # Locates a named binary or one from path, prints to stdout
     local BIN
     for P in "$@" ; do case "$P" in
-        /*) [ -x "$P" ] && BIN="$P" && break;;
-        *) BIN="`which "$P" 2>/dev/null | tail -1`" && [ -n "$BIN" ] && [ -x "$BIN" ] && break || BIN="";;
+        /*) [[ -x "$P" ]] && BIN="$P" && break;;
+        *) BIN="`which "$P" 2>/dev/null | tail -1`" && [[ -n "$BIN" ]] && [[ -x "$BIN" ]] && break || BIN="";;
     esac; done
-    [ -n "$BIN" ] && [ -x "$BIN" ] && echo "$BIN" && return 0
+    [[ -n "$BIN" ]] && [[ -x "$BIN" ]] && echo "$BIN" && return 0
     return 1
 }
 
 # May be passed by caller; also may pass AWK_OPTS *for it* then
-[ -z "$AWK" ] && AWK_OPTS="" && \
+[[ -z "$AWK" ]] && AWK_OPTS="" && \
     AWK="`findbin /usr/xpg4/bin/awk gawk nawk oawk awk`"
 # Error-checked in one optional place it may be needed
 
 # Different OSes have different greps... we like a GNU one
-[ -z "$GGREP" ] && \
+[[ -z "$GGREP" ]] && \
     GGREP="`findbin /{usr,opt}/{gnu,sfw}/bin/grep ggrep /usr/xpg4/bin/grep grep`"
-[ -n "$GGREP" ] && [ -x "$GGREP" ] || throw "No GNU GREP was found!"
+[[ -n "$GGREP" ]] && [[ -x "$GGREP" ]] || throw "No GNU GREP was found!"
 
-[ -z "$GEGREP" ] && \
+[[ -z "$GEGREP" ]] && \
     GEGREP="`findbin /{usr,opt}/{gnu,sfw}/bin/egrep gegrep /usr/xpg4/bin/egrep egrep`"
-[ -n "$GEGREP" ] && [ -x "$GEGREP" ] || throw "No GNU EGREP was found!"
+[[ -n "$GEGREP" ]] && [[ -x "$GEGREP" ]] || throw "No GNU EGREP was found!"
 
-[ -z "$GSORT" ] && \
+[[ -z "$GSORT" ]] && \
     GSORT="`findbin /{usr,opt}/{gnu,sfw}/bin/sort gsort sort /usr/xpg4/bin/sort`"
-[ -n "$GSORT" ] && [ -x "$GSORT" ] || throw "No GNU SORT was found!"
+[[ -n "$GSORT" ]] && [[ -x "$GSORT" ]] || throw "No GNU SORT was found!"
 
-[ -z "$GSED" ] && \
+[[ -z "$GSED" ]] && \
     GSED="`findbin /{usr,opt}/{gnu,sfw}/bin/sed /usr/xpg4/bin/sed gsed sed`"
-[ -n "$GSED" ] && [ -x "$GSED" ] || throw "No GNU SED was found!"
+[[ -n "$GSED" ]] && [[ -x "$GSED" ]] || throw "No GNU SED was found!"
 
 usage() {
   echo
@@ -162,8 +162,8 @@ usage() {
 
 validate_debuglevel() {
     ### Beside command-line, debugging can be enabled by envvars from the caller
-    [ x"$DEBUG" = xy -o x"$DEBUG" = xyes ] && DEBUG=1
-    [ -n "$DEBUG" -a "$DEBUG" -ge 0 ] 2>/dev/null || DEBUG=0
+    { [[ x"$DEBUG" = xy ]] || [[ x"$DEBUG" = xyes ]] ; } && DEBUG=1
+    [[ -n "$DEBUG" ]] && [[ "$DEBUG" -ge 0 ]] 2>/dev/null || DEBUG=0
 }
 
 unquote() {
@@ -174,7 +174,7 @@ unquote() {
 
 ### Empty and non-numeric and non-positive values should be filtered out here
 is_positive() {
-    [ -n "$1" -a "$1" -gt 0 ] 2>/dev/null
+    [[ -n "$1" ]] && [[ "$1" -gt 0 ]] 2>/dev/null
 }
 default_posval() {
     eval is_positive "\$$1" || eval "$1"="$2"
@@ -186,20 +186,20 @@ print_debug() {
     #   $2..	The message to print to stderr (if $DEBUG>=$1)
     local DL="$1"
     shift
-    [ "$DEBUG" -ge "$DL" ] 2>/dev/null && \
+    [[ "$DEBUG" -ge "$DL" ]] 2>/dev/null && \
         echo -E "[$$]DEBUG($DL): $@" >&2
     :
 }
 
 tee_stderr() {
     TEE_TAG="TEE_STDERR: "
-    [ -n "$1" ] && TEE_TAG="$1:"
-    [ -n "$2" -a "$2" -ge 0 ] 2>/dev/null && \
+    [[ -n "$1" ]] && TEE_TAG="$1:"
+    [[ -n "$2" ]] && [[ "$2" -ge 0 ]] 2>/dev/null && \
         TEE_DEBUG="$2" || \
         TEE_DEBUG=$DEBUGLEVEL_PRINTTOKEN_PIPELINE
 
     ### If debug is not enabled, skip tee'ing quickly with little impact
-    [ "$DEBUG" -lt "$TEE_DEBUG" ] 2>/dev/null && cat || \
+    [[ "$DEBUG" -lt "$TEE_DEBUG" ]] 2>/dev/null && cat || \
     while IFS= read -r LINE; do
         echo -E "$LINE"
         print_debug "$TEE_DEBUG" "$TEE_TAG" "$LINE"
@@ -210,7 +210,7 @@ tee_stderr() {
 parse_options() {
   set -- "$@"
   local ARGN=$#
-  while [ $ARGN -ne 0 ]
+  while [[ $ARGN -ne 0 ]]
   do
     case "$1" in
       -h) usage
@@ -303,14 +303,14 @@ parse_options() {
   validate_debuglevel
 
   # For normalized data, we do the whole job and just return the top object
-  [ "$NORMALIZE" -eq 1 ] && BRIEF=0 && LEAFONLY=0 && PRUNE=0
+  [[ "$NORMALIZE" -eq 1 ]] && BRIEF=0 && LEAFONLY=0 && PRUNE=0
 }
 
 awk_egrep () {
   local pattern_string=$1
 
-  [ -z "$AWK" ] && throw "No AWK found!"
-  [ ! -x "$AWK" ] && throw "Not executable AWK='$AWK'!"
+  [[ -z "$AWK" ]] && throw "No AWK found!"
+  [[ ! -x "$AWK" ]] && throw "Not executable AWK='$AWK'!"
 
   ${AWK} $AWK_OPTS '{
     while ($0) {
@@ -344,8 +344,8 @@ strip_newlines() {
     ODD="$(($NUMQ%2))"
     LINENUM="$(($LINENUM+1))"
 
-    if [ "$ODD" -eq 1 -a "$INSTRING" -eq 0 ]; then
-      [ "$TOXIC_NEWLINE" = 1 ] && \
+    if [[ "$ODD" -eq 1 ]] && [[ "$INSTRING" -eq 0 ]]; then
+      [[ "$TOXIC_NEWLINE" = 1 ]] && \
         echo "ERROR: Invalid JSON markup detected: newline in a string value: at line #$LINENUM" >&2 && \
         exit 121
       printf '%s\\n' "$ILINE"
@@ -353,18 +353,18 @@ strip_newlines() {
       continue
     fi
 
-    if [ "$ODD" -eq 1 -a "$INSTRING" -eq 1 ]; then
+    if [[ "$ODD" -eq 1 ]] && [[ "$INSTRING" -eq 1 ]]; then
       printf '%s\n' "$ILINE"
       INSTRING=0
       continue
     fi
 
-    if [ "$ODD" -eq 0 -a "$INSTRING" -eq 1 ]; then
+    if [[ "$ODD" -eq 0 ]] && [[ "$INSTRING" -eq 1 ]]; then
       printf '%s\\n' "$ILINE"
       continue
     fi
 
-    if [ "$ODD" -eq 0 -a "$INSTRING" -eq 0 ]; then
+    if [[ "$ODD" -eq 0 ]] && [[ "$INSTRING" -eq 0 ]]; then
       printf '%s\n' "$ILINE"
       continue
     fi
@@ -377,7 +377,7 @@ cook_a_string() {
     $GGREP '' | $GSED -e 's,\\,\\\\,g' -e 's,\",\\",g' -e 's,\t,\\t,g' 2>/dev/null | \
     { FIRST=''; while IFS="" read -r ILINE; do
       printf '%s%s' "$FIRST" "$ILINE"
-      [ -z "$FIRST" ] && FIRST='\n'
+      [[ -z "$FIRST" ]] && FIRST='\n'
       done; }
     :
 }
@@ -415,7 +415,7 @@ tokenize () {
     GREP_O="$GEGREP -o"
   fi
 
-  if [ -n "$GREP_O" ] && echo "test string" | $GREP_O "test" >/dev/null
+  if [[ -n "$GREP_O" ]] && echo "test string" | $GREP_O "test" >/dev/null
   then
     ESCAPE='(\\[^u[:cntrl:]]|\\u[0-9a-fA-F]{4})'
     CHAR='[^[:cntrl:]"\\]'
@@ -452,8 +452,8 @@ parse_array () {
         parse_value "$1" "$index"
         index=$((index+1))
         ary="$ary""$value"
-        if [ -n "$SORTDATA_ARR" ]; then
-            [ -z "$aryml" ] && aryml="$value" || aryml="$aryml
+        if [[ -n "$SORTDATA_ARR" ]]; then
+            [[ -z "$aryml" ]] && aryml="$value" || aryml="$aryml
 $value"
         fi
         read -r token
@@ -468,10 +468,10 @@ $value"
       done
       ;;
   esac
-  if [ -n "$SORTDATA_ARR" ]; then
+  if [[ -n "$SORTDATA_ARR" ]]; then
     ary="`echo -E "$aryml" | $SORTDATA_ARR | tr '\n' ',' | $GSED 's|,*$||' 2>/dev/null | $GSED 's|^,*||' 2>/dev/null`"
   fi
-  [ "$BRIEF" -eq 0 ] && value=`printf '[%s]' "$ary"` || value=
+  [[ "$BRIEF" -eq 0 ]] && value=`printf '[%s]' "$ary"` || value=
   :
 }
 
@@ -500,8 +500,8 @@ parse_object () {
         print_debug $DEBUGLEVEL_PRINTTOKEN "parse_object(3):" "token='$token'"
         parse_value "$1" "$key"
         obj="$obj$key:$value"
-        if [ -n "$SORTDATA_OBJ" ]; then
-            [ -z "$objml" ] && objml="$key:$value" || objml="$objml
+        if [[ -n "$SORTDATA_OBJ" ]]; then
+            [[ -z "$objml" ]] && objml="$key:$value" || objml="$objml
 $key:$value"
         fi
         read -r token
@@ -516,10 +516,10 @@ $key:$value"
       done
     ;;
   esac
-  if [ -n "$SORTDATA_OBJ" ]; then
+  if [[ -n "$SORTDATA_OBJ" ]]; then
     obj="`echo -E "$objml" | $SORTDATA_OBJ | tr '\n' ',' | $GSED 's|,*$||' 2>/dev/null | $GSED 's|^,*||' 2>/dev/null`"
   fi
-  [ "$BRIEF" -eq 0 ] && value=`printf '{%s}' "$obj"` || value=
+  [[ "$BRIEF" -eq 0 ]] && value=`printf '{%s}' "$obj"` || value=
   :
 }
 
@@ -528,13 +528,13 @@ parse_value () {
   local jpath="${1:+$1,}$2" isleaf=0 isempty=0 print=0
   case "$token" in
     '{') parse_object "$jpath"
-       [ "$value" = '{}' ] && isempty=1
+       [[ "$value" = '{}' ]] && isempty=1
        ;;
     '[') parse_array  "$jpath"
-       [ "$value" = '[]' ] && isempty=1
+       [[ "$value" = '[]' ]] && isempty=1
        ;;
     # At this point, the only valid single-character tokens are digits.
-    ''|[!0-9]) if [ "$ALLOWEMPTYINPUT" = 1 -a -z "$jpath" ] && [ -z "$token" ]; then
+    ''|[!0-9]) if [[ "$ALLOWEMPTYINPUT" = 1 ]] && [[ -z "$jpath" ]] && [[ -z "$token" ]]; then
             print_debug $DEBUGLEVEL_PRINTPATHVAL \
                 'Got a NULL document as input (no jpath, no token)' >&2
             value='{}'
@@ -544,14 +544,14 @@ parse_value () {
     +*|-*|[0-9]*|.*)  # Potential number - separate hit in case for efficiency
        print_debug $DEBUGLEVEL_PRINTPATHVAL \
             "token '$token' is a suspected number" >&2
-       if [ "$NORMALIZE_NUMBERS" = 1 ] && \
+       if [[ "$NORMALIZE_NUMBERS" = 1 ]] && \
          [[ "$token" =~ ${REGEX_NUMBER} ]] \
        ; then
             value="`printf "$NORMALIZE_NUMBERS_FORMAT" "$token"`" || \
             value=$token
             print_debug $DEBUGLEVEL_PRINTPATHVAL "normalized numeric token" \
                 "'$token' into '$value'" >&2
-            if [ "$NORMALIZE_NUMBERS_STRIP" = 1 ]; then
+            if [[ "$NORMALIZE_NUMBERS_STRIP" = 1 ]]; then
                 local valuetmp="`echo "$value" | $GSED -e 's,0*$,,g' -e 's,\.$,,' 2>/dev/null`" && \
                 value="$valuetmp"
                 unset valuetmp
@@ -561,43 +561,43 @@ parse_value () {
        else
         # Not a number or no normalization - process like default
             value=$token
-            [ "$NORMALIZE_SOLIDUS" -eq 1 ] && value=${value//\\\//\/}
+            [[ "$NORMALIZE_SOLIDUS" -eq 1 ]] && value=${value//\\\//\/}
        fi
        isleaf=1
-       [ "$value" = '""' -o "$value" = '' ] && isempty=1
+       { [[ "$value" = '""' ]] || [[ "$value" = '' ]] ; } && isempty=1
        ;;
     *) value=$token
        # if asked, replace solidus ("\/") in json strings with normalized value: "/"
-       [ "$NORMALIZE_SOLIDUS" -eq 1 ] && value=${value//\\\//\/}
+       [[ "$NORMALIZE_SOLIDUS" -eq 1 ]] && value=${value//\\\//\/}
        isleaf=1
-       [ "$value" = '""' ] && isempty=1
+       [[ "$value" = '""' ]] && isempty=1
        ;;
   esac
 
-  if [ "$NORMALIZE" -eq 1 ]; then
+  if [[ "$NORMALIZE" -eq 1 ]]; then
     # Ensure a "true" output from the "if" for "return"
-    if [ "$jpath" != '' ]; then : ; else
+    if [[ "$jpath" != '' ]]; then : ; else
         print_debug $DEBUGLEVEL_PRINTPATHVAL \
             "Non-root keys were skipped due to normalization mode"
-	printf "%s\n" "$value"
+        printf "%s\n" "$value"
     fi
     return
   fi
 
   ### Skip printing larger objects in brief mode
-  [ "$value" = '' ] && return
+  [[ "$value" = '' ]] && return
 
-  [ "$LEAFONLY" -eq 0 ] && [ "$PRUNE" -eq 0 ] && print=1
-  [ "$LEAFONLY" -eq 1 ] && [ "$isleaf" -eq 1 ] && [ $PRUNE -eq 0 ] && print=2
-  [ "$LEAFONLY" -eq 0 ] && [ "$PRUNE" -eq 1 ] && [ "$isempty" -eq 0 ] && print=3
-  [ "$LEAFONLY" -eq 1 ] && [ "$isleaf" -eq 1 ] && \
-    [ $PRUNE -eq 1 ] && [ $isempty -eq 0 ] && print=4
+  [[ "$LEAFONLY" -eq 0 ]] && [[ "$PRUNE" -eq 0 ]] && print=1
+  [[ "$LEAFONLY" -eq 1 ]] && [[ "$isleaf" -eq 1 ]] && [[ $PRUNE -eq 0 ]] && print=2
+  [[ "$LEAFONLY" -eq 0 ]] && [[ "$PRUNE" -eq 1 ]] && [[ "$isempty" -eq 0 ]] && print=3
+  [[ "$LEAFONLY" -eq 1 ]] && [[ "$isleaf" -eq 1 ]] && \
+    [[ $PRUNE -eq 1 ]] && [[ $isempty -eq 0 ]] && print=4
   ### A special case of an empty array or object - for leaf printing
   ### without pruning, we are interested in these:
-  [ "$LEAFONLY" -eq 1 ] && [ "$isleaf" -eq 0 ] && [ "$isempty" -eq 1 ] && \
-    [ $PRUNE -eq 0 ] && print=5
+  [[ "$LEAFONLY" -eq 1 ]] && [[ "$isleaf" -eq 0 ]] && [[ "$isempty" -eq 1 ]] && \
+    [[ $PRUNE -eq 0 ]] && print=5
 
-  if [ "$print" -ne 0 -a -n "$EXTRACT_JPATH" ] ; then
+  if [[ "$print" -ne 0 ]] && [[ -n "$EXTRACT_JPATH" ]] ; then
     ### BASH regex matching:
     [[ ${jpath} =~ ${EXTRACT_JPATH} ]] || print=-1
   fi
@@ -607,7 +607,7 @@ parse_value () {
 	"isleaf='$isleaf'/L='$LEAFONLY' isempty='$isempty'/P='$PRUNE':" \
 	"print='$print'" >&2
 
-  [ "$print" -gt 0 ] && printf "[%s]\t%s\n" "$jpath" "$value"
+  [[ "$print" -gt 0 ]] && printf "[%s]\t%s\n" "$jpath" "$value"
   :
 }
 
@@ -625,7 +625,7 @@ parse () {
 
 smart_parse() {
   strip_newlines | \
-  tokenize | if [ -n "$SORTDATA_OBJ$SORTDATA_ARR" ] ; then
+  tokenize | if [[ -n "$SORTDATA_OBJ$SORTDATA_ARR" ]] ; then
       ### Any type of sort was enabled
       ( NORMALIZE=1 LEAFONLY=0 BRIEF=0 parse ) \
       | tokenize | parse
@@ -638,7 +638,7 @@ JSONSH_DEBUGGING_SETUP=notdone
 JSONSH_DEBUGGING_REPORT=notdone
 JSONSH_DEBUGGING_DEFAULTS=notdone
 jsonsh_debugging_defaults() {
-    [ x"$JSONSH_DEBUGGING_DEFAULTS" = xdone ] && return 0
+    [[ x"$JSONSH_DEBUGGING_DEFAULTS" = xdone ]] && return 0
     ### Caller can disable specific debuggers by setting their level too high
     validate_debuglevel
     default_posval DEBUGLEVEL_PRINTPATHVAL		1
@@ -651,41 +651,41 @@ jsonsh_debugging_defaults() {
 }
 
 jsonsh_debugging_setup() {
-    [ x"$JSONSH_DEBUGGING_SETUP" = xdone ] && return 0
+    [[ x"$JSONSH_DEBUGGING_SETUP" = xdone ]] && return 0
     # Note that the CLI options enable some debug level
 
-    [ "$DEBUG" -ge "$DEBUGLEVEL_MERGE_ERROUT" ] && \
+    [[ "$DEBUG" -ge "$DEBUGLEVEL_MERGE_ERROUT" ]] && \
         exec 2>&1
-    [ "$DEBUG" -ge "$DEBUGLEVEL_TRACE_V" ] && \
+    [[ "$DEBUG" -ge "$DEBUGLEVEL_TRACE_V" ]] && \
         set +v
-    [ "$DEBUG" -ge "$DEBUGLEVEL_TRACE_X" ] && \
+    [[ "$DEBUG" -ge "$DEBUGLEVEL_TRACE_X" ]] && \
         set -x
 
     JSONSH_DEBUGGING_SETUP="done"
 }
 
 jsonsh_debugging_report() {
-    [ x"$JSONSH_DEBUGGING_REPORT" = xdone ] && return 0
+    [[ x"$JSONSH_DEBUGGING_REPORT" = xdone ]] && return 0
     # Note that the CLI options enable some debug level
 
-    [ "$DEBUG" -ge "$DEBUGLEVEL_MERGE_ERROUT" ] && \
+    [[ "$DEBUG" -ge "$DEBUGLEVEL_MERGE_ERROUT" ]] && \
         echo "[$$]DEBUG: Merge stderr and stdout for easier tracing with less" \
         "(DEBUGLEVEL_MERGE_ERROUT=$DEBUGLEVEL_MERGE_ERROUT)" >&2
-    [ "$DEBUG" -gt 0 ] && \
+    [[ "$DEBUG" -gt 0 ]] && \
         echo "[$$]DEBUG: Enabled (debugging level $DEBUG)" >&2
-    [ "$DEBUG" -ge "$DEBUGLEVEL_PRINTPATHVAL" ] && \
+    [[ "$DEBUG" -ge "$DEBUGLEVEL_PRINTPATHVAL" ]] && \
         echo "[$$]DEBUG: Enabled tracing of path:value printing decisions" \
         "(DEBUGLEVEL_PRINTPATHVAL=$DEBUGLEVEL_PRINTPATHVAL)" >&2
-    [ "$DEBUG" -ge "$DEBUGLEVEL_PRINTTOKEN" ] && \
+    [[ "$DEBUG" -ge "$DEBUGLEVEL_PRINTTOKEN" ]] && \
         echo "[$$]DEBUG: Enabled printing of each processed token" \
         "(DEBUGLEVEL_PRINTTOKEN=$DEBUGLEVEL_PRINTTOKEN)" >&2
-    [ "$DEBUG" -ge "$DEBUGLEVEL_PRINTTOKEN_PIPELINE" ] && \
+    [[ "$DEBUG" -ge "$DEBUGLEVEL_PRINTTOKEN_PIPELINE" ]] && \
         echo "[$$]DEBUG: Enabled tracing of read-in token conversions" \
         "(DEBUGLEVEL_PRINTTOKEN_PIPELINE=$DEBUGLEVEL_PRINTTOKEN_PIPELINE)" >&2
-    [ "$DEBUG" -ge "$DEBUGLEVEL_TRACE_V" ] && \
+    [[ "$DEBUG" -ge "$DEBUGLEVEL_TRACE_V" ]] && \
         echo "[$$]DEBUG: Enable execution tracing (-v)" \
         "(DEBUGLEVEL_TRACE_V=$DEBUGLEVEL_TRACE_V)" >&2
-    [ "$DEBUG" -ge "$DEBUGLEVEL_TRACE_X" ] && \
+    [[ "$DEBUG" -ge "$DEBUGLEVEL_TRACE_X" ]] && \
         echo "[$$]DEBUG: Enable execution tracing (-x)" \
         "(DEBUGLEVEL_TRACE_X=$DEBUGLEVEL_TRACE_X)" >&2
 

From 0c0f281b248563cad3766c4d91216859b6190101 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Tue, 7 Feb 2017 14:44:17 +0100
Subject: [PATCH 51/95] Bump copyright for Eaton-paid branch

---
 JSON.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/JSON.sh b/JSON.sh
index ba13f8a..1a3db73 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 #
 # Copyright (C) 2014-2015 Dominic Tarr
-# Copyright (C) 2015 Eaton
+# Copyright (C) 2015-2017 Eaton
 #
 #! \file    JSON.sh
 #  \brief   A json parser written in bash

From 5112ec671efe9fc19af6f0846b97a6fa54021225 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Tue, 7 Feb 2017 17:50:08 +0100
Subject: [PATCH 52/95] Convert all test scripts to use common plain-shell
 syntax

---
 all-tests.sh                   | 20 ++++++++---------
 test/cook-test.sh              | 13 ++++++-----
 test/invalid-test.sh           | 17 ++++++++-------
 test/no-head-test.sh           | 28 ++++++++++++++----------
 test/parse-test.sh             | 22 +++++++++++--------
 test/pedantic-empty-test.sh    | 27 ++++++++++++++---------
 test/solidus-test.sh           | 10 ++++-----
 test/tokenizer-test.sh         | 16 +++++++-------
 test/valid-test.sh             | 31 ++++++++++++++------------
 test/valid/generate-results.sh | 40 +++++++++++++++++-----------------
 10 files changed, 123 insertions(+), 101 deletions(-)

diff --git a/all-tests.sh b/all-tests.sh
index 84f826f..4fdc3ef 100755
--- a/all-tests.sh
+++ b/all-tests.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-cd ${0%/*}
+cd "$(dirname "$0")"
 
 #set -e
 fail=0
@@ -9,25 +9,25 @@ tests=0
 #echo PLAN ${#all_tests}
 for test in test/*.sh ;
 do
-  tests=$((tests+1))
-  echo TEST: $test
-  ./$test
+  tests="$(expr $tests + 1)"
+  echo "TEST: $test"
+  "./$test"
   ret=$?
   if [ $ret -eq 0 ] ; then
-    echo OK: ---- $test
-    passed=$((passed+1))
+    echo "OK: ---- $test"
+    passed="$(expr $passed + 1)"
   else
-    echo FAIL: $test $fail
-    fail=$((fail+ret))
+    echo "FAIL: $test ($ret)"
+    fail="$(expr $fail + 1)"
   fi
 done
 
-if [ $fail -eq 0 ]; then
+if [ "$fail" = 0 ]; then
   echo -n 'SUCCESS '
   exitcode=0
 else
   echo -n 'FAILURE '
   exitcode=1
 fi
-echo   $passed / $tests
+echo " $passed / $tests"
 exit $exitcode
diff --git a/test/cook-test.sh b/test/cook-test.sh
index 2b8b999..2ec9f6f 100755
--- a/test/cook-test.sh
+++ b/test/cook-test.sh
@@ -1,19 +1,20 @@
-#! /usr/bin/env bash
+#!/bin/sh
 
-cd ${0%/*}
+cd "$(dirname "$0")"
 
-. ../JSON.sh
+# Can't detect sourcing in sh, so immediately terminate the attempt to parse
+. ../JSON.sh </dev/null
 
 cooktest() {
     INPUT="$1"
     EXPECT="$2"
-    i=$((i+1))
+    i="$(expr $i + 1)"
     OUT="$(cook_a_string_arg "$INPUT")"
-    if [ $? = 0 -a x"$OUT" = x"$EXPECT" ]; then
+    if [ $? = 0 ] && [ x"$OUT" = x"$EXPECT" ]; then
         echo "ok $i - '$INPUT' => '$OUT'"
     else
         echo "not ok $i - '$INPUT' => '$OUT' (expected '$EXPECT')"
-        fails=$((fails+1))
+        fails="$(expr $fails+1)"
     fi
 }
 
diff --git a/test/invalid-test.sh b/test/invalid-test.sh
index 5d31034..2940f8a 100755
--- a/test/invalid-test.sh
+++ b/test/invalid-test.sh
@@ -1,29 +1,30 @@
 #!/bin/sh
 
-cd ${0%/*}
+cd "$(dirname "$0")"
 
 # make test output TAP compatible
 # http://en.wikipedia.org/wiki/Test_Anything_Protocol
 
 fails=0
-tests=`ls -1 invalid/* | wc -l`
+tests="`ls -1 invalid/* | wc -l`"
 
 echo "1..${tests##* }"
 for input in invalid/*
 do
-  i=$((i+1))
-  if ../JSON.sh < "$input" > /tmp/JSON.sh_outlog 2> /tmp/JSON.sh_errlog 
+  i="$(expr $i + 1)"
+  if ../JSON.sh < "$input" > /tmp/JSON.sh_outlog 2> /tmp/JSON.sh_errlog
   then
-    echo "not ok $i - cat $input | ../JSON.sh should fail"
+    echo "not ok $i - cat $input | ../JSON.sh should have failed"
     #this should be indented with '#' at the start.
     echo "OUTPUT WAS >>>"
     cat /tmp/JSON.sh_outlog
     echo "<<<"
-    fails=$((fails+1))
+    fails="$(expr $fails + 1)"
   else
-    echo "ok $i - $input was rejected"
-    echo "#" `cat /tmp/JSON.sh_errlog`
+    echo "ok $i - $input was rejected as expected"
+    echo "# `cat /tmp/JSON.sh_errlog`"
   fi
 done
+
 echo "$fails test(s) failed"
 exit $fails
diff --git a/test/no-head-test.sh b/test/no-head-test.sh
index a297b92..b69747e 100755
--- a/test/no-head-test.sh
+++ b/test/no-head-test.sh
@@ -1,27 +1,33 @@
 #!/bin/sh
 
-cd ${0%/*}
-tmp=${TEMP:-/tmp}
-tmp=${tmp%%/}/ # Avoid duplicate //
+cd "$(dirname "$0")"
+[ -n "${tmp-}" ] || tmp="/tmp"
+
+# Avoid duplicate // in plain-shell syntax
+tmp="$(echo "$tmp" | sed 's,/+,/,g')"
+case "$tmp" in
+    */) ;;
+    *)  tmp="$tmp/" ;;
+esac
 
 fails=0
 i=0
-tests=`ls valid/*.json | wc -l`
+tests="$(ls -1 valid/*.json | wc -l)"
 echo "1..$tests"
 for input in valid/*.json
 do
-  input_file=${input##*/}
-  expected="${tmp}${input_file%.json}.no-head"
-  egrep -v '^\[]' < ${input%.json}.parsed > $expected
-  i=$((i+1))
-  if ! ../JSON.sh -n < "$input" | diff -u - "$expected" 
+  expected="${tmp}$(basename "$input" .json).no-head"
+  egrep -v '^\[]' < "$(dirname "$input")/$(basename "$input" .json).parsed" > "$expected"
+  i="$(expr $i + 1)"
+  if ! ../JSON.sh -n < "$input" | diff -u - "$expected"
   then
     echo "not ok $i - $input"
-    fails=$((fails+1))
+    fails="$(expr $fails + 1)"
   else
-    echo "ok $i - $input"    
+    echo "ok $i - $input"
   fi
 done
+
 echo "$fails test(s) failed"
 exit $fails
 
diff --git a/test/parse-test.sh b/test/parse-test.sh
index fb2cfdf..bf673fd 100755
--- a/test/parse-test.sh
+++ b/test/parse-test.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-cd ${0%/*}
+cd "$(dirname "$0")"
 
 # Can't detect sourcing in sh, so immediately terminate the attempt to parse
 . ../JSON.sh </dev/null
@@ -12,23 +12,27 @@ ptest () {
 fails=0
 i=0
 echo "1..4"
-for input in '"oooo"  ' '[true, 1, [0, {}]]  ' '{"true": 1}'
-do
-  i=$((i+1))
-  if echo "$input" | ptest 
+for input in \
+    '"oooo"  ' \
+    '[true, 1, [0, {}]]  ' \
+    '{"true": 1}' \
+; do
+  i="$(expr $i + 1)"
+  if echo "$input" | ptest
   then
     echo "ok $i - $input"
   else
     echo "not ok $i - $input"
-    fails=$((fails+1))
+    fails="$(expr $fails + 1)"
   fi
 done
 
+i="$(expr $i + 1)"
 if ! ptest < ../package.json
 then
-  echo "not ok 4 - Parsing package.json failed!"
-  fails=$((fails+1))
-else 
+  echo "not ok $i - Parsing package.json failed!"
+  fails="$(expr $fails + 1)"
+else
   echo "ok $i - package.json"
 fi
 
diff --git a/test/pedantic-empty-test.sh b/test/pedantic-empty-test.sh
index a222758..fef91ca 100755
--- a/test/pedantic-empty-test.sh
+++ b/test/pedantic-empty-test.sh
@@ -1,6 +1,6 @@
-#! /usr/bin/env bash
+#!/bin/sh
 
-cd ${0%/*}
+cd "$(dirname "$0")"
 
 # make test output TAP compatible
 # http://en.wikipedia.org/wiki/Test_Anything_Protocol
@@ -9,26 +9,33 @@ fails=0
 tests=8
 i=0
 
-echo "1..${tests##* }"
+echo "1..${tests}"
 
-for DOC in "" " " "
-" "
+# Note: weird whitespaces and end-of-lines below are intended
+# TODO: populate by printf?
+for DOC in \
+    "" \
+    " " \
+    "
+" \
+"
   
- "; do
-  i=$(($i+1))
+ " \
+; do
+  i="$(expr $i + 1)"
   if echo "$DOC" | ../JSON.sh
   then
     echo "ok $i - empty input '$DOC' is okay in non-pedantic mode"
   else
     echo "not ok $i - empty input '$DOC' was rejected in non-pedantic mode"
-    fails=$((fails+1))
+    fails="$(expr $fails + 1)"
   fi
 
-  i=$(($i+1))
+  i="$(expr $i + 1)"
   if echo "$DOC" | ../JSON.sh -P
   then
     echo "not ok $i - empty input '$DOC' should be rejected in pedantic mode"
-    fails=$((fails+1))
+    fails="$(expr $fails + 1)"
   else
     echo "ok $i - empty input '$DOC' was rejected in pedantic mode"
   fi
diff --git a/test/solidus-test.sh b/test/solidus-test.sh
index 9910937..c349eb3 100755
--- a/test/solidus-test.sh
+++ b/test/solidus-test.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-cd ${0%/*}
+cd "$(dirname "$0")"
 
 INPUT=./solidus/string_with_solidus.json
 OUTPUT_ESCAPED=./solidus/string_with_solidus.with-escaping.parsed
@@ -10,16 +10,16 @@ FAILS=0
 
 echo "1..2"
 
-if ! ../JSON.sh < $INPUT| diff -u - ${OUTPUT_ESCAPED}; then
+if ! ../JSON.sh < "$INPUT" | diff -u - "${OUTPUT_ESCAPED}" ; then
   echo "not ok - JSON.sh run without -s option should leave solidus escaping intact"
-  FAILS=$((FAILS + 1))
+  FAILS="$(expr $FAILS + 1)"
 else
   echo "ok $i - solidus escaping was left intact"
 fi
 
-if ! ../JSON.sh -s < $INPUT| diff -u - ${OUTPUT_WITHOUT_ESCAPING}; then
+if ! ../JSON.sh -s < "$INPUT" | diff -u - "${OUTPUT_WITHOUT_ESCAPING}" ; then
   echo "not ok - JSON.sh run with -s option should remove solidus escaping"
-  FAILS=$((FAILS+1))
+  FAILS="$(expr $FAILS + 1)"
 else
   echo "ok $i - solidus escaping has been removed"
 fi
diff --git a/test/tokenizer-test.sh b/test/tokenizer-test.sh
index 8442a99..1d1a441 100755
--- a/test/tokenizer-test.sh
+++ b/test/tokenizer-test.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-cd ${0%/*}
+cd "$(dirname "$0")"
 
 # Can't detect sourcing in sh, so immediately terminate the attempt to parse
 . ../JSON.sh </dev/null
@@ -8,16 +8,16 @@ cd ${0%/*}
 i=0
 fails=0
 ttest () {
-  i=$((i+1))
-  local input="$1"; shift
-  local expected="$(printf '%s\n' "$@")"
+  i="$(expr $i + 1)"
+  input="$1"; shift
+  expected="$(printf '%s\n' "$@")"
   echo "$expected" > /tmp/json_ttest_expected
   if echo "$input" | tokenize | diff -u - /tmp/json_ttest_expected
   then
-    echo "ok $i - $input"    
-  else 
+    echo "ok $i - $input"
+  else
     echo "not ok $i - $input"
-    fails=$((fails+1))
+    fails="$(expr $fails + 1)"
   fi
 }
 
@@ -48,7 +48,7 @@ ttest '{"e": "string"}'  '{' '"e"' ':' '"string"' '}'
 
 if ! cat ../package.json | tokenize >/dev/null
 then
-  fails=$((fails+1))
+  fails="$(expr $fails + 1)"
   echo "Tokenizing package.json failed!"
 fi
 
diff --git a/test/valid-test.sh b/test/valid-test.sh
index c1dfd51..3c8be73 100755
--- a/test/valid-test.sh
+++ b/test/valid-test.sh
@@ -9,7 +9,8 @@ LANG=C
 LC_ALL=C
 export LANG LC_ALL
 
-cd ${0%/*}
+cd "$(dirname "$0")"
+
 fails=0
 passes=0
 skips=0
@@ -19,13 +20,13 @@ i=0
 CHOMPEXT='\.\(parsed\|sorted\|numnormalized\|normalized\|json\).*$'
 [ $# -gt 0 ] && \
     FILES="$(for F in "$@"; do echo valid/"`basename "$F" | sed "s,${CHOMPEXT},,"`".json ; done | sort | uniq)" || \
-    FILES="`ls -1 valid/*.json`"
+    FILES="$(ls -1 valid/*.json)"
 
 [ -z "$FILES" ] && echo "error - no files found to test!" >&2 && exit 1
 
-tests="`echo "$FILES" | wc -l`"
+tests="$(echo "$FILES" | wc -l)"
 ### We currently have up to 8 extensions to consider per test
-tests="$(expr $tests * 8)"
+tests="$(expr $tests \* 8)"
 echo "1..$tests"
 
 for input in $FILES
@@ -36,13 +37,14 @@ do
   ; do
     if [ ! -f "$input" ]; then
       echo "error - missing input file '$input', assuming all its tests failed"
-      fails=$(expr $fails+8)
+      fails="$(expr $fails + 8)"
       break
     fi
 
-    expected="${input%.json}.$EXT"
+#    expected="${input%.json}.$EXT"
+    expected="$(dirname "$input")/$(basename "$input" .json).$EXT"
     if [ -f "$expected" -o -n "$JSON_TEST_GENERATE" ]; then
-      i=$(expr $i + 1)
+      i="$(expr $i + 1)"
       case "$EXT" in
         sorted) OPTIONS="-S='-n -r'" ;;
         normalized) OPTIONS="-N" ;;
@@ -59,30 +61,31 @@ do
         if ! eval ../JSON.sh $OPTIONS < "$input" > "$expected"
         then
           echo "generation not ok $i - $input $EXT"
-          fails=$((fails+1))
+          fails="$(expr $fails + 1)"
           mv -f "$expected" "$expected.failed"
         else
           echo "generation ok $i - $input $EXT"
-          passes=$(expr $passes + 1)
-          generated=$(expr $generated + 1)
+          passes="$(expr $passes + 1)"
+          generated="$(expr $generated + 1)"
         fi
         continue
       fi
 
-      if ! eval ../JSON.sh $OPTIONS < "$input" | diff -u - "$expected" 
+      if ! eval ../JSON.sh $OPTIONS < "$input" | diff -u - "$expected"
       then
         echo "not ok $i - $input $EXT"
-        fails=$(expr $fails + 1)
+        fails="$(expr $fails + 1)"
       else
         echo "ok $i - $input $EXT"
-        passes=$(expr $passes + 1)
+        passes="$(expr $passes + 1)"
       fi
     else
       # echo "skip (missing result file) - $input $EXT"
-      skips=$(expr $skips + 1)
+      skips="$(expr $skips + 1)"
     fi
   done
 done
+
 [ -n "$JSON_TEST_GENERATE" ] && echo "$generated expected results generated"
 [ -n "$skips" ] && echo "$skips test(s) skipped (missing expected results file)"
 echo "$passes test(s) succeeded"
diff --git a/test/valid/generate-results.sh b/test/valid/generate-results.sh
index 7bdf4a7..5a6b781 100755
--- a/test/valid/generate-results.sh
+++ b/test/valid/generate-results.sh
@@ -10,38 +10,38 @@ export C
 JSONSH=../../JSON.sh
 
 generate() {
-	F="$1"
-	[ -s "$F" ] || return
+    F="$1"
+    [ -s "$F" ] || return
 
-	B="`basename "$F" .json`"
-	echo "=== Generating results for '$F'..."
-	RES=0
+    B="$(basename "$F" .json)"
+    echo "=== Generating results for '$F'..."
+    RES=0
 
-	EXT=parsed
-	$JSONSH < "$F" > "$B.$EXT" || \
-	    { RES=$?; echo "ERROR with $EXT"; }
+    EXT=parsed
+    $JSONSH < "$F" > "$B.$EXT" || \
+        { RES=$?; echo "ERROR with $EXT"; }
 
-	EXT=sorted
-	$JSONSH -S="-n -r" < "$F" > "$B.$EXT" || \
-	    { RES=$?; echo "ERROR with $EXT"; }
+    EXT=sorted
+    $JSONSH -S="-n -r" < "$F" > "$B.$EXT" || \
+        { RES=$?; echo "ERROR with $EXT"; }
 
-	EXT=normalized
-	$JSONSH -N < "$F" > "$B.$EXT" || \
-	    { RES=$?; echo "ERROR with $EXT"; }
+    EXT=normalized
+    $JSONSH -N < "$F" > "$B.$EXT" || \
+        { RES=$?; echo "ERROR with $EXT"; }
 
-	EXT=normalized_sorted
-	$JSONSH -N='-n' < "$F" > "$B.$EXT" || \
-	    { RES=$?; echo "ERROR with $EXT"; }
+    EXT=normalized_sorted
+    $JSONSH -N='-n' < "$F" > "$B.$EXT" || \
+        { RES=$?; echo "ERROR with $EXT"; }
 
-	return $RES
+    return $RES
 }
 
 if [ $# -gt 0 ]; then
     for F in "$@" ; do
-	generate "$F"
+        generate "$F"
     done
 else
     for F in *.json ; do
-	generate "$F"
+        generate "$F"
     done
 fi

From 1fc69fa581bc9a0131043ec63cfd89a25b5ef222 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Tue, 7 Feb 2017 18:15:16 +0100
Subject: [PATCH 53/95] JSON.sh : improve protability to older shells

---
 JSON.sh | 49 +++++++++++++++++++++++++------------------------
 1 file changed, 25 insertions(+), 24 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index 7115f2e..b1e08c2 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -86,7 +86,7 @@ COOKASTRING_INPUT=""
 
 findbin() {
     # Locates a named binary or one from path, prints to stdout
-    local BIN
+    BIN=""
     for P in "$@" ; do case "$P" in
         /*) [ -x "$P" ] && BIN="$P" && break;;
         *) BIN="$(which "$P" 2>/dev/null | tail -1)" && [ -n "$BIN" ] && [ -x "$BIN" ] && break || BIN="";;
@@ -174,8 +174,8 @@ usage() {
 
 validate_debuglevel() {
     ### Beside command-line, debugging can be enabled by envvars from the caller
-    { [[ x"$DEBUG" = xy ]] || [[ x"$DEBUG" = xyes ]] ; } && DEBUG=1
-    [[ -n "$DEBUG" ]] && [[ "$DEBUG" -ge 0 ]] 2>/dev/null || DEBUG=0
+    { [ x"$DEBUG" = xy ] || [ x"$DEBUG" = xyes ] ; } && DEBUG=1
+    [ -n "$DEBUG" ] && [ "$DEBUG" -ge 0 ] 2>/dev/null || DEBUG=0
 }
 
 unquote() {
@@ -186,7 +186,7 @@ unquote() {
 
 ### Empty and non-numeric and non-positive values should be filtered out here
 is_positive() {
-    [[ -n "$1" ]] && [[ "$1" -gt 0 ]] 2>/dev/null
+    [ -n "$1" ] && [ "$1" -gt 0 ] 2>/dev/null
 }
 default_posval() {
     eval is_positive "\$$1" || eval "$1"="$2"
@@ -194,24 +194,24 @@ default_posval() {
 
 print_debug() {
     # Required params:
-    #   $1	Debug level of the message
-    #   $2..	The message to print to stderr (if $DEBUG>=$1)
-    local DL="$1"
+    #   $1    Debug level of the message
+    #   $2..  The message to print to stderr (if $DEBUG>=$1)
+    DL="$1"
     shift
-    [[ "$DEBUG" -ge "$DL" ]] 2>/dev/null && \
+    [ "$DEBUG" -ge "$DL" ] 2>/dev/null && \
         echo -E "[$$]DEBUG($DL): $@" >&2
     :
 }
 
 tee_stderr() {
     TEE_TAG="TEE_STDERR: "
-    [[ -n "$1" ]] && TEE_TAG="$1:"
-    [[ -n "$2" ]] && [[ "$2" -ge 0 ]] 2>/dev/null && \
+    [ -n "$1" ] && TEE_TAG="$1:"
+    [ -n "$2" ] && [ "$2" -ge 0 ] 2>/dev/null && \
         TEE_DEBUG="$2" || \
         TEE_DEBUG=$DEBUGLEVEL_PRINTTOKEN_PIPELINE
 
     ### If debug is not enabled, skip tee'ing quickly with little impact
-    [[ "$DEBUG" -lt "$TEE_DEBUG" ]] 2>/dev/null && cat || \
+    [ "$DEBUG" -lt "$TEE_DEBUG" ] 2>/dev/null && cat || \
     while IFS= read -r LINE; do
         echo -E "$LINE"
         print_debug "$TEE_DEBUG" "$TEE_TAG" "$LINE"
@@ -451,12 +451,13 @@ tokenize () {
   local SPACE='[[:space:]]+'
 
   # Force zsh to expand $A into multiple words
-  local is_wordsplit_disabled=$(unsetopt 2>/dev/null | grep -c '^shwordsplit$')
-  if [ $is_wordsplit_disabled != 0 ]; then setopt shwordsplit; fi
+  is_wordsplit_disabled="$(unsetopt 2>/dev/null | grep -c '^shwordsplit$')"
+  if [ "$is_wordsplit_disabled" != 0 ]; then setopt shwordsplit; fi
   tee_stderr BEFORE_TOKENIZER $DEBUGLEVEL_PRINTTOKEN_PIPELINE | \
   $GREP_O "$STRING|$NUMBER|$KEYWORD|$SPACE|." | $GEGREP -v "^$SPACE$" | \
   tee_stderr AFTER_TOKENIZER $DEBUGLEVEL_PRINTTOKEN_PIPELINE
-  if [ $is_wordsplit_disabled != 0 ]; then unsetopt shwordsplit; fi
+  if [ "$is_wordsplit_disabled" != 0 ]; then unsetopt shwordsplit; fi
+  unset is_wordsplit_disabled
 }
 
 parse_array () {
@@ -491,7 +492,7 @@ $value"
   if [ -n "$SORTDATA_ARR" ]; then
     ary="$(echo -E "$aryml" | $SORTDATA_ARR | tr '\n' ',' | $GSED 's|,*$||' 2>/dev/null | $GSED 's|^,*||' 2>/dev/null)"
   fi
-  [ "$BRIEF" = 0 ] && value=$(printf '[%s]' "$ary") || value=
+  [ "$BRIEF" = 0 ] && value="$(printf '[%s]' "$ary")" || value=""
   :
 }
 
@@ -520,8 +521,8 @@ parse_object () {
         print_debug $DEBUGLEVEL_PRINTTOKEN "parse_object(3):" "token='$token'"
         parse_value "$1" "$key"
         obj="$obj$key:$value"
-        if [[ -n "$SORTDATA_OBJ" ]]; then
-            [[ -z "$objml" ]] && objml="$key:$value" || objml="$objml
+        if [ -n "$SORTDATA_OBJ" ]; then
+            [ -z "$objml" ] && objml="$key:$value" || objml="$objml
 $key:$value"
         fi
         read -r token
@@ -539,7 +540,7 @@ $key:$value"
   if [ -n "$SORTDATA_OBJ" ]; then
     obj="$(echo -E "$objml" | $SORTDATA_OBJ | tr '\n' ',' | $GSED 's|,*$||' 2>/dev/null | $GSED 's|^,*||' 2>/dev/null)"
   fi
-  [ "$BRIEF" = 0 ] && value=$(printf '{%s}' "$obj") || value=
+  [ "$BRIEF" = 0 ] && value="$(printf '{%s}' "$obj")" || value=""
   :
 }
 
@@ -548,10 +549,10 @@ parse_value () {
   local jpath="${1:+$1,}$2" isleaf=0 isempty=0 print=0
   case "$token" in
     '{') parse_object "$jpath"
-       [[ "$value" = '{}' ]] && isempty=1
+       [ "$value" = '{}' ] && isempty=1
        ;;
     '[') parse_array  "$jpath"
-       [[ "$value" = '[]' ]] && isempty=1
+       [ "$value" = '[]' ] && isempty=1
        ;;
     # At this point, the only valid single-character tokens are digits.
     ''|[!0-9]) if [ "$ALLOWEMPTYINPUT" = 1 ] && [ -z "$jpath" ] && [ -z "$token" ]; then
@@ -565,7 +566,7 @@ parse_value () {
        print_debug $DEBUGLEVEL_PRINTPATHVAL \
             "token '$token' is a suspected number" >&2
        # TODO: Bash regex and more if's
-       if [[ "$NORMALIZE_NUMBERS" = 1 ]] && \
+       if [ "$NORMALIZE_NUMBERS" = 1 ] && \
          [[ "$token" =~ ${REGEX_NUMBER} ]] \
        ; then
             value="$(printf "$NORMALIZE_NUMBERS_FORMAT" "$token")" || \
@@ -706,9 +707,9 @@ jsonsh_debugging_report() {
     [ "$DEBUG" -ge "$DEBUGLEVEL_MERGE_ERROUT" ] && \
         echo "[$$]DEBUG: Merge stderr and stdout for easier tracing with less" \
         "(DEBUGLEVEL_MERGE_ERROUT=$DEBUGLEVEL_MERGE_ERROUT)" >&2
-    [ "$DEBUG" -gt 0 ]] && \
+    [ "$DEBUG" -gt 0 ] && \
         echo "[$$]DEBUG: Enabled (debugging level $DEBUG)" >&2
-    [ "$DEBUG" -ge "$DEBUGLEVEL_PRINTPATHVAL" ]] && \
+    [ "$DEBUG" -ge "$DEBUGLEVEL_PRINTPATHVAL" ] && \
         echo "[$$]DEBUG: Enabled tracing of path:value printing decisions" \
         "(DEBUGLEVEL_PRINTPATHVAL=$DEBUGLEVEL_PRINTPATHVAL)" >&2
     [ "$DEBUG" -ge "$DEBUGLEVEL_PRINTTOKEN" ] && \
@@ -736,7 +737,7 @@ jsonsh_cli() {
   parse_options "$@"
   jsonsh_debugging_setup
   jsonsh_debugging_report
-  if [[ "$COOKASTRING" -eq 2 ]]; then
+  if [ "$COOKASTRING" -eq 2 ]; then
     if [ "$DEBUG" -ge "$DEBUGLEVEL_PRINTTOKEN" ] || \
        [ "$DEBUG" -ge "$DEBUGLEVEL_PRINTTOKEN_PIPELINE" ] ; then
         echo "[$$]DEBUG: Cooking an argument into JSON string and exiting:" "$1" >&2

From 282ec906073b55b46135ec51ed106595b698a8cc Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Tue, 7 Feb 2017 18:16:23 +0100
Subject: [PATCH 54/95] Test scripts : use sourced script and jsonsh_cli() so
 as to test in same interpreter as the test-driver script

---
 all-tests.sh                | 4 ++++
 test/invalid-test.sh        | 5 ++++-
 test/no-head-test.sh        | 6 +++++-
 test/pedantic-empty-test.sh | 7 +++++--
 test/solidus-test.sh        | 7 +++++--
 test/valid-test.sh          | 7 +++++--
 6 files changed, 28 insertions(+), 8 deletions(-)

diff --git a/all-tests.sh b/all-tests.sh
index 4fdc3ef..1d09d82 100755
--- a/all-tests.sh
+++ b/all-tests.sh
@@ -11,6 +11,10 @@ for test in test/*.sh ;
 do
   tests="$(expr $tests + 1)"
   echo "TEST: $test"
+  # TODO: find a way to use the current shell-interpreter program to
+  # run sub-tests (simple sourcing fails ATM because scripts start
+  # with "cd `dirname $0`")...
+  #( . "./$test" )
   "./$test"
   ret=$?
   if [ $ret -eq 0 ] ; then
diff --git a/test/invalid-test.sh b/test/invalid-test.sh
index 2940f8a..4c1884b 100755
--- a/test/invalid-test.sh
+++ b/test/invalid-test.sh
@@ -2,6 +2,9 @@
 
 cd "$(dirname "$0")"
 
+# Can't detect sourcing in sh, so immediately terminate the attempt to parse
+. ../JSON.sh </dev/null
+
 # make test output TAP compatible
 # http://en.wikipedia.org/wiki/Test_Anything_Protocol
 
@@ -12,7 +15,7 @@ echo "1..${tests##* }"
 for input in invalid/*
 do
   i="$(expr $i + 1)"
-  if ../JSON.sh < "$input" > /tmp/JSON.sh_outlog 2> /tmp/JSON.sh_errlog
+  if jsonsh_cli < "$input" > /tmp/JSON.sh_outlog 2> /tmp/JSON.sh_errlog
   then
     echo "not ok $i - cat $input | ../JSON.sh should have failed"
     #this should be indented with '#' at the start.
diff --git a/test/no-head-test.sh b/test/no-head-test.sh
index b69747e..6306464 100755
--- a/test/no-head-test.sh
+++ b/test/no-head-test.sh
@@ -1,6 +1,10 @@
 #!/bin/sh
 
 cd "$(dirname "$0")"
+
+# Can't detect sourcing in sh, so immediately terminate the attempt to parse
+. ../JSON.sh </dev/null
+
 [ -n "${tmp-}" ] || tmp="/tmp"
 
 # Avoid duplicate // in plain-shell syntax
@@ -19,7 +23,7 @@ do
   expected="${tmp}$(basename "$input" .json).no-head"
   egrep -v '^\[]' < "$(dirname "$input")/$(basename "$input" .json).parsed" > "$expected"
   i="$(expr $i + 1)"
-  if ! ../JSON.sh -n < "$input" | diff -u - "$expected"
+  if ! jsonsh_cli -n < "$input" | diff -u - "$expected"
   then
     echo "not ok $i - $input"
     fails="$(expr $fails + 1)"
diff --git a/test/pedantic-empty-test.sh b/test/pedantic-empty-test.sh
index fef91ca..efcd46c 100755
--- a/test/pedantic-empty-test.sh
+++ b/test/pedantic-empty-test.sh
@@ -2,6 +2,9 @@
 
 cd "$(dirname "$0")"
 
+# Can't detect sourcing in sh, so immediately terminate the attempt to parse
+. ../JSON.sh </dev/null
+
 # make test output TAP compatible
 # http://en.wikipedia.org/wiki/Test_Anything_Protocol
 
@@ -23,7 +26,7 @@ for DOC in \
  " \
 ; do
   i="$(expr $i + 1)"
-  if echo "$DOC" | ../JSON.sh
+  if echo "$DOC" | jsonsh_cli
   then
     echo "ok $i - empty input '$DOC' is okay in non-pedantic mode"
   else
@@ -32,7 +35,7 @@ for DOC in \
   fi
 
   i="$(expr $i + 1)"
-  if echo "$DOC" | ../JSON.sh -P
+  if echo "$DOC" | jsonsh_cli -P
   then
     echo "not ok $i - empty input '$DOC' should be rejected in pedantic mode"
     fails="$(expr $fails + 1)"
diff --git a/test/solidus-test.sh b/test/solidus-test.sh
index c349eb3..14d35fe 100755
--- a/test/solidus-test.sh
+++ b/test/solidus-test.sh
@@ -2,6 +2,9 @@
 
 cd "$(dirname "$0")"
 
+# Can't detect sourcing in sh, so immediately terminate the attempt to parse
+. ../JSON.sh </dev/null
+
 INPUT=./solidus/string_with_solidus.json
 OUTPUT_ESCAPED=./solidus/string_with_solidus.with-escaping.parsed
 OUTPUT_WITHOUT_ESCAPING=./solidus/string_with_solidus.no-escaping.parsed
@@ -10,14 +13,14 @@ FAILS=0
 
 echo "1..2"
 
-if ! ../JSON.sh < "$INPUT" | diff -u - "${OUTPUT_ESCAPED}" ; then
+if ! jsonsh_cli < "$INPUT" | diff -u - "${OUTPUT_ESCAPED}" ; then
   echo "not ok - JSON.sh run without -s option should leave solidus escaping intact"
   FAILS="$(expr $FAILS + 1)"
 else
   echo "ok $i - solidus escaping was left intact"
 fi
 
-if ! ../JSON.sh -s < "$INPUT" | diff -u - "${OUTPUT_WITHOUT_ESCAPING}" ; then
+if ! jsonsh_cli -s < "$INPUT" | diff -u - "${OUTPUT_WITHOUT_ESCAPING}" ; then
   echo "not ok - JSON.sh run with -s option should remove solidus escaping"
   FAILS="$(expr $FAILS + 1)"
 else
diff --git a/test/valid-test.sh b/test/valid-test.sh
index 3c8be73..0b50143 100755
--- a/test/valid-test.sh
+++ b/test/valid-test.sh
@@ -11,6 +11,9 @@ export LANG LC_ALL
 
 cd "$(dirname "$0")"
 
+# Can't detect sourcing in sh, so immediately terminate the attempt to parse
+. ../JSON.sh </dev/null
+
 fails=0
 passes=0
 skips=0
@@ -58,7 +61,7 @@ do
       if [ "$JSON_TEST_GENERATE" = yes ] || \
          [ "$JSON_TEST_GENERATE" = auto -a ! -s "$expected" ]
       then
-        if ! eval ../JSON.sh $OPTIONS < "$input" > "$expected"
+        if ! eval jsonsh_cli $OPTIONS < "$input" > "$expected"
         then
           echo "generation not ok $i - $input $EXT"
           fails="$(expr $fails + 1)"
@@ -71,7 +74,7 @@ do
         continue
       fi
 
-      if ! eval ../JSON.sh $OPTIONS < "$input" | diff -u - "$expected"
+      if ! eval jsonsh_cli $OPTIONS < "$input" | diff -u - "$expected"
       then
         echo "not ok $i - $input $EXT"
         fails="$(expr $fails + 1)"

From 0a11108a08c3af1a1ae888ae03851e7163ed63b0 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Tue, 7 Feb 2017 18:44:01 +0100
Subject: [PATCH 55/95] all-tests.sh : allow testing with different shells

---
 all-tests.sh | 94 +++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 63 insertions(+), 31 deletions(-)

diff --git a/all-tests.sh b/all-tests.sh
index 1d09d82..a7cef47 100755
--- a/all-tests.sh
+++ b/all-tests.sh
@@ -1,37 +1,69 @@
 #!/bin/sh
 
-cd "$(dirname "$0")"
+# This script can now test with various shell interpreters
+# which you can pass in a space-separated list of SHELL_PROGS
+# To use old behavior : export SHELL_PROGS="-"
 
+cd "$(dirname "$0")"
 #set -e
-fail=0
-tests=0
-#all_tests=${__dirname:}
-#echo PLAN ${#all_tests}
-for test in test/*.sh ;
-do
-  tests="$(expr $tests + 1)"
-  echo "TEST: $test"
-  # TODO: find a way to use the current shell-interpreter program to
-  # run sub-tests (simple sourcing fails ATM because scripts start
-  # with "cd `dirname $0`")...
-  #( . "./$test" )
-  "./$test"
-  ret=$?
-  if [ $ret -eq 0 ] ; then
-    echo "OK: ---- $test"
-    passed="$(expr $passed + 1)"
-  else
-    echo "FAIL: $test ($ret)"
-    fail="$(expr $fail + 1)"
-  fi
+
+overall_exitcode=0
+jsonsh_tests() (
+    [ -z "${SHELL_PROG-}" ] && SHELL_PROG=""
+    fail=0
+    tests=0
+    #all_tests=${__dirname:}
+    #echo PLAN ${#all_tests}
+    for test in test/*.sh ;
+    do
+      tests="$(expr $tests + 1)"
+      echo "TEST: $test"
+      # TODO: find a way to use the current shell-interpreter program to
+      # run sub-tests (simple sourcing fails ATM because scripts start
+      # with "cd `dirname $0`")...
+      #( . "./$test" )
+      $SHELL_PROG "./$test"
+      ret=$?
+      if [ $ret -eq 0 ] ; then
+        echo "OK: ---- $test"
+        passed="$(expr $passed + 1)"
+      else
+        echo "FAIL: $test ($ret)"
+        fail="$(expr $fail + 1)"
+      fi
+    done
+
+    if [ "$fail" = 0 ]; then
+      printf 'SUCCESS '
+      exitcode=0
+    else
+      printf 'FAILURE '
+      exitcode=1
+    fi
+    printf ": $passed / $tests\n"
+    exit $exitcode
+)
+
+[ -n "$SHELL_PROGS" ] || SHELL_PROGS="bash dash ash busybox ksh ksh88 ksh93"
+for SHELL_PROG in $SHELL_PROGS ; do
+    [ "$SHELL_PROG" = "busybox" ] && SHELL_PROG="busybox sh"
+    { [ "$SHELL_PROG" = "-" ] || [ "$SHELL_PROG" = " " ] ; } && \
+        SHELL_PROG=''
+
+    if [ -n "$SHELL_PROG" ] ; then
+        if $SHELL_PROG -c "date" >/dev/null 2>&1 ; then : ; else
+            echo "SKIP missing shell : $SHELL_PROG"
+            continue
+        fi
+        export SHELL_PROG
+        echo "TESTING WITH shell interpreter : $SHELL_PROG"
+    else
+        unset SHELL_PROG
+        echo "TESTING WITH default shell interpreter e.g. likely with /bin/sh, whatever this is in your OS"
+    fi
+
+    jsonsh_tests || overall_exitcode=$?
+    echo ""
 done
 
-if [ "$fail" = 0 ]; then
-  echo -n 'SUCCESS '
-  exitcode=0
-else
-  echo -n 'FAILURE '
-  exitcode=1
-fi
-echo " $passed / $tests"
-exit $exitcode
+exit $overall_exitcode

From 278611f3fec9af2c81751edd963230040ef329b7 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Tue, 7 Feb 2017 18:48:09 +0100
Subject: [PATCH 56/95] all-tests.sh : report which shells were tested and how
 they performed

---
 all-tests.sh | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/all-tests.sh b/all-tests.sh
index a7cef47..c9cab1d 100755
--- a/all-tests.sh
+++ b/all-tests.sh
@@ -44,6 +44,9 @@ jsonsh_tests() (
     exit $exitcode
 )
 
+OKAY_SHELLS=""
+FAIL_SHELLS=""
+SKIP_SHELLS=""
 [ -n "$SHELL_PROGS" ] || SHELL_PROGS="bash dash ash busybox ksh ksh88 ksh93"
 for SHELL_PROG in $SHELL_PROGS ; do
     [ "$SHELL_PROG" = "busybox" ] && SHELL_PROG="busybox sh"
@@ -53,6 +56,7 @@ for SHELL_PROG in $SHELL_PROGS ; do
     if [ -n "$SHELL_PROG" ] ; then
         if $SHELL_PROG -c "date" >/dev/null 2>&1 ; then : ; else
             echo "SKIP missing shell : $SHELL_PROG"
+            SKIP_SHELLS="$SKIP_SHELLS $SHELL_PROG"
             continue
         fi
         export SHELL_PROG
@@ -62,8 +66,13 @@ for SHELL_PROG in $SHELL_PROGS ; do
         echo "TESTING WITH default shell interpreter e.g. likely with /bin/sh, whatever this is in your OS"
     fi
 
-    jsonsh_tests || overall_exitcode=$?
+    jsonsh_tests && OKAY_SHELLS="$OKAY_SHELLS $SHELL_PROG" || \
+        { overall_exitcode=$? ; FAIL_SHELLS="$FAIL_SHELLS $SHELL_PROG" ; }
     echo ""
 done
 
+echo "OVERALL RESULT:"
+echo "OKAY_SHELLS = $OKAY_SHELLS"
+echo "FAIL_SHELLS = $FAIL_SHELLS"
+echo "SKIP_SHELLS = $SKIP_SHELLS"
 exit $overall_exitcode

From d9772af2ba1be86891c2cd5ff6aeaac84b3e6eee Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Tue, 7 Feb 2017 19:03:26 +0100
Subject: [PATCH 57/95] Add support for JSONSH_SOURCED=yes as a way of skipping
 jsonsh_cli() upon startup (non-bash)

---
 JSON.sh                     | 7 ++++++-
 all-tests.sh                | 1 +
 test/cook-test.sh           | 1 +
 test/invalid-test.sh        | 1 +
 test/no-head-test.sh        | 1 +
 test/parse-test.sh          | 1 +
 test/pedantic-empty-test.sh | 1 +
 test/solidus-test.sh        | 1 +
 test/tokenizer-test.sh      | 1 +
 test/valid-test.sh          | 1 +
 10 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/JSON.sh b/JSON.sh
index b1e08c2..a507771 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -15,6 +15,9 @@
 #           Forked and further modified by Eaton / Jim Klimov
 #           https://github.com/jimklimov/JSON.sh
 #
+# NOTE: This script may be used standalone or sourced into your interpreter.
+# For the latter use-case it is recommended to pre-set JSONSH_SOURCED=yes
+#
 # The MIT License (MIT)
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -764,7 +767,9 @@ jsonsh_cli_subshell() (
 jsonsh_debugging_defaults
 
 # If not sourced into a bash script, parse stdin and quit
-if [ "$0" = "$BASH_SOURCE[0]" ] || [ "$0" = "$BASH_SOURCE" ] || [ -z "${BASH-}" ]; \
+# TODO: non-bash shells?
+[ "${JSONSH_SOURCED-}" = yes ] || \
+if  [ "$0" = "$BASH_SOURCE[0]" ] || [ "$0" = "$BASH_SOURCE" ] || [ -z "${BASH-}" ]; \
 then
   jsonsh_cli "$@"
   exit $?
diff --git a/all-tests.sh b/all-tests.sh
index c9cab1d..48f0497 100755
--- a/all-tests.sh
+++ b/all-tests.sh
@@ -56,6 +56,7 @@ for SHELL_PROG in $SHELL_PROGS ; do
     if [ -n "$SHELL_PROG" ] ; then
         if $SHELL_PROG -c "date" >/dev/null 2>&1 ; then : ; else
             echo "SKIP missing shell : $SHELL_PROG"
+            echo ""
             SKIP_SHELLS="$SKIP_SHELLS $SHELL_PROG"
             continue
         fi
diff --git a/test/cook-test.sh b/test/cook-test.sh
index 2ec9f6f..55532e7 100755
--- a/test/cook-test.sh
+++ b/test/cook-test.sh
@@ -3,6 +3,7 @@
 cd "$(dirname "$0")"
 
 # Can't detect sourcing in sh, so immediately terminate the attempt to parse
+JSONSH_SOURCED=yes
 . ../JSON.sh </dev/null
 
 cooktest() {
diff --git a/test/invalid-test.sh b/test/invalid-test.sh
index 4c1884b..9674c77 100755
--- a/test/invalid-test.sh
+++ b/test/invalid-test.sh
@@ -3,6 +3,7 @@
 cd "$(dirname "$0")"
 
 # Can't detect sourcing in sh, so immediately terminate the attempt to parse
+JSONSH_SOURCED=yes
 . ../JSON.sh </dev/null
 
 # make test output TAP compatible
diff --git a/test/no-head-test.sh b/test/no-head-test.sh
index 6306464..d862ad4 100755
--- a/test/no-head-test.sh
+++ b/test/no-head-test.sh
@@ -3,6 +3,7 @@
 cd "$(dirname "$0")"
 
 # Can't detect sourcing in sh, so immediately terminate the attempt to parse
+JSONSH_SOURCED=yes
 . ../JSON.sh </dev/null
 
 [ -n "${tmp-}" ] || tmp="/tmp"
diff --git a/test/parse-test.sh b/test/parse-test.sh
index bf673fd..6b88909 100755
--- a/test/parse-test.sh
+++ b/test/parse-test.sh
@@ -3,6 +3,7 @@
 cd "$(dirname "$0")"
 
 # Can't detect sourcing in sh, so immediately terminate the attempt to parse
+JSONSH_SOURCED=yes
 . ../JSON.sh </dev/null
 
 ptest () {
diff --git a/test/pedantic-empty-test.sh b/test/pedantic-empty-test.sh
index efcd46c..758902c 100755
--- a/test/pedantic-empty-test.sh
+++ b/test/pedantic-empty-test.sh
@@ -3,6 +3,7 @@
 cd "$(dirname "$0")"
 
 # Can't detect sourcing in sh, so immediately terminate the attempt to parse
+JSONSH_SOURCED=yes
 . ../JSON.sh </dev/null
 
 # make test output TAP compatible
diff --git a/test/solidus-test.sh b/test/solidus-test.sh
index 14d35fe..dbdf34d 100755
--- a/test/solidus-test.sh
+++ b/test/solidus-test.sh
@@ -3,6 +3,7 @@
 cd "$(dirname "$0")"
 
 # Can't detect sourcing in sh, so immediately terminate the attempt to parse
+JSONSH_SOURCED=yes
 . ../JSON.sh </dev/null
 
 INPUT=./solidus/string_with_solidus.json
diff --git a/test/tokenizer-test.sh b/test/tokenizer-test.sh
index 1d1a441..0282b10 100755
--- a/test/tokenizer-test.sh
+++ b/test/tokenizer-test.sh
@@ -3,6 +3,7 @@
 cd "$(dirname "$0")"
 
 # Can't detect sourcing in sh, so immediately terminate the attempt to parse
+JSONSH_SOURCED=yes
 . ../JSON.sh </dev/null
 
 i=0
diff --git a/test/valid-test.sh b/test/valid-test.sh
index 0b50143..340106d 100755
--- a/test/valid-test.sh
+++ b/test/valid-test.sh
@@ -12,6 +12,7 @@ export LANG LC_ALL
 cd "$(dirname "$0")"
 
 # Can't detect sourcing in sh, so immediately terminate the attempt to parse
+JSONSH_SOURCED=yes
 . ../JSON.sh </dev/null
 
 fails=0

From f5296ba26ba94f3d62d441c569fdffd8a889df85 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Tue, 7 Feb 2017 19:28:07 +0100
Subject: [PATCH 58/95] JSON.sh : flag support for bash regex with
 SHELL_REGEX=yes and fall back to external utils for other shells

---
 JSON.sh | 63 ++++++++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 44 insertions(+), 19 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index a507771..8fc921c 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -57,6 +57,12 @@
 # (e.g. double-brackets, local keyword, regex expressions...)
 # to either refuse running in a shell or pick an implementation
 # for certain code paths.
+
+SHELL_REGEX=no
+if [ -n "${BASH-}" ] || [ -n "${BASH_VERSION-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
+    SHELL_REGEX=yes
+fi
+
 false && \
 if [ -z "${BASH-}" ] && [ -z "${BASH_VERSION-}" ] && [ -z "${ZSH_VERSION-}" ]; then
     # NOTE: This can break scripts which source this file and are not in bash
@@ -404,12 +410,20 @@ cook_a_string_arg() {
     [ -z "$1" ] && return 0
     # Strangely, for some OSes it does not suffice that all chars must be from
     # the first pattern - should explicitly test that some are not forbidden
-    # TODO: Bash-compatible regex support required for code below.
-    # May need to add support for other shells if this syntax
-    # is not supported there (e.g. revert to sed/grep/awk)...
-    if ! [[ "$1" =~ [\\\"] ]] >/dev/null && \
-         [[ "$1" =~ ^[A-Za-z0-9\ \-\.\+\/\@\:\;\!\%\,\&\(\)\{\}]*$ ]] >/dev/null \
-    ; then
+    IS_TRIVIAL=no
+    if [ "$SHELL_REGEX" = yes ]; then
+        # Bash-compatible regex support required for code below.
+        if ! [[ "$1" =~ [\\\"] ]] >/dev/null && \
+            [[ "$1" =~ ^[A-Za-z0-9\ \-\.\+\/\@\:\;\!\%\,\&\(\)\{\}]*$ ]] >/dev/null \
+        ; then IS_TRIVIAL=yes; fi
+    else
+        # Support for other shells if bash-regex syntax
+        # is not supported there (e.g. revert to sed/grep/awk)...
+        if [ -n "$(echo "$1" | $GEGREP -v '[\\\"]' | $GEGREP '^[A-Za-z0-9\ \-\.\+\/\@\:\;\!\%\,\&\(\)\{\}]*$' )" ] \
+        ; then IS_TRIVIAL=yes; fi
+    fi
+
+    if [ "$IS_TRIVIAL" = yes ] ; then
         print_debug $DEBUGLEVEL_PRINTTOKEN_PIPELINE "cook_a_string_arg(): input trivial, not cooking: '$1'"
         echo "$1"
         return 0
@@ -569,41 +583,48 @@ parse_value () {
        print_debug $DEBUGLEVEL_PRINTPATHVAL \
             "token '$token' is a suspected number" >&2
        # TODO: Bash regex and more if's
-       if [ "$NORMALIZE_NUMBERS" = 1 ] && \
-         [[ "$token" =~ ${REGEX_NUMBER} ]] \
-       ; then
+       DO_NORMALIZE=no
+       if [ "$NORMALIZE_NUMBERS" = 1 ] ; then
+           if [ "$SHELL_REGEX" = yes ]; then
+                [[ "$token" =~ ${REGEX_NUMBER} ]] && DO_NORMALIZE=yes
+           else
+                [ -n "$(echo "$token" | $GEGREP "${REGEX_NUMBER}" )" ] && DO_NORMALIZE=yes
+           fi
+       fi
+
+       if [ "$DO_NORMALIZE" = yes ]; then
             value="$(printf "$NORMALIZE_NUMBERS_FORMAT" "$token")" || \
             value="$token"
             print_debug $DEBUGLEVEL_PRINTPATHVAL "normalized numeric token" \
                 "'$token' into '$value'" >&2
             if [ "$NORMALIZE_NUMBERS_STRIP" = 1 ]; then
-                local valuetmp="$(echo "$value" | $GSED -e 's,0*$,,g' -e 's,\.$,,' 2>/dev/null)" && \
-                value="$valuetmp"
+                valuetmp="$(echo "$value" | $GSED -e 's,0*$,,g' -e 's,\.$,,' 2>/dev/null)" && \
+                    value="$valuetmp"
                 unset valuetmp
                 print_debug $DEBUGLEVEL_PRINTPATHVAL "stripped numeric token" \
                     "'$token' into '$value'" >&2
             fi
        else
-        # Not a number or no normalization - process like default
+            # Not a number or no normalization - process like default
             value="$token"
             if [ "$NORMALIZE_SOLIDUS" = 1 ]; then
                 if [ -n "${BASH-}" ] ; then
-                    value=${value//\\\//\/} ;
+                    value="${value//\\\//\/}"
                 else
-                    value=$(echo "$value" | sed 's#\\/#/#g')
+                    value="$(echo "$value" | $GSED 's#\\/#/#g')"
                 fi
             fi
        fi
        isleaf=1
        { [ "$value" = '""' ] || [ "$value" = '' ] ; } && isempty=1
        ;;
-    *) value=$token
+    *) value="$token"
        # if asked, replace solidus ("\/") in json strings with normalized value: "/"
        if [ "$NORMALIZE_SOLIDUS" = 1 ]; then
             if [ -n "${BASH-}" ] ; then
-                value=${value//\\\//\/} ;
+                value="${value//\\\//\/}"
             else
-                value=$(echo "$value" | sed 's#\\/#/#g')
+                value="$(echo "$value" | $GSED 's#\\/#/#g')"
             fi
        fi
        isleaf=1
@@ -637,8 +658,12 @@ parse_value () {
     [ $PRUNE -eq 0 ] && print=5
 
   if [ "$print" -ne 0 ] && [ -n "$EXTRACT_JPATH" ] ; then
-    ### TODO: BASH regex matching:
-    [[ ${jpath} =~ ${EXTRACT_JPATH} ]] || print=-1
+    if [ "$SHELL_REGEX" = yes ]; then
+        ### BASH regex matching:
+        [[ ${jpath} =~ ${EXTRACT_JPATH} ]] || print=-1
+    else
+        [ -n "$(echo "${jpath}" | $GEGREP "${EXTRACT_JPATH}")" ] || print=-1
+    fi
   fi
 
   print_debug $DEBUGLEVEL_PRINTPATHVAL \

From 9132d57f5b26aa4fb1634aa2811ef789dfa99927 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Tue, 7 Feb 2017 19:29:55 +0100
Subject: [PATCH 59/95] invalid-test.sh : print STDERR on failures too

---
 test/invalid-test.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/invalid-test.sh b/test/invalid-test.sh
index 9674c77..578a88a 100755
--- a/test/invalid-test.sh
+++ b/test/invalid-test.sh
@@ -22,6 +22,8 @@ do
     #this should be indented with '#' at the start.
     echo "OUTPUT WAS >>>"
     cat /tmp/JSON.sh_outlog
+    echo "ERRORS WAS >>>"
+    cat /tmp/JSON.sh_errlog
     echo "<<<"
     fails="$(expr $fails + 1)"
   else

From 54615459034015240484bfd0b7787672fd16393e Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Tue, 7 Feb 2017 19:36:41 +0100
Subject: [PATCH 60/95] JSON.sh : flag support for bash regex with
 SHELL_TWOSLASH=yes and fall back to external utils for other shells

---
 JSON.sh | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index 8fc921c..f2a5620 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -59,10 +59,14 @@
 # for certain code paths.
 
 SHELL_REGEX=no
+SHELL_TWOSLASH=no
 if [ -n "${BASH-}" ] || [ -n "${BASH_VERSION-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
     SHELL_REGEX=yes
+    SHELL_TWOSLASH=yes
 fi
 
+# TODO: detect if busybox - there SHELL_TWOSLASH=yes too, but not in DASH
+
 false && \
 if [ -z "${BASH-}" ] && [ -z "${BASH_VERSION-}" ] && [ -z "${ZSH_VERSION-}" ]; then
     # NOTE: This can break scripts which source this file and are not in bash
@@ -358,10 +362,14 @@ strip_newlines() {
   $GGREP '' | \
   tee_stderr BEFORE_STRIP $DEBUGLEVEL_PRINTTOKEN_PIPELINE | \
   while IFS="" read -r ILINE; do
-    # Remove escaped quotes:
-    LINESTRIP="${ILINE//\\\"}"
-    # Remove all chars but remaining quotes:
-    LINESTRIP="${LINESTRIP//[^\"]}"
+    if [ "$SHELL_TWOSLASH" = yes ]; then
+        # Remove escaped quotes:
+        LINESTRIP="${ILINE//\\\"}"
+        # Remove all chars but remaining quotes:
+        LINESTRIP="${LINESTRIP//[^\"]}"
+    else
+        LINESTRIP="$(echo "$ILINE" | $GSED -e 's,\\\",,g' -e 's,[^\"],,g')"
+    fi
     # Count unescaped quotes:
     NUMQ="${#LINESTRIP}"
     ODD="$(($NUMQ%2))"
@@ -608,7 +616,7 @@ parse_value () {
             # Not a number or no normalization - process like default
             value="$token"
             if [ "$NORMALIZE_SOLIDUS" = 1 ]; then
-                if [ -n "${BASH-}" ] ; then
+                if [ "$SHELL_TWOSLASH" = yes ] ; then
                     value="${value//\\\//\/}"
                 else
                     value="$(echo "$value" | $GSED 's#\\/#/#g')"
@@ -621,7 +629,7 @@ parse_value () {
     *) value="$token"
        # if asked, replace solidus ("\/") in json strings with normalized value: "/"
        if [ "$NORMALIZE_SOLIDUS" = 1 ]; then
-            if [ -n "${BASH-}" ] ; then
+            if [ "$SHELL_TWOSLASH" = yes ] ; then
                 value="${value//\\\//\/}"
             else
                 value="$(echo "$value" | $GSED 's#\\/#/#g')"

From 48b56ead75e36b94cc6d36119b04a4306dc068ea Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Tue, 7 Feb 2017 19:40:37 +0100
Subject: [PATCH 61/95] JSON.sh : dropped ARGN from CLI arg parsing

---
 JSON.sh | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index f2a5620..4c57c86 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -234,8 +234,7 @@ tee_stderr() {
 
 parse_options() {
   set -- "$@"
-  local ARGN=$#
-  while [ "$ARGN" -ne 0 ]
+  while [ "$#" -gt 0 ]
   do
     case "$1" in
       -h) usage
@@ -324,7 +323,6 @@ parse_options() {
       ;;
     esac
     shift 1
-    ARGN=$((ARGN-1))
   done
 
   validate_debuglevel

From ebe55a169d9ee39cb31e9fc74dc8784caf3b670f Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Tue, 7 Feb 2017 19:42:12 +0100
Subject: [PATCH 62/95] JSON.sh : dropped space between func names and "()"

---
 JSON.sh | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index 4c57c86..7227cf7 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -331,8 +331,8 @@ parse_options() {
   [ "$NORMALIZE" = 1 ] && BRIEF=0 && LEAFONLY=0 && PRUNE=0
 }
 
-awk_egrep () {
-  local pattern_string=$1
+awk_egrep() {
+  pattern_string="$1"
 
   [ -z "${AWK-}" ] && throw "No AWK found!"
   [ ! -x "$AWK" ] && throw "Not executable AWK='$AWK'!"
@@ -439,7 +439,7 @@ cook_a_string_arg() {
     echo "$1" | cook_a_string
 }
 
-tokenize () {
+tokenize() {
   local GREP_O
   local ESCAPE
   local CHAR
@@ -483,7 +483,7 @@ tokenize () {
   unset is_wordsplit_disabled
 }
 
-parse_array () {
+parse_array() {
   local index=0
   local ary=''
   local aryml=''
@@ -519,7 +519,7 @@ $value"
   :
 }
 
-parse_object () {
+parse_object() {
   local key
   local obj=''
   local objml=''
@@ -568,7 +568,7 @@ $key:$value"
 }
 
 REGEX_NUMBER='^[+-]?([.][0-9]+|(0+|[1-9][0-9]*)([.][0-9]*)?)([eE][+-]?[0-9]*)?$'
-parse_value () {
+parse_value() {
   local jpath="${1:+$1,}$2" isleaf=0 isempty=0 print=0
   case "$token" in
     '{') parse_object "$jpath"
@@ -681,7 +681,7 @@ parse_value () {
   :
 }
 
-parse () {
+parse() {
   read -r token
   print_debug $DEBUGLEVEL_PRINTTOKEN "parse(1):" "token='$token'"
   parse_value

From 39ee8cc9dcda55cf756953d878e183cb65adfa80 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 10:13:10 +0100
Subject: [PATCH 63/95] .travis.yml : use matrix-testing to try different
 shells and see regressions better

---
 .travis.yml | 28 ++++++++++++++++++++++++----
 1 file changed, 24 insertions(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 9d4afb9..64bd16c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,14 +1,34 @@
 language: python
 
+sudo: false
+
+os:
+  - linux
+  - macos
+
 addons:
   apt:
     packages:
       - bash
       - dash
       - zsh
+      - busybox
+
+env:
+  matrix:
+    - SHELL_PROGS=bash
+    - SHELL_PROGS=ash
+    - SHELL_PROGS=dash
+    - SHELL_PROGS=busybox
+#    - SHELL_PROGS=ksh
+#    - SHELL_PROGS=ksh88
+#    - SHELL_PROGS=ksh93
 
 # Whatever the current shebang, replace with hardcoded shell
-script: >
-  sed -i '1s@.*@#!/usr/bin/env bash@' JSON.sh && ./all-tests.sh &&
-  sed -i '1s@.*@#!/usr/bin/env zsh@'  JSON.sh && ./all-tests.sh &&
-  sed -i '1s@.*@#!/usr/bin/env dash@' JSON.sh && ./all-tests.sh
+#script: >
+#  sed -i '1s@.*@#!/usr/bin/env bash@' JSON.sh && ./all-tests.sh &&
+#  sed -i '1s@.*@#!/usr/bin/env zsh@'  JSON.sh && ./all-tests.sh &&
+#  sed -i '1s@.*@#!/usr/bin/env dash@' JSON.sh && ./all-tests.sh
+
+# This version of the script can use specified shell to source and test JSON.sh
+script: ./all-tests.sh

From 231c64097634cc1a4a7b6ff98f5a35e32aeb4799 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 10:33:09 +0100
Subject: [PATCH 64/95] all-tests.sh : add TEST_PATTERN support and more useful
 failure reports (which sub-tests and in which shell)

---
 all-tests.sh | 27 ++++++++++++++++++++-------
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/all-tests.sh b/all-tests.sh
index 48f0497..ad3954f 100755
--- a/all-tests.sh
+++ b/all-tests.sh
@@ -3,6 +3,8 @@
 # This script can now test with various shell interpreters
 # which you can pass in a space-separated list of SHELL_PROGS
 # To use old behavior : export SHELL_PROGS="-"
+# This script runs one or more actual sub-tests named in TEST_PATTERN
+# (the value may be a shell wildcard).
 
 cd "$(dirname "$0")"
 #set -e
@@ -12,9 +14,10 @@ jsonsh_tests() (
     [ -z "${SHELL_PROG-}" ] && SHELL_PROG=""
     fail=0
     tests=0
+    fail_names=""
     #all_tests=${__dirname:}
     #echo PLAN ${#all_tests}
-    for test in test/*.sh ;
+    for test in $TEST_PATTERN
     do
       tests="$(expr $tests + 1)"
       echo "TEST: $test"
@@ -30,17 +33,25 @@ jsonsh_tests() (
       else
         echo "FAIL: $test ($ret)"
         fail="$(expr $fail + 1)"
+        fail_names="$fail_names $test"
       fi
     done
 
     if [ "$fail" = 0 ]; then
-      printf 'SUCCESS '
+      printf '===== SUCCESS '
       exitcode=0
     else
-      printf 'FAILURE '
+      printf '===== FAILURE '
       exitcode=1
     fi
-    printf ": $passed / $tests\n"
+
+    # Note the leading space if populated
+    [ -z "$fail_names" ] || fail_names=" : failed for$fail_names"
+
+    # Note the leading space if populated
+    [ -z "$SHELL_PROG" ] || fail_names="$fail_names interpreted by '$SHELL_PROG'"
+
+    printf ": passed $passed / $tests tests$fail_names\n"
     exit $exitcode
 )
 
@@ -48,6 +59,8 @@ OKAY_SHELLS=""
 FAIL_SHELLS=""
 SKIP_SHELLS=""
 [ -n "$SHELL_PROGS" ] || SHELL_PROGS="bash dash ash busybox ksh ksh88 ksh93"
+[ -n "$TEST_PATTERN" ] || TEST_PATTERN='test/*.sh'
+export TEST_PATTERN
 for SHELL_PROG in $SHELL_PROGS ; do
     [ "$SHELL_PROG" = "busybox" ] && SHELL_PROG="busybox sh"
     { [ "$SHELL_PROG" = "-" ] || [ "$SHELL_PROG" = " " ] ; } && \
@@ -55,16 +68,16 @@ for SHELL_PROG in $SHELL_PROGS ; do
 
     if [ -n "$SHELL_PROG" ] ; then
         if $SHELL_PROG -c "date" >/dev/null 2>&1 ; then : ; else
-            echo "SKIP missing shell : $SHELL_PROG"
+            echo "=== SKIP missing shell : $SHELL_PROG"
             echo ""
             SKIP_SHELLS="$SKIP_SHELLS $SHELL_PROG"
             continue
         fi
         export SHELL_PROG
-        echo "TESTING WITH shell interpreter : $SHELL_PROG"
+        echo "=== TESTING WITH shell interpreter : $SHELL_PROG"
     else
         unset SHELL_PROG
-        echo "TESTING WITH default shell interpreter e.g. likely with /bin/sh, whatever this is in your OS"
+        echo "=== TESTING WITH default shell interpreter e.g. likely with /bin/sh, whatever this is in your OS"
     fi
 
     jsonsh_tests && OKAY_SHELLS="$OKAY_SHELLS $SHELL_PROG" || \

From bc519f07a2f6aa35cca1ac2e66f3b4e99db39a81 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 10:35:03 +0100
Subject: [PATCH 65/95] all-tests.sh : track the "passed" value from zero
 (saner reports if EVERYTHING failed)

---
 all-tests.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/all-tests.sh b/all-tests.sh
index ad3954f..e3e9ae1 100755
--- a/all-tests.sh
+++ b/all-tests.sh
@@ -14,6 +14,7 @@ jsonsh_tests() (
     [ -z "${SHELL_PROG-}" ] && SHELL_PROG=""
     fail=0
     tests=0
+    passed=0
     fail_names=""
     #all_tests=${__dirname:}
     #echo PLAN ${#all_tests}

From f517a46500a454c687c42b4deeb17c478306de0c Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 11:15:42 +0100
Subject: [PATCH 66/95] Sub-tests: work around lack of pipefail in non-bash by
 explicit call-chains

---
 test/no-head-test.sh |  9 +++++++--
 test/solidus-test.sh | 11 +++++++++--
 test/valid-test.sh   |  6 +++++-
 3 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/test/no-head-test.sh b/test/no-head-test.sh
index d862ad4..50750ca 100755
--- a/test/no-head-test.sh
+++ b/test/no-head-test.sh
@@ -22,9 +22,14 @@ echo "1..$tests"
 for input in valid/*.json
 do
   expected="${tmp}$(basename "$input" .json).no-head"
-  egrep -v '^\[]' < "$(dirname "$input")/$(basename "$input" .json).parsed" > "$expected"
+  # NOTE: The echo trick is required to ensure EOLs for both empty and populated results
+  echo "$(egrep -v '^\[]' < "$(dirname "$input")/$(basename "$input" .json).parsed")" > "$expected"
   i="$(expr $i + 1)"
-  if ! jsonsh_cli -n < "$input" | diff -u - "$expected"
+  # Such explicit chaining is equivalent to "pipefail" in non-Bash interpreters
+  JSONSH_OUT="$(jsonsh_cli -n < "$input")" && \
+    echo "$JSONSH_OUT" | diff -u - "$expected"
+  JSONSH_RES=$?
+  if [ "$JSONSH_RES" != 0 ]
   then
     echo "not ok $i - $input"
     fails="$(expr $fails + 1)"
diff --git a/test/solidus-test.sh b/test/solidus-test.sh
index dbdf34d..aad4a16 100755
--- a/test/solidus-test.sh
+++ b/test/solidus-test.sh
@@ -14,14 +14,21 @@ FAILS=0
 
 echo "1..2"
 
-if ! jsonsh_cli < "$INPUT" | diff -u - "${OUTPUT_ESCAPED}" ; then
+# Such explicit chaining is equivalent to "pipefail" in non-Bash interpreters
+JSONSH_OUT="$(jsonsh_cli < "$INPUT")" && \
+    echo "$JSONSH_OUT" | diff -u - "${OUTPUT_ESCAPED}"
+JSONSH_RES=$?
+if [ "$JSONSH_RES" != 0 ] ; then
   echo "not ok - JSON.sh run without -s option should leave solidus escaping intact"
   FAILS="$(expr $FAILS + 1)"
 else
   echo "ok $i - solidus escaping was left intact"
 fi
 
-if ! jsonsh_cli -s < "$INPUT" | diff -u - "${OUTPUT_WITHOUT_ESCAPING}" ; then
+JSONSH_OUT="$(jsonsh_cli -s < "$INPUT")" && \
+    echo "$JSONSH_OUT" | diff -u - "${OUTPUT_WITHOUT_ESCAPING}"
+JSONSH_RES=$?
+if [ "$JSONSH_RES" != 0 ] ; then
   echo "not ok - JSON.sh run with -s option should remove solidus escaping"
   FAILS="$(expr $FAILS + 1)"
 else
diff --git a/test/valid-test.sh b/test/valid-test.sh
index 340106d..b1aec70 100755
--- a/test/valid-test.sh
+++ b/test/valid-test.sh
@@ -75,7 +75,11 @@ do
         continue
       fi
 
-      if ! eval jsonsh_cli $OPTIONS < "$input" | diff -u - "$expected"
+      # Such explicit chaining is equivalent to "pipefail" in non-Bash interpreters
+      JSONSH_OUT="$(eval jsonsh_cli $OPTIONS < "$input")" && \
+        echo "$JSONSH_OUT" | diff -u - "${expected}"
+      JSONSH_RES=$?
+      if [ "$JSONSH_RES" != 0 ]
       then
         echo "not ok $i - $input $EXT"
         fails="$(expr $fails + 1)"

From 30777d0e4b029aec01f2b2cbf12b121d227d7dc6 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 11:16:09 +0100
Subject: [PATCH 67/95] no-head-test.sh : report details if test-errors happen

---
 test/no-head-test.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/no-head-test.sh b/test/no-head-test.sh
index 50750ca..ec3f2a1 100755
--- a/test/no-head-test.sh
+++ b/test/no-head-test.sh
@@ -33,6 +33,8 @@ do
   then
     echo "not ok $i - $input"
     fails="$(expr $fails + 1)"
+    echo ">>> JSONSH_OUT='$JSONSH_OUT'"
+    echo ">>> EXPECTED : `ls -la $expected`"
   else
     echo "ok $i - $input"
   fi

From 6ecedf2338d323e6fdfc04185a986be1591d34db Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 11:37:32 +0100
Subject: [PATCH 68/95] .travis.yml : install the shells we test in

---
 .travis.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 64bd16c..2333126 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,18 +12,26 @@ addons:
       - bash
       - dash
       - zsh
+      - ash
       - busybox
+      - ksh
 
 env:
+#  global:
+#    - TEST_PATTERN='test/*.sh'
   matrix:
     - SHELL_PROGS=bash
     - SHELL_PROGS=ash
+    - SHELL_PROGS=zsh
     - SHELL_PROGS=dash
     - SHELL_PROGS=busybox
 #    - SHELL_PROGS=ksh
 #    - SHELL_PROGS=ksh88
 #    - SHELL_PROGS=ksh93
 
+before_install:
+- if [ $TRAVIS_OS_NAME == "osx" ] ; then brew update; brew install busybox bash ash dash zsh ksh ; fi
+
 # Whatever the current shebang, replace with hardcoded shell
 #script: >
 #  sed -i '1s@.*@#!/usr/bin/env bash@' JSON.sh && ./all-tests.sh &&

From ac6872f77b4cf97ba92ec7af457c042ee3f084ee Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 11:45:38 +0100
Subject: [PATCH 69/95] .travis.yml : install the shells we test in on various
 platforms

---
 .travis.yml | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 2333126..598f8c2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,9 +2,9 @@ language: python
 
 sudo: false
 
+# Effectively the list below is for Linux; MacOS is kept separately
 os:
   - linux
-  - macos
 
 addons:
   apt:
@@ -25,12 +25,20 @@ env:
     - SHELL_PROGS=zsh
     - SHELL_PROGS=dash
     - SHELL_PROGS=busybox
-#    - SHELL_PROGS=ksh
+    - SHELL_PROGS=ksh
 #    - SHELL_PROGS=ksh88
 #    - SHELL_PROGS=ksh93
 
+# Note that this does not inherit env: or pacages from above
+matrix:
+  include:
+  - os: macos
+    env: SHELL_PROGS=bash
+  - os: macos
+    env: SHELL_PROGS=dash
+
 before_install:
-- if [ $TRAVIS_OS_NAME == "osx" ] ; then brew update; brew install busybox bash ash dash zsh ksh ; fi
+- if [ "$TRAVIS_OS_NAME" = "osx" ] ; then brew update; brew install busybox bash ash dash zsh ksh ; fi
 
 # Whatever the current shebang, replace with hardcoded shell
 #script: >
@@ -39,4 +47,6 @@ before_install:
 #  sed -i '1s@.*@#!/usr/bin/env dash@' JSON.sh && ./all-tests.sh
 
 # This version of the script can use specified shell to source and test JSON.sh
+# Note that some platforms can lack some interpreters, so a test run
+# is effectively skipped and the test looks green if that's the case
 script: ./all-tests.sh

From 9c154cc278c7ac3baab5e9dc5e10379840978bcd Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 11:50:59 +0100
Subject: [PATCH 70/95] Subtests should use printf rather than echo for parsed
 data, to avoid most surprises

---
 test/no-head-test.sh | 2 +-
 test/solidus-test.sh | 4 ++--
 test/valid-test.sh   | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/no-head-test.sh b/test/no-head-test.sh
index ec3f2a1..1652097 100755
--- a/test/no-head-test.sh
+++ b/test/no-head-test.sh
@@ -27,7 +27,7 @@ do
   i="$(expr $i + 1)"
   # Such explicit chaining is equivalent to "pipefail" in non-Bash interpreters
   JSONSH_OUT="$(jsonsh_cli -n < "$input")" && \
-    echo "$JSONSH_OUT" | diff -u - "$expected"
+    printf '%s\n' "$JSONSH_OUT" | diff -u - "$expected"
   JSONSH_RES=$?
   if [ "$JSONSH_RES" != 0 ]
   then
diff --git a/test/solidus-test.sh b/test/solidus-test.sh
index aad4a16..a57f306 100755
--- a/test/solidus-test.sh
+++ b/test/solidus-test.sh
@@ -16,7 +16,7 @@ echo "1..2"
 
 # Such explicit chaining is equivalent to "pipefail" in non-Bash interpreters
 JSONSH_OUT="$(jsonsh_cli < "$INPUT")" && \
-    echo "$JSONSH_OUT" | diff -u - "${OUTPUT_ESCAPED}"
+    printf '%s\n' "$JSONSH_OUT" | diff -u - "${OUTPUT_ESCAPED}"
 JSONSH_RES=$?
 if [ "$JSONSH_RES" != 0 ] ; then
   echo "not ok - JSON.sh run without -s option should leave solidus escaping intact"
@@ -26,7 +26,7 @@ else
 fi
 
 JSONSH_OUT="$(jsonsh_cli -s < "$INPUT")" && \
-    echo "$JSONSH_OUT" | diff -u - "${OUTPUT_WITHOUT_ESCAPING}"
+    printf '%s\n' "$JSONSH_OUT" | diff -u - "${OUTPUT_WITHOUT_ESCAPING}"
 JSONSH_RES=$?
 if [ "$JSONSH_RES" != 0 ] ; then
   echo "not ok - JSON.sh run with -s option should remove solidus escaping"
diff --git a/test/valid-test.sh b/test/valid-test.sh
index b1aec70..11f778e 100755
--- a/test/valid-test.sh
+++ b/test/valid-test.sh
@@ -77,7 +77,7 @@ do
 
       # Such explicit chaining is equivalent to "pipefail" in non-Bash interpreters
       JSONSH_OUT="$(eval jsonsh_cli $OPTIONS < "$input")" && \
-        echo "$JSONSH_OUT" | diff -u - "${expected}"
+        printf '%s\n' "$JSONSH_OUT" | diff -u - "${expected}"
       JSONSH_RES=$?
       if [ "$JSONSH_RES" != 0 ]
       then

From 0c7dd88b275dca5ba8bfeeadd6076346a335aa84 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 11:57:11 +0100
Subject: [PATCH 71/95] solidus-test.sh : track number of test

---
 test/solidus-test.sh | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/test/solidus-test.sh b/test/solidus-test.sh
index a57f306..8a641c4 100755
--- a/test/solidus-test.sh
+++ b/test/solidus-test.sh
@@ -13,23 +13,26 @@ OUTPUT_WITHOUT_ESCAPING=./solidus/string_with_solidus.no-escaping.parsed
 FAILS=0
 
 echo "1..2"
+i=0
 
 # Such explicit chaining is equivalent to "pipefail" in non-Bash interpreters
-JSONSH_OUT="$(jsonsh_cli < "$INPUT")" && \
+i="$(expr $i + 1)"
+JSONSH_OUT="$(eval jsonsh_cli < "$INPUT")" && \
     printf '%s\n' "$JSONSH_OUT" | diff -u - "${OUTPUT_ESCAPED}"
 JSONSH_RES=$?
 if [ "$JSONSH_RES" != 0 ] ; then
-  echo "not ok - JSON.sh run without -s option should leave solidus escaping intact"
+  echo "not ok $i - JSON.sh run without -s option should leave solidus escaping intact"
   FAILS="$(expr $FAILS + 1)"
 else
   echo "ok $i - solidus escaping was left intact"
 fi
 
-JSONSH_OUT="$(jsonsh_cli -s < "$INPUT")" && \
+i="$(expr $i + 1)"
+JSONSH_OUT="$(eval jsonsh_cli -s < "$INPUT")" && \
     printf '%s\n' "$JSONSH_OUT" | diff -u - "${OUTPUT_WITHOUT_ESCAPING}"
 JSONSH_RES=$?
 if [ "$JSONSH_RES" != 0 ] ; then
-  echo "not ok - JSON.sh run with -s option should remove solidus escaping"
+  echo "not ok $i - JSON.sh run with -s option should remove solidus escaping"
   FAILS="$(expr $FAILS + 1)"
 else
   echo "ok $i - solidus escaping has been removed"

From 6c87fb0a22c1fdd8ceb480afd27ef2b3dc1aa6bb Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 11:57:53 +0100
Subject: [PATCH 72/95] .travis.yml : enable JSON.sh debugging to trace what
 fails how in different shells

---
 .travis.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 598f8c2..9baf5d8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,7 +17,8 @@ addons:
       - ksh
 
 env:
-#  global:
+  global:
+    - DEBUG=99
 #    - TEST_PATTERN='test/*.sh'
   matrix:
     - SHELL_PROGS=bash

From 13e6c3229912d24ffa87a61d6f754612f77584de Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 12:05:38 +0100
Subject: [PATCH 73/95] .travis.yml : do not do KSH for now - it hangs (waits
 for input?)

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 9baf5d8..dc9260c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -26,7 +26,7 @@ env:
     - SHELL_PROGS=zsh
     - SHELL_PROGS=dash
     - SHELL_PROGS=busybox
-    - SHELL_PROGS=ksh
+#    - SHELL_PROGS=ksh
 #    - SHELL_PROGS=ksh88
 #    - SHELL_PROGS=ksh93
 

From 5fe8680d4467fe9eea8b29fa30cff62d35227dde Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 12:06:53 +0100
Subject: [PATCH 74/95] .travis.yml : skip blanket debug

---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index dc9260c..eea7e88 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,8 +17,8 @@ addons:
       - ksh
 
 env:
-  global:
-    - DEBUG=99
+#  global:
+#    - DEBUG=99
 #    - TEST_PATTERN='test/*.sh'
   matrix:
     - SHELL_PROGS=bash

From b03c918f83a3d52358977a95317996cd9427f8a6 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 12:08:16 +0100
Subject: [PATCH 75/95] Sub-test scripts - enable tracing of those which fail

---
 test/no-head-test.sh | 3 +++
 test/solidus-test.sh | 2 ++
 test/valid-test.sh   | 2 ++
 3 files changed, 7 insertions(+)

diff --git a/test/no-head-test.sh b/test/no-head-test.sh
index 1652097..2f95316 100755
--- a/test/no-head-test.sh
+++ b/test/no-head-test.sh
@@ -19,6 +19,9 @@ fails=0
 i=0
 tests="$(ls -1 valid/*.json | wc -l)"
 echo "1..$tests"
+
+set -x
+
 for input in valid/*.json
 do
   expected="${tmp}$(basename "$input" .json).no-head"
diff --git a/test/solidus-test.sh b/test/solidus-test.sh
index 8a641c4..736d3ba 100755
--- a/test/solidus-test.sh
+++ b/test/solidus-test.sh
@@ -27,6 +27,8 @@ else
   echo "ok $i - solidus escaping was left intact"
 fi
 
+set -x
+
 i="$(expr $i + 1)"
 JSONSH_OUT="$(eval jsonsh_cli -s < "$INPUT")" && \
     printf '%s\n' "$JSONSH_OUT" | diff -u - "${OUTPUT_WITHOUT_ESCAPING}"
diff --git a/test/valid-test.sh b/test/valid-test.sh
index 11f778e..a04a1dc 100755
--- a/test/valid-test.sh
+++ b/test/valid-test.sh
@@ -33,6 +33,8 @@ tests="$(echo "$FILES" | wc -l)"
 tests="$(expr $tests \* 8)"
 echo "1..$tests"
 
+set -x
+
 for input in $FILES
 do
   for EXT in parsed sorted normalized normalized_sorted \

From d01ed60a1de30499998e97d71da4726cf6294228 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 15:20:17 +0100
Subject: [PATCH 76/95] JSON.sh : try to detect shell type and features of
 current interpreter

---
 JSON.sh | 114 +++++++++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 97 insertions(+), 17 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index 7227cf7..2b8467f 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -58,22 +58,87 @@
 # to either refuse running in a shell or pick an implementation
 # for certain code paths.
 
+get_shellname() {
+    # Linux
+    if [ -x "/proc/$$/exe" ] ; then
+        readlink "/proc/$$/exe" 2>/dev/null && return 0
+        ls -la "/proc/$$/exe" | sed -e 's,^[^\/]*/,/,' -e 's,^.* -> ,,' 2>/dev/null && return 0
+    fi
+
+    # Solaris/illumos
+    if [ -x "/proc/$$/path/a.out" ] ; then
+        readlink "/proc/$$/path/a.out" 2>/dev/null && return 0
+        ls -la "/proc/$$/path/a.out" | sed -e 's,^[^\/]*/,/,' -e 's,^.* -> ,,' 2>/dev/null && return 0
+    fi
+
+    # By far this is most portable approach... although forking makes it slow
+    ps -e -o pid,comm | while read _PID _COMM ; do
+        [ "$$" = "$_PID" ] && echo "$_COMM" && return 0
+    done
+
+    return 1
+}
+
+get_shellbasename() {
+    basename "$(get_shellname)" || echo "sh"
+}
+
+# Flag that can be passed by caller - if we can not detect the interpreter (or
+# know it as unsupported), may we try to re-execute with a more capable one?
+[ -n "${SHELL_CANREEXEC-}" ] || SHELL_CANREEXEC=yes
+SHELL_BASENAME="$(get_shellbasename)"
+
+# Got support for regular expressions in extended-test [[ "$1" =~ ${regex} ]] ?
 SHELL_REGEX=no
+# Got support for pattern substitution in curly braces ${varname/pat/subst} ?
 SHELL_TWOSLASH=no
-if [ -n "${BASH-}" ] || [ -n "${BASH_VERSION-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
-    SHELL_REGEX=yes
-    SHELL_TWOSLASH=yes
-fi
 
-# TODO: detect if busybox - there SHELL_TWOSLASH=yes too, but not in DASH
+case "$SHELL_BASENAME" in
+    bash)
+        SHELL_REGEX=yes
+        SHELL_TWOSLASH=yes
+        ;;
+    dash)
+        SHELL_REGEX=yes
+        ;;
+    busybox*)
+        SHELL_TWOSLASH=yes
+        SHELL_BASENAME=busybox
+        ;;
+    #ash) ;;
+    #ksh93) ;;
+    #ksh88) ;;
+    #ksh) ;;
+    zsh)
+        SHELL_REGEX=yes
+        SHELL_TWOSLASH=yes
+        ;;
+    *)
+        if [ -n "${BASH-}" ] || [ -n "${BASH_VERSION-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
+            SHELL_REGEX=yes
+            SHELL_TWOSLASH=yes
+            SHELL_BASENAME=bash
+        elif [ -n "${ZSH_VERSION-}" ] ; then
+            SHELL_REGEX=yes
+            SHELL_TWOSLASH=yes
+            SHELL_BASENAME=zsh
+        else
+            echo "Unknown shell for JSON.sh: $SHELL_BASENAME" >&2
+            if [ "$SHELL_CANREEXEC" != no ] ; then
+                # NOTE: This can break scripts which source this file and are not in bash
+                for _TRY_SHELL in bash dash zsh ash busybox false ; do
+                    [ "$_TRY_SHELL" = busybox ] && _TRY_SHELL="busybox sh"
+                    ( $_TRY_SHELL -c "date" >/dev/null 2>/dev/null ) && break
+                done
+
+                echo "ERROR: JSON.sh requires to be run with BASH/ZSH/ASH/DASH interpreter! Subshelling due to SHELL_CANREEXEC=$SHELL_CANREEXEC : $_TRY_SHELL ..." >&2
+                ( $_TRY_SHELL "$0" "$@" )
+                exit $?
+            fi
+        fi
+        ;;
+esac
 
-false && \
-if [ -z "${BASH-}" ] && [ -z "${BASH_VERSION-}" ] && [ -z "${ZSH_VERSION-}" ]; then
-    # NOTE: This can break scripts which source this file and are not in bash
-    echo "ERROR: JSON.sh requires to be run with BASH/ZSH/ASH/DASH interpreter! Subshelling..." >&2
-    /usr/bin/bash "$0" "$@"
-    exit $?
-fi
 
 throw() {
   echo "$*" >&2
@@ -666,7 +731,7 @@ parse_value() {
   if [ "$print" -ne 0 ] && [ -n "$EXTRACT_JPATH" ] ; then
     if [ "$SHELL_REGEX" = yes ]; then
         ### BASH regex matching:
-        [[ ${jpath} =~ ${EXTRACT_JPATH} ]] || print=-1
+        [[ "${jpath}" =~ ${EXTRACT_JPATH} ]] || print=-1
     else
         [ -n "$(echo "${jpath}" | $GEGREP "${EXTRACT_JPATH}")" ] || print=-1
     fi
@@ -797,10 +862,25 @@ jsonsh_cli_subshell() (
 ### Active logic
 jsonsh_debugging_defaults
 
-# If not sourced into a bash script, parse stdin and quit
-# TODO: non-bash shells?
-[ "${JSONSH_SOURCED-}" = yes ] || \
-if  [ "$0" = "$BASH_SOURCE[0]" ] || [ "$0" = "$BASH_SOURCE" ] || [ -z "${BASH-}" ]; \
+# If NOT sourced into a bash script, parse stdin and quit
+# TODO: detect having been sourced into non-bash shells?
+if [ -z "${JSONSH_SOURCED-}" ]; then
+    case "$SHELL_BASENAME" in
+        bash)
+            if  [ "$0" = "$BASH_SOURCE[0]" ] || [ "$0" = "$BASH_SOURCE" ] ; then
+                JSONSH_SOURCED=no
+            fi
+            if  [ -n "${BASH-}" ] && [ "$0" = "-bash" ] ; then
+                JSONSH_SOURCED=yes
+            fi
+            ;;
+        *)  JSONSH_SOURCED=no ;;
+    esac
+fi
+
+#[ "${JSONSH_SOURCED-}" != yes ] || \
+#if  [ "$0" = "$BASH_SOURCE[0]" ] || [ "$0" = "$BASH_SOURCE" ] || [ -z "${BASH-}" ] || [ -z "$BASH_SOURCE" ]; \
+if [ "${JSONSH_SOURCED-}" != yes ]
 then
   jsonsh_cli "$@"
   exit $?

From a170d9dd2aac21ec2b518852dd861e2880ad54e6 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 15:32:26 +0100
Subject: [PATCH 77/95] JSON.sh : try to detect if we are sourced and/or can
 reexec into another shell

---
 JSON.sh | 60 +++++++++++++++++++++++++++++++--------------------------
 1 file changed, 33 insertions(+), 27 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index 2b8467f..2b6b353 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -93,6 +93,7 @@ SHELL_REGEX=no
 # Got support for pattern substitution in curly braces ${varname/pat/subst} ?
 SHELL_TWOSLASH=no
 
+SHELL_SUPPORTED=yes
 case "$SHELL_BASENAME" in
     bash)
         SHELL_REGEX=yes
@@ -123,22 +124,42 @@ case "$SHELL_BASENAME" in
             SHELL_TWOSLASH=yes
             SHELL_BASENAME=zsh
         else
-            echo "Unknown shell for JSON.sh: $SHELL_BASENAME" >&2
-            if [ "$SHELL_CANREEXEC" != no ] ; then
-                # NOTE: This can break scripts which source this file and are not in bash
-                for _TRY_SHELL in bash dash zsh ash busybox false ; do
-                    [ "$_TRY_SHELL" = busybox ] && _TRY_SHELL="busybox sh"
-                    ( $_TRY_SHELL -c "date" >/dev/null 2>/dev/null ) && break
-                done
-
-                echo "ERROR: JSON.sh requires to be run with BASH/ZSH/ASH/DASH interpreter! Subshelling due to SHELL_CANREEXEC=$SHELL_CANREEXEC : $_TRY_SHELL ..." >&2
-                ( $_TRY_SHELL "$0" "$@" )
-                exit $?
-            fi
+            SHELL_SUPPORTED=no
         fi
         ;;
 esac
 
+# TODO: detect having been sourced into non-bash shells?
+if [ -z "${JSONSH_SOURCED-}" ]; then
+    case "$SHELL_BASENAME" in
+        bash)
+            if  [ "$0" = "$BASH_SOURCE[0]" ] || [ "$0" = "$BASH_SOURCE" ] ; then
+                JSONSH_SOURCED=no
+            fi
+            if  [ -n "${BASH-}" ] && [ "$0" = "-bash" ] ; then
+                JSONSH_SOURCED=yes
+            fi
+            ;;
+        *)  JSONSH_SOURCED=no ;;
+    esac
+fi
+
+if [ "$SHELL_SUPPORTED" != "yes" ]; then
+    echo "Unknown shell for JSON.sh: $SHELL_BASENAME" >&2
+    if [ "$SHELL_CANREEXEC" != no ] && [ "$JSONSH_SOURCED" != "yes" ] ; then
+        # NOTE: This can break scripts which source this file and are not in bash
+        for _TRY_SHELL in bash dash zsh ash busybox false ; do
+            [ "$_TRY_SHELL" = busybox ] && _TRY_SHELL="busybox sh"
+            ( $_TRY_SHELL -c "date" >/dev/null 2>/dev/null ) && break
+        done
+
+        echo "ERROR: JSON.sh requires to be run with BASH/ZSH/ASH/DASH interpreter! Subshelling due to SHELL_CANREEXEC=$SHELL_CANREEXEC : $_TRY_SHELL ..." >&2
+        ( $_TRY_SHELL "$0" "$@" )
+        exit $?
+    else
+        echo "WARNING: Not changing shell because SHELL_CANREEXEC=$SHELL_CANREEXEC or JSONSH_SOURCED=$JSONSH_SOURCED - but JSON parsing can fail later on" >&2
+    fi
+fi
 
 throw() {
   echo "$*" >&2
@@ -863,21 +884,6 @@ jsonsh_cli_subshell() (
 jsonsh_debugging_defaults
 
 # If NOT sourced into a bash script, parse stdin and quit
-# TODO: detect having been sourced into non-bash shells?
-if [ -z "${JSONSH_SOURCED-}" ]; then
-    case "$SHELL_BASENAME" in
-        bash)
-            if  [ "$0" = "$BASH_SOURCE[0]" ] || [ "$0" = "$BASH_SOURCE" ] ; then
-                JSONSH_SOURCED=no
-            fi
-            if  [ -n "${BASH-}" ] && [ "$0" = "-bash" ] ; then
-                JSONSH_SOURCED=yes
-            fi
-            ;;
-        *)  JSONSH_SOURCED=no ;;
-    esac
-fi
-
 #[ "${JSONSH_SOURCED-}" != yes ] || \
 #if  [ "$0" = "$BASH_SOURCE[0]" ] || [ "$0" = "$BASH_SOURCE" ] || [ -z "${BASH-}" ] || [ -z "$BASH_SOURCE" ]; \
 if [ "${JSONSH_SOURCED-}" != yes ]

From 5d055f964f1461fcac3c2e30c42438dc6399fce5 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 15:39:16 +0100
Subject: [PATCH 78/95] JSON.sh : quote some more vars for portability

---
 JSON.sh | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index 2b6b353..0ffc7b0 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -581,7 +581,7 @@ parse_array() {
       while :
       do
         parse_value "$1" "$index"
-        index=$((index+1))
+        index="$(expr $index + 1)"
         ary="$ary""$value"
         if [ -n "$SORTDATA_ARR" ]; then
             [ -z "$aryml" ] && aryml="$value" || aryml="$aryml
@@ -606,7 +606,7 @@ $value"
 }
 
 parse_object() {
-  local key
+  local key=''
   local obj=''
   local objml=''
   read -r token
@@ -617,7 +617,7 @@ parse_object() {
       while :
       do
         case "$token" in
-          '"'*'"') key=$token ;;
+          '"'*'"') key="$token" ;;
           *) throw "EXPECTED string GOT '${token:-EOF}'" ;;
         esac
         read -r token
@@ -655,7 +655,10 @@ $key:$value"
 
 REGEX_NUMBER='^[+-]?([.][0-9]+|(0+|[1-9][0-9]*)([.][0-9]*)?)([eE][+-]?[0-9]*)?$'
 parse_value() {
-  local jpath="${1:+$1,}$2" isleaf=0 isempty=0 print=0
+  local jpath="${1:+$1,}$2"
+  local isleaf=0
+  local isempty=0
+  local print=0
   case "$token" in
     '{') parse_object "$jpath"
        [ "$value" = '{}' ] && isempty=1

From 2aac0ec261d3fd38f620c0933bb3e922ec264d16 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 15:39:42 +0100
Subject: [PATCH 79/95] JSON.sh : replace "local" with "typeset var" for
 portability

---
 JSON.sh | 50 +++++++++++++++++++++++++-------------------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index 0ffc7b0..0134aec 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -435,12 +435,12 @@ awk_egrep() {
 
 strip_newlines() {
   # replace line returns inside strings in input with \n string
-  local ILINE
-  local LINESTRIP
-  local NUMQ
-  local ODD
-  local INSTRING=0
-  local LINENUM=0
+  typeset var ILINE
+  typeset var LINESTRIP
+  typeset var NUMQ
+  typeset var ODD
+  typeset var INSTRING=0
+  typeset var LINENUM=0
 
   # The first "grep" should ensure that input for "while" has a trailing newline
   $GGREP '' | \
@@ -526,9 +526,9 @@ cook_a_string_arg() {
 }
 
 tokenize() {
-  local GREP_O
-  local ESCAPE
-  local CHAR
+  typeset var GREP_O
+  typeset var ESCAPE
+  typeset var CHAR
 
   if echo "test string" | $GEGREP -ao --color=never "test" >/dev/null 2>&1
   then
@@ -552,12 +552,12 @@ tokenize() {
   fi
 
   # Allow tabs inside strings
-  local CHART="($CHAR|[[:blank:]])"
-  local STRINGVAL="$CHART*($ESCAPE$CHART*)*"
-  local STRING="(\"$STRINGVAL\")"
-  local NUMBER='[+-]?([.][0-9]+|(0+|[1-9][0-9]*)([.][0-9]*)?)([eE][+-]?[0-9]*)?'
-  local KEYWORD='null|false|true'
-  local SPACE='[[:space:]]+'
+  typeset var CHART="($CHAR|[[:blank:]])"
+  typeset var STRINGVAL="$CHART*($ESCAPE$CHART*)*"
+  typeset var STRING="(\"$STRINGVAL\")"
+  typeset var NUMBER='[+-]?([.][0-9]+|(0+|[1-9][0-9]*)([.][0-9]*)?)([eE][+-]?[0-9]*)?'
+  typeset var KEYWORD='null|false|true'
+  typeset var SPACE='[[:space:]]+'
 
   # Force zsh to expand $A into multiple words
   is_wordsplit_disabled="$(unsetopt 2>/dev/null | grep -c '^shwordsplit$')"
@@ -570,9 +570,9 @@ tokenize() {
 }
 
 parse_array() {
-  local index=0
-  local ary=''
-  local aryml=''
+  typeset var index=0
+  typeset var ary=''
+  typeset var aryml=''
   read -r token
   print_debug $DEBUGLEVEL_PRINTTOKEN "parse_array(1):" "token='$token'"
   case "$token" in
@@ -606,9 +606,9 @@ $value"
 }
 
 parse_object() {
-  local key=''
-  local obj=''
-  local objml=''
+  typeset var key=''
+  typeset var obj=''
+  typeset var objml=''
   read -r token
   print_debug $DEBUGLEVEL_PRINTTOKEN "parse_object(1):" "token='$token'"
   case "$token" in
@@ -655,10 +655,10 @@ $key:$value"
 
 REGEX_NUMBER='^[+-]?([.][0-9]+|(0+|[1-9][0-9]*)([.][0-9]*)?)([eE][+-]?[0-9]*)?$'
 parse_value() {
-  local jpath="${1:+$1,}$2"
-  local isleaf=0
-  local isempty=0
-  local print=0
+  typeset var jpath="${1:+$1,}$2"
+  typeset var isleaf=0
+  typeset var isempty=0
+  typeset var print=0
   case "$token" in
     '{') parse_object "$jpath"
        [ "$value" = '{}' ] && isempty=1

From 30d919cf3f55fc50ce1489d2403796f76069bdce Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 15:55:58 +0100
Subject: [PATCH 80/95] JSON.sh : quote some more vars for portability

---
 JSON.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/JSON.sh b/JSON.sh
index 0134aec..0af9c38 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -456,7 +456,7 @@ strip_newlines() {
     fi
     # Count unescaped quotes:
     NUMQ="${#LINESTRIP}"
-    ODD="$(($NUMQ%2))"
+    ODD="$(expr $NUMQ % 2)"
     LINENUM="$(expr $LINENUM + 1)"
 
     if [ "$ODD" -eq 1 ] && [ "$INSTRING" -eq 0 ]; then

From 40ba0dc9e39a8e86e89d425f56d2d7ac8c6a3c36 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 15:58:12 +0100
Subject: [PATCH 81/95] Revert "JSON.sh : replace "local" with "typeset var"
 for portability"

This reverts commit 2aac0ec261d3fd38f620c0933bb3e922ec264d16.
While "local" is unknown to "ksh", the "typeset" is unknown in "dash/ash"... bummer...
---
 JSON.sh | 50 +++++++++++++++++++++++++-------------------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index 0af9c38..cf83e5b 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -435,12 +435,12 @@ awk_egrep() {
 
 strip_newlines() {
   # replace line returns inside strings in input with \n string
-  typeset var ILINE
-  typeset var LINESTRIP
-  typeset var NUMQ
-  typeset var ODD
-  typeset var INSTRING=0
-  typeset var LINENUM=0
+  local ILINE
+  local LINESTRIP
+  local NUMQ
+  local ODD
+  local INSTRING=0
+  local LINENUM=0
 
   # The first "grep" should ensure that input for "while" has a trailing newline
   $GGREP '' | \
@@ -526,9 +526,9 @@ cook_a_string_arg() {
 }
 
 tokenize() {
-  typeset var GREP_O
-  typeset var ESCAPE
-  typeset var CHAR
+  local GREP_O
+  local ESCAPE
+  local CHAR
 
   if echo "test string" | $GEGREP -ao --color=never "test" >/dev/null 2>&1
   then
@@ -552,12 +552,12 @@ tokenize() {
   fi
 
   # Allow tabs inside strings
-  typeset var CHART="($CHAR|[[:blank:]])"
-  typeset var STRINGVAL="$CHART*($ESCAPE$CHART*)*"
-  typeset var STRING="(\"$STRINGVAL\")"
-  typeset var NUMBER='[+-]?([.][0-9]+|(0+|[1-9][0-9]*)([.][0-9]*)?)([eE][+-]?[0-9]*)?'
-  typeset var KEYWORD='null|false|true'
-  typeset var SPACE='[[:space:]]+'
+  local CHART="($CHAR|[[:blank:]])"
+  local STRINGVAL="$CHART*($ESCAPE$CHART*)*"
+  local STRING="(\"$STRINGVAL\")"
+  local NUMBER='[+-]?([.][0-9]+|(0+|[1-9][0-9]*)([.][0-9]*)?)([eE][+-]?[0-9]*)?'
+  local KEYWORD='null|false|true'
+  local SPACE='[[:space:]]+'
 
   # Force zsh to expand $A into multiple words
   is_wordsplit_disabled="$(unsetopt 2>/dev/null | grep -c '^shwordsplit$')"
@@ -570,9 +570,9 @@ tokenize() {
 }
 
 parse_array() {
-  typeset var index=0
-  typeset var ary=''
-  typeset var aryml=''
+  local index=0
+  local ary=''
+  local aryml=''
   read -r token
   print_debug $DEBUGLEVEL_PRINTTOKEN "parse_array(1):" "token='$token'"
   case "$token" in
@@ -606,9 +606,9 @@ $value"
 }
 
 parse_object() {
-  typeset var key=''
-  typeset var obj=''
-  typeset var objml=''
+  local key=''
+  local obj=''
+  local objml=''
   read -r token
   print_debug $DEBUGLEVEL_PRINTTOKEN "parse_object(1):" "token='$token'"
   case "$token" in
@@ -655,10 +655,10 @@ $key:$value"
 
 REGEX_NUMBER='^[+-]?([.][0-9]+|(0+|[1-9][0-9]*)([.][0-9]*)?)([eE][+-]?[0-9]*)?$'
 parse_value() {
-  typeset var jpath="${1:+$1,}$2"
-  typeset var isleaf=0
-  typeset var isempty=0
-  typeset var print=0
+  local jpath="${1:+$1,}$2"
+  local isleaf=0
+  local isempty=0
+  local print=0
   case "$token" in
     '{') parse_object "$jpath"
        [ "$value" = '{}' ] && isempty=1

From aabbf967e35db9cc57aaa5468e6dabd210bcfe48 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 16:08:03 +0100
Subject: [PATCH 82/95] JSON.sh : use none of the advanced features in dash/ash

---
 JSON.sh | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index cf83e5b..266de34 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -99,14 +99,12 @@ case "$SHELL_BASENAME" in
         SHELL_REGEX=yes
         SHELL_TWOSLASH=yes
         ;;
-    dash)
-        SHELL_REGEX=yes
+    dash|ash) # The spartan bare minimum
         ;;
     busybox*)
         SHELL_TWOSLASH=yes
         SHELL_BASENAME=busybox
         ;;
-    #ash) ;;
     #ksh93) ;;
     #ksh88) ;;
     #ksh) ;;

From 4813133bb9e406e5b96d1ebbc826426ac56643ac Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 16:09:09 +0100
Subject: [PATCH 83/95] Sub-tests : take care of shwordsplit for zsh, wherever
 we iterate "for $LISTVAR"

---
 all-tests.sh       | 7 +++++++
 test/valid-test.sh | 5 +++++
 2 files changed, 12 insertions(+)

diff --git a/all-tests.sh b/all-tests.sh
index e3e9ae1..a201e6f 100755
--- a/all-tests.sh
+++ b/all-tests.sh
@@ -62,6 +62,11 @@ SKIP_SHELLS=""
 [ -n "$SHELL_PROGS" ] || SHELL_PROGS="bash dash ash busybox ksh ksh88 ksh93"
 [ -n "$TEST_PATTERN" ] || TEST_PATTERN='test/*.sh'
 export TEST_PATTERN
+
+# Force zsh to expand $A into multiple words
+is_wordsplit_disabled="$(unsetopt 2>/dev/null | grep -c '^shwordsplit$')"
+if [ "$is_wordsplit_disabled" != 0 ]; then setopt shwordsplit; fi
+
 for SHELL_PROG in $SHELL_PROGS ; do
     [ "$SHELL_PROG" = "busybox" ] && SHELL_PROG="busybox sh"
     { [ "$SHELL_PROG" = "-" ] || [ "$SHELL_PROG" = " " ] ; } && \
@@ -86,6 +91,8 @@ for SHELL_PROG in $SHELL_PROGS ; do
     echo ""
 done
 
+if [ "${is_wordsplit_disabled-}" != 0 ]; then unsetopt shwordsplit; is_wordsplit_disabled=0; fi
+
 echo "OVERALL RESULT:"
 echo "OKAY_SHELLS = $OKAY_SHELLS"
 echo "FAIL_SHELLS = $FAIL_SHELLS"
diff --git a/test/valid-test.sh b/test/valid-test.sh
index a04a1dc..b33ba29 100755
--- a/test/valid-test.sh
+++ b/test/valid-test.sh
@@ -35,8 +35,13 @@ echo "1..$tests"
 
 set -x
 
+# Force zsh to expand $A into multiple words
+is_wordsplit_disabled="$(unsetopt 2>/dev/null | grep -c '^shwordsplit$')"
+if [ "$is_wordsplit_disabled" != 0 ]; then setopt shwordsplit; fi
+
 for input in $FILES
 do
+  if [ "${is_wordsplit_disabled-}" != 0 ]; then unsetopt shwordsplit; is_wordsplit_disabled=0; fi
   for EXT in parsed sorted normalized normalized_sorted \
         numnormalized numnormalized_stripped \
         normalized_numnormalized normalized_numnormalized_stripped \

From b9cc99a0027ce369aa3f5044b0b3e48a0d388d97 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 16:13:04 +0100
Subject: [PATCH 84/95] Sub-tests : use common "${tmp}"

---
 test/invalid-test.sh   | 17 +++++++++++++----
 test/tokenizer-test.sh | 13 +++++++++++--
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/test/invalid-test.sh b/test/invalid-test.sh
index 578a88a..84b434e 100755
--- a/test/invalid-test.sh
+++ b/test/invalid-test.sh
@@ -12,23 +12,32 @@ JSONSH_SOURCED=yes
 fails=0
 tests="`ls -1 invalid/* | wc -l`"
 
+[ -n "${tmp-}" ] || tmp="/tmp"
+
+# Avoid duplicate // in plain-shell syntax
+tmp="$(echo "$tmp" | sed 's,/+,/,g')"
+case "$tmp" in
+    */) ;;
+    *)  tmp="$tmp/" ;;
+esac
+
 echo "1..${tests##* }"
 for input in invalid/*
 do
   i="$(expr $i + 1)"
-  if jsonsh_cli < "$input" > /tmp/JSON.sh_outlog 2> /tmp/JSON.sh_errlog
+  if jsonsh_cli < "$input" > "${tmp}"JSON.sh_outlog 2> "${tmp}"JSON.sh_errlog
   then
     echo "not ok $i - cat $input | ../JSON.sh should have failed"
     #this should be indented with '#' at the start.
     echo "OUTPUT WAS >>>"
-    cat /tmp/JSON.sh_outlog
+    cat "${tmp}"JSON.sh_outlog
     echo "ERRORS WAS >>>"
-    cat /tmp/JSON.sh_errlog
+    cat "${tmp}"JSON.sh_errlog
     echo "<<<"
     fails="$(expr $fails + 1)"
   else
     echo "ok $i - $input was rejected as expected"
-    echo "# `cat /tmp/JSON.sh_errlog`"
+    echo "# `cat "${tmp}"JSON.sh_errlog`"
   fi
 done
 
diff --git a/test/tokenizer-test.sh b/test/tokenizer-test.sh
index 0282b10..21c77f6 100755
--- a/test/tokenizer-test.sh
+++ b/test/tokenizer-test.sh
@@ -6,14 +6,23 @@ cd "$(dirname "$0")"
 JSONSH_SOURCED=yes
 . ../JSON.sh </dev/null
 
+[ -n "${tmp-}" ] || tmp="/tmp"
+
+# Avoid duplicate // in plain-shell syntax
+tmp="$(echo "$tmp" | sed 's,/+,/,g')"
+case "$tmp" in
+    */) ;;
+    *)  tmp="$tmp/" ;;
+esac
+
 i=0
 fails=0
 ttest () {
   i="$(expr $i + 1)"
   input="$1"; shift
   expected="$(printf '%s\n' "$@")"
-  echo "$expected" > /tmp/json_ttest_expected
-  if echo "$input" | tokenize | diff -u - /tmp/json_ttest_expected
+  echo "$expected" > "${tmp}"json_ttest_expected
+  if echo "$input" | tokenize | diff -u - "${tmp}"json_ttest_expected
   then
     echo "ok $i - $input"
   else

From 84b01680ff33aa63fc2e52b137353debfd339652 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 16:13:59 +0100
Subject: [PATCH 85/95] Sub-tests : no-head-test.sh : report contents of
 "$expected"

---
 test/no-head-test.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/no-head-test.sh b/test/no-head-test.sh
index 2f95316..c51cd34 100755
--- a/test/no-head-test.sh
+++ b/test/no-head-test.sh
@@ -38,6 +38,7 @@ do
     fails="$(expr $fails + 1)"
     echo ">>> JSONSH_OUT='$JSONSH_OUT'"
     echo ">>> EXPECTED : `ls -la $expected`"
+    [ -s "$expected" ] || cat "$expected"
   else
     echo "ok $i - $input"
   fi

From 64a00aaa8ef1fe38f99e9ca131ac6846e3ad77a1 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 16:17:36 +0100
Subject: [PATCH 86/95] tokenizer-test.sh : optimize a bit and update for
 portability

---
 test/tokenizer-test.sh | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/test/tokenizer-test.sh b/test/tokenizer-test.sh
index 21c77f6..1342048 100755
--- a/test/tokenizer-test.sh
+++ b/test/tokenizer-test.sh
@@ -22,7 +22,12 @@ ttest () {
   input="$1"; shift
   expected="$(printf '%s\n' "$@")"
   echo "$expected" > "${tmp}"json_ttest_expected
-  if echo "$input" | tokenize | diff -u - "${tmp}"json_ttest_expected
+
+  # Such explicit chaining is equivalent to "pipefail" in non-Bash interpreters
+  JSONSH_OUT="$(echo "$input" | tokenize)" && \
+    printf '%s\n' "$JSONSH_OUT" | diff -u - "${tmp}"json_ttest_expected
+  JSONSH_RES=$?
+  if [ "$JSONSH_RES" = 0 ]
   then
     echo "ok $i - $input"
   else
@@ -56,7 +61,7 @@ ttest '[ null   ,  -110e10, "null" ]' \
 ttest '{"e": false}'     '{' '"e"' ':' 'false' '}'
 ttest '{"e": "string"}'  '{' '"e"' ':' '"string"' '}'
 
-if ! cat ../package.json | tokenize >/dev/null
+if tokenize < ../package.json >/dev/null
 then
   fails="$(expr $fails + 1)"
   echo "Tokenizing package.json failed!"

From 12fbdd18ecb5b303a5cdcc201f4f5daad784b45b Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 16:19:01 +0100
Subject: [PATCH 87/95] Sub-tests : no-head-test.sh : report contents of
 "$expected"

---
 test/no-head-test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/no-head-test.sh b/test/no-head-test.sh
index c51cd34..2012426 100755
--- a/test/no-head-test.sh
+++ b/test/no-head-test.sh
@@ -38,7 +38,7 @@ do
     fails="$(expr $fails + 1)"
     echo ">>> JSONSH_OUT='$JSONSH_OUT'"
     echo ">>> EXPECTED : `ls -la $expected`"
-    [ -s "$expected" ] || cat "$expected"
+    cat "$expected"
   else
     echo "ok $i - $input"
   fi

From d84bf3d0c458a76b5d4773c78ed8da14dafa40c9 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 16:33:21 +0100
Subject: [PATCH 88/95] Sub-tests : no-head-test.sh : use printf when preparing
 (and displaying) results so as to not choke on escaped chard we are testing

---
 test/no-head-test.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/no-head-test.sh b/test/no-head-test.sh
index 2012426..5231a97 100755
--- a/test/no-head-test.sh
+++ b/test/no-head-test.sh
@@ -26,7 +26,7 @@ for input in valid/*.json
 do
   expected="${tmp}$(basename "$input" .json).no-head"
   # NOTE: The echo trick is required to ensure EOLs for both empty and populated results
-  echo "$(egrep -v '^\[]' < "$(dirname "$input")/$(basename "$input" .json).parsed")" > "$expected"
+  printf '%s\n' "$(egrep -v '^\[]' < "$(dirname "$input")/$(basename "$input" .json).parsed")" > "$expected"
   i="$(expr $i + 1)"
   # Such explicit chaining is equivalent to "pipefail" in non-Bash interpreters
   JSONSH_OUT="$(jsonsh_cli -n < "$input")" && \
@@ -36,7 +36,7 @@ do
   then
     echo "not ok $i - $input"
     fails="$(expr $fails + 1)"
-    echo ">>> JSONSH_OUT='$JSONSH_OUT'"
+    printf ">>> JSONSH_OUT='%s'\n" "$JSONSH_OUT"
     echo ">>> EXPECTED : `ls -la $expected`"
     cat "$expected"
   else

From e0fdb2fbda8ea41cf4e257b0d3838542102c5259 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 16:52:41 +0100
Subject: [PATCH 89/95] JSON.sh : be more careful about return codes of the
 tokenize() call

---
 JSON.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/JSON.sh b/JSON.sh
index 266de34..089de38 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -563,8 +563,10 @@ tokenize() {
   tee_stderr BEFORE_TOKENIZER $DEBUGLEVEL_PRINTTOKEN_PIPELINE | \
   $GREP_O "$STRING|$NUMBER|$KEYWORD|$SPACE|." | $GEGREP -v "^$SPACE$" | \
   tee_stderr AFTER_TOKENIZER $DEBUGLEVEL_PRINTTOKEN_PIPELINE
+  RES=$?
   if [ "$is_wordsplit_disabled" != 0 ]; then unsetopt shwordsplit; fi
-  unset is_wordsplit_disabled
+  unset is_wordsplit_disabled || true
+  return $RES
 }
 
 parse_array() {

From a83531652c705a70167f50f1d4575f5282222a71 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 16:54:16 +0100
Subject: [PATCH 90/95] tokenizer-test.sh : fix the test (logic error) and
 improve reports for file-parsing step

---
 test/tokenizer-test.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/test/tokenizer-test.sh b/test/tokenizer-test.sh
index 1342048..7ca0fc2 100755
--- a/test/tokenizer-test.sh
+++ b/test/tokenizer-test.sh
@@ -61,10 +61,14 @@ ttest '[ null   ,  -110e10, "null" ]' \
 ttest '{"e": false}'     '{' '"e"' ':' 'false' '}'
 ttest '{"e": "string"}'  '{' '"e"' ':' '"string"' '}'
 
+i="$(expr $i + 1)"
+input="Tokenizing the 'package.json' file"
 if tokenize < ../package.json >/dev/null
 then
+  echo "ok $i - $input"
+else
+  echo "not ok $i - $input"
   fails="$(expr $fails + 1)"
-  echo "Tokenizing package.json failed!"
 fi
 
 echo "$fails test(s) failed"

From cd3ce67af679ac9dd4f1b14f353b29fdf8ca5cf5 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 17:08:26 +0100
Subject: [PATCH 91/95] valid-test.sh : report details of failures, if any

---
 test/valid-test.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/valid-test.sh b/test/valid-test.sh
index b33ba29..a32507f 100755
--- a/test/valid-test.sh
+++ b/test/valid-test.sh
@@ -90,6 +90,9 @@ do
       then
         echo "not ok $i - $input $EXT"
         fails="$(expr $fails + 1)"
+        printf ">>> JSONSH_OUT='%s'\n" "$JSONSH_OUT"
+        echo ">>> EXPECTED : `ls -la $expected`"
+        cat "$expected"
       else
         echo "ok $i - $input $EXT"
         passes="$(expr $passes + 1)"

From a7f6899996855cdc50e1554ad345cc22f8a4fa7e Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 17:19:11 +0100
Subject: [PATCH 92/95] JSON.sh : replace "echo -E" with "printf %s\n"

---
 JSON.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index 089de38..ccc2d64 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -296,7 +296,7 @@ print_debug() {
     DL="$1"
     shift
     [ "$DEBUG" -ge "$DL" ] 2>/dev/null && \
-        echo -E "[$$]DEBUG($DL): $@" >&2
+        printf '[%s]DEBUG(%s): %s\n' "$$" "$DL" "$*" >&2
     :
 }
 
@@ -310,7 +310,7 @@ tee_stderr() {
     ### If debug is not enabled, skip tee'ing quickly with little impact
     [ "$DEBUG" -lt "$TEE_DEBUG" ] 2>/dev/null && cat || \
     while IFS= read -r LINE; do
-        echo -E "$LINE"
+        printf '%s\n' "$LINE"
         print_debug "$TEE_DEBUG" "$TEE_TAG" "$LINE"
     done
     :
@@ -599,7 +599,7 @@ $value"
       ;;
   esac
   if [ -n "$SORTDATA_ARR" ]; then
-    ary="$(echo -E "$aryml" | $SORTDATA_ARR | tr '\n' ',' | $GSED 's|,*$||' 2>/dev/null | $GSED 's|^,*||' 2>/dev/null)"
+    ary="$(printf '%s\n' "$aryml" | $SORTDATA_ARR | tr '\n' ',' | $GSED 's|,*$||' 2>/dev/null | $GSED 's|^,*||' 2>/dev/null)"
   fi
   [ "$BRIEF" = 0 ] && value="$(printf '[%s]' "$ary")" || value=""
   :
@@ -647,7 +647,7 @@ $key:$value"
     ;;
   esac
   if [ -n "$SORTDATA_OBJ" ]; then
-    obj="$(echo -E "$objml" | $SORTDATA_OBJ | tr '\n' ',' | $GSED 's|,*$||' 2>/dev/null | $GSED 's|^,*||' 2>/dev/null)"
+    obj="$(printf '%s\n' "$objml" | $SORTDATA_OBJ | tr '\n' ',' | $GSED 's|,*$||' 2>/dev/null | $GSED 's|^,*||' 2>/dev/null)"
   fi
   [ "$BRIEF" = 0 ] && value="$(printf '{%s}' "$obj")" || value=""
   :

From d7ab0c26d179b5cbd536a5265f006473ad050ad8 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 17:21:58 +0100
Subject: [PATCH 93/95] all-tests.sh : do not test with KSH by default so far

---
 all-tests.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/all-tests.sh b/all-tests.sh
index a201e6f..141645f 100755
--- a/all-tests.sh
+++ b/all-tests.sh
@@ -59,7 +59,8 @@ jsonsh_tests() (
 OKAY_SHELLS=""
 FAIL_SHELLS=""
 SKIP_SHELLS=""
-[ -n "$SHELL_PROGS" ] || SHELL_PROGS="bash dash ash busybox ksh ksh88 ksh93"
+[ -n "$SHELL_PROGS" ] || SHELL_PROGS="bash dash ash zsh busybox"
+#SHELL_PROGS="$SHELL_PROGS ksh ksh88 ksh93"
 [ -n "$TEST_PATTERN" ] || TEST_PATTERN='test/*.sh'
 export TEST_PATTERN
 

From 8b1357b53ed03eb3a7a0ffac375ca6b10b9dd184 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 17:32:50 +0100
Subject: [PATCH 94/95] JSON.sh : avoid potential errors due to unset DEBUG
 value

---
 JSON.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/JSON.sh b/JSON.sh
index ccc2d64..2ab0a51 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -271,8 +271,8 @@ usage() {
 
 validate_debuglevel() {
     ### Beside command-line, debugging can be enabled by envvars from the caller
-    { [ x"$DEBUG" = xy ] || [ x"$DEBUG" = xyes ] ; } && DEBUG=1
-    [ -n "$DEBUG" ] && [ "$DEBUG" -ge 0 ] 2>/dev/null || DEBUG=0
+    { [ x"${DEBUG-}" = xy ] || [ x"${DEBUG-}" = xyes ] ; } && DEBUG=1
+    [ -n "${DEBUG-}" ] && [ "${DEBUG-}" -ge 0 ] 2>/dev/null || DEBUG=0
 }
 
 unquote() {

From 288bfd26b6b9eb3ce01520065b796b133b22d581 Mon Sep 17 00:00:00 2001
From: Jim Klimov <jim@jimklimov.com>
Date: Wed, 8 Feb 2017 17:42:58 +0100
Subject: [PATCH 95/95] JSON.sh : fix fails when DEBUG is enabled and input
 does not end with newline

---
 JSON.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/JSON.sh b/JSON.sh
index 2ab0a51..532a751 100755
--- a/JSON.sh
+++ b/JSON.sh
@@ -308,8 +308,9 @@ tee_stderr() {
         TEE_DEBUG=$DEBUGLEVEL_PRINTTOKEN_PIPELINE
 
     ### If debug is not enabled, skip tee'ing quickly with little impact
+    ### The first "grep" should ensure that input for "while" has a trailing newline
     [ "$DEBUG" -lt "$TEE_DEBUG" ] 2>/dev/null && cat || \
-    while IFS= read -r LINE; do
+    $GGREP '' | while IFS= read -r LINE; do
         printf '%s\n' "$LINE"
         print_debug "$TEE_DEBUG" "$TEE_TAG" "$LINE"
     done