Merge pull request #1 from dhkolf/master

updated to dkjson 2.4

dist.info

@@ -1,12 +1,12 @@
 --- This file is part of LuaDist project
 
 name = "dkjson"
-version = "2.2"
+version = "2.4"
 
 desc = "dkjson is a module for encoding and decoding JSON data. It supports UTF-8."
 author = "David Kolf"
 license = "MIT/X11"
-url = "http://chiselapp.com/user/dhkolf/repository/dkjson/"
+url = "http://dkolf.de/src/dkjson-lua.fsl/"
 maintainer = "Peter Drahoš"
 
 depends = {

dkjson-2.2-1.rockspec

@@ -1,30 +0,0 @@
-package = "dkjson"
-version = "2.2-1"
-source = {
-  url = "http://chiselapp.com/user/dhkolf/repository/dkjson/tarball/dkjson-2.2.tar.gz?uuid=release_2_2",
-  file = "dkjson-2.2.tar.gz"
-}
-description = {
-  summary = "David Kolf's JSON module for Lua",
-  detailed = [[
-dkjson is a module for encoding and decoding JSON data. It supports UTF-8.
-
-JSON (JavaScript Object Notation) is a format for serializing data based
-on the syntax for JavaScript data structures.
-
-dkjson is written in Lua without any dependencies, but
-when LPeg is available dkjson uses it to speed up decoding.
-]],
-  homepage = "http://chiselapp.com/user/dhkolf/repository/dkjson/",
-  license = "MIT/X11"
-}
-dependencies = {
-  "lua >= 5.1"
-}
-build = {
-  type = "builtin",
-  modules = {
-    dkjson = "dkjson.lua"
-  }
-}
-

dkjson.lua

@@ -1,18 +1,26 @@
     -- Module options:
     local always_try_using_lpeg = true
+    local register_global_module_table = false
+    local global_module_name = 'json'
 
     --[==[
 
 David Kolf's JSON module for Lua 5.1/5.2
 ========================================
 
-*Version 2.2*
+*Version 2.4*
 
-This module writes no global values, not even the module table.
+In the default configuration this module writes no global values, not even
-Import it using
+the module table. Import it using
 
     json = require ("dkjson")
 
+In environments where `require` or a similiar function are not available
+and you cannot receive the return value of the module, you can set the
+option `register_global_module_table` to `true`.  The module table will
+then be saved in the global variable with the name given by the option
+`global_module_name`.
+
 Exported functions and values:
 
 `json.encode (object [, state])`
@@ -105,7 +113,7 @@ You can use this value for setting explicit `null` values.
 `json.version`
 --------------
 
-Set to `"dkjson 2.2"`.
+Set to `"dkjson 2.4"`.
 
 `json.quotestring (string)`
 ---------------------------
@@ -159,12 +167,12 @@ This variable is set to `true` when LPeg was loaded successfully.
 Contact
 -------
 
-You can contact the author by sending an e-mail to 'kolf' at the
+You can contact the author by sending an e-mail to 'david' at the
-e-mail provider 'gmx.de'.
+domain 'dkolf.de'.
 
 ---------------------------------------------------------------------
 
-*Copyright (C) 2010, 2011, 2012 David Heiko Kolf*
+*Copyright (C) 2010-2013 David Heiko Kolf*
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the
@@ -202,11 +210,16 @@ local floor, huge = math.floor, math.huge
 local strrep, gsub, strsub, strbyte, strchar, strfind, strlen, strformat =
       string.rep, string.gsub, string.sub, string.byte, string.char,
       string.find, string.len, string.format
+local strmatch = string.match
 local concat = table.concat
 
-local _ENV = nil -- blocking globals in Lua 5.2
+local json = { version = "dkjson 2.4" }
 
-local json = { version = "dkjson 2.2" }
+if register_global_module_table then
+  _G[global_module_name] = json
+end
+
+local _ENV = nil -- blocking globals in Lua 5.2
 
 pcall (function()
   -- Enable access to blocked metatables.
@@ -297,15 +310,48 @@ local function quotestring (value)
     value = fsub (value, "\216[\128-\132]", escapeutf8)
     value = fsub (value, "\220\143", escapeutf8)
     value = fsub (value, "\225\158[\180\181]", escapeutf8)
-    value = fsub (value, "\226\128[\140-\143\168\175]", escapeutf8)
+    value = fsub (value, "\226\128[\140-\143\168-\175]", escapeutf8)
     value = fsub (value, "\226\129[\160-\175]", escapeutf8)
     value = fsub (value, "\239\187\191", escapeutf8)
-    value = fsub (value, "\239\191[\176\191]", escapeutf8)
+    value = fsub (value, "\239\191[\176-\191]", escapeutf8)
   end
   return "\"" .. value .. "\""
 end
 json.quotestring = quotestring
 
+local function replace(str, o, n)
+  local i, j = strfind (str, o, 1, true)
+  if i then
+    return strsub(str, 1, i-1) .. n .. strsub(str, j+1, -1)
+  else
+    return str
+  end
+end
+
+-- locale independent num2str and str2num functions
+local decpoint, numfilter
+
+local function updatedecpoint ()
+  decpoint = strmatch(tostring(0.5), "([^05+])")
+  -- build a filter that can be used to remove group separators
+  numfilter = "[^0-9%-%+eE" .. gsub(decpoint, "[%^%$%(%)%%%.%[%]%*%+%-%?]", "%%%0") .. "]+"
+end
+
+updatedecpoint()
+
+local function num2str (num)
+  return replace(fsub(tostring(num), numfilter, ""), decpoint, ".")
+end
+
+local function str2num (str)
+  local num = tonumber(replace(str, ".", decpoint))
+  if not num then
+    updatedecpoint()
+    num = tonumber(replace(str, ".", decpoint))
+  end
+  return num
+end
+
 local function addnewline2 (level, buffer, buflen)
   buffer[buflen+1] = "\n"
   buffer[buflen+2] = strrep ("  ", level)
@@ -370,7 +416,7 @@ encode2 = function (value, indent, level, buffer, buflen, tables, globalorder)
       -- This is the behaviour of the original JSON implementation.
       s = "null"
     else
-      s = tostring (value)
+      s = num2str (value)
     end
     buflen = buflen + 1
     buffer[buflen] = s
@@ -452,6 +498,7 @@ function json.encode (value, state)
   state = state or {}
   local oldbuffer = state.buffer
   local buffer = oldbuffer or {}
+  updatedecpoint()
   local ret, msg = encode2 (value, state.indent, state.level or 0,
                    buffer, state.bufferlen or 0, state.tables or {}, state.keyorder)
   if not ret then
@@ -647,7 +694,7 @@ scanvalue = function (str, pos, nullval, objectmeta, arraymeta)
   else
     local pstart, pend = strfind (str, "^%-?[%d%.]+[eE]?[%+%-]?%d*", pos)
     if pstart then
-      local number = tonumber (strsub (str, pstart, pend))
+      local number = str2num (strsub (str, pstart, pend))
       if number then
         return number, pend + 1
       end
@@ -682,8 +729,13 @@ end
 
 function json.use_lpeg ()
   local g = require ("lpeg")
+
+  if g.version() == "0.11" then
+    error "due to a bug in LPeg 0.11, it cannot be used for JSON matching"
+  end
+
   local pegmatch = g.match
-  local P, S, R, V = g.P, g.S, g.R, g.V
+  local P, S, R = g.P, g.S, g.R
 
   local function ErrorCall (str, pos, msg, state)
     if not state.msg then
@@ -720,7 +772,7 @@ function json.use_lpeg ()
   local Integer = P"-"^(-1) * (P"0" + (R"19" * R"09"^0))
   local Fractal = P"." * R"09"^0
   local Exponent = (S"eE") * (S"+-")^(-1) * R"09"^1
-  local Number = (Integer * Fractal^(-1) * Exponent^(-1))/tonumber
+  local Number = (Integer * Fractal^(-1) * Exponent^(-1))/str2num
   local Constant = P"true" * g.Cc (true) + P"false" * g.Cc (false) + P"null" * g.Carg (1)
   local SimpleValue = Number + String + Constant
   local ArrayContent, ObjectContent

jsontest.lua

@@ -19,6 +19,18 @@ local test_module, opt = ... -- command line argument
 
 --local opt = "refcycle" -- What happens when a reference cycle gets encoded?
 
+local testlocale = "de_DE.UTF8"
+
+local function inlocale(fn)
+  local oldloc = os.setlocale(nil, 'numeric')
+  if not os.setlocale(testlocale, 'numeric') then
+    print("test could not switch to locale "..testlocale)
+  else
+    fn()
+  end
+  os.setlocale(oldloc, 'numeric')
+end
+
 if test_module == 'dkjson-nopeg' then
   test_module = 'dkjson'
   package.preload["lpeg"] = function () error "lpeg disabled" end
@@ -26,6 +38,11 @@ if test_module == 'dkjson-nopeg' then
   lpeg = nil
 end
 
+if test_module == 'dkjson-lulpeg' then
+  test_module = 'dkjson'
+  package.loaded["lpeg"] = require "lulpeg"
+end
+
 do
   -- http://chiselapp.com/user/dhkolf/repository/dkjson/
   local dkjson = require "dkjson"
@@ -34,6 +51,7 @@ do
 end
 
 if test_module == 'cmj-json' then
+  -- https://github.com/craigmj/json4lua/
   -- http://json.luaforge.net/
   local json = require "cmjjson" -- renamed, the original file was just 'json'
   encode = json.encode
@@ -72,6 +90,7 @@ elseif test_module == 'sb-json' then
   encode = json.Encode
   decode = json.Decode
 elseif test_module == 'th-json' then
+  -- https://github.com/harningt/luajson
   -- http://luaforge.net/projects/luajson/
   local json = require "json"
   encode = json.encode
@@ -86,24 +105,30 @@ if not encode then
 else
   local x, r
 
-  local function test (x, s)
+  local escapecodes = {
-    return string.match(x, "^%s*%[%s*%\"" .. s .. "%\"%s*%]%s*$")
+    ["\""] = "\\\"", ["\\"] = "\\\\", ["\b"] = "\\b", ["\f"] = "\\f",
+    ["\n"] = "\\n",  ["\r"] = "\\r",  ["\t"] = "\\t", ["/"] = "\\/"
+  }
+  local function test (x, n, expect)
+    local enc = encode{ x }:match("^%s*%[%s*%\"(.-)%\"%s*%]%s*$")
+    if not enc or (escapecodes[x] ~= enc
+        and ("\\u%04x"):format(n) ~= enc:gsub("[A-F]", string.lower)
+        and not (expect and enc:match("^"..expect.."$"))) then
+      print(("U+%04X isn't encoded correctly: %q"):format(n, enc))
+    end
   end
 
-  x = encode{ "'" }
+  -- necessary escapes for JSON:
-  if not test(x, "%'") then
+  for i = 0,31 do
-    print("\"'\" isn't encoded correctly:", x)
+    test(string.char(i), i)
-  end
-
-  x = encode{ "\011" }
-  if not test(x, "%\\u000[bB]") then
-    print("\\u000b isn't encoded correctly:", x)
-  end
-
-  x = encode{ "\000" }
-  if not test(x, "%\\u0000") then
-    print("\\u0000 isn't encoded correctly")
   end
+  test("\"", ("\""):byte())
+  test("\\", ("\\"):byte())
+  -- necessary escapes for JavaScript:
+  test("\226\128\168", 0x2028)
+  test("\226\128\169", 0x2029)
+  -- invalid escapes that were seen in the wild:
+  test("'", ("'"):byte(), "%'")
 
   r,x = pcall (encode, { [1000] = "x" })
   if not r then
@@ -175,6 +200,15 @@ else
       end
     end
   end
+
+  inlocale(function ()
+    local r, x = pcall(encode, { 0.5 })
+    if not r then
+      print("encoding 0.5 in locale raises an error:", x)
+    elseif not x:find(".", 1, true) then
+      print("In locale 0.5 isn't converted into valid JSON:", x)
+    end
+  end)
 end
 
 if not decode then
@@ -253,13 +287,26 @@ else
     print ("1e+2 decoded incorrectly:", r[1])
   end
 
+  inlocale(function ()
+    local r, x = pcall(decode, "[0.5]")
+    if not r then
+      print("decoding 0.5 in locale raises an error:", x)
+    elseif not x then
+      print("cannot decode 0.5 in locale")
+    elseif x[1] ~= 0.5 then
+      print("decoded 0.5 incorrectly in locale:", x[1])
+    end
+  end)
+
   -- special tests for dkjson:
   if test_module == 'dkjson' then
     x = dkdecode[=[ [{"x":0}] ]=]
-    if getmetatable(x).__jsontype ~= 'array' then
+    local m = getmetatable(x)
+    if not m or m.__jsontype ~= 'array' then
       print ("<metatable>.__jsontype ~= array")
     end
-    if getmetatable(x[1]).__jsontype ~= 'object' then
+    local m = getmetatable(x[1])
+    if not m or m.__jsontype ~= 'object' then
       print ("<metatable>.__jsontype ~= object")
     end
     
@@ -339,6 +386,29 @@ local function escapeutf8 (uchar)
   end
 end
 
+  local isspecial = {}
+  local unifile = io.open("UnicodeData.txt")
+  if unifile then
+    -- <http://www.unicode.org/Public/UNIDATA/UnicodeData.txt>
+    -- each line consists of 15 parts for each defined codepoints
+    local pat = {}
+    for i = 1,14 do
+      pat[i] = "[^;]*;"
+    end
+    pat[1] = "([^;]*);" -- Codepoint
+    pat[3] = "([^;]*);" -- Category
+    pat[15] = "[^;]*"
+    pat = table.concat(pat)
+
+    for line in unifile:lines() do
+      local cp, cat = line:match(pat)
+      if cat:match("^C[^so]") or cat:match("^Z[lp]") then
+        isspecial[tonumber(cp, 16)] = cat
+      end
+    end
+    unifile:close()
+  end
+
   local x,xe
 
   local t = {}
@@ -360,7 +430,6 @@ end
     elseif x == escapecodes[t[1]] then
       esc[i] = 'c'
     elseif x:sub(1,1) == "\\" then
-      --print ("Invalid escape code for "..i..":", x)
       escerr[i] = xe
     end
   end
@@ -368,17 +437,34 @@ end
     local i = 0
     while i <= range do
       local first
-      while i <= range and not esc[i] do i = i + 1 end
+      while i <= range and not (esc[i] or isspecial[i]) do i = i + 1 end
-      if not esc[i] then break end
+      if i > range then break end
       first = i
-      while esc[i] do i = i + 1 end
+      local special = isspecial[i]
+      if esc[i] and special then
+        while esc[i] and isspecial[i] == special do i = i + 1 end
         if i-1 > first then
-        print ("Escaped from "..first.." to "..i-1)
+          print (("Escaped %s characters from U+%04X to U+%04X"):format(special,first,i-1))
+        else
+          print (("Escaped %s character U+%04X"):format(special,first))
+        end
+      elseif esc[i] then
+        while esc[i] and not isspecial[i] do i = i + 1 end
+        if i-1 > first then
+          print (("Escaped from U+%04X to U+%04X"):format(first,i-1))
         else
           if first >= 32 and first <= 127 then
-          print ("Escaped "..first.." ("..string.char(first)..")")
+            print (("Escaped U+%04X (%c)"):format(first,first))
           else
-          print ("Escaped "..first)
+            print (("Escaped U+%04X"):format(first))
+          end
+        end
+      elseif special then
+        while not esc[i] and isspecial[i] == special do i = i + 1 end
+        if i-1 > first then
+          print (("Unescaped %s characters from U+%04X to U+%04X"):format(special,first,i-1))
+        else
+          print (("Unescaped %s character U+%04X"):format(special,first))
         end
       end
     end
@@ -392,9 +478,9 @@ end
       first = i
       while escerr[i] do i = i + 1 end
       if i-1 > first then
-        print ("Errors while escaping from "..first.." to "..i-1)
+        print (("Errors while escaping from U+%04X to U+%04X"):format(first, i-1))
       else
-        print ("Errors while escaping "..first)
+        print (("Errors while escaping U+%04X"):format(first))
       end
     end
   end

versions.txt

@@ -1,3 +1,41 @@
+Version 2.4 (2013-09-28)
+===========
+
+Changes since version 2.3:
+
+  *  Fixed encoding and decoding of numbers in different numeric locales.
+  *  Prevent using version 0.11 of LPeg (causes segmentation faults on
+     some systems).
+
+Version 1.3 (2013-09-28)
+===========
+
+Changes since version 1.2:
+
+  *  Fixed encoding and decoding of numbers in different numeric locales.
+
+Version 2.3 (2013-04-14)
+===========
+
+Changes since version 2.2:
+
+  *  Corrected the range of escaped characters.  Among other characters
+     U+2029 was missing, which would cause trouble when parsed by a
+     JavaScript interpreter.
+  *  Added options to register the module table in a global variable.
+     This is useful in environments where functions similar to require are
+     not available.
+
+Version 1.2 (2013-04-14)
+===========
+
+Changes since version 1.1:
+
+  *  Corrected the range of escaped characters.  Among other characters
+     U+2029 was missing, which would cause trouble when parsed by a
+     JavaScript interpreter.
+  *  Locations for error messages were off by one in the first line.
+
 Version 2.2 (2012-04-28)
 ===========