Merge pull request #1 from dhkolf/master

updated to dkjson 2.4
2013-10-16 04:21:31 -07:00 · 2013-10-16 04:21:31 -07:00 · 7ba6b81eb2
commit 7ba6b81eb2
parent 41a18483ae 9627735c66
5 changed files with 223 additions and 77 deletions
--- a/dist.info
+++ b/dist.info
@ -1,12 +1,12 @@
 --- This file is part of LuaDist project

 name = "dkjson"
-version = "2.2"
+version = "2.4"

 desc = "dkjson is a module for encoding and decoding JSON data. It supports UTF-8."
 author = "David Kolf"
 license = "MIT/X11"
-url = "http://chiselapp.com/user/dhkolf/repository/dkjson/"
+url = "http://dkolf.de/src/dkjson-lua.fsl/"
 maintainer = "Peter Drahoš"

 depends = {
--- a/dkjson-2.2-1.rockspec
+++ b/dkjson-2.2-1.rockspec
@ -1,30 +0,0 @@
-package = "dkjson"
-version = "2.2-1"
-source = {
-  url = "http://chiselapp.com/user/dhkolf/repository/dkjson/tarball/dkjson-2.2.tar.gz?uuid=release_2_2",
-  file = "dkjson-2.2.tar.gz"
-}
-description = {
-  summary = "David Kolf's JSON module for Lua",
-  detailed = [[
-dkjson is a module for encoding and decoding JSON data. It supports UTF-8.
-
-JSON (JavaScript Object Notation) is a format for serializing data based
-on the syntax for JavaScript data structures.
-
-dkjson is written in Lua without any dependencies, but
-when LPeg is available dkjson uses it to speed up decoding.
-]],
-  homepage = "http://chiselapp.com/user/dhkolf/repository/dkjson/",
-  license = "MIT/X11"
-}
-dependencies = {
-  "lua >= 5.1"
-}
-build = {
-  type = "builtin",
-  modules = {
-    dkjson = "dkjson.lua"
-  }
-}
-
--- a/dkjson.lua
+++ b/dkjson.lua
@ -1,18 +1,26 @@
    -- Module options:
    local always_try_using_lpeg = true
+    local register_global_module_table = false
+    local global_module_name = 'json'

    --[==[

 David Kolf's JSON module for Lua 5.1/5.2
 ========================================

-*Version 2.2*
+*Version 2.4*

-This module writes no global values, not even the module table.
-Import it using
+In the default configuration this module writes no global values, not even
+the module table. Import it using

    json = require ("dkjson")

+In environments where `require` or a similiar function are not available
+and you cannot receive the return value of the module, you can set the
+option `register_global_module_table` to `true`.  The module table will
+then be saved in the global variable with the name given by the option
+`global_module_name`.
+
 Exported functions and values:

 `json.encode (object [, state])`
@ -105,7 +113,7 @@ You can use this value for setting explicit `null` values.
 `json.version`
 --------------

-Set to `"dkjson 2.2"`.
+Set to `"dkjson 2.4"`.

 `json.quotestring (string)`
 ---------------------------
@ -159,12 +167,12 @@ This variable is set to `true` when LPeg was loaded successfully.
 Contact
 -------

-You can contact the author by sending an e-mail to 'kolf' at the
-e-mail provider 'gmx.de'.
+You can contact the author by sending an e-mail to 'david' at the
+domain 'dkolf.de'.

 ---------------------------------------------------------------------

-*Copyright (C) 2010, 2011, 2012 David Heiko Kolf*
+*Copyright (C) 2010-2013 David Heiko Kolf*

 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the
@ -192,7 +200,7 @@ SOFTWARE.
     it isn't a valid HTML comment (and wastes space).
  -->

-  <!--]==]
+<!--]==]

 -- global dependencies:
 local pairs, type, tostring, tonumber, getmetatable, setmetatable, rawset =
@ -202,11 +210,16 @@ local floor, huge = math.floor, math.huge
 local strrep, gsub, strsub, strbyte, strchar, strfind, strlen, strformat =
      string.rep, string.gsub, string.sub, string.byte, string.char,
      string.find, string.len, string.format
+local strmatch = string.match
 local concat = table.concat

-local _ENV = nil -- blocking globals in Lua 5.2
+local json = { version = "dkjson 2.4" }

-local json = { version = "dkjson 2.2" }
+if register_global_module_table then
+  _G[global_module_name] = json
+end
+
+local _ENV = nil -- blocking globals in Lua 5.2

 pcall (function()
  -- Enable access to blocked metatables.
@ -297,15 +310,48 @@ local function quotestring (value)
    value = fsub (value, "\216[\128-\132]", escapeutf8)
    value = fsub (value, "\220\143", escapeutf8)
    value = fsub (value, "\225\158[\180\181]", escapeutf8)
-    value = fsub (value, "\226\128[\140-\143\168\175]", escapeutf8)
+    value = fsub (value, "\226\128[\140-\143\168-\175]", escapeutf8)
    value = fsub (value, "\226\129[\160-\175]", escapeutf8)
    value = fsub (value, "\239\187\191", escapeutf8)
-    value = fsub (value, "\239\191[\176\191]", escapeutf8)
+    value = fsub (value, "\239\191[\176-\191]", escapeutf8)
  end
  return "\"" .. value .. "\""
 end
 json.quotestring = quotestring

+local function replace(str, o, n)
+  local i, j = strfind (str, o, 1, true)
+  if i then
+    return strsub(str, 1, i-1) .. n .. strsub(str, j+1, -1)
+  else
+    return str
+  end
+end
+
+-- locale independent num2str and str2num functions
+local decpoint, numfilter
+
+local function updatedecpoint ()
+  decpoint = strmatch(tostring(0.5), "([^05+])")
+  -- build a filter that can be used to remove group separators
+  numfilter = "[^0-9%-%+eE" .. gsub(decpoint, "[%^%$%(%)%%%.%[%]%*%+%-%?]", "%%%0") .. "]+"
+end
+
+updatedecpoint()
+
+local function num2str (num)
+  return replace(fsub(tostring(num), numfilter, ""), decpoint, ".")
+end
+
+local function str2num (str)
+  local num = tonumber(replace(str, ".", decpoint))
+  if not num then
+    updatedecpoint()
+    num = tonumber(replace(str, ".", decpoint))
+  end
+  return num
+end
+
 local function addnewline2 (level, buffer, buflen)
  buffer[buflen+1] = "\n"
  buffer[buflen+2] = strrep ("  ", level)
@ -370,7 +416,7 @@ encode2 = function (value, indent, level, buffer, buflen, tables, globalorder)
      -- This is the behaviour of the original JSON implementation.
      s = "null"
    else
-      s = tostring (value)
+      s = num2str (value)
    end
    buflen = buflen + 1
    buffer[buflen] = s
@ -452,6 +498,7 @@ function json.encode (value, state)
  state = state or {}
  local oldbuffer = state.buffer
  local buffer = oldbuffer or {}
+  updatedecpoint()
  local ret, msg = encode2 (value, state.indent, state.level or 0,
                   buffer, state.bufferlen or 0, state.tables or {}, state.keyorder)
  if not ret then
@ -647,7 +694,7 @@ scanvalue = function (str, pos, nullval, objectmeta, arraymeta)
  else
    local pstart, pend = strfind (str, "^%-?[%d%.]+[eE]?[%+%-]?%d*", pos)
    if pstart then
-      local number = tonumber (strsub (str, pstart, pend))
+      local number = str2num (strsub (str, pstart, pend))
      if number then
        return number, pend + 1
      end
@ -682,8 +729,13 @@ end

 function json.use_lpeg ()
  local g = require ("lpeg")
+
+  if g.version() == "0.11" then
+    error "due to a bug in LPeg 0.11, it cannot be used for JSON matching"
+  end
+
  local pegmatch = g.match
-  local P, S, R, V = g.P, g.S, g.R, g.V
+  local P, S, R = g.P, g.S, g.R

  local function ErrorCall (str, pos, msg, state)
    if not state.msg then
@ -720,7 +772,7 @@ function json.use_lpeg ()
  local Integer = P"-"^(-1) * (P"0" + (R"19" * R"09"^0))
  local Fractal = P"." * R"09"^0
  local Exponent = (S"eE") * (S"+-")^(-1) * R"09"^1
-  local Number = (Integer * Fractal^(-1) * Exponent^(-1))/tonumber
+  local Number = (Integer * Fractal^(-1) * Exponent^(-1))/str2num
  local Constant = P"true" * g.Cc (true) + P"false" * g.Cc (false) + P"null" * g.Carg (1)
  local SimpleValue = Number + String + Constant
  local ArrayContent, ObjectContent
--- a/jsontest.lua
+++ b/jsontest.lua
@ -19,6 +19,18 @@ local test_module, opt = ... -- command line argument

 --local opt = "refcycle" -- What happens when a reference cycle gets encoded?

+local testlocale = "de_DE.UTF8"
+
+local function inlocale(fn)
+  local oldloc = os.setlocale(nil, 'numeric')
+  if not os.setlocale(testlocale, 'numeric') then
+    print("test could not switch to locale "..testlocale)
+  else
+    fn()
+  end
+  os.setlocale(oldloc, 'numeric')
+end
+
 if test_module == 'dkjson-nopeg' then
  test_module = 'dkjson'
  package.preload["lpeg"] = function () error "lpeg disabled" end
@ -26,6 +38,11 @@ if test_module == 'dkjson-nopeg' then
  lpeg = nil
 end

+if test_module == 'dkjson-lulpeg' then
+  test_module = 'dkjson'
+  package.loaded["lpeg"] = require "lulpeg"
+end
+
 do
  -- http://chiselapp.com/user/dhkolf/repository/dkjson/
  local dkjson = require "dkjson"
@ -34,6 +51,7 @@ do
 end

 if test_module == 'cmj-json' then
+  -- https://github.com/craigmj/json4lua/
  -- http://json.luaforge.net/
  local json = require "cmjjson" -- renamed, the original file was just 'json'
  encode = json.encode
@ -72,6 +90,7 @@ elseif test_module == 'sb-json' then
  encode = json.Encode
  decode = json.Decode
 elseif test_module == 'th-json' then
+  -- https://github.com/harningt/luajson
  -- http://luaforge.net/projects/luajson/
  local json = require "json"
  encode = json.encode
@ -86,24 +105,30 @@ if not encode then
 else
  local x, r

-  local function test (x, s)
-    return string.match(x, "^%s*%[%s*%\"" .. s .. "%\"%s*%]%s*$")
+  local escapecodes = {
+    ["\""] = "\\\"", ["\\"] = "\\\\", ["\b"] = "\\b", ["\f"] = "\\f",
+    ["\n"] = "\\n",  ["\r"] = "\\r",  ["\t"] = "\\t", ["/"] = "\\/"
+  }
+  local function test (x, n, expect)
+    local enc = encode{ x }:match("^%s*%[%s*%\"(.-)%\"%s*%]%s*$")
+    if not enc or (escapecodes[x] ~= enc
+        and ("\\u%04x"):format(n) ~= enc:gsub("[A-F]", string.lower)
+        and not (expect and enc:match("^"..expect.."$"))) then
+      print(("U+%04X isn't encoded correctly: %q"):format(n, enc))
+    end
  end

-  x = encode{ "'" }
-  if not test(x, "%'") then
-    print("\"'\" isn't encoded correctly:", x)
-  end
-
-  x = encode{ "\011" }
-  if not test(x, "%\\u000[bB]") then
-    print("\\u000b isn't encoded correctly:", x)
-  end
-
-  x = encode{ "\000" }
-  if not test(x, "%\\u0000") then
-    print("\\u0000 isn't encoded correctly")
+  -- necessary escapes for JSON:
+  for i = 0,31 do
+    test(string.char(i), i)
  end
+  test("\"", ("\""):byte())
+  test("\\", ("\\"):byte())
+  -- necessary escapes for JavaScript:
+  test("\226\128\168", 0x2028)
+  test("\226\128\169", 0x2029)
+  -- invalid escapes that were seen in the wild:
+  test("'", ("'"):byte(), "%'")

  r,x = pcall (encode, { [1000] = "x" })
  if not r then
@ -175,6 +200,15 @@ else
      end
    end
  end
+
+  inlocale(function ()
+    local r, x = pcall(encode, { 0.5 })
+    if not r then
+      print("encoding 0.5 in locale raises an error:", x)
+    elseif not x:find(".", 1, true) then
+      print("In locale 0.5 isn't converted into valid JSON:", x)
+    end
+  end)
 end

 if not decode then
@ -253,13 +287,26 @@ else
    print ("1e+2 decoded incorrectly:", r[1])
  end

+  inlocale(function ()
+    local r, x = pcall(decode, "[0.5]")
+    if not r then
+      print("decoding 0.5 in locale raises an error:", x)
+    elseif not x then
+      print("cannot decode 0.5 in locale")
+    elseif x[1] ~= 0.5 then
+      print("decoded 0.5 incorrectly in locale:", x[1])
+    end
+  end)
+
  -- special tests for dkjson:
  if test_module == 'dkjson' then
    x = dkdecode[=[ [{"x":0}] ]=]
-    if getmetatable(x).__jsontype ~= 'array' then
+    local m = getmetatable(x)
+    if not m or m.__jsontype ~= 'array' then
      print ("<metatable>.__jsontype ~= array")
    end
-    if getmetatable(x[1]).__jsontype ~= 'object' then
+    local m = getmetatable(x[1])
+    if not m or m.__jsontype ~= 'object' then
      print ("<metatable>.__jsontype ~= object")
    end
    
@ -339,6 +386,29 @@ local function escapeutf8 (uchar)
  end
 end

+  local isspecial = {}
+  local unifile = io.open("UnicodeData.txt")
+  if unifile then
+    -- <http://www.unicode.org/Public/UNIDATA/UnicodeData.txt>
+    -- each line consists of 15 parts for each defined codepoints
+    local pat = {}
+    for i = 1,14 do
+      pat[i] = "[^;]*;"
+    end
+    pat[1] = "([^;]*);" -- Codepoint
+    pat[3] = "([^;]*);" -- Category
+    pat[15] = "[^;]*"
+    pat = table.concat(pat)
+
+    for line in unifile:lines() do
+      local cp, cat = line:match(pat)
+      if cat:match("^C[^so]") or cat:match("^Z[lp]") then
+        isspecial[tonumber(cp, 16)] = cat
+      end
+    end
+    unifile:close()
+  end
+
  local x,xe

  local t = {}
@ -360,7 +430,6 @@ end
    elseif x == escapecodes[t[1]] then
      esc[i] = 'c'
    elseif x:sub(1,1) == "\\" then
-      --print ("Invalid escape code for "..i..":", x)
      escerr[i] = xe
    end
  end
@ -368,17 +437,34 @@ end
    local i = 0
    while i <= range do
      local first
-      while i <= range and not esc[i] do i = i + 1 end
-      if not esc[i] then break end
+      while i <= range and not (esc[i] or isspecial[i]) do i = i + 1 end
+      if i > range then break end
      first = i
-      while esc[i] do i = i + 1 end
-      if i-1 > first then
-        print ("Escaped from "..first.." to "..i-1)
-      else
-        if first >= 32 and first <= 127 then
-          print ("Escaped "..first.." ("..string.char(first)..")")
+      local special = isspecial[i]
+      if esc[i] and special then
+        while esc[i] and isspecial[i] == special do i = i + 1 end
+        if i-1 > first then
+          print (("Escaped %s characters from U+%04X to U+%04X"):format(special,first,i-1))
        else
-          print ("Escaped "..first)
+          print (("Escaped %s character U+%04X"):format(special,first))
+        end
+      elseif esc[i] then
+        while esc[i] and not isspecial[i] do i = i + 1 end
+        if i-1 > first then
+          print (("Escaped from U+%04X to U+%04X"):format(first,i-1))
+        else
+          if first >= 32 and first <= 127 then
+            print (("Escaped U+%04X (%c)"):format(first,first))
+          else
+            print (("Escaped U+%04X"):format(first))
+          end
+        end
+      elseif special then
+        while not esc[i] and isspecial[i] == special do i = i + 1 end
+        if i-1 > first then
+          print (("Unescaped %s characters from U+%04X to U+%04X"):format(special,first,i-1))
+        else
+          print (("Unescaped %s character U+%04X"):format(special,first))
        end
      end
    end
@ -392,9 +478,9 @@ end
      first = i
      while escerr[i] do i = i + 1 end
      if i-1 > first then
-        print ("Errors while escaping from "..first.." to "..i-1)
+        print (("Errors while escaping from U+%04X to U+%04X"):format(first, i-1))
      else
-        print ("Errors while escaping "..first)
+        print (("Errors while escaping U+%04X"):format(first))
      end
    end
  end
--- a/versions.txt
+++ b/versions.txt
@ -1,3 +1,41 @@
+Version 2.4 (2013-09-28)
+===========
+
+Changes since version 2.3:
+
+  *  Fixed encoding and decoding of numbers in different numeric locales.
+  *  Prevent using version 0.11 of LPeg (causes segmentation faults on
+     some systems).
+
+Version 1.3 (2013-09-28)
+===========
+
+Changes since version 1.2:
+
+  *  Fixed encoding and decoding of numbers in different numeric locales.
+
+Version 2.3 (2013-04-14)
+===========
+
+Changes since version 2.2:
+
+  *  Corrected the range of escaped characters.  Among other characters
+     U+2029 was missing, which would cause trouble when parsed by a
+     JavaScript interpreter.
+  *  Added options to register the module table in a global variable.
+     This is useful in environments where functions similar to require are
+     not available.
+
+Version 1.2 (2013-04-14)
+===========
+
+Changes since version 1.1:
+
+  *  Corrected the range of escaped characters.  Among other characters
+     U+2029 was missing, which would cause trouble when parsed by a
+     JavaScript interpreter.
+  *  Locations for error messages were off by one in the first line.
+
 Version 2.2 (2012-04-28)
 ===========