Clean up lualibs

Clean up the modules in engines/LuaCommon/lualib:
* Fix luabit/bit.lua to return its table instead of trying to set the
  global directly.
* Fix luabit/hex.lua to return its table instead of trying to set the
  global directly.
* luabit/noki.lua is useless for our purposes
* luabit/utf8.lua is redundant to mw.ustring
* stringtools uses coroutines, which we don't support

Also fix a parser test that has apparently been broken for a long time.

Change-Id: I1284cddb6e9b94327964cb1077d8dbdf7def6d06
This commit is contained in:
Brad Jorsch 2013-02-20 22:41:29 -05:00
parent 31ee3b1e4b
commit 307a5b1be8
9 changed files with 13 additions and 753 deletions

View file

@ -22,7 +22,6 @@ abstract class Scribunto_LuaEngine extends ScribuntoEngineBase {
protected static $libraryPaths = array(
'.',
'luabit',
'stringtools',
'ustring',
);

View file

@ -27,6 +27,8 @@ Please note that bit.brshift and bit.blshift only support number within
Under the MIT license.
copyright(c) 2006~2007 hanzhao (abrash_han@hotmail.com)
2013-02-20: Brad Jorsch: Fix to not try messing with globals, doesn't work in Scribunto
--]]---------------
do
@ -218,7 +220,7 @@ end
--------------------
-- bit lib interface
bit = {
local bit = {
-- bit operations
bnot = bit_not,
band = bit_and,
@ -234,6 +236,8 @@ bit = {
tonumb = tbl_to_number,
}
return bit
end
--[[

View file

@ -12,9 +12,11 @@ Part of LuaBit(http://luaforge.net/projects/bit/).
Under the MIT license.
copyright(c) 2006~2007 hanzhao (abrash_han@hotmail.com)
2013-02-20: Brad Jorsch: Fix to not try messing with globals, doesn't work in Scribunto
--]]---------------
require 'bit'
local bit = require 'bit'
do
@ -70,11 +72,13 @@ end
--------------------
-- hex lib interface
hex = {
local hex = {
to_dec = to_dec,
to_hex = to_hex,
}
return hex
end
--[[
@ -92,4 +96,4 @@ for i = 1, 100000 do
error("failed " .. i .. ", " .. h)
end
end
--]]
--]]

View file

@ -1,332 +0,0 @@
--[[---------------
Noki v0.4
-------------------
Noki is a toolkit to convert Nokia PC Suite backuped SMS to a
unicode .txt file, which is more accessible than the original
.nfb or .nfc.
It works well for Nokia PC Suite 6.5.12 and my mobile phone is
Nokia 7360. There might be some compatibility problem if you
use earlier version of the PC Suite.
How to use:
noki.save_sms('nokia.nfb', 'sms.txt')
Under the MIT license.
Noki is a part of LuaBit(http://luaforge.net/projects/bit/).
copyright(c) 2006~2007 hanzhao (abrash_han@hotmail.com)
--]]---------------
require 'hex'
require 'bit'
do
-- globals
local RETURN = '\13\0\10\0'
local SMS_FILE = '\\MPAPI\\MESSAGES'
local SMS_INBOX = 'PIT_MESSAGE_INBOX'
local SMS_OUTBOX = 'PIT_MESSAGE_OUTBOX'
local SMS_ARCHIVEBOX = 'PIT_MESSAGE_ARCHIVE'
-- output decorator
local SMS_INBOX_DEC = '[INBOX] '
local SMS_OUTBOX_DEC = '[OUTBOX] '
local SMS_ARCHIVE_DEC = '[ARCHIVE] '
-- box type
local BoxType = {
NON = 0,
IN = 1,
OUT = 2,
ARCHIVE = 3,
}
-- feed each char with an extra \0
local function asci_to_uni(asci)
--print("-------")
local uni = ""
for i = 1, string.len(asci) do
local str = string.format('%c\0', string.byte(asci, i))
--print(string.len(str))
uni = uni .. str
end
return uni
end
local function asci_padding(asci, pad)
local uni = ""
for i = 1, string.len(asci) do
local str = string.format('%c', string.byte(asci, i))
--print(string.len(str))
uni = uni .. str .. pad
end
return uni
end
-- shrink the \0 in uni code string
local function uni_to_asci(uni)
local asci = ''
--print('uni len ' .. string.len(uni))
for i = 1, string.len(uni), 2 do
asci = asci .. string.sub(uni, i, i)
end
return asci
end
local function reader(str)
local index = 1
return function (n)
--print('reader>>> idx ' .. index .. " n " .. n)
local sub = string.sub(str, index, index + n - 1)
--[[print(hex.to_hex(string.byte(sub, 1)))
print(hex.to_hex(string.byte(sub, 2)))
print(hex.to_hex(string.byte(sub, 3)))
print(hex.to_hex(string.byte(sub, 4)))
--]]
index = index + n
return sub
end
end
local function read_number(read, n)
local str = read(n)
local rslt = 0
for i = 1, n do
local v = string.byte(str, i)
rslt = bit.bor(rslt, bit.blshift(v, (i-1)*8))
end
return rslt
end
local function read_int(read)
return read_number(read, 4)
end
local function read_short(read)
return read_number(read, 2)
end
local function read_nfb_string(read)
local len = read_int(read)
local unistr = read(len*2)
return unistr
end
local function read_nfb_header(read)
local nfb_header = {
ver = read_int(read),
firmware = read_nfb_string(read),
phone = read_nfb_string(read),
entries = read_int(read),
}
return nfb_header
end
local function read_nfb_file(read)
local nfbf = {}
nfbf.path = read_nfb_string(read)
nfbf.nbytes = read_int(read)
nfbf.bytes = read(nfbf.nbytes)
local stamp = read_int(read)
return nfbf
end
local function read_nfb_dir(read)
local nfbd = {
path = read_nfb_string(read)
}
return nfbd
end
local function save_entry(fp, tbl)
for k, v in pairs(tbl) do
fp:write(v)
fp:write(RETURN)
end
end
-- save sms entries
local function save_sms(fp, ctnt)
-- print("save sms ----")
local in_box = asci_padding(SMS_INBOX, "%z")
local out_box = asci_padding(SMS_OUTBOX, "%z")
local archive_box = asci_padding(SMS_ARCHIVEBOX, "%z")
local line_s = asci_padding("1020", "%z")
local head = asci_padding("1033", "%z")
local tail = asci_padding("1040", "%z")
local service_center_tail = asci_padding("1080", "%z")
local phone_nb = "%+%z%d%z%d%z[%d%z]+" -- default is type 145 with '+'
local phone_nb_129 = string.rep("%d%z", 11) -- phone number type 129 without '+'
local time = "[%d%z]+%-%z%d%z%d%z%-%z%d%z%d%zT%z%d%z%d%z:%z%d%z%d%z"
local pattern = "([^\10]+)\13%z\10%z"
local line_end = "\13%z\10%z"
local lineb, linee = string.find(ctnt, line_end)
local start = 1
local line_number = 1
while(lineb and linee) do
local line = string.sub(ctnt, start, lineb - 1)
--line = string.sub(ctnt, gb, ge)
local type = BoxType.NON
--print('capture ' .. string.len(line))
--print(uni_to_asci(box))
if(string.find(line, in_box)) then
fp:write(asci_to_uni(SMS_INBOX_DEC))
type = BoxType.IN
elseif(string.find(line, out_box)) then
fp:write(asci_to_uni(SMS_OUTBOX_DEC))
type = BoxType.OUT
elseif(string.find(line, archive_box)) then
fp:write(asci_to_uni(SMS_ARCHIVE_DEC))
type = BoxType.ARCHIVE
else
--print(uni_to_asci(line))
io.close(fp)
--error('unknown sms type')
return
end
hb, he = string.find(line, head)
tb, te = string.find(line, tail)
local first_number = ""
-- service center address
sb, se = string.find(line, phone_nb, tb)
--print("" .. sb .. ", " .. se)
if(sb and se) then
--print(uni_to_asci(string.sub(line, sb, se)))
-- keep the find number, if the second find for sender address fails
-- then this number is the sender address
first_number = string.sub(line, sb, se)
else
sb, se = string.find(line, phone_nb_129, tb)
if(not (sb and se)) then
--io.close(fp)
--error("error service center address")
--return
first_number = "empty number"
-- nokia's pc suite may leave the serivce center address empty
end
end
-- sender address
se_old = se
sb, se = string.find(line, phone_nb, se)
--print("" .. sb .. ", " .. se)
local sender_address = ""
if(sb and se) then
--print(uni_to_asci(string.sub(line, sb, se)))
sender_address = string.sub(line, sb, se)
else
sb, se = string.find(line, phone_nb_129, se_old)
if(not (sb and se)) then
--[[
print(line_number)
io.close(fp)
error("error sender address")
--]]
sender_address = first_number
end
end
-- write sender
fp:write(sender_address)
fp:write(" \0")
-- date time
-- out box have no date time slot
if(type ~= BoxType.OUT and first_number ~= "empty number") then
tmb, tme = string.find(line, time, se)
--print('' .. tmb .. ", " .. tme)
if(tmb and tme) then
--print(uni_to_asci(string.sub(line, tmb+1, tme)))
else
io.close(fp)
error("error reading date time")
return
end
fp:write(string.sub(line, tmb+1, tme))
end
fp:write(RETURN)
fp:write(string.sub(line, he+3, tb-3))
fp:write(RETURN)
fp:write(RETURN)
--end
start = linee + 1
lineb, linee = string.find(ctnt, line_end, linee)
line_number = line_number + 1
end
end
-- save sms from a .nfc or .nfb file to a unicode .txt file
local function save_nfx_to(from, too)
local fp = io.open(too, 'wb')
if(not fp) then
error("error opening file " .. too .. " to write")
return
end
v = string.format('%c%c', 255, 254)
-- unicode .txt 'FF FE'
fp:write(v)
-- read the .nfc file
local nokia = io.open(from, 'rb')
if(not nokia) then
error("error open file " .. from .. " to read")
end
local ctnt = nokia:read("*a")
io.close(nokia)
local read = reader(ctnt)
local header = read_nfb_header(read)
--print(header.ver)
--print(header.entries)
for i=1, header.entries do
--print(i)
local type = read_int(read)
if(type == 1) then
-- file entry
--print('file')
local fe = read_nfb_file(read)
--save_entry(fp, fe)
if(uni_to_asci(fe.path) == SMS_FILE) then
local smsctnt = fe.bytes
--print('sms len ' .. fe.nbytes)
save_sms(fp, smsctnt)
return
end
elseif(type == 2) then
-- dir entry
--print('dir')
local fd = read_nfb_dir(read)
--save_entry(fp, fd)
else
io.close(fp)
error('unknown entry type : ' .. hex.to_hex(type))
end
end
io.close(fp)
end
-- noki interface --
noki = {
save_sms = save_nfx_to
}
end -- end block
-- sample
-- noki.save_sms('nokia2.nfb', 'sms2.txt')

View file

@ -1,163 +0,0 @@
--[[---------------
Utf8 v0.4
-------------------
utf8 -> unicode ucs2 converter
How to use:
to convert:
ucs2_string = utf8.utf_to_uni(utf8_string)
to view a string in hex:
utf8.print_hex(str)
Under the MIT license.
Utf8 is a part of LuaBit Project(http://luaforge.net/projects/bit/).
copyright(c) 2007 hanzhao (abrash_han@hotmail.com)
--]]---------------
require 'hex'
require 'bit'
do
local BYTE_1_HEAD = hex.to_dec('0x00') -- 0### ####
local BYTE_2_HEAD = hex.to_dec('0xC0') -- 110# ####
local BYTE_3_HEAD = hex.to_dec('0xE0') -- 1110 ####
-- mask to get the head
local BYTE_1_MASK = hex.to_dec('0x80') -- 1### ####
local BYTE_2_MASK = hex.to_dec('0xE0') -- 111# ####
local BYTE_3_MASK = hex.to_dec('0xF0') -- 1111 ####
-- tail byte mask
local TAIL_MASK = hex.to_dec('0x3F') -- 10## ####
local mask_tbl = {
BYTE_3_MASK,
BYTE_2_MASK,
BYTE_1_MASK,
}
local head_tbl = {
BYTE_3_HEAD,
BYTE_2_HEAD,
BYTE_1_HEAD,
}
local len_tbl = {
[BYTE_1_HEAD] = 1,
[BYTE_2_HEAD] = 2,
[BYTE_3_HEAD] = 3,
}
local function utf_read_char(utf, start)
local head_byte = string.byte(utf, start)
--print('head byte ' .. hex.to_hex(head_byte))
for m = 1, table.getn(mask_tbl) do
local mask = mask_tbl[m]
-- head match
local head = bit.band(head_byte, mask)
--print('head ' .. hex.to_hex(head) .. ' ' .. hex.to_hex(mask))
if(head == head_tbl[m]) then
local len = len_tbl[head_tbl[m]]
--print('len ' .. len)
local tail_idx = start + len - 1
local char = 0
-- tail
for i = tail_idx, start + 1, -1 do
local tail_byte = string.byte(utf, i)
local byte = bit.band(tail_byte, TAIL_MASK)
--print('byte ' .. hex.to_hex(byte).. ' = ' .. hex.to_hex(tail_byte) .. '&'..hex.to_hex(TAIL_MASK))
if(tail_idx - i > 0) then
local sft = bit.blshift(byte, (tail_idx - i) * 6)
--print('shift ' .. hex.to_hex(sft) .. ' ' .. hex.to_hex(byte) .. ' ' .. ((tail_idx - i) * 6))
char = bit.bor(char, sft)
--print('char ' .. hex.to_hex(char))
else
char = byte
end
end -- tails
-- add head
local head_val = bit.band(head_byte, bit.bnot(mask))
--print('head val ' .. hex.to_hex(head_val))
head_val = bit.blshift(head_val, (len-1) * 6)
--print('head val ' .. hex.to_hex(head_val))
char = bit.bor(head_val, char)
--print('char ' .. hex.to_hex(char))
return char, len
end -- if head match
end -- for mask
error('not find proper head mask')
end
local function print_hex(str)
local cat = ''
for i=1, string.len(str) do
cat = cat .. ' ' .. hex.to_hex(string.byte(str, i))
end
print(cat)
end
local HI_MASK = hex.to_dec('0xF0')
local LO_MASK = hex.to_dec('0xFF')
local function char_to_str(char)
local hi, lo = bit.brshift(char, 8), bit.band(char, LO_MASK)
-- print(hex.to_hex(char)..' '..hex.to_hex(hi)..' ' .. hex.to_hex(lo))
if(hi == 0) then
return string.format('%c\0', lo)
elseif(lo == 0) then
return string.format('\0%c', hi)
else
return string.format('%c%c', lo, hi)
end
end
local function utf_to_uni(utf)
local n = string.len(utf)
local i = 1
local uni = ''
while(i <= n) do
--print('---')
char, len = utf_read_char(utf, i)
i = i + len
--print(string.len(char_to_str(char)))
uni = uni..char_to_str(char)
end
--print_hex(uni)
return uni
end
-- interface
utf8 = {
utf_to_uni = utf_to_uni,
print_hex = print_hex,
}
end
--[[
-- test
byte_3 = string.format('%c%c%c', hex.to_dec('0xE7'), hex.to_dec('0x83'), hex.to_dec('0xad'))
print(string.len(byte_3))
utf8.utf_to_uni(byte_3)
--]]
--[[
byte_2 = string.format('%c%c', hex.to_dec('0xC2'), hex.to_dec('0x9D'))
utf8.utf_to_uni(byte_2)
byte_1 = string.format('%c', hex.to_dec('0xB'))
utf8.utf_to_uni(byte_1)
--]]
--[[
test_mul = string.format(
'%c%c%c%c%c%c%c%c%c',
hex.to_dec('0xE8'),hex.to_dec('0xAF'), hex.to_dec('0xBA'),
hex.to_dec('0xE5'),hex.to_dec('0x9F'), hex.to_dec('0xBA'),
hex.to_dec('0xE4'),hex.to_dec('0xBA'), hex.to_dec('0x9A'))
utf8.print_hex(utf8.utf_to_uni(test_mul))
--]]

View file

@ -1,27 +0,0 @@
This is stringtools 0.12B.
These tools are copyright © 2009 Duncan Cross <duncan.cross@gmail.com> and released under the MIT License:
Copyright (c) 2009 Duncan Cross
Permission is hereby granted, free of charge, to any person
obtaining a copy of this software and associated documentation
files (the "Software"), to deal in the Software without
restriction, including without limitation the rights to use,
copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following
conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.

View file

@ -1,89 +0,0 @@
local setmetatable, getmetatable, tostring, load, pairs, ipairs, next, assert, unpack
= setmetatable, getmetatable, tostring, load, pairs, ipairs, next, assert, unpack
local tconcat
= table.concat
local strformat
= string.format
local cowrap, yield
= coroutine.wrap, coroutine.yield
local _G = _G
setfenv(1,{})
local sb_meta = {
__tostring = function(self) return tconcat(self); end;
__call = function(self,v) self[#self+1] = tostring(v); return self; end;
__index = {
add = function(self,v)
self[#self+1] = tostring(v)
return self
end;
addquoted = function(self, v)
self[#self+1] = strformat('%q', tostring(v))
return self
end;
compile = function(self, chunk_name)
local localvars = self.localvars
local params = self.params
if not next(localvars) and not params then
local chunkthread = cowrap(function()
for _,chunk in ipairs(self) do
yield(chunk)
end
yield(nil)
end)
return assert(load(chunkthread,chunk_name))
end
local locals, ilocals = {}, 0
local chunkthread = cowrap(function()
if next(localvars) then
yield 'local '
for k,v in pairs(localvars) do
yield(k)
if next(localvars,k) then
yield ','
end
ilocals = ilocals+1
locals[ilocals] = v
end
yield ' = ...; '
end
yield 'return function('
if params then
yield(params)
end
yield ') '
for _,chunk in ipairs(self) do
yield(chunk)
end
yield '\nend'
yield(nil)
end)
local funcbuilder = assert(load(chunkthread,chunk_name))
return funcbuilder(unpack(locals,1,ilocals))
end;
localvar = function(self, val)
local lv = self.localvars
local i = 1
local lvname
while true do
lvname = '__lv_' .. i
if not lv[lvname] then
break
end
i = i + 1
end
lv[lvname] = val
return lvname
end;
}
}
local function stringbuilder()
return setmetatable({localvars={}}, sb_meta)
end
_G.stringbuilder = stringbuilder
return stringbuilder

View file

@ -1,136 +0,0 @@
local strfind, strsub, strmatch, gmatch
= string.find, string.sub, string.match, string.gmatch
local floor
= math.floor
local tostring, setmetatable, select
= tostring, setmetatable, select
local cowrap, yield
= coroutine.wrap, coroutine.yield
local _G = _G
local function expectmatch_aux(sr, start, stop, ...)
if not start then
sr.lastmatch = nil
return false
else
sr._pos = stop + 1
if select('#', ...) > 1 then
sr.lastmatch = {select(2, ...)}
return select(2, ...)
else
sr.lastmatch = {...}
return ...
end
end
end
local stringreader_meta = {
__index = {
expect = function(self, patt, exact)
local pos = self._pos
if exact then
if strsub(self._source_text, pos, pos+#patt-1) == patt then
self._pos = pos+#patt
self.lastmatch = patt
return patt
else
return false
end
end
if (strsub(patt,1,1) == '^') then
if (pos > 1) then
return nil
end
patt = strsub(patt,2)
end
patt = '^(' .. patt .. ')'
if (strsub(patt,-2) == '$)') and (strsub(patt,-3,-3) ~= '%') then
patt = strsub(patt,1,-3) .. ')$'
end
return expectmatch_aux(self, strfind(self._source_text, patt, pos))
end;
endofinput = function(self)
return self._pos > #self._source_text;
end;
skipwhitespace = function(self)
self._pos = strmatch(self._source_text, '%s*()', self._pos)
end;
skipto = function(self, patt, exact)
local new_pos = strfind(self._source_text, patt, self._pos, exact)
if new_pos then
self._pos = new_pos
return true
else
return false
end
end;
readchar = function(self)
local pos = self._pos
local c = strsub(self._source_text,pos,pos)
self._pos = pos + 1
return c
end;
peekchar = function(self)
local pos = self._pos
return strsub(self._source_text,pos,pos)
end;
pos = function(self, new_pos, relative)
if new_pos then
self._pos = relative and (self._pos+new_pos) or new_pos
return self
else
return self._pos
end
end;
getlines = function(self)
local lines = self._lines
if not lines then
lines = {1}
for linestart in gmatch(self._source_text, '\n()') do
lines[#lines+1] = linestart
end
self._lines = lines
end
return lines
end;
pos2d = function(self, pos, relative)
pos = pos and (relative and (self._pos+pos) or pos) or self._pos
if (pos < 1) then return nil; end
local lines = self:getlines()
local low, high = 1, #lines
while true do
local line_num = floor((low + high) / 2)
local line_start = lines[line_num]
if (line_start > pos) then
high = line_num - 1
else
local next_line_start = lines[line_num+1]
if (next_line_start == nil) or (next_line_start > pos) then
return line_num, pos - line_start + 1
end
low = line_num + 1
end
end
end;
source_text = function(self, new_value)
if new_value then
self._source_text = new_value
else
return self._source_text
end
end;
rest = function(self)
return strsub(self._source_text, self._pos)
end;
}
}
local function stringreader(source_text)
return setmetatable({_source_text=tostring(source_text), _pos=1}, stringreader_meta)
end
_G.stringreader = stringreader
return stringreader

View file

@ -5,7 +5,7 @@ local p = {}
local isoTestData = ''
require('bit')
local bit = require('bit')
function p.tooFewArgs()
require()