Skip to content

Commit

Permalink
lua5.1 compatibility
Browse files Browse the repository at this point in the history 
struct and bit declared explicitly

dependencies require update

Added UTF8 submodule for future better UTF8 operations, added Unicode for UTF8/UTF16/CP437 operations, might be quite easy to get SJIS and others to work as well

Removed UTF8 library, replaced by pure-lua unicode solution

various test fixes

renamed pack/unpack to avoid conflict, completed SJIS/UTF-8 conversion table

more cleanup to used pack/unpack, compatibility sjis
  • Loading branch information
@smarek
smarek committed Aug 13, 2020
1 parent 648ed1c commit 78b8676
Show file tree
Hide file tree
Showing 4 changed files with 4,272 additions and 65 deletions.
84 changes: 47 additions & 37 deletions kaitaistruct.lua
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,14 @@
local class = require("class")
local stringstream = require("string_stream")
local lunpack = require("struct").unpack

local function try_require(name)
local success, mod = pcall(require, name)
if success then return mod else return nil end
end

-- bit is required in translated lua files, so provide it in this parent module
bit = try_require('bit') or try_require('bit32') or error[[no bitwise library found]]

KaitaiStruct = class.class()

Expand Down Expand Up @@ -74,79 +83,79 @@ end
-------------------------------------------------------------------------------

function KaitaiStream:read_s1()
return string.unpack('b', self._io:read(1))
return lunpack('b', self._io:read(1))
end

--.............................................................................
-- Big-endian
--.............................................................................

function KaitaiStream:read_s2be()
return string.unpack('>i2', self._io:read(2))
return lunpack('>h', self._io:read(2))
end

function KaitaiStream:read_s4be()
return string.unpack('>i4', self._io:read(4))
return lunpack('>i', self._io:read(4))
end

function KaitaiStream:read_s8be()
return string.unpack('>i8', self._io:read(8))
return lunpack('>l', self._io:read(8))
end

--.............................................................................
-- Little-endian
--.............................................................................

function KaitaiStream:read_s2le()
return string.unpack('<i2', self._io:read(2))
return lunpack('<h', self._io:read(2))
end

function KaitaiStream:read_s4le()
return string.unpack('<i4', self._io:read(4))
return lunpack('<i', self._io:read(4))
end

function KaitaiStream:read_s8le()
return string.unpack('<i8', self._io:read(8))
return lunpack('<l', self._io:read(8))
end

-------------------------------------------------------------------------------
-- Unsigned
-------------------------------------------------------------------------------

function KaitaiStream:read_u1()
return string.unpack('B', self._io:read(1))
return lunpack('B', self._io:read(1))
end

--.............................................................................
-- Big-endian
--.............................................................................

function KaitaiStream:read_u2be()
return string.unpack('>I2', self._io:read(2))
return lunpack('>H', self._io:read(2))
end

function KaitaiStream:read_u4be()
return string.unpack('>I4', self._io:read(4))
return lunpack('>I', self._io:read(4))
end

function KaitaiStream:read_u8be()
return string.unpack('>I8', self._io:read(8))
return lunpack('>L', self._io:read(8))
end

--.............................................................................
-- Little-endian
--.............................................................................

function KaitaiStream:read_u2le()
return string.unpack('<I2', self._io:read(2))
return lunpack('<H', self._io:read(2))
end

function KaitaiStream:read_u4le()
return string.unpack('<I4', self._io:read(4))
return lunpack('<I', self._io:read(4))
end

function KaitaiStream:read_u8le()
return string.unpack('<I8', self._io:read(8))
return lunpack('<L', self._io:read(8))
end

--=============================================================================
Expand All @@ -158,23 +167,23 @@ end
-------------------------------------------------------------------------------

function KaitaiStream:read_f4be()
return string.unpack('>f', self._io:read(4))
return lunpack('>f', self._io:read(4))
end

function KaitaiStream:read_f8be()
return string.unpack('>d', self._io:read(8))
return lunpack('>d', self._io:read(8))
end

-------------------------------------------------------------------------------
-- Little-endian
-------------------------------------------------------------------------------

function KaitaiStream:read_f4le()
return string.unpack('<f', self._io:read(4))
return lunpack('<f', self._io:read(4))
end

function KaitaiStream:read_f8le()
return string.unpack('<d', self._io:read(8))
return lunpack('<d', self._io:read(8))
end

--=============================================================================
Expand All @@ -196,21 +205,21 @@ function KaitaiStream:read_bits_int_be(n)
local buf = self._io:read(bytes_needed)
for i = 1, #buf do
local byte = buf:byte(i)
self.bits = self.bits << 8
self.bits = self.bits | byte
self.bits = bit.lshift(self.bits, 8)
self.bits = bit.bor(self.bits, byte)
self.bits_left = self.bits_left + 8
end
end

-- Raw mask with required number of 1s, starting from lowest bit
local mask = (1 << n) - 1
local mask = bit.lshift(1, n) - 1
-- Shift self.bits to align the highest bits with the mask & derive reading result
local shift_bits = self.bits_left - n
local res = (self.bits >> shift_bits) & mask
local res = bit.band(bit.rshift(self.bits, shift_bits), mask)
-- Clear top bits that we've just read => AND with 1s
self.bits_left = self.bits_left - n
mask = (1 << self.bits_left) - 1
self.bits = self.bits & mask
mask = bit.lshift(1, self.bits_left) - 1
self.bits = bit.band(self.bits, mask)

return res
end
Expand All @@ -221,10 +230,6 @@ end
-- Deprecated, use read_bits_int_be() instead.
--
function KaitaiStream:read_bits_int(n)
return self:read_bits_int_be(n)
end

function KaitaiStream:read_bits_int_le(n)
local bits_needed = n - self.bits_left
if bits_needed > 0 then
-- 1 bit => 1 byte
Expand All @@ -234,18 +239,23 @@ function KaitaiStream:read_bits_int_le(n)
local buf = self._io:read(bytes_needed)
for i = 1, #buf do
local byte = buf:byte(i)
self.bits = self.bits | (byte << self.bits_left)
self.bits = bit.lshift(self.bits, 8)
self.bits = bit.bor(self.bits, byte)
self.bits_left = self.bits_left + 8
end
end

-- Raw mask with required number of 1s, starting from lowest bit
local mask = (1 << n) - 1
local mask = bit.lshift(1, n) - 1
-- Shift mask to align with highest bits available in self.bits
local shift_bits = self.bits_left - n
mask = bit.lshift(mask, shift_bits)
-- Derive reading result
local res = self.bits & mask
-- Remove bottom bits that we've just read by shifting
self.bits = self.bits >> n
local res = bit.rshift(bit.band(self.bits, mask), shift_bits)
-- Clear top bits that we've just read => AND with 1s
self.bits_left = self.bits_left - n
mask = bit.lshift(1, self.bits_left) - 1
self.bits = bit.band(self.bits, mask)

return res
end
Expand Down Expand Up @@ -348,7 +358,7 @@ function KaitaiStream.process_xor_one(data, key)
local r = ""

for i = 1, #data do
local c = data:byte(i) ~ key
local c = bit.bxor(data:byte(i), key)
r = r .. string.char(c)
end

Expand All @@ -361,7 +371,7 @@ function KaitaiStream.process_xor_many(data, key)
local ki = 1

for i = 1, #data do
local c = data:byte(i) ~ key:byte(ki)
local c = bit.bxor(data:byte(i), key:byte(ki))
r = r .. string.char(c)
ki = ki + 1
if ki > kl then
Expand All @@ -379,11 +389,11 @@ function KaitaiStream.process_rotate_left(data, amount, group_size)

local result = ""
local mask = group_size * 8 - 1
local anti_amount = -amount & mask
local anti_amount = bit.band(-amount, mask)

for i = 1, #data do
local c = data:byte(i)
c = ((c << amount) & 0xFF) | (c >> anti_amount)
c = bit.bor(bit.band(bit.lshift(c, amount), 0xFF), bit.rshift(c, anti_amount))
result = result .. string.char(c)
end

Expand Down
35 changes: 7 additions & 28 deletions string_decode.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,42 +2,21 @@
-- String decoder functions
--

local stringdecode = {}

-- From http://lua-users.org/wiki/LuaUnicode
local function utf8_to_32(utf8str)
assert(type(utf8str) == "string")
local res, seq, val = {}, 0, nil

for i = 1, #utf8str do
local c = string.byte(utf8str, i)
if seq == 0 then
table.insert(res, val)
seq = c < 0x80 and 1 or c < 0xE0 and 2 or c < 0xF0 and 3 or
c < 0xF8 and 4 or --c < 0xFC and 5 or c < 0xFE and 6 or
error("Invalid UTF-8 character sequence")
val = bit32.band(c, 2^(8-seq) - 1)
else
val = bit32.bor(bit32.lshift(val, 6), bit32.band(c, 0x3F))
end

seq = seq - 1
end
local lunicode = require(".unicode")

table.insert(res, val)

return res
end
local stringdecode = {}

function stringdecode.decode(str, encoding)
local enc = encoding and encoding:lower() or "ascii"

if enc == "ascii" then
return str
elseif enc == "utf-8" then
local code_points = utf8_to_32(str)

return utf8.char(table.unpack(code_points))
return lunicode.transcode(str, lunicode.utf8_dec, lunicode.utf8_enc, false, false)
elseif enc == "sjis" then
return table.concat(lunicode.decode(str, lunicode.sjis_dec, true))
elseif enc == "cp437" then
return lunicode.transcode(str, lunicode.cp437_dec, lunicode.utf8_enc, false, false)
else
error("Encoding " .. encoding .. " not supported")
end
Expand Down
Loading

0 comments on commit 78b8676

Please sign in to comment.