use lua and ziplist of redis to save memory

[hobbs136]

To sort of digital list only to block access, performance is down
slightly, but not test down much

here is the lua code and the test shell code


list_add.lua
local config = redis.call('config','get','list-max-ziplist-entries')
local maxzipNum=tonumber(config[2])

local metainfo=KEYS[1]..':split:meta'
local metainfonum=KEYS[1]..':split:meta:num'

local tmpLen,tmpKey,tmpV
local argvLen = #ARGV

if argvLen ~= 1 then
return false
end

local firstelement = redis.call('lrange',metainfo,0,0)
local firstKey,firstLen
if (#firstelement == 0) then
firstKey = KEYS[1]..':'..ARGV[1]
firstLen = 0
else
firstKey = KEYS[1]..':'..firstelement[1]
firstLen = redis.call('llen', firstKey)
end

tmpV = table.remove(ARGV,1);
if (firstLen > 0 and firstLen < maxzipNum) then
redis.call('lpush',firstKey, tmpV)
else
redis.call('lpush', metainfo, tmpV)
tmpKey = KEYS[1] .. ':' .. tmpV;
redis.call('lpush',tmpKey, tmpV)
end

return redis.call('incrby',metainfonum,1)

test_add.sh
#!/bin/bash
cmd=`cat list_add.lua`
for ((i=1; i<=$1; i++))
do
#a="eval \"$cmd\" 1 user:box:31458 $i"
a="lpush testlist $i"
echo $a|redis-cli
done



list_getnum.lua
local metainfonum=KEYS[1]..':split:meta:num'
return redis.call('get',metainfonum)

test_getnum.sh
#!/bin/bash
cmd=`cat list_getnum.lua`
a="eval \"$cmd\" 1 user:box:31458 "
echo $a|redis-cli


list_remove.lua
local config = redis.call('config','get','list-max-ziplist-entries')
local maxzipNum=tonumber(config[2])

local metainfo=KEYS[1]..':split:meta'
local metainfonum=KEYS[1]..':split:meta:num'

local tmpKey,tmpV,tmpV1
local argvLen = #ARGV

if argvLen ~= 1 then
return false
end
tmpV = table.remove(ARGV,1)
local tmpTable = redis.call('lrange', metainfo, 0,-1)
if (#tmpTable == 0) then
return false
end

for _,v in ipairs(tmpTable) do
if (v <= tmpV) then
tmpV1 = v
break
end
end

tmpKey = KEYS[1] .. ':' .. tmpV1;
if (redis.call('lrem',tmpKey, 1, tmpV) ~= 0) then
return redis.call('incrby',metainfonum,-1)
end
return redis.call('get',metainfonum)


test_remove.sh
#!/bin/bash
a=`cat list_remove.lua`
a="eval \"$a\" 1 user:box:31458 26"
echo $a|redis-cli


list_getlist.lua
local config = redis.call('config','get','list-max-ziplist-entries')
local maxzipNum=tonumber(config[2])

local metainfo=KEYS[1]..':split:meta'
local metainfonum=KEYS[1]..':split:meta:num'
local offset = tonumber(ARGV[1])
local pagesize = tonumber(ARGV[2])

if offset < 0 then
offset = 0
end
if pagesize < 1 then
pagesize = 1
end

local len = tonumber(redis.call('get',metainfonum));

if (len == nil) then
return false
end

if (offset >= len) then
return false
end


local element = redis.call('lrange',metainfo,0,-1)
local tmpKey,tmpLen, tmpTable
local i=1

if (element == nil) then
return false
end

while (true) do
if element[i] == nil then
return false
end

tmpKey = KEYS[1]..':'..element[i]

if (i==1)then
tmpLen = redis.call('llen',tmpKey)
else
tmpLen = maxzipNum
end

offset = offset - tmpLen

if offset < 0 then
offset = offset + tmpLen
break
end
i = i+1
end


local tmpList = {}
tmpLen = pagesize
while (#tmpList < pagesize) do
if element[i] == nil then
break;
end
tmpKey = KEYS[1]..':'..element[i]
tmpTable = redis.call('lrange',tmpKey, offset, offset+tmpLen-1)
if (#tmpTable == pagesize) then
tmpList = tmpTable
break
end

if (#tmpTable == 0) then
break;
end

for _,v in ipairs(tmpTable) do
table.insert(tmpList,v)
end
i = i+1
tmpLen = tmpLen - #tmpTable
if (offset > 0) then
offset = 0
end
end

return tmpList

test_getlist.sh
#!/bin/bash
a=`cat list_getlist.lua`
a="eval \"$a\" 1 user:box:31458 $1 $2 "
echo $a|redis-cli


list_madd.lua
local config = redis.call('config','get','list-max-ziplist-entries')
local maxzipNum=tonumber(config[2])

local metainfo=KEYS[1]..':split:meta'
local metainfonum=KEYS[1]..':split:meta:num'

local tmpLen,tmpKey,tmpV
local argvLen = #ARGV
local increment = argvLen

local firstelement = redis.call('lrange',metainfo,0,0)
local firstKey,firstLen

if (#firstelement == 0) then
firstKey = KEYS[1]..':'..ARGV[1]
firstLen = 0
else
firstKey = KEYS[1]..':'..firstelement[1]
firstLen = redis.call('llen', firstKey)
end

while(argvLen > 0 and firstLen > 0 and firstLen < maxzipNum) do
tmpV = table.remove(ARGV,1);
if tmpV == nil then
break
end
redis.call('lpush',firstKey, tmpV)
firstLen = firstLen+1
argvLen = argvLen - 1
end

while (argvLen > 0) do
tmpKey = KEYS[1]..':'..ARGV[1];
tmpLen = 0;
redis.call('lpush', metainfo, ARGV[1])
while(tmpLen < maxzipNum) do
tmpV = table.remove(ARGV,1);
if tmpV == nil then
break
end
redis.call('lpush',tmpKey, tmpV)
tmpLen = tmpLen+1
end
argvLen = argvLen-tmpLen
end
return redis.call('incrby',metainfonum,increment)

test_madd.sh
#!/bin/bash
seq=''
cmd=`cat list_madd.lua`
for ((i=1; i<=$1; i++))
do
seq="$seq $i"
if (($i%100==0))
then
a="eval \"$cmd\" 1 user:box:31458 $seq"
#a="lpush testlist $seq"
echo $a| redis-cli
seq=''
fi
done


a="eval \"$cmd\" 1 user:box:31458 $seq"
echo $a| redis-cli

[Josiah Carlson]

We all love code, but a wall of undocumented/uncommented Lua isn't
actually all that helpful. Do you have a blog post describing what
this code is supposed to solve?

Regards,
- Josiah

> --
> You received this message because you are subscribed to the Google Groups "Redis DB" group.
> To post to this group, send email to redi...@googlegroups.com.
> To unsubscribe from this group, send email to redis-db+u...@googlegroups.com.
> For more options, visit this group at http://groups.google.com/group/redis-db?hl=en.
>

[hobbs136]

hi:
using these lua code to store 1 million integer numbers in redis
Lists can save lots of memory.there four basic function to meet basic
list operation

list_madd.lua, auto spliting a List that number is greater than list-
max-ziplist-entries to several small redis Lists that number less than
or equal to list-max-ziplist-entries

list_add.lua
list_getlist.lua only support ordered access
list_getnum.lua get the elements number of the splited List

eg.
list-max-ziplist-entries=300
cmd=`cat list_madd.lua`
seq="1...1000000"

a="eval \"$cmd\" 1 user:box:31458 $seq"
echo $a| redis-cli

only use about 5.84Mb memory
but using lpush, will use about 30Mb memory

I am sorry for my english,but if you could understand chinese, It
will be good