do not transmit on first frame

Reviewed-on: #9

browser.lua

@@ -1,15 +1,22 @@
-local leaderboard = nil
+local MapRecords
 local maps
 local mapIndex = 1
 local scrollPos = 1
 local modes = nil
 local mode = 1
 local prefMode = nil
+local ModeSep
+
+---- Imported functions ----
 
--- Imported functions -------
 -- lb_common.lua
 local ZoneAct = lb_ZoneAct
 local TicsToTime = lb_TicsToTime
+local mapChecksum = lb_map_checksum
+
+-- lb_store.lua
+local GetMapRecords = lb_get_map_records
+
 -----------------------------
 
 local cv_kartencore
@@ -25,11 +32,9 @@ end
 local function updateModes()
 	-- set available modes for this map
 	modes = {}
-	for mode, scoreTable in pairs(leaderboard) do
-		if scoreTable[getMap()] then
+	for mode, _ in pairs(MapRecords) do
 		table.insert(modes, mode)
 	end
-	end
 	table.sort(modes)
 
 	mode = 1
@@ -46,6 +51,8 @@ local function updateMapIndex(n)
 	mapIndex = mapIndexOffset(n)
 	scrollPos = 1
 
+	MapRecords = GetMapRecords(maps[mapIndex], mapChecksum(maps[mapIndex]), ModeSep)
+
 	updateModes()
 end
 
@@ -337,7 +344,7 @@ local function drawScore(v, i, pos, score, highlight)
 end
 
 local function drawBrowser(v, player)
-	if not leaderboard then return end
+	if not MapRecords then return end
 
 	v.fadeScreen(0xFF00, 16)
 
@@ -358,23 +365,20 @@ local function drawBrowser(v, player)
 
 	if not modes then return end
 
-	local gamemode = leaderboard[modes[mode]]
-	if not gamemode then return end
+	local records = MapRecords[modes[mode]]
+	if not records then return end
 
-	local scoreTable = gamemode[getMap()]
-	if not scoreTable then return end
-
-	local scores = #scoreTable
-	scrollPos = max(min(scrollPos, scores - 3), 1)
-	local endi = min(scrollPos + 7, scores)
+	local record_count = #records
+	scrollPos = max(min(scrollPos, record_count - 3), 1)
+	local endi = min(scrollPos + 7, record_count)
 	for i = scrollPos, endi do
-		drawScore(v, i - scrollPos + 1, i, scoreTable[i], scoreTable[i].name == player.name)
+		drawScore(v, i - scrollPos + 1, i, records[i], records[i].name == player.name)
 	end
 end
 rawset(_G, "DrawBrowser", drawBrowser)
 
-local function initBrowser(lb)
-	leaderboard = lb
+local function initBrowser(modeSep)
+	ModeSep = modeSep
 
 	-- set mapIndex to current map
 	for i, m in ipairs(maps) do
@@ -384,6 +388,9 @@ local function initBrowser(lb)
 		end
 	end
 
+	-- initialize MapRecords
+	MapRecords = GetMapRecords(gamemap, mapChecksum(gamemap), ModeSep)
+
 	scrollPos = 1
 	updateModes()
 end
@@ -485,6 +492,7 @@ local function netvars(net)
 	mode = net($)
 	prefMode = net($)
 	scrollPos = net($)
-	leaderboard = net($)
+	MapRecords = net($)
+	ModeSep = net($)
 end
 addHook("NetVars", netvars)

ghost.lua

@@ -0,0 +1,808 @@
+local DEBUG = false
+
+freeslot("MT_PLAYER_GHOST")
+
+mobjinfo[MT_PLAYER_GHOST] = {
+	-1,             -- doomednum
+	S_KART_STND1,    -- spawnstate
+	1,              -- spawnhealth
+	S_KART_WALK1,    -- seestate
+	sfx_None,       -- seesound
+	0,              -- reactiontime
+	sfx_thok,       -- attacksound
+	S_KART_PAIN,    -- painstate
+	MT_THOK,        -- painchance
+	sfx_None,       -- painsound
+	S_NULL,         -- meleestate
+	S_NULL,         -- missilestate
+	S_KART_PAIN,    -- deathstate
+	S_NULL,         -- xdeathstate
+	sfx_None,       -- deathsound
+	1,              -- speed
+	16*FRACUNIT,    -- radius
+	48*FRACUNIT,    -- height
+	0,              -- display offset
+	1000,           -- mass
+	MT_THOK,        -- damage
+	sfx_None,       -- activesound
+	MF_NOCLIP,	-- flags
+	MT_THOK		-- raisestate
+}
+
+local nenc = lb_base62_encode
+local ndec = lb_base62_decode
+local mapChecksum = lb_map_checksum
+local Transmitter = lb_transmitter
+local Reciever = lb_reciever
+
+local Player
+
+local Client
+local Server
+
+local Index
+local Ghost
+local Ghosts
+local Recording
+
+local PREFIX -- Dir
+local INDEX = "ghosts" -- Filename
+
+local cv_disable = CV_RegisterVar({
+	name = "lb_ghost_disable",
+	defaultvalue = "No",
+	flags = CV_NETVAR | CV_CALL,
+	PossibleValue = CV_YesNo,
+	func = function(v)
+		Recording = nil
+	end
+})
+
+
+local function Columns(line, sep)
+	local t = {}
+	sep = sep or " "
+	for str in (line..sep):gmatch("(.-)"..sep) do
+		table.insert(t, str)
+	end
+	return {
+		index = 0,
+		items = t,
+		next = function(this)
+			this.index = $ + 1
+			return this.items[this.index]
+		end
+	}
+end
+
+local function compareTables(a, b, ...)
+	for _, key in ipairs({...}) do
+		if a[key] != b[key] then
+			return false
+		end
+	end
+	return true
+end
+
+Ghosts = {
+	map,
+	players = {},
+	headers = {},
+	data = {},
+
+	dirty = true,
+
+	getData = function(this, player)
+		return this.data[player]
+	end,
+
+	set = function(this, header, data)
+		if not this.headers[header.player] then
+			table.insert(this.players, header.player)
+		end
+
+		this.data[header.player] = data
+		this.headers[header.player] = header
+	end,
+
+	iterate = function(this)
+		local i = 0
+		local m = #this.players
+		local player
+		return function()
+			i = i + 1
+			if i <= m then
+				player = this.players[i]
+				return i, this.headers[player], this.data[player]
+			end
+		end
+	end,
+
+	init = function(this)
+		if this.map == gamemap and not this.dirty then return end
+		this:reset()
+
+		local headers = Index:getMap(gamemap, mapChecksum(gamemap))
+		local ok, data
+		for _, header in ipairs(headers) do
+			ok, data = pcall(Ghost.read, header, Ghost.tableReader)
+			if not ok then
+				print(data)
+				print("\x85\ERROR:\x80 invalid ghost data, removing entry")
+				Index:remove(header)
+				Index:write()
+				continue
+			end
+			this:set(header, data)
+		end
+	end,
+
+	reset = function(this)
+		this.players = {}
+		this.headers = {}
+		this.data = {}
+		this.map = gamemap
+		this.dirty = false
+	end,
+
+	play = function(this)
+		for _, header in pairs(this.headers) do
+			local ghost = P_SpawnMobj(0, 0, 0, MT_PLAYER_GHOST)
+			ghost.playerName = header.player
+			ghost.skin = header.skin
+			ghost.color = header.color
+		end
+	end
+}
+
+local function GhostTable(header)
+	local function frame(this, frameNum)
+		return this.frames[frameNum]
+	end
+
+	local function append(this, frame)
+		table.insert(this.frames, frame)
+	end
+
+	-- Calculate the momentum of the last frame
+	local function thrust(this, frameNum)
+		local a = this:frame(frameNum-1)
+		local b = this:frame(frameNum)
+		local angle = R_PointToAngle2(a.x, a.y, b.x, b.y)
+		local momentum = R_PointToDist2(a.x, a.y, b.x, b.y) / FRACUNIT
+
+		local x = momentum * cos(angle)
+		local y = momentum * sin(angle)
+		return x, y
+	end
+
+	local t = {
+		frames = {},
+		frame = frame,
+		append = append,
+		thrust = thrust,
+	}
+
+	if header then
+		for key, value in pairs(header) do
+			t[key] = value
+		end
+	end
+
+	return t
+end
+
+local transProximity = 150
+local function calcTrans(mobj)
+	local dist = R_PointToDist2(mobj.x, mobj.y, Player.mo.x, Player.mo.y)
+	local i = dist / transProximity
+	local tr = 9 - min(9, FixedInt(i * 9))
+	return tr << FF_TRANSSHIFT
+end
+
+CV_RegisterVar({
+	name = "lb_ghost_trans_prox",
+	defaultvalue = 150,
+	flags = CV_CALL,
+	PossibleValue = CV_Natural,
+	func = function(cv)
+		transProximity = cv.value
+	end
+})
+
+local function ghost_think(mobj)
+	if mobj.done then return end
+	if leveltime < 1 then return end
+
+	if not Recording then
+		P_RemoveMobj(mobj)
+		return
+	end
+
+	local ghost = Ghosts:getData(mobj.playerName)
+	local frame = ghost:frame(leveltime)
+	if frame == nil then
+		mobj.done = true
+		mobj.state = S_KART_STND1
+		mobj.momx, mobj.momy = ghost:thrust(leveltime-1)
+		return
+	end
+
+	P_MoveOrigin(mobj, frame.x, frame.y, frame.z)
+	mobj.angle = frame.angle
+	mobj.state = frame.state
+	mobj.frame
+		= mobj.frame
+		& (~FF_TRANSMASK)
+		| calcTrans(mobj)
+
+	return true
+end
+addHook("MobjThinker", ghost_think, MT_PLAYER_GHOST)
+
+local function drawGhostProgress(v)
+	local x, y = 0, 0
+	local flags = V_SNAPTOTOP | V_SNAPTOLEFT
+
+	local function fmt(rec)
+		local stale = rec.tics > (TICRATE * 10) and "(STALE) " or ""
+		if not rec.len then return stale.."0%" end
+		local n = #rec.packets
+		local p = ((n * 100) / rec.len).."%"
+		return stale.."["..n.."/"..rec.len.."] "..p
+	end
+
+	for _, rec in pairs(Client.data.recievers) do
+		v.drawString(x, y, fmt(rec), flags, "small")
+		y = y + 5
+	end
+end
+
+hud.add(function(v)
+	if not Recording then return end
+
+	drawGhostProgress(v)
+
+	local leveltime = leveltime
+	local frame, patch, skin, color
+	for i, header, data in Ghosts:iterate() do
+		frame = data:frame(leveltime)
+		if not frame then continue end
+
+		skin = skins[header.skin] or skins["sonic"]
+		patch = v.cachePatch(skin.facemmap)
+		color = v.getColormap(header.skin, header.color)
+
+		v.drawOnMinimap(frame.x, frame.y, FRACUNIT, patch, color)
+	end
+end)
+
+if lb_hook then
+lb_hook("Finish", function(data)
+	if not data.position then
+		return
+	end
+
+	if data.score.flags & 0x1 then
+		Recording = nil
+		return
+	end
+
+	local score = data.score
+
+	local header = {
+		version = 1,
+		encoding = "b62",
+		player = score.name,
+		skin = score.skin,
+		color = score.color,
+		map = score.map,
+		checksum = score.checksum,
+		time = score.time,
+		flags = score.flags,
+	}
+
+	Ghost.write(header, Ghost.tableWriter(Recording))
+	Index:insert(header)
+	Index:write()
+	Ghosts.dirty = true
+	Recording = nil
+end)
+end
+
+--COM_AddCommand("save_ghost", function(player)
+--	if not Recording then return end
+--
+--	print("Saving Ghost")
+--
+--	local header = {
+--		version = 1,
+--		encoding = "b62",
+--		player = player.name,
+--		skin = player.mo.skin,
+--		color = player.skincolor,
+--		map = gamemap,
+--		checksum = mapChecksum(gamemap),
+--		time = leveltime,
+--		flags = 0,
+--	}
+--
+--	Ghost.write(header, Ghost.tableWriter(Recording))
+--	Index:insert(header)
+--	Index:write()
+--
+--	Recording = nil
+--end)
+
+local function singlePlayer()
+	local player
+	for p in players.iterate do
+		if p.valid and not p.spectator then
+			if player then
+				return nil
+			end
+			player = p
+		end
+	end
+
+	return player
+end
+
+addHook("ThinkFrame", function()
+	if not Recording then return end
+
+	Player = singlePlayer()
+	local p = Player
+	if not p then
+		Server:stop()
+		Client:stop()
+		Recording = nil
+		return
+	end
+
+	local frame = {
+		x = p.mo.x,
+		y = p.mo.y,
+		z = p.mo.z,
+		angle = p.frameangle,
+		state = p.mo.state
+	}
+
+	Recording:append(frame)
+end)
+
+Server = {
+	header = {
+		transmitter,
+		send = function(this)
+			if this.transmitter
+			or #Server.command.transmitters
+			then
+				return
+			end
+
+			this.transmitter = Transmitter("GM", {free = this.free, handle = this})
+			local mapindex = Index:getMap(gamemap, mapChecksum(gamemap))
+
+			local s = ""
+			for _, header in ipairs(mapindex) do
+				s = s..Index.headerFmt(header)
+			end
+
+			this.transmitter:transmit(s)
+		end,
+
+		free = function(tr, this)
+			this.transmitter = nil
+		end,
+
+		stop = function(this)
+			if this.transmitter then
+				this.transmitter:close()
+			end
+		end
+	},
+
+	command = {
+		reciever,
+		listen = function(this)
+			if this.reciever then return end
+			this.reciever = Reciever("GC", this.callback, {stream = true, handle = this})
+			this.reciever:listen()
+		end,
+
+		transmitters = {},
+		callback = function(cmd, handle)
+			if handle.transmitters[cmd] then
+				return
+			end
+
+			local mapindex = Index:getMap(gamemap, mapChecksum(gamemap))
+			local header = assert(mapindex[tonumber(cmd)])
+			local data = Ghost.read(header, Ghost.stringReader)
+
+			local tr = Transmitter(
+				"GC"..cmd,
+				{
+					free = handle.free,
+					handle = {
+						this = handle,
+						cmd = cmd
+					}
+				}
+			)
+			handle.transmitters[cmd] = tr
+			tr:transmit(data)
+		end,
+
+		free = function(tr, handle)
+			handle.this.transmitters[handle.cmd] = nil
+		end,
+
+		stop = function(this)
+			for _, tr in pairs(this.transmitters) do
+				tr:close()
+			end
+		end
+	},
+
+	stop = function(this)
+		this.header:stop()
+		this.command:stop()
+	end
+}
+
+Client = {
+	header = {
+		reciever,
+		listen = function(this)
+			if this.reciever then return end
+			this.reciever = Reciever(
+				"GM",
+				this.callback,
+				{
+					free = this.free,
+					handle = this
+				}
+			)
+			this.reciever:listen()
+		end,
+
+		callback = function(data, this)
+			local header, storedHeader
+			local cmd = 1
+			for str in data:gmatch("(.-)\n") do
+				if Client.data:busy(cmd) then
+					continue
+				end
+
+				header = Index.parseHeader(str)
+				storedHeader = Index:find(header)
+				if not (
+					storedHeader
+					and compareTables(
+						header,
+						storedHeader,
+						"time",
+						"flags",
+						"skin",
+						"color"
+					)
+				) then
+					local tr = Transmitter("GC", {stream = true})
+					tr:transmit(cmd)
+					Client.data:listen(header, cmd)
+				end
+				cmd = $ + 1
+			end
+		end,
+
+		free = function(rc, handle)
+			handle.reciever = nil
+		end,
+
+		stop = function(this)
+			if this.reciever then
+				this.reciever:close()
+			end
+		end
+	},
+
+	data = {
+		recievers = {},
+		listen = function(this, header, cmd)
+			local rc = Reciever(
+				"GC"..cmd,
+				this.callback,
+				{
+					free = this.free,
+					handle = {
+						this = this,
+						header = header,
+						cmd = cmd
+					},
+				}
+			)
+
+			this.recievers[cmd] = rc
+			rc:listen()
+		end,
+
+		callback = function(data, handle)
+			Ghost.write(handle.header, Ghost.stringWriter(data))
+			Index:insert(handle.header)
+			Index:write()
+
+			Ghosts.dirty = true
+		end,
+
+		free = function(rc, handle)
+			handle.this.recievers[handle.cmd] = nil
+		end,
+
+		busy = function(this, cmd)
+			for com in pairs(this.recievers) do
+				if com == cmd then return true end
+			end
+		end,
+
+		stop = function(this)
+			for _, rec in pairs(this.recievers) do
+				rec:close()
+			end
+		end
+	},
+
+	stop = function(this)
+		this.header:stop()
+		this.data:stop()
+	end
+}
+
+addHook("MapLoad", function(num)
+	if cv_disable.value
+	or not singlePlayer()
+	then
+		Recording = nil
+		return
+	end
+
+	Recording = GhostTable()
+
+	if isserver then
+		Server.header:send()
+		Server.command:listen()
+		return
+	end
+
+	Ghosts:init()
+	Client.header:listen()
+	Ghosts:play()
+end)
+
+local function open(filename, mode, fn)
+	local f, err = io.open(filename, mode)
+	if err then
+		return nil, err
+	end
+
+	local ok, err = pcall(fn, f)
+	f:close()
+
+	if not ok then
+		err = string.format("%s\n(\x82%s\x80 (%s))", err, filename, mode)
+	end
+
+	return ok, err
+end
+
+Index = {
+	headers,
+
+	insert = function(this, header)
+		local map = this:getMap(header.map, header.checksum)
+		for i, h in ipairs(map) do
+			if h.player == header.player then
+				map[i] = header
+				this:setMap(header.map, header.checksum, map)
+				return
+			end
+		end
+
+		table.insert(map, header)
+		this:setMap(header.map, header.checksum, map)
+	end,
+
+	remove = function(this, header)
+		local map = this:getMap(header.map, header.checksum)
+		for i, h in ipairs(map) do
+			if h.player == header.player then
+				map[i] = nil
+				this:setMap(header.map, header.checksum, map)
+				return
+			end
+		end
+	end,
+
+	find = function(this, header)
+		local index = this:getMap(header.map, header.checksum)
+		for i, h in ipairs(index) do
+			if h.player == header.player then
+				return h
+			end
+		end
+	end,
+
+	join = function(sep, ...)
+		local t = {...}
+		local ret = ""..t[1]
+		local v
+		for i = 2, #t do
+			v = t[i]
+			ret = $..sep..(v ~= nil and v or "")
+		end
+		return ret
+	end,
+
+	headerFmt = function(h)
+		return Index.join(
+			"\t",
+			h.version,
+			h.encoding,
+			h.map,
+			h.checksum,
+			h.player,
+			h.time,
+			h.flags or "0",
+			h.skin,
+			h.color or "0"
+		).."\n"
+	end,
+
+	parseHeader = function(line)
+		local c = Columns(line, "\t")
+		return {
+			version = c:next(),
+			encoding = c:next(),
+			map = c:next(),
+			checksum = c:next(),
+			player = c:next(),
+			time = tonumber(c:next()),
+			flags = tonumber(c:next()),
+			skin = c:next(),
+			color = tonumber(c:next())
+		}
+	end,
+
+	filename = function(prefix, filename)
+		return string.format(
+			"%s/%s.txt",
+			prefix,
+			filename
+		)
+	end,
+
+	write = function(this)
+		local filename = this.filename(PREFIX, INDEX)
+		assert(open(filename, "w", function(f)
+			for _, map in pairs(this.headers) do
+				for _, header in ipairs(map) do
+					f:write(this.headerFmt(header))
+				end
+			end
+		end))
+	end,
+
+	read = function(this)
+		local filename = this.filename(PREFIX, INDEX)
+		this.headers = {}
+		assert(open(filename, "r", function(f)
+			for line in f:lines() do
+				this:insert(
+					this.parseHeader(line)
+				)
+			end
+		end))
+	end,
+
+	getMap = function(this, map, checksum)
+		if not this.headers then
+			this:read()
+		end
+
+		return this.headers[map..checksum] or {}
+	end,
+
+	setMap = function(this, map, checksum, entry)
+		assert(this.headers, "Index: header unread")
+		this.headers[map..checksum] = entry
+	end
+}
+
+Ghost = {
+	stringReader = function(f)
+		return f:read("*a")
+	end,
+
+	tableReader = function(f)
+		local data = GhostTable(header)
+		local c
+		f:read() -- skip first frame
+		for line in f:lines() do
+			c = Columns(line)
+			data:append({
+				state = ndec(c:next()),
+				angle = ndec(c:next()),
+				x = ndec(c:next()),
+				y = ndec(c:next()),
+				z = ndec(c:next()),
+			})
+		end
+		return data
+	end,
+
+	filename = function(prefix, header)
+		return string.format(
+			"%s/%d_%s_%s.txt",
+			prefix,
+			header.map,
+			header.checksum,
+			header.player
+		)
+	end,
+
+	read = function(header, reader)
+		local filename = Ghost.filename(PREFIX, header)
+		local data
+		assert(open(filename, "r", function(f)
+			data = reader(f)
+		end))
+
+		return data
+	end,
+
+	stringWriter = function(dataStr)
+		return function(f)
+			f:setvbuf("full")
+			f:write(dataStr)
+		end
+	end,
+
+	frameFmt = function(frame)
+		return string.format(
+			"%s %s %s %s %s\n",
+			nenc(frame.state),
+			nenc(frame.angle),
+			nenc(frame.x),
+			nenc(frame.y),
+			nenc(frame.z)
+		)
+	end,
+
+	tableWriter = function(data)
+		return function(f)
+			f:setvbuf("line")
+			for _, frame in ipairs(data.frames) do
+				f:write(Ghost.frameFmt(frame))
+			end
+		end
+	end,
+
+	write = function(header, writer)
+		local filename = Ghost.filename(PREFIX, header)
+		assert(open(filename, "w", writer))
+	end,
+}
+
+local cv_prefix = CV_RegisterVar({
+	name = "lb_ghost_dir",
+	defaultvalue = "ghosts",
+	flags = CV_CALL | CV_NETVAR,
+	func = function(cv)
+		PREFIX = cv.string
+		if DEBUG and not isserver then
+			PREFIX = $.."_client"
+		end
+		Index.headers = nil
+	end
+})

lb_common.lua

@@ -1,3 +1,17 @@
+rawset(_G, "lb_score_t", function(map, name, skin, color, time, splits, flags, stat, checksum)
+	return {
+		["map"]    = map,
+		["name"]   = name,
+		["skin"]   = skin,
+		["color"]  = color,
+		["time"]   = time,
+		["splits"] = splits,
+		["flags"]  = flags,
+		["stat"] = stat,
+		["checksum"] = checksum
+	}
+end)
+
 rawset(_G, "lb_TicsToTime", function(tics, pure)
 	if tics == 0 and pure then
 		return "-:--:--"
@@ -24,3 +38,130 @@ rawset(_G, "lb_ZoneAct", function(map)
 
 	return z
 end)
+
+rawset(_G, "lb_stat_t", function(speed, weight)
+	if speed and weight then
+		return (speed << 4) | weight
+	end
+	return 0
+end)
+
+
+local F_SPBBIG = 0x4
+local F_SPBEXP = 0x8
+-- True if a is better than b
+rawset(_G, "lb_comp", function(a, b)
+	-- Calculates the difficulty, harder has higher priority
+	-- if s is positive then a is harder
+	-- if s is negative then b is harder
+	-- if s is 0 then compare time
+	local s = (a.flags & (F_SPBEXP | F_SPBBIG)) - (b.flags & (F_SPBEXP | F_SPBBIG))
+	return s > 0 or not(s < 0 or a.time >= b.time)
+end)
+
+local function djb2(message)
+	local digest = 5381
+	for c in message:gmatch(".") do
+		digest = (($ << 5) + $) + string.byte(c)
+	end
+
+	return digest
+end
+
+-- Produce a checksum by using the maps title, subtitle and zone
+rawset(_G, "lb_map_checksum", function(mapnum)
+	local mh = mapheaderinfo[mapnum]
+	if not mh then
+		return nil
+	end
+
+	local digest = string.format("%04x", djb2(mh.lvlttl..mh.subttl..mh.zonttl))
+	return string.sub(digest, #digest - 3)
+end)
+
+rawset(_G, "lb_mapnum_from_extended", function(map)
+	local p, q = map:upper():match("MAP(%w)(%w)$", 1)
+	if not (p and q) then
+		return nil
+	end
+
+	local mapnum = 0
+	local A = string.byte("A")
+
+	if tonumber(p) != nil then
+		-- Non extended map numbers
+		if tonumber(q) == nil then
+			return nil
+		end
+		mapnum = tonumber(p) * 10 + tonumber(q)
+	else
+		--Extended map numbers
+		p = string.byte(p) - A
+		local qn = tonumber(q)
+		if qn == nil then
+			qn = string.byte(q) - A + 10
+		end
+
+		mapnum = 36 * p + qn + 100
+	end
+
+	return mapnum
+end)
+
+local eventHandler = {}
+
+rawset(_G, "lb_hook", function(event, callback)
+	local handle = eventHandler[event] or {}
+	table.insert(handle, callback)
+	eventHandler[event] = handle
+end)
+
+rawset(_G, "lb_fire_event", function(event, ...)
+	local handle = eventHandler[event]
+	if not handle then return end
+
+	for _, callback in ipairs(handle) do
+		pcall(callback, ...)
+	end
+end)
+
+local b62 = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+local function base62_encode(n)
+	if n <= 0 then return "0" end
+	local b62 = b62
+	local q = n
+	local r = ""
+	local t
+	while q > 0 do
+		t = q % 62 + 1
+		r = b62:sub(t, t)..r
+		q = q / 62
+	end
+	return r
+end
+
+local function base62_decode(s)
+	local n = b62:find(s:sub(1,1)) - 1
+	for i = 2, #s do
+		n = n * 62 + b62:find(s:sub(i, i)) - 1
+	end
+
+	return n
+end
+
+local function neg_base62_encode(n)
+	if n == INT32_MIN then
+		return "-2lkCB2"
+	end
+
+	if n < 0 then return "-"..base62_encode(abs(n)) end
+	return base62_encode(n)
+end
+
+local function neg_base62_decode(s)
+	if s:sub(1, 1) == "-" then return -base62_decode(s:sub(2)) end
+	return base62_decode(s)
+end
+
+rawset(_G, "lb_base62_encode", neg_base62_encode)
+rawset(_G, "lb_base62_decode", neg_base62_decode)

lb_store.lua

@@ -0,0 +1,394 @@
+-- This file handles the storage and related netvars of the leaderboard
+
+---- Imported functions ----
+
+-- lb_common.lua
+local stat_t = lb_stat_t
+local lbComp = lb_comp
+local score_t = lb_score_t
+local mapChecksum = lb_map_checksum
+local mapnumFromExtended = lb_mapnum_from_extended
+
+----------------------------
+
+local LEADERBOARD_FILE = "leaderboard.txt"
+local COLDSTORE_FILE = "leaderboard.coldstore.txt"
+
+-- ColdStore are records loaded from lua addons
+-- this table should never be modified outside of the AddColdStore function
+local ColdStore = {}
+
+-- Livestore are new records nad records loaded from leaderboard.txt file
+local LiveStore = {}
+
+-- parse score function
+local parseScore
+
+local MSK_SPEED = 0xF0
+local MSK_WEIGHT = 0xF
+local function stat_str(stat)
+	if stat then
+		return string.format("%d%d", (stat & MSK_SPEED) >> 4, stat & MSK_WEIGHT)
+	end
+
+	return "0"
+end
+
+local function isSameRecord(a, b, modeSep)
+	return a.name == b.name and (a.flags & modeSep) == (b.flags & modeSep)
+end
+
+-- insert or replace the score in dest
+local function insertOrReplace(dest, score, modeSep)
+	for i, record in ipairs(dest) do
+		if isSameRecord(record, score, modeSep) then
+			if lbComp(score, record) then
+				dest[i] = score
+			end
+			return
+		end
+	end
+
+	table.insert(dest, score)
+end
+
+local function dumpStoreToFile(filename, store)
+	local f = assert(
+		io.open(filename, "w"),
+		"Failed to open file for writing: "..filename
+	)
+
+	f:setvbuf("line")
+
+	for mapid, checksums in pairs(store) do
+		for checksum, records in pairs(checksums) do
+			for _, record in ipairs(records) do
+				if not record.checksum or record.checksum == "" then
+					record.checksum = mapChecksum(record.map) or ""
+				end
+
+				f:write(
+					mapid, "\t",
+					record.name, "\t",
+					record.skin, "\t",
+					record.color, "\t",
+					record.time, "\t",
+					table.concat(record.splits, " "), "\t",
+					record.flags, "\t",
+					stat_str(record.stat), "\t",
+					record.checksum, "\n"
+				)
+			end
+		end
+	end
+
+	f:close()
+end
+
+-- GLOBAL
+-- Returns a list of all maps with records
+local function MapList()
+	local maps = {}
+	for mapid, checksums in pairs(ColdStore) do
+		maps[mapid] = $ or {}
+		for checksum in pairs(checksums) do
+			maps[mapid][checksum] = true
+		end
+	end
+	for mapid, checksums in pairs(LiveStore) do
+		maps[mapid] = $ or {}
+		for checksum in pairs(checksums) do
+			maps[mapid][checksum] = true
+		end
+	end
+
+	local maplist = {}
+	for mapid, checksums in pairs(maps) do
+		for checksum in pairs(checksums) do
+			table.insert(maplist, {["id"] = mapid, ["checksum"] = checksum})
+		end
+	end
+	table.sort(maplist, function(a, b) return a.id < b.id end)
+
+	return maplist
+end
+rawset(_G, "lb_map_list", MapList)
+
+-- GLOBAL
+-- Function for adding a single record from lua
+local function AddColdStore(record)
+	ColdStore[record.map] = $ or {}
+	ColdStore[record.map][record.checksum] = $ or {}
+
+	table.insert(ColdStore[record.map][record.checksum], record)
+end
+rawset(_G, "lb_add_coldstore_record", AddColdStore)
+
+-- GLOBAL
+-- Function for adding a single record in string form from lua
+local function AddColdStoreString(record)
+	AddColdStore(parseScore(record))
+end
+rawset(_G, "lb_add_coldstore_record_string", AddColdStoreString)
+
+-- Insert mode separated records from the flat sourceTable into dest
+local function insertRecords(dest, sourceTable, checksum, modeSep)
+	if not sourceTable then return end
+	if not sourceTable[checksum] then return end
+
+	local mode = nil
+	for _, record in ipairs(sourceTable[checksum]) do
+		mode = record.flags & modeSep
+		dest[mode] = $ or {}
+		table.insert(dest[mode], record)
+	end
+end
+
+-- GLOBAL
+-- Construct the leaderboard table of the supplied mapid
+-- combines the ColdStore and LiveStore records
+local function GetMapRecords(map, checksum, modeSep)
+	local mapRecords = {}
+
+	-- Insert ColdStore records
+	insertRecords(mapRecords, ColdStore[map], checksum, modeSep)
+
+	-- Insert LiveStore records
+	insertRecords(mapRecords, LiveStore[map], checksum, modeSep)
+
+	-- Sort records
+	for _, records in pairs(mapRecords) do
+		table.sort(records, lbComp)
+	end
+
+	-- Remove duplicate entries
+	for _, records in pairs(mapRecords) do
+		local players = {}
+		local i = 1
+		while i <= #records do
+			if players[records[i].name] then
+				table.remove(records, i)
+			else
+				players[records[i].name] = true
+				i = i + 1
+			end
+		end
+	end
+
+	return mapRecords
+end
+rawset(_G, "lb_get_map_records", GetMapRecords)
+
+-- GLOBAL
+-- Save a record to the LiveStore and write to disk
+-- SaveRecord will replace the record holders previous record
+local function SaveRecord(score, map, modeSep)
+	local checksum = mapChecksum(map)
+	LiveStore[map] = $ or {}
+	LiveStore[map][checksum] = $ or {}
+	insertOrReplace(LiveStore[map][checksum], score, modeSep)
+
+	print("Saving score")
+	if isserver then 
+		dumpStoreToFile(LEADERBOARD_FILE, LiveStore)
+	end
+end
+rawset(_G, "lb_save_record", SaveRecord)
+
+local function netvars(net)
+	LiveStore = net($)
+end
+
+addHook("NetVars", netvars)
+
+function parseScore(str)
+	-- Leaderboard is stored in the following tab separated format
+	-- mapnum, name, skin, color, time, splits, flags, stat
+	local t = {}
+	for word in (str.."\t"):gmatch("(.-)\t") do
+		table.insert(t, word)
+	end
+
+	local splits = {}
+	if t[6] != nil then
+		for str in t[6]:gmatch("([^ ]+)") do
+			table.insert(splits, tonumber(str))
+		end
+	end
+
+	local flags = 0
+	if t[7] != nil then
+		flags = tonumber(t[7])
+	end
+
+	local stats = nil
+	if t[8] != nil then
+		if #t[8] >= 2 then
+			local speed = tonumber(string.sub(t[8], 1, 1))
+			local weight = tonumber(string.sub(t[8], 2, 2))
+			stats = stat_t(speed, weight)
+		end
+	end
+
+	local checksum = t[9] or ""
+
+	return score_t(
+		tonumber(t[1]), -- Map
+		t[2],		-- Name
+		t[3],		-- Skin
+		t[4],		-- Color
+		tonumber(t[5]),	-- Time
+		splits,
+		flags,
+		stats,
+		checksum:lower()
+	)
+end
+rawset(_G, "lb_parse_score", parseScore)
+
+-- Read and parse a store file
+local function loadStoreFile(filename)
+	local f = assert(
+		io.open(filename, "r"),
+		"Failed to open file for reading: "..filename
+	)
+
+	local store = {}
+
+	for l in f:lines() do
+		local score = parseScore(l)
+		store[score.map] = $ or {}
+		store[score.map][score.checksum] = $ or {}
+		table.insert(store[score.map][score.checksum], score)
+	end
+
+	f:close()
+
+	return store
+end
+
+-- GLOBAL
+-- Command for moving records from one map to another
+local function moveRecords(from, to, modeSep)
+	local function moveRecordsInStore(store)
+		if not (store[from.id] and store[from.id][from.checksum]) then
+			return 0
+		end
+
+		store[to.id] = $ or {}
+		store[to.id][to.checksum] = $ or {}
+		for i, score in ipairs(store[from.id][from.checksum]) do
+			score.map = to.id
+			score.checksum = to.checksum
+			insertOrReplace(store[to.id][to.checksum], score, modeSep)
+		end
+
+		-- Destroy the original table
+		store[from.id][from.checksum] = nil
+	end
+
+	-- move livestore records and write to disk
+	moveRecordsInStore(LiveStore)
+
+	if isserver then
+		dumpStoreToFile(LEADERBOARD_FILE, LiveStore)
+
+		-- move coldstore records
+		local ok, coldstore = pcall(loadStoreFile, COLDSTORE_FILE)
+		if ok and coldstore then
+			moveRecordsInStore(coldstore)
+			dumpStoreToFile(COLDSTORE_FILE, coldstore)
+		end
+	end
+end
+rawset(_G, "lb_move_records", moveRecords)
+
+-- Helper function for those upgrading from 1.2 to 1.3
+COM_AddCommand("lb_write_checksums", function(player)
+	local count = 0
+	local moved = {}
+
+	-- Gather movable records (no checksum, map loaded)
+	for map, checksums in pairs(LiveStore) do
+		for checksum, records in pairs(checksums) do
+			if checksum == "" then
+				local sum = mapChecksum(map)
+
+				if not sum then continue end
+
+				moved[map] = {}
+				moved[map][sum] = {}
+
+				for i, record in ipairs(records) do
+					record.checksum = sum
+					table.insert(moved[map][sum], record)
+				end
+			end
+		end
+	end
+
+	-- Write moved to livestore
+	for map, checksums in pairs(moved) do
+		LiveStore[map] = $ or {}
+		for checksum, records in pairs(checksums) do
+			LiveStore[map][checksum] = $ or {}
+			for i, score in ipairs(records) do
+				table.insert(LiveStore[map][checksum], score)
+			end
+			count = $ + #records
+		end
+		LiveStore[map][""] = nil
+	end
+
+	if isserver then
+		dumpStoreToFile(LEADERBOARD_FILE, LiveStore)
+	end
+
+	CONS_Printf(player, string.format("Successful operation on %d records", count))
+end, COM_ADMIN)
+
+COM_AddCommand("lb_known_maps", function(player, map)
+	local mapnum = gamemap
+	if map then
+		mapnum = mapnumFromExtended(map)
+		if not mapnum then
+			CONS_Printf(player, string.format("invalid map '%s'", map))
+			return
+		end
+	end
+
+	local known = {}
+
+	if LiveStore[mapnum] then
+		for checksum, records in pairs(LiveStore[mapnum]) do
+			known[checksum] = #records
+		end
+	end
+	if ColdStore[mapnum] then
+		for checksum, records in pairs(ColdStore[mapnum]) do
+			known[checksum] = $ or 0 + #records
+		end
+	end
+
+	CONS_Printf(player, "Map     Chck   Records")
+	for checksum, count in pairs(known) do
+		CONS_Printf(player, string.format("%s   %s   %d", G_BuildMapName(mapnum), checksum, count))
+	end
+end)
+
+COM_AddCommand("lb_download_live_records", function(player, filename)
+	if not filename then
+		CONS_Printf(player, "Usage: lb_download_live_records <filename>")
+		return
+	end
+
+	if filename:sub(#filename-3) != ".txt" then
+		filename = $..".txt"
+	end
+	dumpStoreToFile(filename, LiveStore)
+end, COM_LOCAL)
+
+-- Load the livestore
+if isserver then
+	LiveStore = loadStoreFile(LEADERBOARD_FILE)
+end

leaderboard.lua

@@ -1,18 +1,40 @@
 -- Leaderboards written by Not
 -- Reusable
 
--- Leaderboard Table
--- [mode][mapnum][scoreTable]
-local lb = {}
+---------- Imported functions -------------
+-- lb_common.lua
+local ticsToTime = lb_TicsToTime
+local zoneAct = lb_ZoneAct
+local stat_t = lb_stat_t
+local lbComp = lb_comp
+local mapChecksum = lb_map_checksum
+local score_t = lb_score_t
+local mapnumFromExtended = lb_mapnum_from_extended
+local fireEvent = lb_fire_event
 
-local timeFinished = 0
+-- browser.lua
+local InitBrowser = InitBrowser
+local DrawBrowser = DrawBrowser
+local BrowserController = BrowserController
+
+-- lb_store.lua
+local GetMapRecords = lb_get_map_records
+local SaveRecord = lb_save_record
+local MapList = lb_map_list
+local MoveRecords = lb_move_records
+--------------------------------------------
+
+-- Holds the current maps records table including all modes
+local MapRecords = {}
+
+local TimeFinished = 0
 local disable = false
 local prevLap = 0
 local splits = {}
 local PATCH = nil
 local help = true
 local EncoreInitial = nil
-local scoreTable
+local ScoreTable
 
 
 -- Text flash on finish
@@ -29,9 +51,6 @@ local RedFlash = {
 	[1] = 0
 }
 
--- Tracks if stats have been written or not
-local StatTrack = false
-
 local UNCLAIMED = "Unclaimed Record"
 local HELP_MESSAGE = "\x89Leaderboard Commands:\nretry exit findmap changelevel spba_clearcheats lb_gui rival scroll encore records levelselect"
 local FILENAME = "leaderboard.txt"
@@ -89,17 +108,9 @@ local scroll_to
 
 local allowJoin
 
--- Imported functions --
+-- Events
+local EVENT_FINISH = "Finish"
 
--- lb_common.lua
-local ticsToTime = lb_TicsToTime
-local zoneAct = lb_ZoneAct
-
--- browser.lua
-local InitBrowser = InitBrowser
-local DrawBrowser = DrawBrowser
-local BrowserController = BrowserController
----------------
 
 -- cvars
 local cv_teamchange
@@ -164,170 +175,23 @@ local cv_interrupt = CV_RegisterVar({
 	end
 })
 
-local function setST(t, map, flags, scoreTable)
-	local mode = flags & ST_SEP
-	t[mode] = t[mode] or {}
-	t[mode][map] = scoreTable
-end
-
-local function getST(t, map, flags)
-	local mode = flags & ST_SEP
-	return t[mode] and t[mode][map] or nil
-end
-
-local function setScoreTable(map, flags, scoreTable)
-	setST(lb, map, flags, scoreTable)
-end
-
-local function getScoreTable(map, flags)
-	return getST(lb, map, flags)
-end
-
--- True if a is better than b
-local function lbComp(a, b)
-	-- Calculates the difficulty, harder has higher priority
-	-- if s is positive then a is harder
-	-- if s is negative then b is harder
-	-- if s is 0 then compare time
-	local s = (a["flags"] & (F_SPBEXP | F_SPBBIG)) - (b["flags"] & (F_SPBEXP | F_SPBBIG))
-	return s > 0 or not(s < 0 or a["time"] >= b["time"])
-end
-
-local function sortScores()
-	for mode, t in pairs(lb) do
-		for map, scoreTable in pairs(t) do
-			table.sort(scoreTable, lbComp)
-			setScoreTable(map, mode, scoreTable)
-		end
-	end
-end
-
--- Reinitialize lb based on new ST_SEP value
-local function reinit_lb()
-	local nlb = {}
-
-	for mode, t in pairs(lb) do
-		for map, scoreTable in pairs(t) do
-			for i, score in ipairs(scoreTable) do
-				local st = getST(nlb, map, score["flags"]) or {}
-				table.insert(st, score)
-				setST(nlb, map, score["flags"], st)
-			end
-		end
-	end
-
-	lb = nlb
-	sortScores()
-end
-
 local cv_spb_separate = CV_RegisterVar({
 	name = "lb_spb_combined",
 	defaultvalue = 1,
 	flags = CV_NETVAR | CV_CALL | CV_NOINIT,
 	PossibleValue = CV_YesNo,
 	func = function(v)
-		local curSep = ST_SEP
-
 		if v.value then
 			ST_SEP = F_SPBATK
 		else
 			ST_SEP = F_SPBATK | F_SPBBIG | F_SPBEXP
 		end
-
-		if curSep != ST_SEP then
-			reinit_lb()
-		end
 	end
 })
 
-local function score_t(map, name, skin, color, time, splits, flags, stat)
-	return {
-		["map"]    = map,
-		["name"]   = name,
-		["skin"]   = skin,
-		["color"]  = color,
-		["time"]   = time,
-		["splits"] = splits,
-		["flags"]  = flags,
-		["stat"] = stat
-	}
-end
-
 local MSK_SPEED = 0xF0
 local MSK_WEIGHT = 0xF
 
-local function stat_t(speed, weight)
-	if speed and weight then
-		return (speed << 4) | weight
-	end
-	return 0
-end
-
-local function stat_str(stat)
-	if stat then
-		return string.format("%d%d", (stat & MSK_SPEED) >> 4, stat & MSK_WEIGHT)
-	end
-
-	return "0"
-end
-
--- Read the leaderboard
-local f = io.open(FILENAME, "r")
-if f then
-	for l in f:lines() do
-		-- Leaderboard is stored in the following tab separated format
-		-- mapnum, name, skin, color, time, splits, flags, stat
-		local t = {}
-		for word in (l+"\t"):gmatch("(.-)\t") do
-			table.insert(t, word)
-		end
-
-		local flags = 0
-		if t[7] != nil then
-			flags = tonumber(t[7])
-		end
-
-		scoreTable = getScoreTable(tonumber(t[1]), flags) or {}
-
-		local spl = {}
-		if t[6] != nil then
-			for str in t[6]:gmatch("([^ ]+)") do
-				table.insert(spl, tonumber(str))
-			end
-		end
-
-		local stats = nil
-		if t[8] != nil then
-			if #t[8] >= 2 then
-				local speed = tonumber(string.sub(t[8], 1, 1))
-				local weight = tonumber(string.sub(t[8], 2, 2))
-				stats = stat_t(speed, weight)
-			end
-		end
-
-		table.insert(
-			scoreTable,
-			score_t(
-				tonumber(t[1]),
-				t[2],
-				t[3],
-				t[4],
-				tonumber(t[5]),
-				spl,
-				flags,
-				stats
-			)
-		)
-
-		setScoreTable(tonumber(t[1]), flags, scoreTable)
-	end
-
-	sortScores()
-	f:close()
-else
-	print("Failed to open file: ", FILENAME)
-end
-
 function allowJoin(v)
 	if not cv_interrupt.value then
 		local y
@@ -421,7 +285,7 @@ local function initBrowser(player)
 		return
 	end
 
-	InitBrowser(lb)
+	InitBrowser(ST_SEP)
 	drawState = DS_BROWSER
 
 	player.afkTime = leveltime
@@ -473,35 +337,6 @@ local function findMap(player, ...)
 end
 COM_AddCommand("findmap", findMap)
 
-local function mapnumFromExtended(map)
-	local p, q = map:upper():match("MAP(%w)(%w)$", 1)
-	if not (p and q) then
-		return nil
-	end
-
-	local mapnum = 0
-	local A = string.byte("A")
-
-	if tonumber(p) != nil then
-		-- Non extended map numbers
-		if tonumber(q) == nil then
-			return nil
-		end
-		mapnum = tonumber(p) * 10 + tonumber(q)
-	else
-		--Extended map numbers
-		p = string.byte(p) - A
-		local qn = tonumber(q)
-		if qn == nil then
-			qn = string.byte(q) - A + 10
-		end
-
-		mapnum = 36 * p + qn + 100
-	end
-
-	return mapnum
-end
-
 local SPBModeSym = {
 	[F_SPBEXP] = "X",
 	[F_SPBBIG] = "B",
@@ -525,6 +360,7 @@ end
 local function records(player, ...)
 	local mapid = ...
 	local mapnum = gamemap
+	local mapRecords = MapRecords
 
 	if mapid then
 		mapnum = mapnumFromExtended(mapid)
@@ -532,6 +368,8 @@ local function records(player, ...)
 			CONS_Printf(player, string.format("Invalid map name: %s", mapid))
 			return
 		end
+
+		mapRecords = GetMapRecords(mapnum, mapChecksum(mapnum), ST_SEP)
 	end
 
 	local map = mapheaderinfo[mapnum]
@@ -559,35 +397,33 @@ local function records(player, ...)
 		CONS_Printf(player, "\x85UNKNOWN MAP")
 	end
 
-	for mode, maps in pairs(lb) do
-		local maptbl = maps[mapnum]
-		if not maptbl then continue end
+	for mode, records in pairs(mapRecords) do
 		CONS_Printf(player, "")
 		CONS_Printf(player, modeToString(mode))
 
 		-- don't print flags for time attack
 		if mode then
-			for i, tbl in ipairs(maptbl) do
+			for i, score in ipairs(records) do
 				CONS_Printf(
 					player,
 					string.format(
 						"%2d %-21s \x89%8s \x80%s",
 						i,
-						tbl["name"],
-						ticsToTime(tbl["time"]),
-						modeToString(tbl["flags"])
+						score["name"],
+						ticsToTime(score["time"]),
+						modeToString(score["flags"])
 					)
 				)
 			end
 		else
-			for i, tbl in ipairs(maptbl) do
+			for i, score in ipairs(records) do
 				CONS_Printf(
 					player,
 					string.format(
 						"%2d %-21s \x89%8s",
 						i,
-						tbl["name"],
-						ticsToTime(tbl["time"])
+						score["name"],
+						ticsToTime(score["time"])
 					)
 				)
 			end
@@ -691,21 +527,26 @@ local function findRival(player, ...)
 	local totalScores = 0
 	local totalDiff = 0
 
-
 	CONS_Printf(player, string.format("\x89%s's times:", rival))
-	CONS_Printf(player, "MAP  	 Time    	  Diff    	Mode")
+	CONS_Printf(player, "MAP   CHCK	 Time    	  Diff    	Mode")
 
-	for mode, tbl in pairs(lb) do
-		scores[mode] = {}
+	local maplist = MapList()
+	local mapRecords
+	local rivalScore
+	local yourScore
+	for i = 1, #maplist do
+		mapRecords = GetMapRecords(maplist[i].id, maplist[i].checksum, ST_SEP)
 
-		for map, scoreTable in pairs(tbl) do
-			local rivalScore = nil
-			local yourScore = nil
+		for mode, records in pairs(mapRecords) do
+			scores[mode] = $ or {}
 
-			for _, score in pairs(scoreTable) do
-				if score["name"] == player.name then
+			rivalScore = nil
+			yourScore = nil
+
+			for _, score in ipairs(records) do
+				if score.name == player.name then
 					yourScore = score
-				elseif score["name"] == rival then
+				elseif score.name == rival then
 					rivalScore = score
 				end
 
@@ -715,7 +556,7 @@ local function findRival(player, ...)
 			end
 
 			if rivalScore and yourScore then
-				totalDiff = totalDiff + yourScore["time"] - rivalScore["time"]
+				totalDiff = totalDiff + yourScore.time - rivalScore.time
 			end
 
 			if rivalScore then
@@ -723,8 +564,8 @@ local function findRival(player, ...)
 				table.insert(
 					scores[mode],
 					{
-						["rival"] = rivalScore,
-						["your"] = yourScore
+						rival = rivalScore,
+						your = yourScore
 					}
 				)
 			end
@@ -761,9 +602,10 @@ local function findRival(player, ...)
 				CONS_Printf(
 					player,
 					string.format(
-						"%s	%8s	%s%9s	\x80%s",
-						G_BuildMapName(score["rival"]["map"]),
-						ticsToTime(score["rival"]["time"]),
+						"%s %4s	%8s	%s%9s	\x80%s",
+						G_BuildMapName(score.rival.map),
+						score.rival.checksum,
+						ticsToTime(score.rival.time),
 						color,
 						sym[diff<0] + ticsToTime(abs(diff)),
 						modestr
@@ -773,9 +615,10 @@ local function findRival(player, ...)
 				CONS_Printf(
 					player,
 					string.format(
-						"%s	%8s	%9s	%s",
-						G_BuildMapName(score["rival"]["map"]),
-						ticsToTime(score["rival"]["time"]),
+						"%s %4s	%8s	%9s	%s",
+						G_BuildMapName(score.rival.map),
+						score.rival.checksum,
+						ticsToTime(score.rival.time),
 						ticsToTime(0, true),
 						modestr
 					)
@@ -805,6 +648,64 @@ local function findRival(player, ...)
 end
 COM_AddCommand("rival", findRival)
 
+local function moveRecords(player, from_map, from_checksum, to_map, to_checksum)
+	if not(from_map and from_checksum and to_map) then
+		CONS_Printf(player, "Usage: lb_move_records <from_map> <from_checksum> <to_map> [<to_checksum>]")
+		CONS_Printf(
+			player,
+			string.format(
+				"Summary: Move records from one map to another.\n"..
+				"If no <to_checksum> is supplied then the checksum of the current loaded map %s is used.\n"..
+				"Hint: Use lb_known_maps to find checksums",
+				to_map or "<to_map>"
+			)
+		)
+		return
+	end
+
+	local from = {
+		["id"] = mapnumFromExtended(from_map),
+		["checksum"] = from_checksum:lower()
+	}
+
+	local to = {
+		["id"] = mapnumFromExtended(to_map),
+	}
+	to.checksum = to_checksum or mapChecksum(to.id)
+
+	if not to.checksum then
+		CONS_Printf(player, string.format("error: %s is not loaded; provide to_checksum to continue", to_map:upper()))
+		return
+	end
+	if #to.checksum != 4 or to.checksum:match("[^a-f0-9]") then
+		CONS_Printf(player, string.format("error: %s is an invalid checksum; checksums are of length 4 and can contain only 0-9a-f", to.checksum))
+		return
+	end
+
+	to.checksum = $:lower()
+
+	local mapRecords = GetMapRecords(from.id, from.checksum, F_SPBATK | F_SPBBIG | F_SPBEXP)
+	local recordCount = 0
+	for mode, records in pairs(mapRecords) do
+		recordCount = $ + #records
+	end
+
+	MoveRecords(from, to, ST_SEP)
+
+	CONS_Printf(
+		player,
+		string.format(
+			"%d records have been moved from\x82 %s %s\x80 to\x88 %s %s",
+			recordCount,
+			from_map, from.checksum,
+			to_map, to.checksum
+		)
+	)
+
+	CONS_Printf(player, "Please repack coldstore and restart the server for changes to take effect.")
+end
+COM_AddCommand("lb_move_records", moveRecords, COM_ADMIN)
+
 --DEBUGGING
 --local function printTable(tb)
 --	for mode, tbl in pairs(tb) do
@@ -828,7 +729,7 @@ COM_AddCommand("rival", findRival)
 --end
 
 addHook("MapLoad", function()
-		timeFinished = 0
+		TimeFinished = 0
 		splits = {}
 		prevLap = 0
 		drawState = DS_DEFAULT
@@ -838,6 +739,8 @@ addHook("MapLoad", function()
 
 		allowJoin(true)
 		--printTable(lb)
+
+		MapRecords = GetMapRecords(gamemap, mapChecksum(gamemap), ST_SEP)
 	end
 )
 
@@ -1147,20 +1050,22 @@ local function drawScoreboard(v, player)
 
 	cachePatches(v)
 
-	local gui = cv_gui.value
+	local gui = cv_gui.value or drawState == DS_BROWSER
+
+	-- Force enable gui at start and end of the race
 	if leveltime < START_TIME or player.exiting or player.lives == 0 then
 		gui = GUI_ON
 	end
 
 	if gui then
-		stateFunctions[drawState](v, player, scoreTable, gui)
-	end
+		stateFunctions[drawState](v, player, ScoreTable, gui)
 
 		local pos = 0
 		-- Draw current active modes bottom left
 		pos = drawMode(v, pos, F_SPBJUS)
 		pos = drawMode(v, pos, F_SPBBIG)
 		pos = drawMode(v, pos, F_SPBEXP)
+	end
 end
 hud.add(drawScoreboard, "game")
 
@@ -1193,7 +1098,7 @@ end
 
 -- Find location of player and scroll to it
 function scroll_to(player)
-	local m = scoreTable or {}
+	local m = ScoreTable or {}
 
 	scrollToPos = 2
 	for pos, score in ipairs(m) do
@@ -1207,19 +1112,19 @@ function scroll_to(player)
 end
 
 -- Write skin stats to each score where there are none
-local function writeStats()
-	for _, t in pairs(lb) do
-		for _, scoreTable in pairs(t) do
-			for _, score in ipairs(scoreTable) do
-				local skin = skins[score["skin"]]
-				if skin and not score["stat"] then
-					local stats = stat_t(skin.kartspeed, skin.kartweight)
-					score["stat"] = stats
-				end
-			end
-		end
-	end
-end
+--local function writeStats()
+--	for _, t in pairs(lb) do
+--		for _, scoreTable in pairs(t) do
+--			for _, score in ipairs(scoreTable) do
+--				local skin = skins[score["skin"]]
+--				if skin and not score["stat"] then
+--					local stats = stat_t(skin.kartspeed, skin.kartweight)
+--					score["stat"] = stats
+--				end
+--			end
+--		end
+--	end
+--end
 
 local function checkFlags(p)
 	local flags = 0
@@ -1256,10 +1161,13 @@ local function saveTime(player)
 	if checkFlags(player) != Flags then
 		print("Game mode change detected! Time has been disqualified.")
 		S_StartSound(nil, 110)
+		fireEvent(EVENT_FINISH, {
+			disqualified = true,
+		})
 		return
 	end
 
-	scoreTable = $ or {}
+	ScoreTable = $ or {}
 
 	local pskin = skins[player.mo.skin]
 	local newscore = score_t(
@@ -1267,90 +1175,65 @@ local function saveTime(player)
 		player.name,
 		player.mo.skin,
 		player.skincolor,
-		timeFinished,
+		TimeFinished,
 		splits,
 		Flags,
-		stat_t(player.HMRs or pskin.kartspeed, player.HMRw or pskin.kartweight)
+		stat_t(player.HMRs or pskin.kartspeed, player.HMRw or pskin.kartweight),
+		mapChecksum(gamemap)
 	)
 
 	-- Check if you beat your previous best
-	for i = 1, #scoreTable do
-		if scoreTable[i]["name"] == player.name then
-			if lbComp(newscore, scoreTable[i]) then
-				table.remove(scoreTable, i)
-				S_StartSound(nil, 130)
-				FlashTics = leveltime + TICRATE * 3
-				FlashRate = 1
-				FlashVFlags = YellowFlash
-				break
-			else
+	for i = 1, #ScoreTable do
+		if ScoreTable[i].name == player.name then
+			if not lbComp(newscore, ScoreTable[i]) then
 				-- You suck lol
 				S_StartSound(nil, 201)
 				FlashTics = leveltime + TICRATE * 3
 				FlashRate = 3
 				FlashVFlags = RedFlash
 				scroll_to(player)
+				fireEvent(EVENT_FINISH, {score = newscore})
 				return
 			end
 		end
 	end
 
-	print("Saving score")
-	table.insert(
-		scoreTable,
-		newscore
-	)
 
-	table.sort(scoreTable, lbComp)
-	while #scoreTable > cv_saves.value do
-		table.remove(scoreTable)
+	-- Save the record
+	SaveRecord(newscore, gamemap, ST_SEP)
+
+	-- Set players text flash and play chime sfx
+	S_StartSound(nil, 130)
+	FlashTics = leveltime + TICRATE * 3
+	FlashRate = 1
+	FlashVFlags = YellowFlash
+
+	-- Reload the MapRecords
+	MapRecords = GetMapRecords(gamemap, mapChecksum(gamemap), ST_SEP)
+
+	-- Set the updated ScoreTable
+	ScoreTable = MapRecords[ST_SEP & Flags]
+
+	for i, score in ipairs(ScoreTable) do
+		if score.name != player.name then continue end
+		fireEvent(EVENT_FINISH, {position = i, score = newscore})
+		break
 	end
 
+	-- Scroll the gui to the player entry
 	scroll_to(player)
-
-	setScoreTable(gamemap, Flags, scoreTable)
-
-	if not StatTrack then
-		writeStats()
-		StatTrack = true
-	end
-
-	local f = assert(io.open(FILENAME, "w"))
-	if f == nil then
-		print("Failed to open file for writing: " + FILENAME)
-		return
-	end
-
-	for _, tbl in pairs(lb) do
-		for _, scoreTable in pairs(tbl) do
-			for _, score in ipairs(scoreTable) do
-				f:write(
-					score["map"], "\t",
-					score["name"], "\t",
-					score["skin"], "\t",
-					score["color"], "\t",
-					score["time"], "\t",
-					table.concat(score["splits"], " "), "\t",
-					score["flags"], "\t",
-					stat_str(score["stat"]), "\n"
-				)
-			end
-		end
-	end
-
-	f:close()
 end
 
 -- DEBUGGING
 --local function saveLeaderboard(player, ...)
---	timeFinished = tonumber(... or player.realtime)
+--	TimeFinished = tonumber(... or player.realtime)
 --	splits = {1000, 2000, 3000}
 --	saveTime(player)
 --end
 --COM_AddCommand("save", saveLeaderboard)
 
 local function regLap(player)
-	if player.laps > prevLap and timeFinished == 0 then
+	if player.laps > prevLap and TimeFinished == 0 then
 		prevLap = player.laps
 		table.insert(splits, player.realtime)
 		showSplit = 5 * TICRATE
@@ -1457,7 +1340,7 @@ local function think()
 		end
 	end
 
-	scoreTable = getScoreTable(gamemap, Flags)
+	ScoreTable = MapRecords[ST_SEP & Flags]
 
 	if not cv_teamchange then
 		cv_teamchange = CV_FindVar("allowteamchange")
@@ -1465,8 +1348,8 @@ local function think()
 
 	if p then
 		-- must be done before browser control
-		if p.laps >= mapheaderinfo[gamemap].numlaps and timeFinished == 0 then
-			timeFinished = p.realtime
+		if p.laps >= mapheaderinfo[gamemap].numlaps and TimeFinished == 0 then
+			TimeFinished = p.realtime
 			saveTime(p)
 		end
 
@@ -1556,8 +1439,8 @@ local function netvars(net)
 	splits = net($)
 	prevLap = net($)
 	drawState = net($)
-	StatTrack = net($)
 	EncoreInitial = net($)
-	lb = net($)
+	MapRecords = net($)
+	TimeFinished = net($)
 end
 addHook("NetVars", netvars)

tools/coldstore.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+import sys
+from os import path
+linesep = "\n"
+
+if len(sys.argv) != 3 or not sys.argv[1] or not sys.argv[2]:
+	print("Usage: coldstore.py <game_directory> <leaderboard_records.lua>")
+	print("\t<game_directory>\t\tthe game directory where wads and luafiles reside. Usually at '$HOME/.srb2kart'.")
+	print("\t<leaderboard_records.lua>\tthe output name for the records packed lua file. It will be saved within <game_directory>.")
+	quit()
+
+if not sys.argv[2].endswith(".lua"):
+	print("{} must end with .lua".format(sys.argv[2]))
+	quit()
+
+game_dir = sys.argv[1]
+leaderboard_txt = path.join(game_dir, "luafiles", "leaderboard.txt")
+coldstore_txt = path.join(game_dir, "luafiles", "leaderboard.coldstore.txt")
+records_lua = path.join(game_dir, sys.argv[2])
+
+def ParseScore(score):
+	# Map Name Skin Color Time Splits Flags Stat
+	split = score.split("\t")
+
+	checksum = ""
+	if len(split) > 8:
+		checksum = split[8]
+
+	return {
+		"map": split[0],
+		"name": split[1],
+		"skin": split[2],
+		"color": split[3],
+		"time": int(split[4]),
+		"splits": split[5],
+		"flags": int(split[6]),
+		"stat": split[7],
+		"checksum": checksum
+	}
+
+# Compare scores
+def CompareScore(a, b):
+	return a["time"] < b["time"]
+
+F_SEP = 0xF
+def SameScore(a, b):
+	return a["name"] == b["name"] and a["checksum"] == b["checksum"] and (a["flags"] & F_SEP) == (b["flags"] & F_SEP)
+
+def LoadRecordsFromFile(path):
+	records = []
+	try:
+		with open(path, "r") as f:
+			for line in f.readlines():
+				line = line.strip()
+				if line != "":
+					records.append(ParseScore(line))
+	except FileNotFoundError:
+		pass
+
+	return records
+
+def AddScore(records, score):
+	mapid = score["map"]
+	mapTable = records.get(mapid) or []
+
+	for i in range(len(mapTable)):
+		scoreb = mapTable[i]
+		if SameScore(score, scoreb):
+			if CompareScore(score, scoreb):
+				mapTable[i] = score
+				records[mapid] = mapTable
+			return
+	mapTable.append(score)
+	records[mapid] = mapTable
+
+# load leaderboard.txt and coldstore.txt
+recordsList = LoadRecordsFromFile(leaderboard_txt)
+recordsList.extend(LoadRecordsFromFile(coldstore_txt))
+
+
+# construct the map tables
+records = {}
+for score in recordsList:
+	AddScore(records, score)
+
+# convert records to flat list
+recordsList = []
+rejected = []
+for mapTable in records.values():
+	for score in mapTable:
+		scoreStr = "\t".join([str(v) for v in list(score.values())])
+		# only allow records with checksums
+		if score["checksum"] != "":
+			recordsList.append(scoreStr)
+		else:
+			rejected.append(scoreStr)
+
+# truncate and write records to coldstore
+with open(coldstore_txt, "w") as f:
+	for score in recordsList:
+			f.write(score + linesep)
+
+luaA = """do
+	local AddColdStore = lb_add_coldstore_record_string
+	local records = {
+"""
+luaB = """	}
+	for _, str in ipairs(records) do
+		AddColdStore(str)
+	end
+end
+"""
+# pack the records.lua file
+with open(records_lua, "w") as f:
+	f.write(luaA)
+	for score in recordsList:
+		score = score.replace("\\", "\\\\")
+		score = score.replace("\"", "\\\"")
+		f.write("\t\t\"{}\",{}".format(score, linesep))
+	f.write(luaB)
+
+# truncate and rewrite rejected scores to leaderboard.txt
+with open(leaderboard_txt, "w") as f:
+	for score in rejected:
+		f.write(score + linesep)

transmission.lua

@@ -0,0 +1,249 @@
+local MSG = "<<~"
+local CHN = "~>>"
+
+local PACKET_MAX_SIZE = 200
+local MAX_TICS = TICRATE * 2
+
+local function encode(data)
+	data = string.gsub(data, "\n", "\\n")
+	data = string.gsub(data, "\t", "\\t")
+	return data
+end
+
+local function decode(data)
+	data = string.gsub(data, "\\n", "\n")
+	data = string.gsub(data, "\\t", "\t")
+	return data
+end
+
+local transmitters = {}
+
+local function Transmitter(channel, opts)
+	assert(channel, "Transmitter: channel is required")
+	opts = $ or {}
+
+	return {
+		packets = {},
+
+		push = function(this, packet)
+			table.insert(this.packets, MSG..channel..CHN..packet)
+		end,
+
+		pop = function(this)
+			return table.remove(this.packets)
+		end,
+
+		sendPacket = function(this)
+			local sender = consoleplayer or server
+			COM_BufInsertText(sender, "say \""..this:pop().."\"")
+			return not #this.packets
+		end,
+
+		writeHeader = function(this)
+			this:push(#this.packets)
+		end,
+
+		close = function(this)
+			for i, tr in ipairs(transmitters) do
+				if tr == this then
+					table.remove(transmitters, i)
+					break
+				end
+			end
+
+			if opts.free then
+				opts.free(this, opts.handle)
+			end
+		end,
+
+		enqueue = function(this)
+			table.insert(transmitters, this)
+		end,
+
+		transmit = function(this, data)
+			assert(data, "Transmitter: nil data")
+
+			data = encode(data)
+
+			if opts.stream then
+				assert(
+					#data < PACKET_MAX_SIZE,
+					"Transmitter: data packet too large for stream"
+				)
+
+				this:push(data)
+				this:enqueue()
+				return
+			end
+
+			local sub
+			for i = 1, #data, PACKET_MAX_SIZE do
+				sub = data:sub(i, min(#data, i + PACKET_MAX_SIZE-1))
+				this:push(sub)
+			end
+
+			this:writeHeader()
+			this:enqueue()
+		end
+	}
+end
+rawset(_G, "lb_transmitter", Transmitter)
+
+addHook("ThinkFrame", function()
+	if not (#transmitters and leveltime) then return end
+
+	local index = (leveltime % #transmitters) + 1
+	local transmitter = transmitters[index]
+
+	if transmitter:sendPacket() then
+		transmitter:close()
+	end
+end)
+
+local Channels = {
+	channel = {},
+
+	add = function(this, ch, reciever)
+		this.channel[ch] = $ or {}
+		table.insert(this.channel[ch], reciever)
+	end,
+
+	chan = function(this, ch)
+		local c = this.channel[ch]
+		local i = c and #c + 1 or 0
+		return function()
+			if i > 1 then
+				i = i - 1
+				return i, c[i]
+			end
+		end
+	end,
+
+	remove = function(this, ch, index)
+		table.remove(this.channel[ch], index)
+	end,
+
+	recieve = function(this)
+		return function(packet, ch)
+			for i, reciever in this:chan(ch) do
+				if reciever:push(packet) then
+					reciever:close()
+				end
+			end
+		end
+	end
+}
+
+addHook("ThinkFrame", function()
+	for _, ch in pairs(Channels.channel) do
+		for _, rec in pairs(ch) do
+			if rec:tick() then
+				rec:close()
+			end
+		end
+	end
+end)
+
+local function Reciever(channel, callback, opts)
+	assert(callback, "Reciever: callback is required")
+	opts = $ or {}
+
+	local ticker, pusher
+	local MAX_TICS = MAX_TICS
+	if opts.stream then
+		ticker = function(this) end
+
+		pusher = function(this, packet)
+			callback(decode(packet), opts.handle)
+		end
+	else
+		ticker = function(this)
+			this.tics = $ + 1
+			return this.tics > MAX_TICS
+		end
+
+		pusher = function(this, packet)
+			if not this.len then
+				this:recieveHeader(packet)
+				return
+			end
+
+			table.insert(this.packets, packet)
+
+			if opts.progress then
+				opts.progress(#this.packets, this.len, opts.handle)
+			end
+
+			if #this.packets >= this.len then
+				this:finish()
+				return true
+			end
+
+			this.tics = 0
+		end
+	end
+
+	return {
+		len,
+		packets = {},
+		tics = 0,
+		tick = ticker,
+
+		close = function(this)
+			for i, rec in Channels:chan(channel) do
+				if rec == this then
+					Channels:remove(channel, i)
+					if opts.free then
+						opts.free(this, opts.handle)
+					end
+					return
+				end
+			end
+		end,
+
+		push = pusher,
+
+		recieveHeader = function(this, header)
+			local len = tonumber(header)
+			assert(len ~= nil,
+				"Reciever: invalid header '"..(header or "nil").."'")
+			this.len = len
+		end,
+
+		listen = function(this)
+			Channels:add(channel, this)
+		end,
+
+		pop = function(this)
+			return table.remove(this.packets)
+		end,
+
+		finish = function(this)
+			local data = ""
+
+			local s = this:pop()
+			while s do
+				data = $..s
+				s = this:pop()
+			end
+
+			callback(decode(data), opts.handle)
+		end
+	}
+end
+rawset(_G, "lb_reciever", Reciever)
+
+local function scan(sym, msg, fn)
+	if msg:find(sym) == 1 then
+		msg = msg:sub(#sym+1)
+		local chi = msg:find(CHN)
+		local ch = msg:sub(1, chi-1)
+		msg = msg:sub(chi+#CHN)
+		fn(msg, ch)
+		return true
+	end
+end
+
+addHook("PlayerMsg", function(source, type, target, msg)
+	return scan(MSG, msg, Channels:recieve())
+end)