Merge pull request 'Lua loaded records' (#9) from partition into master

Reviewed-on: #9

browser.lua

@@ -1,15 +1,22 @@
-local leaderboard = nil
+local MapRecords
 local maps
 local mapIndex = 1
 local scrollPos = 1
 local modes = nil
 local mode = 1
 local prefMode = nil
+local ModeSep
+
+---- Imported functions ----
 
--- Imported functions -------
 -- lb_common.lua
 local ZoneAct = lb_ZoneAct
 local TicsToTime = lb_TicsToTime
+local mapChecksum = lb_map_checksum
+
+-- lb_store.lua
+local GetMapRecords = lb_get_map_records
+
 -----------------------------
 
 local cv_kartencore
@@ -25,10 +32,8 @@ end
 local function updateModes()
 	-- set available modes for this map
 	modes = {}
-	for mode, scoreTable in pairs(leaderboard) do
+	for mode, _ in pairs(MapRecords) do
-		if scoreTable[getMap()] then
+		table.insert(modes, mode)
-			table.insert(modes, mode)
-		end
 	end
 	table.sort(modes)
 
@@ -46,6 +51,8 @@ local function updateMapIndex(n)
 	mapIndex = mapIndexOffset(n)
 	scrollPos = 1
 
+	MapRecords = GetMapRecords(maps[mapIndex], mapChecksum(maps[mapIndex]), ModeSep)
+
 	updateModes()
 end
 
@@ -337,7 +344,7 @@ local function drawScore(v, i, pos, score, highlight)
 end
 
 local function drawBrowser(v, player)
-	if not leaderboard then return end
+	if not MapRecords then return end
 
 	v.fadeScreen(0xFF00, 16)
 
@@ -358,23 +365,20 @@ local function drawBrowser(v, player)
 
 	if not modes then return end
 
-	local gamemode = leaderboard[modes[mode]]
+	local records = MapRecords[modes[mode]]
-	if not gamemode then return end
+	if not records then return end
 
-	local scoreTable = gamemode[getMap()]
+	local record_count = #records
-	if not scoreTable then return end
+	scrollPos = max(min(scrollPos, record_count - 3), 1)
-
+	local endi = min(scrollPos + 7, record_count)
-	local scores = #scoreTable
-	scrollPos = max(min(scrollPos, scores - 3), 1)
-	local endi = min(scrollPos + 7, scores)
 	for i = scrollPos, endi do
-		drawScore(v, i - scrollPos + 1, i, scoreTable[i], scoreTable[i].name == player.name)
+		drawScore(v, i - scrollPos + 1, i, records[i], records[i].name == player.name)
 	end
 end
 rawset(_G, "DrawBrowser", drawBrowser)
 
-local function initBrowser(lb)
+local function initBrowser(modeSep)
-	leaderboard = lb
+	ModeSep = modeSep
 
 	-- set mapIndex to current map
 	for i, m in ipairs(maps) do
@@ -384,6 +388,9 @@ local function initBrowser(lb)
 		end
 	end
 
+	-- initialize MapRecords
+	MapRecords = GetMapRecords(gamemap, mapChecksum(gamemap), ModeSep)
+
 	scrollPos = 1
 	updateModes()
 end
@@ -485,6 +492,7 @@ local function netvars(net)
 	mode = net($)
 	prefMode = net($)
 	scrollPos = net($)
-	leaderboard = net($)
+	MapRecords = net($)
+	ModeSep = net($)
 end
 addHook("NetVars", netvars)

lb_common.lua

@@ -1,3 +1,17 @@
+rawset(_G, "lb_score_t", function(map, name, skin, color, time, splits, flags, stat, checksum)
+	return {
+		["map"]    = map,
+		["name"]   = name,
+		["skin"]   = skin,
+		["color"]  = color,
+		["time"]   = time,
+		["splits"] = splits,
+		["flags"]  = flags,
+		["stat"] = stat,
+		["checksum"] = checksum
+	}
+end)
+
 rawset(_G, "lb_TicsToTime", function(tics, pure)
 	if tics == 0 and pure then
 		return "-:--:--"
@@ -24,3 +38,72 @@ rawset(_G, "lb_ZoneAct", function(map)
 
 	return z
 end)
+
+rawset(_G, "lb_stat_t", function(speed, weight)
+	if speed and weight then
+		return (speed << 4) | weight
+	end
+	return 0
+end)
+
+
+local F_SPBBIG = 0x4
+local F_SPBEXP = 0x8
+-- True if a is better than b
+rawset(_G, "lb_comp", function(a, b)
+	-- Calculates the difficulty, harder has higher priority
+	-- if s is positive then a is harder
+	-- if s is negative then b is harder
+	-- if s is 0 then compare time
+	local s = (a.flags & (F_SPBEXP | F_SPBBIG)) - (b.flags & (F_SPBEXP | F_SPBBIG))
+	return s > 0 or not(s < 0 or a.time >= b.time)
+end)
+
+local function djb2(message)
+	local digest = 5381
+	for c in message:gmatch(".") do
+		digest = (($ << 5) + $) + string.byte(c)
+	end
+
+	return digest
+end
+
+-- Produce a checksum by using the maps title, subtitle and zone
+rawset(_G, "lb_map_checksum", function(mapnum)
+	local mh = mapheaderinfo[mapnum]
+	if not mh then
+		return nil
+	end
+
+	local digest = string.format("%04x", djb2(mh.lvlttl..mh.subttl..mh.zonttl))
+	return string.sub(digest, #digest - 3)
+end)
+
+rawset(_G, "lb_mapnum_from_extended", function(map)
+	local p, q = map:upper():match("MAP(%w)(%w)$", 1)
+	if not (p and q) then
+		return nil
+	end
+
+	local mapnum = 0
+	local A = string.byte("A")
+
+	if tonumber(p) != nil then
+		-- Non extended map numbers
+		if tonumber(q) == nil then
+			return nil
+		end
+		mapnum = tonumber(p) * 10 + tonumber(q)
+	else
+		--Extended map numbers
+		p = string.byte(p) - A
+		local qn = tonumber(q)
+		if qn == nil then
+			qn = string.byte(q) - A + 10
+		end
+
+		mapnum = 36 * p + qn + 100
+	end
+
+	return mapnum
+end)

lb_store.lua

@@ -0,0 +1,394 @@
+-- This file handles the storage and related netvars of the leaderboard
+
+---- Imported functions ----
+
+-- lb_common.lua
+local stat_t = lb_stat_t
+local lbComp = lb_comp
+local score_t = lb_score_t
+local mapChecksum = lb_map_checksum
+local mapnumFromExtended = lb_mapnum_from_extended
+
+----------------------------
+
+local LEADERBOARD_FILE = "leaderboard.txt"
+local COLDSTORE_FILE = "leaderboard.coldstore.txt"
+
+-- ColdStore are records loaded from lua addons
+-- this table should never be modified outside of the AddColdStore function
+local ColdStore = {}
+
+-- Livestore are new records nad records loaded from leaderboard.txt file
+local LiveStore = {}
+
+-- parse score function
+local parseScore
+
+local MSK_SPEED = 0xF0
+local MSK_WEIGHT = 0xF
+local function stat_str(stat)
+	if stat then
+		return string.format("%d%d", (stat & MSK_SPEED) >> 4, stat & MSK_WEIGHT)
+	end
+
+	return "0"
+end
+
+local function isSameRecord(a, b, modeSep)
+	return a.name == b.name and (a.flags & modeSep) == (b.flags & modeSep)
+end
+
+-- insert or replace the score in dest
+local function insertOrReplace(dest, score, modeSep)
+	for i, record in ipairs(dest) do
+		if isSameRecord(record, score, modeSep) then
+			if lbComp(score, record) then
+				dest[i] = score
+			end
+			return
+		end
+	end
+
+	table.insert(dest, score)
+end
+
+local function dumpStoreToFile(filename, store)
+	local f = assert(
+		io.open(filename, "w"),
+		"Failed to open file for writing: "..filename
+	)
+
+	f:setvbuf("line")
+
+	for mapid, checksums in pairs(store) do
+		for checksum, records in pairs(checksums) do
+			for _, record in ipairs(records) do
+				if not record.checksum or record.checksum == "" then
+					record.checksum = mapChecksum(record.map) or ""
+				end
+
+				f:write(
+					mapid, "\t",
+					record.name, "\t",
+					record.skin, "\t",
+					record.color, "\t",
+					record.time, "\t",
+					table.concat(record.splits, " "), "\t",
+					record.flags, "\t",
+					stat_str(record.stat), "\t",
+					record.checksum, "\n"
+				)
+			end
+		end
+	end
+
+	f:close()
+end
+
+-- GLOBAL
+-- Returns a list of all maps with records
+local function MapList()
+	local maps = {}
+	for mapid, checksums in pairs(ColdStore) do
+		maps[mapid] = $ or {}
+		for checksum in pairs(checksums) do
+			maps[mapid][checksum] = true
+		end
+	end
+	for mapid, checksums in pairs(LiveStore) do
+		maps[mapid] = $ or {}
+		for checksum in pairs(checksums) do
+			maps[mapid][checksum] = true
+		end
+	end
+
+	local maplist = {}
+	for mapid, checksums in pairs(maps) do
+		for checksum in pairs(checksums) do
+			table.insert(maplist, {["id"] = mapid, ["checksum"] = checksum})
+		end
+	end
+	table.sort(maplist, function(a, b) return a.id < b.id end)
+
+	return maplist
+end
+rawset(_G, "lb_map_list", MapList)
+
+-- GLOBAL
+-- Function for adding a single record from lua
+local function AddColdStore(record)
+	ColdStore[record.map] = $ or {}
+	ColdStore[record.map][record.checksum] = $ or {}
+
+	table.insert(ColdStore[record.map][record.checksum], record)
+end
+rawset(_G, "lb_add_coldstore_record", AddColdStore)
+
+-- GLOBAL
+-- Function for adding a single record in string form from lua
+local function AddColdStoreString(record)
+	AddColdStore(parseScore(record))
+end
+rawset(_G, "lb_add_coldstore_record_string", AddColdStoreString)
+
+-- Insert mode separated records from the flat sourceTable into dest
+local function insertRecords(dest, sourceTable, checksum, modeSep)
+	if not sourceTable then return end
+	if not sourceTable[checksum] then return end
+
+	local mode = nil
+	for _, record in ipairs(sourceTable[checksum]) do
+		mode = record.flags & modeSep
+		dest[mode] = $ or {}
+		table.insert(dest[mode], record)
+	end
+end
+
+-- GLOBAL
+-- Construct the leaderboard table of the supplied mapid
+-- combines the ColdStore and LiveStore records
+local function GetMapRecords(map, checksum, modeSep)
+	local mapRecords = {}
+
+	-- Insert ColdStore records
+	insertRecords(mapRecords, ColdStore[map], checksum, modeSep)
+
+	-- Insert LiveStore records
+	insertRecords(mapRecords, LiveStore[map], checksum, modeSep)
+
+	-- Sort records
+	for _, records in pairs(mapRecords) do
+		table.sort(records, lbComp)
+	end
+
+	-- Remove duplicate entries
+	for _, records in pairs(mapRecords) do
+		local players = {}
+		local i = 1
+		while i <= #records do
+			if players[records[i].name] then
+				table.remove(records, i)
+			else
+				players[records[i].name] = true
+				i = i + 1
+			end
+		end
+	end
+
+	return mapRecords
+end
+rawset(_G, "lb_get_map_records", GetMapRecords)
+
+-- GLOBAL
+-- Save a record to the LiveStore and write to disk
+-- SaveRecord will replace the record holders previous record
+local function SaveRecord(score, map, modeSep)
+	local checksum = mapChecksum(map)
+	LiveStore[map] = $ or {}
+	LiveStore[map][checksum] = $ or {}
+	insertOrReplace(LiveStore[map][checksum], score, modeSep)
+
+	print("Saving score")
+	if isserver then 
+		dumpStoreToFile(LEADERBOARD_FILE, LiveStore)
+	end
+end
+rawset(_G, "lb_save_record", SaveRecord)
+
+local function netvars(net)
+	LiveStore = net($)
+end
+
+addHook("NetVars", netvars)
+
+function parseScore(str)
+	-- Leaderboard is stored in the following tab separated format
+	-- mapnum, name, skin, color, time, splits, flags, stat
+	local t = {}
+	for word in (str.."\t"):gmatch("(.-)\t") do
+		table.insert(t, word)
+	end
+
+	local splits = {}
+	if t[6] != nil then
+		for str in t[6]:gmatch("([^ ]+)") do
+			table.insert(splits, tonumber(str))
+		end
+	end
+
+	local flags = 0
+	if t[7] != nil then
+		flags = tonumber(t[7])
+	end
+
+	local stats = nil
+	if t[8] != nil then
+		if #t[8] >= 2 then
+			local speed = tonumber(string.sub(t[8], 1, 1))
+			local weight = tonumber(string.sub(t[8], 2, 2))
+			stats = stat_t(speed, weight)
+		end
+	end
+
+	local checksum = t[9] or ""
+
+	return score_t(
+		tonumber(t[1]), -- Map
+		t[2],		-- Name
+		t[3],		-- Skin
+		t[4],		-- Color
+		tonumber(t[5]),	-- Time
+		splits,
+		flags,
+		stats,
+		checksum:lower()
+	)
+end
+rawset(_G, "lb_parse_score", parseScore)
+
+-- Read and parse a store file
+local function loadStoreFile(filename)
+	local f = assert(
+		io.open(filename, "r"),
+		"Failed to open file for reading: "..filename
+	)
+
+	local store = {}
+
+	for l in f:lines() do
+		local score = parseScore(l)
+		store[score.map] = $ or {}
+		store[score.map][score.checksum] = $ or {}
+		table.insert(store[score.map][score.checksum], score)
+	end
+
+	f:close()
+
+	return store
+end
+
+-- GLOBAL
+-- Command for moving records from one map to another
+local function moveRecords(from, to, modeSep)
+	local function moveRecordsInStore(store)
+		if not (store[from.id] and store[from.id][from.checksum]) then
+			return 0
+		end
+
+		store[to.id] = $ or {}
+		store[to.id][to.checksum] = $ or {}
+		for i, score in ipairs(store[from.id][from.checksum]) do
+			score.map = to.id
+			score.checksum = to.checksum
+			insertOrReplace(store[to.id][to.checksum], score, modeSep)
+		end
+
+		-- Destroy the original table
+		store[from.id][from.checksum] = nil
+	end
+
+	-- move livestore records and write to disk
+	moveRecordsInStore(LiveStore)
+
+	if isserver then
+		dumpStoreToFile(LEADERBOARD_FILE, LiveStore)
+
+		-- move coldstore records
+		local ok, coldstore = pcall(loadStoreFile, COLDSTORE_FILE)
+		if ok and coldstore then
+			moveRecordsInStore(coldstore)
+			dumpStoreToFile(COLDSTORE_FILE, coldstore)
+		end
+	end
+end
+rawset(_G, "lb_move_records", moveRecords)
+
+-- Helper function for those upgrading from 1.2 to 1.3
+COM_AddCommand("lb_write_checksums", function(player)
+	local count = 0
+	local moved = {}
+
+	-- Gather movable records (no checksum, map loaded)
+	for map, checksums in pairs(LiveStore) do
+		for checksum, records in pairs(checksums) do
+			if checksum == "" then
+				local sum = mapChecksum(map)
+
+				if not sum then continue end
+
+				moved[map] = {}
+				moved[map][sum] = {}
+
+				for i, record in ipairs(records) do
+					record.checksum = sum
+					table.insert(moved[map][sum], record)
+				end
+			end
+		end
+	end
+
+	-- Write moved to livestore
+	for map, checksums in pairs(moved) do
+		LiveStore[map] = $ or {}
+		for checksum, records in pairs(checksums) do
+			LiveStore[map][checksum] = $ or {}
+			for i, score in ipairs(records) do
+				table.insert(LiveStore[map][checksum], score)
+			end
+			count = $ + #records
+		end
+		LiveStore[map][""] = nil
+	end
+
+	if isserver then
+		dumpStoreToFile(LEADERBOARD_FILE, LiveStore)
+	end
+
+	CONS_Printf(player, string.format("Successful operation on %d records", count))
+end, COM_ADMIN)
+
+COM_AddCommand("lb_known_maps", function(player, map)
+	local mapnum = gamemap
+	if map then
+		mapnum = mapnumFromExtended(map)
+		if not mapnum then
+			CONS_Printf(player, string.format("invalid map '%s'", map))
+			return
+		end
+	end
+
+	local known = {}
+
+	if LiveStore[mapnum] then
+		for checksum, records in pairs(LiveStore[mapnum]) do
+			known[checksum] = #records
+		end
+	end
+	if ColdStore[mapnum] then
+		for checksum, records in pairs(ColdStore[mapnum]) do
+			known[checksum] = $ or 0 + #records
+		end
+	end
+
+	CONS_Printf(player, "Map     Chck   Records")
+	for checksum, count in pairs(known) do
+		CONS_Printf(player, string.format("%s   %s   %d", G_BuildMapName(mapnum), checksum, count))
+	end
+end)
+
+COM_AddCommand("lb_download_live_records", function(player, filename)
+	if not filename then
+		CONS_Printf(player, "Usage: lb_download_live_records <filename>")
+		return
+	end
+
+	if filename:sub(#filename-3) != ".txt" then
+		filename = $..".txt"
+	end
+	dumpStoreToFile(filename, LiveStore)
+end, COM_LOCAL)
+
+-- Load the livestore
+if isserver then
+	LiveStore = loadStoreFile(LEADERBOARD_FILE)
+end

leaderboard.lua

@@ -1,18 +1,39 @@
 -- Leaderboards written by Not
 -- Reusable
 
--- Leaderboard Table
+---------- Imported functions -------------
--- [mode][mapnum][scoreTable]
+-- lb_common.lua
-local lb = {}
+local ticsToTime = lb_TicsToTime
+local zoneAct = lb_ZoneAct
+local stat_t = lb_stat_t
+local lbComp = lb_comp
+local mapChecksum = lb_map_checksum
+local score_t = lb_score_t
+local mapnumFromExtended = lb_mapnum_from_extended
 
-local timeFinished = 0
+-- browser.lua
+local InitBrowser = InitBrowser
+local DrawBrowser = DrawBrowser
+local BrowserController = BrowserController
+
+-- lb_store.lua
+local GetMapRecords = lb_get_map_records
+local SaveRecord = lb_save_record
+local MapList = lb_map_list
+local MoveRecords = lb_move_records
+--------------------------------------------
+
+-- Holds the current maps records table including all modes
+local MapRecords = {}
+
+local TimeFinished = 0
 local disable = false
 local prevLap = 0
 local splits = {}
 local PATCH = nil
 local help = true
 local EncoreInitial = nil
-local scoreTable
+local ScoreTable
 
 
 -- Text flash on finish
@@ -29,9 +50,6 @@ local RedFlash = {
 	[1] = 0
 }
 
--- Tracks if stats have been written or not
-local StatTrack = false
-
 local UNCLAIMED = "Unclaimed Record"
 local HELP_MESSAGE = "\x89Leaderboard Commands:\nretry exit findmap changelevel spba_clearcheats lb_gui rival scroll encore records levelselect"
 local FILENAME = "leaderboard.txt"
@@ -89,17 +107,6 @@ local scroll_to
 
 local allowJoin
 
--- Imported functions --
-
--- lb_common.lua
-local ticsToTime = lb_TicsToTime
-local zoneAct = lb_ZoneAct
-
--- browser.lua
-local InitBrowser = InitBrowser
-local DrawBrowser = DrawBrowser
-local BrowserController = BrowserController
----------------
 
 -- cvars
 local cv_teamchange
@@ -164,172 +171,23 @@ local cv_interrupt = CV_RegisterVar({
 	end
 })
 
-local function setST(t, map, flags, scoreTable)
-	local mode = flags & ST_SEP
-	t[mode] = t[mode] or {}
-	t[mode][map] = scoreTable
-end
-
-local function getST(t, map, flags)
-	local mode = flags & ST_SEP
-	return t[mode] and t[mode][map] or nil
-end
-
-local function setScoreTable(map, flags, scoreTable)
-	setST(lb, map, flags, scoreTable)
-end
-
-local function getScoreTable(map, flags)
-	return getST(lb, map, flags)
-end
-
--- True if a is better than b
-local function lbComp(a, b)
-	-- Calculates the difficulty, harder has higher priority
-	-- if s is positive then a is harder
-	-- if s is negative then b is harder
-	-- if s is 0 then compare time
-	local s = (a["flags"] & (F_SPBEXP | F_SPBBIG)) - (b["flags"] & (F_SPBEXP | F_SPBBIG))
-	return s > 0 or not(s < 0 or a["time"] >= b["time"])
-end
-
-local function sortScores()
-	for mode, t in pairs(lb) do
-		for map, scoreTable in pairs(t) do
-			table.sort(scoreTable, lbComp)
-			setScoreTable(map, mode, scoreTable)
-		end
-	end
-end
-
--- Reinitialize lb based on new ST_SEP value
-local function reinit_lb()
-	local nlb = {}
-
-	for mode, t in pairs(lb) do
-		for map, scoreTable in pairs(t) do
-			for i, score in ipairs(scoreTable) do
-				local st = getST(nlb, map, score["flags"]) or {}
-				table.insert(st, score)
-				setST(nlb, map, score["flags"], st)
-			end
-		end
-	end
-
-	lb = nlb
-	sortScores()
-end
-
 local cv_spb_separate = CV_RegisterVar({
 	name = "lb_spb_combined",
 	defaultvalue = 1,
 	flags = CV_NETVAR | CV_CALL | CV_NOINIT,
 	PossibleValue = CV_YesNo,
 	func = function(v)
-		local curSep = ST_SEP
-
 		if v.value then
 			ST_SEP = F_SPBATK
 		else
 			ST_SEP = F_SPBATK | F_SPBBIG | F_SPBEXP
 		end
-
-		if curSep != ST_SEP then
-			reinit_lb()
-		end
 	end
 })
 
-local function score_t(map, name, skin, color, time, splits, flags, stat)
-	return {
-		["map"]    = map,
-		["name"]   = name,
-		["skin"]   = skin,
-		["color"]  = color,
-		["time"]   = time,
-		["splits"] = splits,
-		["flags"]  = flags,
-		["stat"] = stat
-	}
-end
-
 local MSK_SPEED = 0xF0
 local MSK_WEIGHT = 0xF
 
-local function stat_t(speed, weight)
-	if speed and weight then
-		return (speed << 4) | weight
-	end
-	return 0
-end
-
-local function stat_str(stat)
-	if stat then
-		return string.format("%d%d", (stat & MSK_SPEED) >> 4, stat & MSK_WEIGHT)
-	end
-
-	return "0"
-end
-
--- Read the leaderboard
-if isserver then
-	local f = io.open(FILENAME, "r")
-	if f then
-		for l in f:lines() do
-			-- Leaderboard is stored in the following tab separated format
-			-- mapnum, name, skin, color, time, splits, flags, stat
-			local t = {}
-			for word in (l+"\t"):gmatch("(.-)\t") do
-				table.insert(t, word)
-			end
-
-			local flags = 0
-			if t[7] != nil then
-				flags = tonumber(t[7])
-			end
-
-			scoreTable = getScoreTable(tonumber(t[1]), flags) or {}
-
-			local spl = {}
-			if t[6] != nil then
-				for str in t[6]:gmatch("([^ ]+)") do
-					table.insert(spl, tonumber(str))
-				end
-			end
-
-			local stats = nil
-			if t[8] != nil then
-				if #t[8] >= 2 then
-					local speed = tonumber(string.sub(t[8], 1, 1))
-					local weight = tonumber(string.sub(t[8], 2, 2))
-					stats = stat_t(speed, weight)
-				end
-			end
-
-			table.insert(
-				scoreTable,
-				score_t(
-					tonumber(t[1]),
-					t[2],
-					t[3],
-					t[4],
-					tonumber(t[5]),
-					spl,
-					flags,
-					stats
-				)
-			)
-
-			setScoreTable(tonumber(t[1]), flags, scoreTable)
-		end
-
-		sortScores()
-		f:close()
-	else
-		print("Failed to open file: ", FILENAME)
-	end
-end
-
 function allowJoin(v)
 	if not cv_interrupt.value then
 		local y
@@ -423,7 +281,7 @@ local function initBrowser(player)
 		return
 	end
 
-	InitBrowser(lb)
+	InitBrowser(ST_SEP)
 	drawState = DS_BROWSER
 
 	player.afkTime = leveltime
@@ -475,35 +333,6 @@ local function findMap(player, ...)
 end
 COM_AddCommand("findmap", findMap)
 
-local function mapnumFromExtended(map)
-	local p, q = map:upper():match("MAP(%w)(%w)$", 1)
-	if not (p and q) then
-		return nil
-	end
-
-	local mapnum = 0
-	local A = string.byte("A")
-
-	if tonumber(p) != nil then
-		-- Non extended map numbers
-		if tonumber(q) == nil then
-			return nil
-		end
-		mapnum = tonumber(p) * 10 + tonumber(q)
-	else
-		--Extended map numbers
-		p = string.byte(p) - A
-		local qn = tonumber(q)
-		if qn == nil then
-			qn = string.byte(q) - A + 10
-		end
-
-		mapnum = 36 * p + qn + 100
-	end
-
-	return mapnum
-end
-
 local SPBModeSym = {
 	[F_SPBEXP] = "X",
 	[F_SPBBIG] = "B",
@@ -527,6 +356,7 @@ end
 local function records(player, ...)
 	local mapid = ...
 	local mapnum = gamemap
+	local mapRecords = MapRecords
 
 	if mapid then
 		mapnum = mapnumFromExtended(mapid)
@@ -534,6 +364,8 @@ local function records(player, ...)
 			CONS_Printf(player, string.format("Invalid map name: %s", mapid))
 			return
 		end
+
+		mapRecords = GetMapRecords(mapnum, mapChecksum(mapnum), ST_SEP)
 	end
 
 	local map = mapheaderinfo[mapnum]
@@ -561,35 +393,33 @@ local function records(player, ...)
 		CONS_Printf(player, "\x85UNKNOWN MAP")
 	end
 
-	for mode, maps in pairs(lb) do
+	for mode, records in pairs(mapRecords) do
-		local maptbl = maps[mapnum]
-		if not maptbl then continue end
 		CONS_Printf(player, "")
 		CONS_Printf(player, modeToString(mode))
 
 		-- don't print flags for time attack
 		if mode then
-			for i, tbl in ipairs(maptbl) do
+			for i, score in ipairs(records) do
 				CONS_Printf(
 					player,
 					string.format(
 						"%2d %-21s \x89%8s \x80%s",
 						i,
-						tbl["name"],
+						score["name"],
-						ticsToTime(tbl["time"]),
+						ticsToTime(score["time"]),
-						modeToString(tbl["flags"])
+						modeToString(score["flags"])
 					)
 				)
 			end
 		else
-			for i, tbl in ipairs(maptbl) do
+			for i, score in ipairs(records) do
 				CONS_Printf(
 					player,
 					string.format(
 						"%2d %-21s \x89%8s",
 						i,
-						tbl["name"],
+						score["name"],
-						ticsToTime(tbl["time"])
+						ticsToTime(score["time"])
 					)
 				)
 			end
@@ -693,21 +523,26 @@ local function findRival(player, ...)
 	local totalScores = 0
 	local totalDiff = 0
 
-
 	CONS_Printf(player, string.format("\x89%s's times:", rival))
-	CONS_Printf(player, "MAP  	 Time    	  Diff    	Mode")
+	CONS_Printf(player, "MAP   CHCK	 Time    	  Diff    	Mode")
 
-	for mode, tbl in pairs(lb) do
+	local maplist = MapList()
-		scores[mode] = {}
+	local mapRecords
+	local rivalScore
+	local yourScore
+	for i = 1, #maplist do
+		mapRecords = GetMapRecords(maplist[i].id, maplist[i].checksum, ST_SEP)
 
-		for map, scoreTable in pairs(tbl) do
+		for mode, records in pairs(mapRecords) do
-			local rivalScore = nil
+			scores[mode] = $ or {}
-			local yourScore = nil
 
-			for _, score in pairs(scoreTable) do
+			rivalScore = nil
-				if score["name"] == player.name then
+			yourScore = nil
+
+			for _, score in ipairs(records) do
+				if score.name == player.name then
 					yourScore = score
-				elseif score["name"] == rival then
+				elseif score.name == rival then
 					rivalScore = score
 				end
 
@@ -717,7 +552,7 @@ local function findRival(player, ...)
 			end
 
 			if rivalScore and yourScore then
-				totalDiff = totalDiff + yourScore["time"] - rivalScore["time"]
+				totalDiff = totalDiff + yourScore.time - rivalScore.time
 			end
 
 			if rivalScore then
@@ -725,8 +560,8 @@ local function findRival(player, ...)
 				table.insert(
 					scores[mode],
 					{
-						["rival"] = rivalScore,
+						rival = rivalScore,
-						["your"] = yourScore
+						your = yourScore
 					}
 				)
 			end
@@ -763,9 +598,10 @@ local function findRival(player, ...)
 				CONS_Printf(
 					player,
 					string.format(
-						"%s	%8s	%s%9s	\x80%s",
+						"%s %4s	%8s	%s%9s	\x80%s",
-						G_BuildMapName(score["rival"]["map"]),
+						G_BuildMapName(score.rival.map),
-						ticsToTime(score["rival"]["time"]),
+						score.rival.checksum,
+						ticsToTime(score.rival.time),
 						color,
 						sym[diff<0] + ticsToTime(abs(diff)),
 						modestr
@@ -775,9 +611,10 @@ local function findRival(player, ...)
 				CONS_Printf(
 					player,
 					string.format(
-						"%s	%8s	%9s	%s",
+						"%s %4s	%8s	%9s	%s",
-						G_BuildMapName(score["rival"]["map"]),
+						G_BuildMapName(score.rival.map),
-						ticsToTime(score["rival"]["time"]),
+						score.rival.checksum,
+						ticsToTime(score.rival.time),
 						ticsToTime(0, true),
 						modestr
 					)
@@ -807,6 +644,64 @@ local function findRival(player, ...)
 end
 COM_AddCommand("rival", findRival)
 
+local function moveRecords(player, from_map, from_checksum, to_map, to_checksum)
+	if not(from_map and from_checksum and to_map) then
+		CONS_Printf(player, "Usage: lb_move_records <from_map> <from_checksum> <to_map> [<to_checksum>]")
+		CONS_Printf(
+			player,
+			string.format(
+				"Summary: Move records from one map to another.\n"..
+				"If no <to_checksum> is supplied then the checksum of the current loaded map %s is used.\n"..
+				"Hint: Use lb_known_maps to find checksums",
+				to_map or "<to_map>"
+			)
+		)
+		return
+	end
+
+	local from = {
+		["id"] = mapnumFromExtended(from_map),
+		["checksum"] = from_checksum:lower()
+	}
+
+	local to = {
+		["id"] = mapnumFromExtended(to_map),
+	}
+	to.checksum = to_checksum or mapChecksum(to.id)
+
+	if not to.checksum then
+		CONS_Printf(player, string.format("error: %s is not loaded; provide to_checksum to continue", to_map:upper()))
+		return
+	end
+	if #to.checksum != 4 or to.checksum:match("[^a-f0-9]") then
+		CONS_Printf(player, string.format("error: %s is an invalid checksum; checksums are of length 4 and can contain only 0-9a-f", to.checksum))
+		return
+	end
+
+	to.checksum = $:lower()
+
+	local mapRecords = GetMapRecords(from.id, from.checksum, F_SPBATK | F_SPBBIG | F_SPBEXP)
+	local recordCount = 0
+	for mode, records in pairs(mapRecords) do
+		recordCount = $ + #records
+	end
+
+	MoveRecords(from, to, ST_SEP)
+
+	CONS_Printf(
+		player,
+		string.format(
+			"%d records have been moved from\x82 %s %s\x80 to\x88 %s %s",
+			recordCount,
+			from_map, from.checksum,
+			to_map, to.checksum
+		)
+	)
+
+	CONS_Printf(player, "Please repack coldstore and restart the server for changes to take effect.")
+end
+COM_AddCommand("lb_move_records", moveRecords, COM_ADMIN)
+
 --DEBUGGING
 --local function printTable(tb)
 --	for mode, tbl in pairs(tb) do
@@ -830,7 +725,7 @@ COM_AddCommand("rival", findRival)
 --end
 
 addHook("MapLoad", function()
-		timeFinished = 0
+		TimeFinished = 0
 		splits = {}
 		prevLap = 0
 		drawState = DS_DEFAULT
@@ -840,6 +735,8 @@ addHook("MapLoad", function()
 
 		allowJoin(true)
 		--printTable(lb)
+
+		MapRecords = GetMapRecords(gamemap, mapChecksum(gamemap), ST_SEP)
 	end
 )
 
@@ -1150,12 +1047,14 @@ local function drawScoreboard(v, player)
 	cachePatches(v)
 
 	local gui = cv_gui.value
+
+	-- Force enable gui at start and end of the race
 	if leveltime < START_TIME or player.exiting or player.lives == 0 then
 		gui = GUI_ON
 	end
 
 	if gui then
-		stateFunctions[drawState](v, player, scoreTable, gui)
+		stateFunctions[drawState](v, player, ScoreTable, gui)
 	end
 
 	local pos = 0
@@ -1195,7 +1094,7 @@ end
 
 -- Find location of player and scroll to it
 function scroll_to(player)
-	local m = scoreTable or {}
+	local m = ScoreTable or {}
 
 	scrollToPos = 2
 	for pos, score in ipairs(m) do
@@ -1209,19 +1108,19 @@ function scroll_to(player)
 end
 
 -- Write skin stats to each score where there are none
-local function writeStats()
+--local function writeStats()
-	for _, t in pairs(lb) do
+--	for _, t in pairs(lb) do
-		for _, scoreTable in pairs(t) do
+--		for _, scoreTable in pairs(t) do
-			for _, score in ipairs(scoreTable) do
+--			for _, score in ipairs(scoreTable) do
-				local skin = skins[score["skin"]]
+--				local skin = skins[score["skin"]]
-				if skin and not score["stat"] then
+--				if skin and not score["stat"] then
-					local stats = stat_t(skin.kartspeed, skin.kartweight)
+--					local stats = stat_t(skin.kartspeed, skin.kartweight)
-					score["stat"] = stats
+--					score["stat"] = stats
-				end
+--				end
-			end
+--			end
-		end
+--		end
-	end
+--	end
-end
+--end
 
 local function checkFlags(p)
 	local flags = 0
@@ -1261,7 +1160,7 @@ local function saveTime(player)
 		return
 	end
 
-	scoreTable = $ or {}
+	ScoreTable = $ or {}
 
 	local pskin = skins[player.mo.skin]
 	local newscore = score_t(
@@ -1269,23 +1168,17 @@ local function saveTime(player)
 		player.name,
 		player.mo.skin,
 		player.skincolor,
-		timeFinished,
+		TimeFinished,
 		splits,
 		Flags,
-		stat_t(player.HMRs or pskin.kartspeed, player.HMRw or pskin.kartweight)
+		stat_t(player.HMRs or pskin.kartspeed, player.HMRw or pskin.kartweight),
+		mapChecksum(gamemap)
 	)
 
 	-- Check if you beat your previous best
-	for i = 1, #scoreTable do
+	for i = 1, #ScoreTable do
-		if scoreTable[i]["name"] == player.name then
+		if ScoreTable[i].name == player.name then
-			if lbComp(newscore, scoreTable[i]) then
+			if not lbComp(newscore, ScoreTable[i]) then
-				table.remove(scoreTable, i)
-				S_StartSound(nil, 130)
-				FlashTics = leveltime + TICRATE * 3
-				FlashRate = 1
-				FlashVFlags = YellowFlash
-				break
-			else
 				-- You suck lol
 				S_StartSound(nil, 201)
 				FlashTics = leveltime + TICRATE * 3
@@ -1297,64 +1190,36 @@ local function saveTime(player)
 		end
 	end
 
-	print("Saving score")
-	table.insert(
-		scoreTable,
-		newscore
-	)
 
-	table.sort(scoreTable, lbComp)
+	-- Save the record
-	while #scoreTable > cv_saves.value do
+	SaveRecord(newscore, gamemap, ST_SEP)
-		table.remove(scoreTable)
-	end
 
+	-- Set players text flash and play chime sfx
+	S_StartSound(nil, 130)
+	FlashTics = leveltime + TICRATE * 3
+	FlashRate = 1
+	FlashVFlags = YellowFlash
+
+	-- Reload the MapRecords
+	MapRecords = GetMapRecords(gamemap, mapChecksum(gamemap), ST_SEP)
+
+	-- Set the updated ScoreTable
+	ScoreTable = MapRecords[Flags]
+
+	-- Scroll the gui to the player entry
 	scroll_to(player)
-
-	setScoreTable(gamemap, Flags, scoreTable)
-
-	if not StatTrack then
-		writeStats()
-		StatTrack = true
-	end
-
-	if isserver then
-		local f = assert(io.open(FILENAME, "w"))
-		if f == nil then
-			print("Failed to open file for writing: " + FILENAME)
-			return
-		end
-
-		for _, tbl in pairs(lb) do
-			for _, scoreTable in pairs(tbl) do
-				for _, score in ipairs(scoreTable) do
-					f:write(
-						score["map"], "\t",
-						score["name"], "\t",
-						score["skin"], "\t",
-						score["color"], "\t",
-						score["time"], "\t",
-						table.concat(score["splits"], " "), "\t",
-						score["flags"], "\t",
-						stat_str(score["stat"]), "\n"
-					)
-				end
-			end
-		end
-
-		f:close()
-	end
 end
 
 -- DEBUGGING
 --local function saveLeaderboard(player, ...)
---	timeFinished = tonumber(... or player.realtime)
+--	TimeFinished = tonumber(... or player.realtime)
 --	splits = {1000, 2000, 3000}
 --	saveTime(player)
 --end
 --COM_AddCommand("save", saveLeaderboard)
 
 local function regLap(player)
-	if player.laps > prevLap and timeFinished == 0 then
+	if player.laps > prevLap and TimeFinished == 0 then
 		prevLap = player.laps
 		table.insert(splits, player.realtime)
 		showSplit = 5 * TICRATE
@@ -1461,7 +1326,7 @@ local function think()
 		end
 	end
 
-	scoreTable = getScoreTable(gamemap, Flags)
+	ScoreTable = MapRecords[ST_SEP & Flags]
 
 	if not cv_teamchange then
 		cv_teamchange = CV_FindVar("allowteamchange")
@@ -1469,8 +1334,8 @@ local function think()
 
 	if p then
 		-- must be done before browser control
-		if p.laps >= mapheaderinfo[gamemap].numlaps and timeFinished == 0 then
+		if p.laps >= mapheaderinfo[gamemap].numlaps and TimeFinished == 0 then
-			timeFinished = p.realtime
+			TimeFinished = p.realtime
 			saveTime(p)
 		end
 
@@ -1560,8 +1425,8 @@ local function netvars(net)
 	splits = net($)
 	prevLap = net($)
 	drawState = net($)
-	StatTrack = net($)
 	EncoreInitial = net($)
-	lb = net($)
+	MapRecords = net($)
+	TimeFinished = net($)
 end
 addHook("NetVars", netvars)

tools/coldstore.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+import sys
+from os import linesep, path
+
+if len(sys.argv) != 3 or not sys.argv[1] or not sys.argv[2]:
+	print("Usage: coldstore.py <game_directory> <leaderboard_records.lua>")
+	print("\t<game_directory>\t\tthe game directory where wads and luafiles reside. Usually at '$HOME/.srb2kart'.")
+	print("\t<leaderboard_records.lua>\tthe output name for the records packed lua file. It will be saved within <game_directory>.")
+	quit()
+
+if not sys.argv[2].endswith(".lua"):
+	print("{} must end with .lua".format(sys.argv[2]))
+	quit()
+
+game_dir = sys.argv[1]
+leaderboard_txt = path.join(game_dir, "luafiles", "leaderboard.txt")
+coldstore_txt = path.join(game_dir, "luafiles", "leaderboard.coldstore.txt")
+records_lua = path.join(game_dir, sys.argv[2])
+
+def ParseScore(score):
+	# Map Name Skin Color Time Splits Flags Stat
+	split = score.split("\t")
+
+	checksum = ""
+	if len(split) > 8:
+		checksum = split[8]
+
+	return {
+		"map": split[0],
+		"name": split[1],
+		"skin": split[2],
+		"color": split[3],
+		"time": int(split[4]),
+		"splits": split[5],
+		"flags": int(split[6]),
+		"stat": split[7],
+		"checksum": checksum
+	}
+
+# Compare scores
+def CompareScore(a, b):
+	return a["time"] < b["time"]
+
+F_SEP = 0xF
+def SameScore(a, b):
+	return a["name"] == b["name"] and a["checksum"] == b["checksum"] and (a["flags"] & F_SEP) == (b["flags"] & F_SEP)
+
+def LoadRecordsFromFile(path):
+	records = []
+	with open(path, "r") as f:
+		for line in f.readlines():
+			records.append(ParseScore(line.strip()))
+	return records
+
+def AddScore(records, score):
+	mapid = score["map"]
+	mapTable = records.get(mapid) or []
+
+	for i in range(len(mapTable)):
+		scoreb = mapTable[i]
+		if SameScore(score, scoreb):
+			if CompareScore(score, scoreb):
+				mapTable[i] = score
+				records[mapid] = mapTable
+			return
+	mapTable.append(score)
+	records[mapid] = mapTable
+
+# load leaderboard.txt and coldstore.txt
+recordsList = LoadRecordsFromFile(leaderboard_txt)
+recordsList.extend(LoadRecordsFromFile(coldstore_txt))
+
+
+# construct the map tables
+records = {}
+for score in recordsList:
+	AddScore(records, score)
+
+# convert records to flat list
+recordsList = []
+rejected = []
+for mapTable in records.values():
+	for score in mapTable:
+		scoreStr = "\t".join([str(v) for v in list(score.values())])
+		# only allow records with checksums
+		if score["checksum"] != "":
+			recordsList.append(scoreStr)
+		else:
+			rejected.append(scoreStr)
+
+# truncate and write records to coldstore
+with open(coldstore_txt, "w") as f:
+	for score in recordsList:
+			f.write(score + linesep)
+
+luaA = """do
+	local AddColdStore = lb_add_coldstore_record_string
+	local records = {
+"""
+luaB = """	}
+	for _, str in ipairs(records) do
+		AddColdStore(str)
+	end
+end
+"""
+# pack the records.lua file
+with open(records_lua, "w") as f:
+	f.write(luaA)
+	for score in recordsList:
+		score = score.replace("\\", "\\\\")
+		score = score.replace("\"", "\\\"")
+		f.write("\t\t\"{}\",{}".format(score, linesep))
+	f.write(luaB)
+
+# truncate and rewrite rejected scores to leaderboard.txt
+with open(leaderboard_txt, "w") as f:
+	for score in rejected:
+		f.write(score + linesep)