Apply a large number of optimizations

code/__defines/_compile_options.dm

@@ -1,3 +1,16 @@
 // The default value for all uses of set background. Set background can cause gradual lag and is recommended you only turn this on if necessary.
 // 1 will enable set background. 0 will disable set background.
 #define BACKGROUND_ENABLED 0
+
+// If REFTRACK_IN_CI is defined, the reftracker will run in CI.
+#define REFTRACK_IN_CI
+#if defined(REFTRACK_IN_CI) && defined(UNIT_TEST) && !defined(SPACEMAN_DMM)
+#define REFTRACKING_ENABLED
+#define GC_FAILURE_HARD_LOOKUP
+#define FIND_REF_NO_CHECK_TICK
+#endif
+
+// parity with previous behavior where TESTING enabled reftracking
+#ifdef TESTING
+#define REFTRACKING_ENABLED
+#endif

code/__defines/qdel.dm

@@ -5,8 +5,8 @@
 #define QDEL_HINT_IWILLGC		2 //functionally the same as the above. qdel should assume the object will gc on its own, and not check it.
 #define QDEL_HINT_HARDDEL		3 //qdel should assume this object won't gc, and queue a hard delete using a hard reference.
 #define QDEL_HINT_HARDDEL_NOW	4 //qdel should assume this object won't gc, and hard del it post haste.
-#define QDEL_HINT_FINDREFERENCE	5 //functionally identical to QDEL_HINT_QUEUE if TESTING is not enabled in _compiler_options.dm.
-								  //if TESTING is enabled, qdel will call this object's find_references() verb.
+#define QDEL_HINT_FINDREFERENCE	5 //functionally identical to QDEL_HINT_QUEUE if REFTRACKING_ENABLED is not enabled in _compiler_options.dm.
+								  //if REFTRACKING_ENABLED is enabled, qdel will call this object's find_references() verb.
 #define QDEL_HINT_IFFAIL_FINDREFERENCE 6		//Above but only if gc fails.
 //defines for the gc_destroyed var
 
@@ -15,6 +15,12 @@
 #define GC_QUEUE_HARDDELETE 3
 #define GC_QUEUE_COUNT 3 //increase this when adding more steps.
 
+// Defines for the ssgarbage queue items
+#define GC_QUEUE_ITEM_QUEUE_TIME 1 //! Time this item entered the queue
+#define GC_QUEUE_ITEM_REF 2 //! Ref to the item
+#define GC_QUEUE_ITEM_GCD_DESTROYED 3 //! Item's gc_destroyed var value. Used to detect ref reuse.
+#define GC_QUEUE_ITEM_INDEX_COUNT 3 //! Number of item indexes, used for allocating the nested lists. Don't forget to increase this if you add a new queue item index
+
 #define GC_QUEUED_FOR_HARD_DEL -1
 #define GC_CURRENTLY_BEING_QDELETED -2
 

code/controllers/subsystems/garbage.dm

@@ -37,7 +37,7 @@ SUBSYSTEM_DEF(garbage)
 	//Queue
 	var/list/queues
 
-	#ifdef TESTING
+	#ifdef REFTRACKING_ENABLED
 	var/list/reference_find_on_fail = list()
 	#endif
 
@@ -130,45 +130,53 @@ SUBSYSTEM_DEF(garbage)
 
 	lastlevel = level
 
+// 1 from the hard reference in the queue, and 1 from `D` in the code below
+#define REFS_WE_EXPECT 2
+
 	//We do this rather then for(var/refID in queue) because that sort of for loop copies the whole list.
 	//Normally this isn't expensive, but the gc queue can grow to 40k items, and that gets costly/causes overrun.
-	for (var/refidx in 1 to length(queue))
-		var/refID = queue[refidx]
-		if (isnull(refID))
+	for (var/i in 1 to length(queue))
+		var/list/L = queue[i]
+		if (length(L) < GC_QUEUE_ITEM_INDEX_COUNT)
 			count++
 			if (MC_TICK_CHECK)
 				return
 			continue
 
-		var/GCd_at_time = queue[refID]
-		if(GCd_at_time > cut_off_time)
+		var/queued_at_time = L[GC_QUEUE_ITEM_QUEUE_TIME]
+		if(queued_at_time > cut_off_time)
 			break // Everything else is newer, skip them
 		count++
 
-		var/datum/D
-		D = locate(refID)
+		var/datum/D = L[GC_QUEUE_ITEM_REF]
 
-		if (isnull(D) || D.gc_destroyed != GCd_at_time) // So if something else coincidently gets the same ref, it's not deleted by mistake
+		// If that's all we've got, send er off
+		if (refcount(D) == REFS_WE_EXPECT)
 			++gcedlasttick
 			++totalgcs
 			pass_counts[level]++
-			#ifdef TESTING
-			reference_find_on_fail -= refID		//It's deleted we don't care anymore.
+			#ifdef REFTRACKING_ENABLED
+			reference_find_on_fail -= ref(D) //It's deleted we don't care anymore.
 			#endif
 			if (MC_TICK_CHECK)
 				return
 			continue
 
 		// Something's still referring to the qdel'd object.
 		fail_counts[level]++
+
 		switch (level)
 			if (GC_QUEUE_CHECK)
-				#ifdef TESTING
+				#ifdef REFTRACKING_ENABLED
+				// Decides how many refs to look for (potentially)
+				// Based off the remaining and the ones we can account for
+				var/remaining_refs = refcount(D) - REFS_WE_EXPECT
+				var/refID = ref(D)
 				if(reference_find_on_fail[refID])
-					D.find_references()
+					D.find_references(remaining_refs)
 				#ifdef GC_FAILURE_HARD_LOOKUP
 				else
-					D.find_references()
+					D.find_references(remaining_refs)
 				#endif
 				reference_find_on_fail -= refID
 				#endif
@@ -201,12 +209,14 @@ SUBSYSTEM_DEF(garbage)
 	if (level > GC_QUEUE_COUNT)
 		HardDelete(D)
 		return
-	var/gctime = world.time
-	var/refid = "\ref[D]"
+	var/queue_time = world.time
 
-	D.gc_destroyed = gctime
+	if(D.gc_destroyed <= 0) // hasn't been queued yet, or is queued for harddel/actively being qdeleted
+		D.gc_destroyed = queue_time
 	var/list/queue = queues[level]
-	queue[refid] = gctime
+	// not += for byond reasons
+	// we include D.gc_destroyed to skip things under the cutoff
+	queue[++queue.len] = list(queue_time, D, D.gc_destroyed)
 
 //this is mainly to separate things profile wise.
 /datum/controller/subsystem/garbage/proc/HardDelete(datum/D)
@@ -240,11 +250,6 @@ SUBSYSTEM_DEF(garbage)
 		message_admins("Error: [type]([refID]) took longer than 1 second to delete (took [time/10] seconds to delete).")
 		postpone(time)
 
-/datum/controller/subsystem/garbage/proc/HardQueue(datum/D)
-	if (D.gc_destroyed == GC_CURRENTLY_BEING_QDELETED)
-		queues[GC_QUEUE_FILTER] += D
-		D.gc_destroyed = GC_QUEUED_FOR_HARD_DEL
-
 /datum/controller/subsystem/garbage/Recover()
 	if (istype(SSgarbage.queues))
 		for (var/i in 1 to SSgarbage.queues.len)
@@ -270,7 +275,7 @@ SUBSYSTEM_DEF(garbage)
 /datum/qdel_item/New(mytype)
 	name = "[mytype]"
 
-#ifdef TESTING
+#ifdef REFTRACKING_ENABLED
 /proc/qdel_and_find_ref_if_fail(datum/D, force = FALSE)
 	SSgarbage.reference_find_on_fail["\ref[D]"] = TRUE
 	qdel(D, force)
@@ -328,7 +333,7 @@ SUBSYSTEM_DEF(garbage)
 					return
 				// Returning LETMELIVE after being told to force destroy
 				// indicates the objects Destroy() does not respect force
-				#ifdef TESTING
+				#ifdef REFTRACKING_ENABLED
 				if(!I.no_respect_force)
 					PRINT_STACK_TRACE("WARNING: [D.type] has been force deleted, but is \
 						returning an immortal QDEL_HINT, indicating it does \
@@ -341,21 +346,22 @@ SUBSYSTEM_DEF(garbage)
 				SSgarbage.Queue(D)
 			if (QDEL_HINT_HARDDEL)		//qdel should assume this object won't gc, and queue a hard delete using a hard reference to save time from the locate()
 				GC_CHECK_AM_NULLSPACE(D, "QDEL_HINT_HARDDEL")
-				SSgarbage.HardQueue(D)
+				SSgarbage.Queue(D, GC_QUEUE_HARDDELETE)
 			if (QDEL_HINT_HARDDEL_NOW)	//qdel should assume this object won't gc, and hard del it post haste.
 				SSgarbage.HardDelete(D)
-			if (QDEL_HINT_FINDREFERENCE)//qdel will, if TESTING is enabled, display all references to this object, then queue the object for deletion.
+			if (QDEL_HINT_FINDREFERENCE)//qdel will, if REFTRACKING_ENABLED is enabled, display all references to this object, then queue the object for deletion.
 				SSgarbage.Queue(D)
-				#ifdef TESTING
-				D.find_references()
+				#ifdef REFTRACKING_ENABLED
+				var/remaining_refs = refcount(D) - REFS_WE_EXPECT
+				D.find_references(remaining_refs)
 				#endif
 			if (QDEL_HINT_IFFAIL_FINDREFERENCE)
 				SSgarbage.Queue(D)
-				#ifdef TESTING
+				#ifdef REFTRACKING_ENABLED
 				SSgarbage.reference_find_on_fail["\ref[D]"] = TRUE
 				#endif
 			else
-				#ifdef TESTING
+				#ifdef REFTRACKING_ENABLED
 				if(!I.no_hint)
 					PRINT_STACK_TRACE("WARNING: [D.type] is not returning a qdel hint. It is being placed in the queue. Further instances of this type will also be queued.")
 				#endif
@@ -364,7 +370,7 @@ SUBSYSTEM_DEF(garbage)
 	else if(D.gc_destroyed == GC_CURRENTLY_BEING_QDELETED)
 		CRASH("[D.type] destroy proc was called multiple times, likely due to a qdel loop in the Destroy logic")
 
-#ifdef TESTING
+#ifdef REFTRACKING_ENABLED
 
 /datum/verb/find_refs()
 	set category = "Debug"
@@ -378,8 +384,9 @@ SUBSYSTEM_DEF(garbage)
 		return
 	find_references()
 
-/datum/proc/find_references()
+/datum/proc/find_references(references_to_clear = INFINITY)
 	running_find_references = type
+	src.references_to_clear = references_to_clear
 	if(usr && usr.client)
 		if(usr.client.running_find_references)
 			testing("CANCELLED search for references to a [usr.client.running_find_references].")
@@ -406,20 +413,28 @@ SUBSYSTEM_DEF(garbage)
 		normal_globals[global_var] = global.vars[global_var]
 	DoSearchVar(normal_globals, "(global) -> ") //globals
 	testing("Finished searching globals")
+	if(src.references_to_clear == 0) // Found all expected references!
+		return
 
 	for(var/atom/atom_thing) //atoms
 		DoSearchVar(atom_thing, "World -> [atom_thing]")
+		if(src.references_to_clear == 0) // Found all expected references!
+			return
 	testing("Finished searching atoms")
 
 	for (var/datum/datum_thing) //datums
 		DoSearchVar(datum_thing, "World -> [datum_thing]")
+		if(src.references_to_clear == 0) // Found all expected references!
+			return
 	testing("Finished searching datums")
 
 #ifndef FIND_REF_SKIP_CLIENTS
 	// DO NOT RUN THIS ON A LIVE SERVER
 	// IT WILL CRASH!!!
 	for (var/client/client_thing) //clients
 		DoSearchVar(client_thing, "World -> [client_thing]")
+		if(src.references_to_clear == 0) // Found all expected references!
+			return
 	testing("Finished searching clients")
 #endif
 
@@ -455,37 +470,47 @@ SUBSYSTEM_DEF(garbage)
 #define GET_TYPEID(ref) ( ( (length(ref) <= 10) ? "TYPEID_NULL" : copytext(ref, 4, length(ref)-6) ) )
 #define IS_NORMAL_LIST(L) (GET_TYPEID("\ref[L]") == TYPEID_NORMAL_LIST)
 
-/datum/proc/DoSearchVar(X, Xname, recursive_limit = 128)
+/datum/proc/DoSearchVar(X, container_name, recursive_limit = 128)
 	if(usr && usr.client && !usr.client.running_find_references)
 		return
 	if (!recursive_limit)
 		return
+	if(references_to_clear == 0)
+		return
 
 	#ifndef FIND_REF_NO_CHECK_TICK
 	CHECK_TICK
 	#endif
 
 	if(istype(X, /datum))
-		var/datum/D = X
-		if(D.last_find_references == last_find_references)
+		var/datum/datum_container = X
+		if(datum_container.last_find_references == last_find_references)
 			return
 
-		D.last_find_references = last_find_references
-		var/list/L = D.vars
+		datum_container.last_find_references = last_find_references
+		var/list/vars_list = datum_container.vars
 
-		for(var/varname in L)
+		var/is_atom = FALSE
+		var/is_area = FALSE
+		if(isatom(datum_container))
+			is_atom = TRUE
+			if(isarea(datum_container))
+				is_area = TRUE
+		for(var/varname in vars_list)
 			#ifndef FIND_REF_NO_CHECK_TICK
 			CHECK_TICK
 			#endif
-			if (varname == "vars")
+			//Fun fact, vis_locs don't count for references
+			if(varname == "vars" || (is_atom && (varname == "vis_locs" || varname == "overlays" || varname == "underlays" || varname == "filters" || varname == "verbs" || (is_area && varname == "contents"))))
 				continue
-			var/variable = L[varname]
+			var/variable = vars_list[varname]
 
 			if(variable == src)
-				testing("Found [src.type] \ref[src] in [D.type]'s [varname] var. [Xname]")
+				testing("Found [src.type] \ref[src] in [datum_container.type]'s [varname] var. [container_name]")
+				references_to_clear -= 1
 
 			else if(islist(variable))
-				DoSearchVar(variable, "[Xname] -> [varname] (list)", recursive_limit-1)
+				DoSearchVar(variable, "[container_name] -> [varname] (list)", recursive_limit-1)
 
 	else if(islist(X))
 		var/normal = IS_NORMAL_LIST(X)
@@ -494,16 +519,33 @@ SUBSYSTEM_DEF(garbage)
 			CHECK_TICK
 			#endif
 			if (I == src)
-				testing("Found [src.type] \ref[src] in list [Xname].")
+				testing("Found [src.type] \ref[src] in list [container_name].")
+
+				// This is dumb as hell I'm sorry
+				// I don't want the garbage subsystem to count as a ref for the purposes of this number
+				// If we find all other refs before it I want to early exit, and if we don't I want to keep searching past it
+				var/ignore_ref = FALSE
+				var/list/queues = SSgarbage.queues
+				for(var/list/queue in queues)
+					if(X in queue)
+						ignore_ref = TRUE
+						break
+				if(ignore_ref)
+					testing("[container_name] does not count as a ref for our count")
+				else
+					references_to_clear -= 1
+				if(references_to_clear == 0)
+					testing("All references to [type] \ref[src] found, exiting.")
+					return
 
 			else if (I && !isnum(I) && normal)
 				if(X[I] == src)
-					testing("Found [src.type] \ref[src] in list [Xname]\[[I]\]")
+					testing("Found [src.type] \ref[src] in list [container_name]\[[I]\]")
 				else if(islist(X[I]))
-					DoSearchVar(X[I], "[Xname]\[[I]\]", recursive_limit-1)
+					DoSearchVar(X[I], "[container_name]\[[I]\]", recursive_limit-1)
 
 			else if (islist(I))
 				var/list/Xlist = X
-				DoSearchVar(I, "[Xname]\[[Xlist.Find(I)]\] -> list", recursive_limit-1)
+				DoSearchVar(I, "[container_name]\[[Xlist.Find(I)]\] -> list", recursive_limit-1)
 
 #endif