fix VIS cleanup function bugs

some leaves were not being remapped properly in the resulting VIS data

src/bsp/Bsp.cpp

@@ -1258,72 +1258,76 @@ int Bsp::remove_unused_lightmaps(bool* usedFaces) {
 	return oldLightdataSize - newLightDataSize;
 }
 
-int Bsp::remove_unused_visdata(STRUCTREMAP* remap, BSPLEAF* oldLeaves, int oldLeafCount, int oldWorldspawnLeafCount) {
+int Bsp::remove_unused_visdata(STRUCTREMAP* remap, BSPLEAF* oldLeaves, int oldLeafCount, int oldWorldspawnLeafCount) {	
 	int oldVisLength = visDataLength;
 
 	// exclude solid leaf
 	int oldVisLeafCount = oldLeafCount - 1;
 	int newVisLeafCount = (header.lump[LUMP_LEAVES].nLength / sizeof(BSPLEAF)) - 1;
 
-	int oldWorldLeaves = oldWorldspawnLeafCount; // TODO: allow deleting world leaves
-	//int oldWorldLeaves = ((BSPMODEL*)lumps[LUMP_MODELS])->nVisLeafs;
-	int newWorldLeaves = ((BSPMODEL*)lumps[LUMP_MODELS])->nVisLeafs;
+	if (oldVisLeafCount == newVisLeafCount) {
+		return 0; // VIS data needs updating only when leaves are added/removed
+	}
 
 	uint oldVisRowSize = ((oldVisLeafCount + 63) & ~63) >> 3;
 	uint newVisRowSize = ((newVisLeafCount + 63) & ~63) >> 3;
 
-	int decompressedVisSize = oldLeafCount * oldVisRowSize;
-	byte* decompressedVis = new byte[decompressedVisSize];
-	memset(decompressedVis, 0, decompressedVisSize);
-	decompress_vis_lump(oldLeaves, lumps[LUMP_VISIBILITY], decompressedVis, 
-		oldWorldLeaves, oldVisLeafCount, oldVisLeafCount);
+	int oldDecompressedVisSize = oldLeafCount * oldVisRowSize;
+	byte* oldDecompressedVis = new byte[oldDecompressedVisSize];
+	memset(oldDecompressedVis, 0, oldDecompressedVisSize);
+	decompress_vis_lump(oldLeaves, lumps[LUMP_VISIBILITY], oldDecompressedVis, oldVisLeafCount);
 
-	if (oldVisRowSize != newVisRowSize) {
-		int newDecompressedVisSize = newWorldLeaves * newVisRowSize;
-		byte* newDecompressedVis = new byte[decompressedVisSize];
-		memset(newDecompressedVis, 0, newDecompressedVisSize);
+	int newDecompressedVisSize = newVisLeafCount * newVisRowSize;
+	byte* newDecompressedVis = new byte[oldDecompressedVisSize];
+	memset(newDecompressedVis, 0, newDecompressedVisSize);
 
-		if (newVisRowSize > oldVisRowSize) {
-			logf("ERROR: New vis row size larger than old size. VIS will likely be broken\n");
-		}
+	if (newVisRowSize > oldVisRowSize) {
+		logf("ERROR: New vis row size larger than old size. VIS will likely be broken\n");
+	}
 
-		for (int i = 0; i < newWorldLeaves; i++) {
-			// find the source leaf to load vis data from
-			int sourceLeaf = 0;
-			for (int k = 0; k < oldWorldLeaves; k++) {
-				if (remap->leaves[k] == i) {
-					sourceLeaf = k;
-					//logf("Load source leaf %d row into new leaf %d row\n", k, i);
-					break;
-				}
-			}
+	int* oldLeafs = new int[newVisLeafCount];
 
-			byte* oldVisRow = decompressedVis + sourceLeaf * oldVisRowSize;
+	for (int i = 1; i < newVisLeafCount; i++) {
+		int oldLeafIdx = 0;
 
-			// remove deleted leaves from the old visibility list
-			for (int k = oldWorldLeaves-1; k > 0; k--) {
-				if (remap->leaves[k] == 0) {
-					shiftVis(oldVisRow, oldVisRowSize, k, -1);
-				}
+		for (int k = 1; k < oldVisLeafCount; k++) {
+			if (remap->leaves[k] == i) {
+				oldLeafs[i] = k;
+				break;
 			}
-
-			memcpy(newDecompressedVis + i * newVisRowSize, oldVisRow, newVisRowSize);
 		}
+	}
 
-		delete[] decompressedVis;
-		decompressedVis = newDecompressedVis;
+	for (int i = 1; i < newVisLeafCount; i++) {
+		byte* oldVisRow = oldDecompressedVis + (oldLeafs[i] - 1) * oldVisRowSize;
+		byte* newVisRow = newDecompressedVis + (i - 1) * newVisRowSize;
+
+		for (int k = 1; k < newVisLeafCount; k++) {
+			int oldLeafIdx = oldLeafs[k] - 1;
+			int oldByteOffset = oldLeafIdx / 8;
+			int oldBitOffset = 1 << (oldLeafIdx % 8);
+
+			if (oldVisRow[oldByteOffset] & oldBitOffset) {
+				int newLeafIdx = k-1;
+				int newByteOffset = newLeafIdx / 8;
+				int newBitOffset = 1 << (newLeafIdx % 8);
+				newVisRow[newByteOffset] |= newBitOffset;
+			}
+		}
 	}
 
-	byte* compressedVis = new byte[decompressedVisSize];
-	memset(compressedVis, 0, decompressedVisSize);
-	int newVisLen = CompressAll(leaves, decompressedVis, compressedVis, newVisLeafCount, newWorldLeaves, decompressedVisSize);
+	delete[] oldLeafs;
+	delete[] oldDecompressedVis;
+
+	byte* compressedVis = new byte[newDecompressedVisSize]; // assuming compressed will reduce size
+	memset(compressedVis, 0, newDecompressedVisSize);
+	int newVisLen = CompressAll(leaves, newDecompressedVis, compressedVis, newVisLeafCount, newDecompressedVisSize);
 
 	byte* compressedVisResized = new byte[newVisLen];
 	memcpy(compressedVisResized, compressedVis, newVisLen);
 
 	replace_lump(LUMP_VISIBILITY, compressedVisResized, newVisLen);
 
-	delete[] decompressedVis;
 	delete[] compressedVis;
 
 	return oldVisLength - newVisLen;
@@ -1333,9 +1337,6 @@ STRUCTCOUNT Bsp::remove_unused_model_structures() {
 	int oldVisLeafCount = 0;
 	count_leaves(models[0].iHeadnodes[0], oldVisLeafCount);
 	//oldVisLeafCount = models[0].nVisLeafs;
-	//if (leafCount != models[0].nVisLeafs)
-	//	logf("WARNING: old leaf count doesn't match worldpsawn leaf count %d != %d\n", 
-	//		leafCount, models[0].nVisLeafs);
 
 	// marks which structures should not be moved
 	STRUCTUSAGE usedStructures(this);
@@ -3412,20 +3413,6 @@ bool Bsp::validate() {
 
 			isValid = false;
 		}
-		if (visDataLength > 0 && leaves[i].nVisOffset < -1 || leaves[i].nVisOffset >= visDataLength) {
-			logf("Bad vis offset in leaf %d: %d / %d\n", i, leaves[i].nVisOffset, visDataLength);
-			isValid = false;
-		}
-
-		for (int k = 0; k < leaves[i].nMarkSurfaces; k++) {
-			//if (marksurfs[leaves[i].iFirstMarkSurface + k] == 3697) { // after subdivide
-			if (marksurfs[leaves[i].iFirstMarkSurface + k] == 3696) {
-				logf("Found face in leaf %d (mins %d %d %d) (maxs %d %d %d)\n", i,
-					(int)leaves[i].nMins[0], (int)leaves[i].nMins[1], (int)leaves[i].nMins[2],
-					(int)leaves[i].nMaxs[0], (int)leaves[i].nMaxs[1], (int)leaves[i].nMaxs[2]);
-			}
-		}
-
 		//logf("Leaf %d: %d %d %d\n", i, marksurfs[leaves[i].iFirstMarkSurface], leaves[i].nMarkSurfaces);
 	}
 	for (int i = 0; i < edgeCount; i++) {
@@ -3930,6 +3917,63 @@ const char* Bsp::getLeafContentsName(int32_t contents) {
 	}
 }
 
+int Bsp::get_leaf(vec3 pos) {
+	int iNode = models->iHeadnodes[0];
+
+	while (iNode >= 0)
+	{
+		BSPNODE& node = nodes[iNode];
+		BSPPLANE& plane = planes[node.iPlane];
+
+		float d = dotProduct(plane.vNormal, pos) - plane.fDist;
+		if (d < 0) {
+			iNode = node.iChildren[1];
+		}
+		else {
+			iNode = node.iChildren[0];
+		}
+	}
+
+	return ~iNode;
+}
+
+bool Bsp::is_leaf_visible(int ileaf, vec3 pos) {
+	int ipvsLeaf = get_leaf(pos);
+	BSPLEAF& pvsLeaf = leaves[ipvsLeaf];
+
+	int p = pvsLeaf.nVisOffset; // pvs offset
+	byte* pvs = lumps[LUMP_VISIBILITY];
+
+	bool isVisible = false;
+	int numVisible = 0;
+
+	//logf("leaf %d can see:", ipvsLeaf);
+
+	for (int lf = 1; lf < leafCount; p++)
+	{
+		if (pvs[p] == 0) // prepare to skip leafs
+			lf += 8 * pvs[++p]; // next byte holds number of leafs to skip
+		else
+		{
+			for (byte bit = 1; bit != 0; bit *= 2, lf++)
+			{
+				if ((pvs[p] & bit) && lf < leafCount) // leaf is flagged as visible
+				{
+					numVisible++;
+					//logf(" %d", lf);
+					if (lf == ileaf) {
+						isVisible = true;
+					}
+				}
+			}
+		}
+	}
+
+	//logf("\n");
+
+	return isVisible;
+}
+
 void Bsp::mark_face_structures(int iFace, STRUCTUSAGE* usage) {
 	BSPFACE& face = faces[iFace];
 	usage->faces[iFace] = true;

src/bsp/Bsp.h

@@ -47,6 +47,14 @@ class Bsp
 	int32_t* surfedges;
 	BSPEDGE* edges;
 	uint16* marksurfs;
+
+	// VIS data is a compressed 2D array.
+	// Example binary for uncompressed vis data in a map with 4 leaves:
+	//     0000 ... (no leaves are visible from leaf 1)
+	//     1001 ... (leaves 1 and 4 are visible from leaf 2)
+	//     1111 ... (all leaves are visible from leaf 3)
+	// There are only 3 rows because the shared solid leaf 0 is excluded from both columns and rows.
+	// Dots "..." indicate padding. Rows are padded to multiples of 8 bytes/64 leaves.
 	byte* visdata;
 
 	int planeCount;
@@ -86,6 +94,12 @@ class Bsp
 	void traceHull(vec3 start, vec3 end, int hull, TraceResult* ptr);
 	const char* getLeafContentsName(int32_t contents);
 
+	// returns true if leaf is in the PVS from the given position
+	bool is_leaf_visible(int ileaf, vec3 pos);
+
+	// get leaf index from world position
+	int get_leaf(vec3 pos);
+
 	// strips a collision hull from the given model index
 	// and redirects to the given hull, if redirect>0
 	void delete_hull(int hull_number, int modelIdx, int redirect);

src/bsp/BspMerger.cpp

@@ -1624,7 +1624,7 @@ void BspMerger::merge_vis(Bsp& mapA, Bsp& mapB) {
 	// recompress the combined vis data
 	byte* compressedVis = new byte[decompressedVisSize];
 	memset(compressedVis, 0, decompressedVisSize);
-	int newVisLen = CompressAll(allLeaves, decompressedVis, compressedVis, totalVisLeaves, mergedWorldLeafCount, decompressedVisSize);
+	int newVisLen = CompressAll(allLeaves, decompressedVis, compressedVis, mergedWorldLeafCount, decompressedVisSize);
 	int oldLen = mapA.header.lump[LUMP_VISIBILITY].nLength;
 
 	byte* compressedVisResize = new byte[newVisLen];

src/editor/Gui.cpp

@@ -1442,6 +1442,29 @@ void Gui::drawDebugWidget() {
 						ImGui::Text("Texture: %s (%dx%d)", tex.szName, tex.nWidth, tex.nHeight);
 					}
 					ImGui::Text("Lightmap Offset: %d", face.nLightmapOffset);
+
+					static int lastFaceIdx = -1;
+					static string leafList;
+					static int leafPick = 0;
+
+					if (app->pickInfo.faceIdx != lastFaceIdx) {
+						lastFaceIdx = app->pickInfo.faceIdx;
+						leafList = "";
+						leafPick = -1;
+						for (int i = 1; i < map->leafCount; i++) {
+							BSPLEAF& leaf = map->leaves[i];
+							for (int k = 0; k < leaf.nMarkSurfaces; k++) {
+								if (map->marksurfs[leaf.iFirstMarkSurface + k] == app->pickInfo.faceIdx) {
+									leafList += " " + to_string(i);
+									leafPick = i;
+								}
+							}
+						}
+					}
+
+					const char* isVis = map->is_leaf_visible(leafPick, app->cameraOrigin) ? " (visible!)" : "";
+
+					ImGui::Text("Leaf IDs:%s%s", leafList.c_str(), isVis);
 				}
 			}
 

src/qtools/vis.cpp

@@ -50,7 +50,7 @@ bool shiftVis(byte* vis, int len, int offsetLeaf, int shift) {
 
 	if (shift < 0) {
 		// TODO: memcpy for negative shifts
-		bitShifts += byteShifts * 8;
+		bitShifts = abs(shift);
 		byteShifts = 0;
 	}
 
@@ -139,6 +139,11 @@ bool shiftVis(byte* vis, int len, int offsetLeaf, int shift) {
 	return overflow;
 }
 
+void decompress_vis_lump(BSPLEAF* leafLump, byte* visLump, byte* output, int visDataLeafCount)
+{
+	decompress_vis_lump(leafLump, visLump, output, visDataLeafCount, visDataLeafCount, visDataLeafCount);
+}
+
 // decompress this map's vis data into arrays of bits where each bit indicates if a leaf is visible or not
 // iterationLeaves = number of leaves to decompress vis for
 // visDataLeafCount = total leaves in this map (exluding the shared solid leaf 0)
@@ -281,7 +286,7 @@ int CompressVis(const byte* const src, const unsigned int src_length, byte* dest
 	return dest_p - dest;
 }
 
-int CompressAll(BSPLEAF* leafs, byte* uncompressed, byte* output, int numLeaves, int iterLeaves, int bufferSize)
+int CompressAll(BSPLEAF* leafs, byte* uncompressed, byte* output, int numLeaves, int bufferSize)
 {
 	int x = 0;
 	byte* dest;
@@ -291,8 +296,8 @@ int CompressAll(BSPLEAF* leafs, byte* uncompressed, byte* output, int numLeaves,
 
 	byte* vismap_p = output;
 
-	int* sharedRows = new int[iterLeaves];
-	for (int i = 0; i < iterLeaves; i++) {
+	int* sharedRows = new int[numLeaves];
+	for (int i = 0; i < numLeaves; i++) {
 		byte* src = uncompressed + i * g_bitbytes;
 
 		sharedRows[i] = i;
@@ -309,7 +314,7 @@ int CompressAll(BSPLEAF* leafs, byte* uncompressed, byte* output, int numLeaves,
 		g_progress.tick();
 	}
 
-	for (int i = 0; i < iterLeaves; i++)
+	for (int i = 0; i < numLeaves; i++)
 	{
 		if (sharedRows[i] != i) {
 			leafs[i + 1].nVisOffset = leafs[sharedRows[i] + 1].nVisOffset;

src/qtools/vis.h

@@ -4,17 +4,22 @@ struct BSPLEAF;
 
 bool shiftVis(byte* vis, int len, int offsetLeaf, int shift);
 
+// decompression function for use with merging multiple sets of vis data
 // decompress the given vis data into arrays of bits where each bit indicates if a leaf is visible or not
 // iterationLeaves = number of leaves to decompress vis for
 // visDataLeafCount = total leaves in the map (exluding the shared solid leaf 0)
 // newNumLeaves = total leaves that will be in the map after merging is finished (again, excluding solid leaf 0)
 void decompress_vis_lump(BSPLEAF* leafLump, byte* visLump, byte* output,
 	int iterationLeaves, int visDataLeafCount, int newNumLeaves);
 
+// visDataLeafCount should exclude the solid leaf 0
+void decompress_vis_lump(BSPLEAF* leafLump, byte* visLump, byte* output, int visDataLeafCount);
+
 void DecompressVis(const byte* src, byte* const dest, const unsigned int dest_length, uint numLeaves);
 
 int CompressVis(const byte* const src, const unsigned int src_length, byte* dest, unsigned int dest_length);
 
-int CompressAll(BSPLEAF* leafs, byte* uncompressed, byte* output, int numLeaves, int iterLeaves, int bufferSize);
+// compress all VIS data for every leaf. numLeaves should exclude the shared solid leaf 0
+int CompressAll(BSPLEAF* leafs, byte* uncompressed, byte* output, int numLeaves, int bufferSize);
 
 extern bool g_debug_shift;