foxygit / doom Log in
commit c52a135e5a606fa1fc91e68845d3eae6ac6adefc
Author:     Noseey <omega_1986@hotmail.com>
AuthorDate: Wed Sep 10 09:40:05 2025 +0200
Commit:     GitHub <noreply@github.com>
CommitDate: Wed Sep 10 09:40:05 2025 +0200

    Reject overflow emulation for Heretic/Hexen and moving of PadRejectArray (#1759)

    * Move PadRejectArray and align P_LoadReject

    Move PadRejectArray into own sourcefiles and add P_LoadReject to heretic/hexen, to be aligned with the other games and support empty reject lumps.

    * Add missing include string.h to p_rejectpad.h

    * Fixing typo in Makefile.am

    * Move includes to p_rejectpad.c

    .. not needed in .h
---
 src/CMakeLists.txt    |  1 +
 src/Makefile.am       |  1 +
 src/doom/p_setup.c    | 58 ++----------------------------------
 src/heretic/p_setup.c | 44 ++++++++++++++++++++++++----
 src/hexen/p_setup.c   | 43 +++++++++++++++++++++++----
 src/p_rejectpad.c     | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++
 src/p_rejectpad.h     | 28 ++++++++++++++++++
 src/strife/p_setup.c  | 59 ++-----------------------------------
 8 files changed, 190 insertions(+), 125 deletions(-)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index aa9d0a4b..336a5241 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -108,6 +108,7 @@ set(GAME_SOURCE_FILES
     net_sdl.c           net_sdl.h
     net_server.c        net_server.h
     net_structrw.c      net_structrw.h
+    p_rejectpad.c       p_rejectpad.h
     sha1.c              sha1.h
     memio.c             memio.h
     tables.c            tables.h
diff --git a/src/Makefile.am b/src/Makefile.am
index 29ed0d72..4fbade07 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -105,6 +105,7 @@ net_query.c          net_query.h           \
 net_sdl.c            net_sdl.h             \
 net_server.c         net_server.h          \
 net_structrw.c       net_structrw.h        \
+p_rejectpad.c        p_rejectpad.h         \
 sha1.c               sha1.h                \
 memio.c              memio.h               \
 tables.c             tables.h              \
diff --git a/src/doom/p_setup.c b/src/doom/p_setup.c
index 1a66fa42..fbb1a180 100644
--- a/src/doom/p_setup.c
+++ b/src/doom/p_setup.c
@@ -36,6 +36,7 @@

 #include "doomdef.h"
 #include "p_local.h"
+#include "p_rejectpad.h"

 #include "s_sound.h"

@@ -677,61 +678,6 @@ void P_GroupLines (void)

 }

-// Pad the REJECT lump with extra data when the lump is too small,
-// to simulate a REJECT buffer overflow in Vanilla Doom.
-
-static void PadRejectArray(byte *array, unsigned int len)
-{
-    unsigned int i;
-    unsigned int byte_num;
-    byte *dest;
-    unsigned int padvalue;
-
-    // Values to pad the REJECT array with:
-
-    unsigned int rejectpad[4] =
-    {
-        0,                                    // Size
-        0,                                    // Part of z_zone block header
-        50,                                   // PU_LEVEL
-        0x1d4a11                              // DOOM_CONST_ZONEID
-    };
-
-    rejectpad[0] = ((totallines * 4 + 3) & ~3) + 24;
-
-    // Copy values from rejectpad into the destination array.
-
-    dest = array;
-
-    for (i=0; i<len && i<sizeof(rejectpad); ++i)
-    {
-        byte_num = i % 4;
-        *dest = (rejectpad[i / 4] >> (byte_num * 8)) & 0xff;
-        ++dest;
-    }
-
-    // We only have a limited pad size.  Print a warning if the
-    // REJECT lump is too small.
-
-    if (len > sizeof(rejectpad))
-    {
-        fprintf(stderr, "PadRejectArray: REJECT lump too short to pad! (%u > %i)\n",
-                        len, (int) sizeof(rejectpad));
-
-        // Pad remaining space with 0 (or 0xff, if specified on command line).
-
-        if (M_CheckParm("-reject_pad_with_ff"))
-        {
-            padvalue = 0xff;
-        }
-        else
-        {
-            padvalue = 0x00;
-        }
-
-        memset(array + sizeof(rejectpad), padvalue, len - sizeof(rejectpad));
-    }
-}

 static void P_LoadReject(int lumpnum)
 {
@@ -757,7 +703,7 @@ static void P_LoadReject(int lumpnum)
         rejectmatrix = Z_Malloc(minlength, PU_LEVEL, &rejectmatrix);
         W_ReadLump(lumpnum, rejectmatrix);

-        PadRejectArray(rejectmatrix + lumplen, minlength - lumplen);
+        PadRejectArray(rejectmatrix + lumplen, minlength - lumplen, totallines);
     }
 }

diff --git a/src/heretic/p_setup.c b/src/heretic/p_setup.c
index 2acb0167..64517555 100644
--- a/src/heretic/p_setup.c
+++ b/src/heretic/p_setup.c
@@ -25,6 +25,7 @@
 #include "m_argv.h"
 #include "m_bbox.h"
 #include "p_local.h"
+#include "p_rejectpad.h"
 #include "s_sound.h"

 void P_SpawnMapThing(mapthing_t * mthing);
@@ -50,6 +51,8 @@ line_t *lines;
 int numsides;
 side_t *sides;

+static int totallines;
+
 short *blockmaplump;            // offsets in blockmap are from here
 short *blockmap;
 int bmapwidth, bmapheight;      // in mapblocks
@@ -467,7 +470,7 @@ void P_LoadBlockMap(int lump)
 void P_GroupLines(void)
 {
     line_t **linebuffer;
-    int i, j, total;
+    int i, j;
     line_t *li;
     sector_t *sector;
     subsector_t *ss;
@@ -485,20 +488,20 @@ void P_GroupLines(void)

 // count number of lines in each sector
     li = lines;
-    total = 0;
+    totallines = 0;
     for (i = 0; i < numlines; i++, li++)
     {
-        total++;
+        totallines++;
         li->frontsector->linecount++;
         if (li->backsector && li->backsector != li->frontsector)
         {
             li->backsector->linecount++;
-            total++;
+            totallines++;
         }
     }

 // build line tables for each sector
-    linebuffer = Z_Malloc(total * sizeof(line_t *), PU_LEVEL, 0);
+    linebuffer = Z_Malloc(totallines * sizeof(line_t *), PU_LEVEL, 0);
     sector = sectors;
     for (i = 0; i < numsectors; i++, sector++)
     {
@@ -541,6 +544,35 @@ void P_GroupLines(void)

 }

+
+static void P_LoadReject(int lumpnum)
+{
+    int minlength;
+    int lumplen;
+
+    // Calculate the size that the REJECT lump *should* be.
+
+    minlength = (numsectors * numsectors + 7) / 8;
+
+    // If the lump meets the minimum length, it can be loaded directly.
+    // Otherwise, we need to allocate a buffer of the correct size
+    // and pad it with appropriate data.
+
+    lumplen = W_LumpLength(lumpnum);
+
+    if (lumplen >= minlength)
+    {
+        rejectmatrix = W_CacheLumpNum(lumpnum, PU_LEVEL);
+    }
+    else
+    {
+        rejectmatrix = Z_Malloc(minlength, PU_LEVEL, &rejectmatrix);
+        W_ReadLump(lumpnum, rejectmatrix);
+
+        PadRejectArray(rejectmatrix + lumplen, minlength - lumplen, totallines);
+    }
+}
+
 //=============================================================================

 lumpinfo_t *maplumpinfo;
@@ -600,8 +632,8 @@ void P_SetupLevel(int episode, int map, int playermask, skill_t skill)
     P_LoadNodes(lumpnum + ML_NODES);
     P_LoadSegs(lumpnum + ML_SEGS);

-    rejectmatrix = W_CacheLumpNum(lumpnum + ML_REJECT, PU_LEVEL);
     P_GroupLines();
+    P_LoadReject(lumpnum + ML_REJECT);

     bodyqueslot = 0;
     deathmatch_p = deathmatchstarts;
diff --git a/src/hexen/p_setup.c b/src/hexen/p_setup.c
index 79d09e1a..8fb1ea2e 100644
--- a/src/hexen/p_setup.c
+++ b/src/hexen/p_setup.c
@@ -27,6 +27,7 @@
 #include "i_swap.h"
 #include "s_sound.h"
 #include "p_local.h"
+#include "p_rejectpad.h"

 // MACROS ------------------------------------------------------------------

@@ -103,6 +104,7 @@ int numlines;
 line_t *lines;
 int numsides;
 side_t *sides;
+static int totallines;
 short *blockmaplump;            // offsets in blockmap are from here
 short *blockmap;
 int bmapwidth, bmapheight;      // in mapblocks
@@ -579,7 +581,7 @@ void P_LoadBlockMap(int lump)
 void P_GroupLines(void)
 {
     line_t **linebuffer;
-    int i, j, total;
+    int i, j;
     line_t *li;
     sector_t *sector;
     subsector_t *ss;
@@ -597,20 +599,20 @@ void P_GroupLines(void)

 // count number of lines in each sector
     li = lines;
-    total = 0;
+    totallines = 0;
     for (i = 0; i < numlines; i++, li++)
     {
-        total++;
+        totallines++;
         li->frontsector->linecount++;
         if (li->backsector && li->backsector != li->frontsector)
         {
             li->backsector->linecount++;
-            total++;
+            totallines++;
         }
     }

 // build line tables for each sector
-    linebuffer = Z_Malloc(total * sizeof(line_t *), PU_LEVEL, 0);
+    linebuffer = Z_Malloc(totallines * sizeof(line_t *), PU_LEVEL, 0);
     sector = sectors;
     for (i = 0; i < numsectors; i++, sector++)
     {
@@ -653,6 +655,35 @@ void P_GroupLines(void)

 }

+
+static void P_LoadReject(int lumpnum)
+{
+    int minlength;
+    int lumplen;
+
+    // Calculate the size that the REJECT lump *should* be.
+
+    minlength = (numsectors * numsectors + 7) / 8;
+
+    // If the lump meets the minimum length, it can be loaded directly.
+    // Otherwise, we need to allocate a buffer of the correct size
+    // and pad it with appropriate data.
+
+    lumplen = W_LumpLength(lumpnum);
+
+    if (lumplen >= minlength)
+    {
+        rejectmatrix = W_CacheLumpNum(lumpnum, PU_LEVEL);
+    }
+    else
+    {
+        rejectmatrix = Z_Malloc(minlength, PU_LEVEL, &rejectmatrix);
+        W_ReadLump(lumpnum, rejectmatrix);
+
+        PadRejectArray(rejectmatrix + lumplen, minlength - lumplen, totallines);
+    }
+}
+
 //=============================================================================

 lumpinfo_t *maplumpinfo;
@@ -710,8 +741,8 @@ void P_SetupLevel(int episode, int map, int playermask, skill_t skill)
     P_LoadSubsectors(lumpnum + ML_SSECTORS);
     P_LoadNodes(lumpnum + ML_NODES);
     P_LoadSegs(lumpnum + ML_SEGS);
-    rejectmatrix = W_CacheLumpNum(lumpnum + ML_REJECT, PU_LEVEL);
     P_GroupLines();
+    P_LoadReject(lumpnum + ML_REJECT);
     bodyqueslot = 0;
     po_NumPolyobjs = 0;
     deathmatch_p = deathmatchstarts;
diff --git a/src/p_rejectpad.c b/src/p_rejectpad.c
new file mode 100644
index 00000000..b336464c
--- /dev/null
+++ b/src/p_rejectpad.c
@@ -0,0 +1,81 @@
+//
+// Copyright(C) 1993-1996 Id Software, Inc.
+// Copyright(C) 2005-2014 Simon Howard
+//
+// This program is free software; you can redistribute it and/or
+// modify it under the terms of the GNU General Public License
+// as published by the Free Software Foundation; either version 2
+// of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// DESCRIPTION:
+// Padding of Reject Lump
+//
+
+
+#include <stdio.h>
+#include <string.h>
+
+#include "m_argv.h"
+#include "p_rejectpad.h"
+
+
+// Pad the REJECT lump with extra data when the lump is too small,
+// to simulate a REJECT buffer overflow in Vanilla Doom.
+
+void PadRejectArray(byte *array, unsigned int len, int totallines)
+{
+    unsigned int i;
+    unsigned int byte_num;
+    byte *dest;
+    unsigned int padvalue;
+
+    // Values to pad the REJECT array with:
+
+    unsigned int rejectpad[4] =
+    {
+        0,                                    // Size
+        0,                                    // Part of z_zone block header
+        50,                                   // PU_LEVEL
+        0x1d4a11                              // DOOM_CONST_ZONEID
+    };
+
+    rejectpad[0] = ((totallines * 4 + 3) & ~3) + 24;
+
+    // Copy values from rejectpad into the destination array.
+
+    dest = array;
+
+    for (i=0; i<len && i<sizeof(rejectpad); ++i)
+    {
+        byte_num = i % 4;
+        *dest = (rejectpad[i / 4] >> (byte_num * 8)) & 0xff;
+        ++dest;
+    }
+
+    // We only have a limited pad size.  Print a warning if the
+    // REJECT lump is too small.
+
+    if (len > sizeof(rejectpad))
+    {
+        fprintf(stderr, "PadRejectArray: REJECT lump too short to pad! (%u > %i)\n",
+                        len, (int) sizeof(rejectpad));
+
+        // Pad remaining space with 0 (or 0xff, if specified on command line).
+
+        if (M_CheckParm("-reject_pad_with_ff"))
+        {
+            padvalue = 0xff;
+        }
+        else
+        {
+            padvalue = 0x00;
+        }
+
+        memset(array + sizeof(rejectpad), padvalue, len - sizeof(rejectpad));
+    }
+}
diff --git a/src/p_rejectpad.h b/src/p_rejectpad.h
new file mode 100644
index 00000000..78ab8407
--- /dev/null
+++ b/src/p_rejectpad.h
@@ -0,0 +1,28 @@
+//
+// Copyright(C) 1993-1996 Id Software, Inc.
+// Copyright(C) 2005-2014 Simon Howard
+//
+// This program is free software; you can redistribute it and/or
+// modify it under the terms of the GNU General Public License
+// as published by the Free Software Foundation; either version 2
+// of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// DESCRIPTION:
+// Padding of Reject Lump
+//
+
+
+#ifndef __P_REJECTPAD__
+#define __P_REJECTPAD__
+
+#include "doomtype.h"
+
+// Padding of Reject Lump
+void PadRejectArray(byte *array, unsigned int len, int totallines);
+
+#endif
diff --git a/src/strife/p_setup.c b/src/strife/p_setup.c
index 323b32f7..d6c3057b 100644
--- a/src/strife/p_setup.c
+++ b/src/strife/p_setup.c
@@ -35,6 +35,7 @@

 #include "doomdef.h"
 #include "p_local.h"
+#include "p_rejectpad.h"

 #include "s_sound.h"

@@ -656,62 +657,6 @@ void P_GroupLines (void)

 }

-// Pad the REJECT lump with extra data when the lump is too small,
-// to simulate a REJECT buffer overflow in Vanilla Doom.
-
-static void PadRejectArray(byte *array, unsigned int len)
-{
-    unsigned int i;
-    unsigned int byte_num;
-    byte *dest;
-    unsigned int padvalue;
-
-    // Values to pad the REJECT array with:
-
-    unsigned int rejectpad[4] =
-    {
-        0,                                    // Size
-        0,                                    // Part of z_zone block header
-        50,                                   // PU_LEVEL
-        0x1d4a11                              // DOOM_CONST_ZONEID
-    };
-
-    rejectpad[0] = ((totallines * 4 + 3) & ~3) + 24;
-
-    // Copy values from rejectpad into the destination array.
-
-    dest = array;
-
-    for (i=0; i<len && i<sizeof(rejectpad); ++i)
-    {
-        byte_num = i % 4;
-        *dest = (rejectpad[i / 4] >> (byte_num * 8)) & 0xff;
-        ++dest;
-    }
-
-    // We only have a limited pad size.  Print a warning if the
-    // REJECT lump is too small.
-
-    if (len > sizeof(rejectpad))
-    {
-        fprintf(stderr,
-                "PadRejectArray: REJECT lump too short to pad! (%u > %i)\n",
-                len, (int) sizeof(rejectpad));
-
-        // Pad remaining space with 0 (or 0xff, if specified on command line).
-
-        if (M_CheckParm("-reject_pad_with_ff"))
-        {
-            padvalue = 0xff;
-        }
-        else
-        {
-            padvalue = 0xf00;
-        }
-
-        memset(array + sizeof(rejectpad), padvalue, len - sizeof(rejectpad));
-    }
-}

 static void P_LoadReject(int lumpnum)
 {
@@ -737,7 +682,7 @@ static void P_LoadReject(int lumpnum)
         rejectmatrix = Z_Malloc(minlength, PU_LEVEL, &rejectmatrix);
         W_ReadLump(lumpnum, rejectmatrix);

-        PadRejectArray(rejectmatrix + lumplen, minlength - lumplen);
+        PadRejectArray(rejectmatrix + lumplen, minlength - lumplen, totallines);
     }
 }