shuffler: make the new shuffler actually work

Make the new shuffler actually work.  This includes changing
rllpack.inc to run in 32-bit mode (since simple_pm_call now switches
to 32-bit mode) and changing the new shuffler interface to move the
shuffle list before actually doing any work.

Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>

11 files changed, 271 insertions, 164 deletions

diff --git a/com32/include/syslinux/movebits.h b/com32/include/syslinux/movebits.h
index f35ef228..ff3711ee 100644
--- a/com32/include/syslinux/movebits.h
+++ b/com32/include/syslinux/movebits.h
@@ -59,8 +59,13 @@ int syslinux_add_movelist(struct syslinux_movelist **,
 			  addr_t dst, addr_t src, addr_t len);
 int syslinux_allocate_from_list(struct syslinux_movelist **freelist,
 				addr_t dst, addr_t len);
-int syslinux_prepare_shuffle(struct syslinux_movelist *fraglist,
-			     struct syslinux_memmap *memmap);
+int syslinux_do_shuffle(struct syslinux_movelist *fraglist,
+			struct syslinux_memmap *memmap,
+			addr_t entry_point, addr_t entry_type,
+			uint16_t bootflags);
+struct syslinux_memmap *
+syslinux_target_memmap(struct syslinux_movelist *fraglist,
+		       struct syslinux_memmap *memmap);
 
 /* Operatons on struct syslinux_memmap */
 struct syslinux_memmap *syslinux_init_memmap(void);
diff --git a/com32/lib/syslinux/memmap.c b/com32/lib/syslinux/memmap.c
index 105c9473..4c5738f8 100644
--- a/com32/lib/syslinux/memmap.c
+++ b/com32/lib/syslinux/memmap.c
@@ -1,6 +1,6 @@
 /* ----------------------------------------------------------------------- *
  *
- *   Copyright 2007-2008 H. Peter Anvin - All Rights Reserved
+ *   Copyright 2007-2009 H. Peter Anvin - All Rights Reserved
  *
  *   Permission is hereby granted, free of charge, to any person
  *   obtaining a copy of this software and associated documentation
@@ -63,11 +63,11 @@ struct syslinux_memmap *syslinux_memory_map(void)
   if (!mmap)
     goto bail;
 
-  /* Use INT 12h to get DOS memory above 0x7c00 */
+  /* Use INT 12h to get DOS memory 0x600 */
   __intcall(0x12, &zireg, &oreg);
   if (oreg.eax.w[0] > 31 && oreg.eax.w[0] <= 640) {
-    addr_t dosmem = (oreg.eax.w[0] << 10) - 0x7c00;
-    if (syslinux_add_memmap(&mmap, 0x7c00, dosmem, SMT_FREE))
+    addr_t dosmem = (oreg.eax.w[0] << 10) - 0x600;
+    if (syslinux_add_memmap(&mmap, 0x600, dosmem, SMT_FREE))
       goto bail;
   }
 
diff --git a/com32/lib/syslinux/shuffle.c b/com32/lib/syslinux/shuffle.c
index b891722d..b29f0043 100644
--- a/com32/lib/syslinux/shuffle.c
+++ b/com32/lib/syslinux/shuffle.c
@@ -62,33 +62,42 @@ struct shuffle_descriptor {
   uint32_t dst, src, len;
 };
 
-static int desc_block_size;
+static int shuffler_size;
 
-static void __constructor __syslinux_get_desc_block_size(void)
+static void __constructor __syslinux_get_shuffer_size(void)
 {
   static com32sys_t reg;
 
-  reg.eax.w[0] = 0x0011;
+  reg.eax.w[0] = 0x0023;
   __intcall(0x22, &reg, &reg);
 
-  desc_block_size = (reg.eflags.l & EFLAGS_CF) ? 64 : reg.ecx.w[0];
+  shuffler_size = (reg.eflags.l & EFLAGS_CF) ? 2048 : reg.ecx.w[0];
 }
 
-/* Allocate descriptor memory in these chunks */
-#define DESC_BLOCK_SIZE	desc_block_size
+/*
+ * Allocate descriptor memory in these chunks; if this is large we may
+ * waste memory, if it is small we may get slow convergence.
+ */
+#define DESC_BLOCK_SIZE	256
 
-int syslinux_prepare_shuffle(struct syslinux_movelist *fraglist,
-			     struct syslinux_memmap *memmap)
+int syslinux_do_shuffle(struct syslinux_movelist *fraglist,
+			struct syslinux_memmap *memmap,
+			addr_t entry_point, addr_t entry_type,
+			uint16_t bootflags)
 {
+  int rv = -1;
   struct syslinux_movelist *moves = NULL, *mp;
   struct syslinux_memmap *rxmap = NULL, *ml;
   struct shuffle_descriptor *dp, *dbuf;
-  int np, nb, nl, rv = -1;
+  int np;
   int desc_blocks, need_blocks;
   int need_ptrs;
   addr_t desczone, descfree, descaddr, descoffs;
   int nmoves, nzero;
-  struct shuffle_descriptor primaries[2];
+  com32sys_t ireg;
+
+  descaddr = 0;
+  dp = dbuf = NULL;
 
   /* Count the number of zero operations */
   nzero = 0;
@@ -98,11 +107,15 @@ int syslinux_prepare_shuffle(struct syslinux_movelist *fraglist,
   }
 
   /* Find the largest contiguous region unused by input *and* output;
-     this is where we put the move descriptor list */
+     this is where we put the move descriptor list and safe area */
 
   rxmap = syslinux_dup_memmap(memmap);
   if (!rxmap)
     goto bail;
+  /* Avoid using the low 1 MB for the shuffle area -- this avoids
+     possible interference with the real mode code or stack */
+  if (syslinux_add_memmap(&rxmap, 0, 1024*1024, SMT_ALLOC))
+    goto bail;
   for (mp = fraglist; mp; mp = mp->next) {
     if (syslinux_add_memmap(&rxmap, mp->src, mp->len, SMT_ALLOC) ||
 	syslinux_add_memmap(&rxmap, mp->dst, mp->len, SMT_ALLOC))
@@ -119,10 +132,13 @@ int syslinux_prepare_shuffle(struct syslinux_movelist *fraglist,
   if (!rxmap)
     goto bail;
 
-  desc_blocks = (nzero+DESC_BLOCK_SIZE-1)/(DESC_BLOCK_SIZE-1);
+  desc_blocks = (nzero+DESC_BLOCK_SIZE-1)/DESC_BLOCK_SIZE;
   for (;;) {
+    /* We want (desc_blocks) allocation blocks, plus the terminating
+       descriptor, plus the shuffler safe area. */
     addr_t descmem = desc_blocks*
-      sizeof(struct shuffle_descriptor)*DESC_BLOCK_SIZE;
+      sizeof(struct shuffle_descriptor)*DESC_BLOCK_SIZE
+      + sizeof(struct shuffle_descriptor) + shuffler_size;
 
     if (descfree < descmem)
       goto bail;		/* No memory block large enough */
@@ -143,7 +159,7 @@ int syslinux_prepare_shuffle(struct syslinux_movelist *fraglist,
     for (mp = moves; mp; mp = mp->next)
       nmoves++;
 
-    need_blocks = (nmoves+nzero+DESC_BLOCK_SIZE-1)/(DESC_BLOCK_SIZE-1);
+    need_blocks = (nmoves+nzero+DESC_BLOCK_SIZE-1)/DESC_BLOCK_SIZE;
 
     if (desc_blocks >= need_blocks)
       break;			/* Sufficient memory, yay */
@@ -159,7 +175,7 @@ int syslinux_prepare_shuffle(struct syslinux_movelist *fraglist,
   syslinux_free_memmap(rxmap);
   rxmap = NULL;
 
-  need_ptrs = nmoves+nzero+desc_blocks-1;
+  need_ptrs = nmoves+nzero+1;
   dbuf = malloc(need_ptrs*sizeof(struct shuffle_descriptor));
   if (!dbuf)
     goto bail;
@@ -173,71 +189,38 @@ int syslinux_prepare_shuffle(struct syslinux_movelist *fraglist,
 
   /* Copy the move sequence into the descriptor buffer */
   np = 0;
-  nb = 0;
-  nl = nmoves+nzero;
   dp = dbuf;
   for (mp = moves; mp; mp = mp->next) {
-    if (nb == DESC_BLOCK_SIZE-1) {
-      dp->dst = -1;		/* Load new descriptors */
-      dp->src = (addr_t)(dp+1) + descoffs;
-      dp->len = sizeof(*dp)*min(nl, DESC_BLOCK_SIZE);
-      dprintf("[ %08x %08x %08x ]\n", dp->dst, dp->src, dp->len);
-      dp++; np++;
-      nb = 0;
-    }
-
     dp->dst = mp->dst;
     dp->src = mp->src;
     dp->len = mp->len;
     dprintf2("[ %08x %08x %08x ]\n", dp->dst, dp->src, dp->len);
-    dp++; np++; nb++; nl--;
+    dp++; np++;
   }
 
   /* Copy bzero operations into the descriptor buffer */
   for (ml = memmap; ml->type != SMT_END; ml = ml->next) {
     if (ml->type == SMT_ZERO) {
-      if (nb == DESC_BLOCK_SIZE-1) {
-	dp->dst = (addr_t)-1;	/* Load new descriptors */
-	dp->src = (addr_t)(dp+1) + descoffs;
-	dp->len = sizeof(*dp)*min(nl, DESC_BLOCK_SIZE);
-	dprintf("[ %08x %08x %08x ]\n", dp->dst, dp->src, dp->len);
-	dp++; np++;
-	nb = 0;
-      }
-
       dp->dst = ml->start;
       dp->src = (addr_t)-1;	/* bzero region */
       dp->len = ml->next->start - ml->start;
       dprintf2("[ %08x %08x %08x ]\n", dp->dst, dp->src, dp->len);
-      dp++; np++; nb++; nl--;
+      dp++; np++;
     }
   }
 
+  /* Finally, record the termination entry */
+  dp->dst = entry_point;
+  dp->src = entry_type;
+  dp->len = 0;
+  dp++; np++;
+
   if (np != need_ptrs) {
     dprintf("!!! np = %d : nmoves = %d, nzero = %d, desc_blocks = %d\n",
 	    np, nmoves, nzero, desc_blocks);
   }
 
-  /* Set up the primary descriptors in the bounce buffer.
-     The first one moves the descriptor list into its designated safe
-     zone, the second one loads the first descriptor block. */
-  dp = primaries;
-
-  dp->dst = descaddr;
-  dp->src = (addr_t)dbuf;
-  dp->len = np*sizeof(*dp);
-  dprintf("< %08x %08x %08x >\n", dp->dst, dp->src, dp->len);
-  dp++;
-
-  dp->dst = (addr_t)-1;
-  dp->src = descaddr;
-  dp->len = sizeof(*dp)*min(np, DESC_BLOCK_SIZE);
-  dprintf("< %08x %08x %08x >\n", dp->dst, dp->src, dp->len);
-  dp++;
-
-  memcpy(__com32.cs_bounce, primaries, 2*sizeof(*dp));
-
-  rv = 2;			/* Always two primaries */
+  rv = 0;
 
  bail:
   /* This is safe only because free() doesn't use the bounce buffer!!!! */
@@ -246,5 +229,42 @@ int syslinux_prepare_shuffle(struct syslinux_movelist *fraglist,
   if (rxmap)
     syslinux_free_memmap(rxmap);
 
-  return rv;
+  if (rv)
+    return rv;
+
+  /* Actually do it... */
+  memset(&ireg, 0, sizeof ireg);
+  ireg.edi.l = descaddr;
+  ireg.esi.l = (addr_t)dbuf;
+  ireg.ecx.l = (addr_t)dp-(addr_t)dbuf;
+  ireg.edx.w[0] = bootflags;
+  ireg.eax.w[0] = 0x0024;
+  __intcall(0x22, &ireg, NULL);
+
+  return -1;			/* Shouldn't have returned! */
+}
+
+/*
+ * Common helper routine: takes a memory map and blots out the
+ * zones which are used in the destination of a fraglist
+ */
+struct syslinux_memmap *
+syslinux_target_memmap(struct syslinux_movelist *fraglist,
+		       struct syslinux_memmap *memmap)
+{
+  struct syslinux_memmap *tmap;
+  struct syslinux_movelist *mp;
+
+  tmap = syslinux_dup_memmap(memmap);
+  if (!tmap)
+    return NULL;
+
+  for (mp = fraglist; mp; mp = mp->next) {
+    if (syslinux_add_memmap(&tmap, mp->dst, mp->len, SMT_ALLOC)) {
+      syslinux_free_memmap(tmap);
+      return NULL;
+    }
+  }
+
+  return tmap;
 }
diff --git a/com32/lib/syslinux/shuffle_pm.c b/com32/lib/syslinux/shuffle_pm.c
index d19d01bb..00bb4e4b 100644
--- a/com32/lib/syslinux/shuffle_pm.c
+++ b/com32/lib/syslinux/shuffle_pm.c
@@ -46,24 +46,38 @@ int syslinux_shuffle_boot_pm(struct syslinux_movelist *fraglist,
   int nd;
   com32sys_t ireg;
   char *regbuf;
+  uint8_t handoff_code[9*5], *p;
+  const uint32_t *rp;
+  int i, rv;
+  struct syslinux_memmap *tmap;
+  addr_t regstub, stublen, safe;
 
-  nd = syslinux_prepare_shuffle(fraglist, memmap);
-  if (nd < 0)
+  tmap = syslinux_target_memmap(fraglist, memmap);
+  if (!tmap)
     return -1;
 
-  regbuf = (char *)__com32.cs_bounce + (12*nd);
-  memcpy(regbuf, regs, sizeof(*regs));
+  regstub = 0x800;		/* Locate anywhere above this point */
+  stublen = sizeof handoff_code;
+  rv = syslinux_memmap_find(tmap, SMT_FREE, &regstub, &stublen);
+  syslinux_free_memmap(tmap);
+  if (rv)
+    return -1;
 
-  memset(&ireg, 0, sizeof ireg);
+  /* Build register-setting stub */
+  p = handoff_code;
+  rp = (const uint32_t *)regs;
+  for (i = 0; i < 8; i++) {
+    *p = 0xb8 + i;		/* MOV gpr,imm32 */
+    *(uint32_t *)(p+1) = *rp++;
+    p += 5;
+  }
+  *p = 0xe9;			/* JMP */
+  *(uint32_t *)(p+1) = regs->eip - regstub - sizeof handoff_code;
 
-  ireg.eax.w[0] = 0x001a;
-  ireg.edx.w[0] = bootflags;
-  ireg.es       = SEG(__com32.cs_bounce);
-  ireg.edi.l    = OFFS(__com32.cs_bounce);
-  ireg.ecx.l    = nd;
-  ireg.ds       = SEG(regbuf);
-  ireg.esi.l    = OFFS(regbuf);
-  __intcall(0x22, &ireg, NULL);
+  /* Add register-setting stub to shuffle list */
+  if (syslinux_add_movelist(&fraglist, regstub, (addr_t)handoff_code,
+			    sizeof handoff_code))
+    return -1;
 
-  return -1;			/* Too many descriptors? */
+  return syslinux_do_shuffle(fraglist, memmap, regstub, 1, bootflags);
 }
diff --git a/com32/lib/syslinux/shuffle_rm.c b/com32/lib/syslinux/shuffle_rm.c
index e9226a82..4852d3c5 100644
--- a/com32/lib/syslinux/shuffle_rm.c
+++ b/com32/lib/syslinux/shuffle_rm.c
@@ -46,24 +46,72 @@ int syslinux_shuffle_boot_rm(struct syslinux_movelist *fraglist,
   int nd;
   com32sys_t ireg;
   char *regbuf;
+  const struct syslinux_rm_regs_alt {
+    uint16_t seg[6];
+    uint32_t gpr[8];
+    uint32_t csip;
+  } *rp;
+  int i, rv;
+  uint8_t handoff_code[5*5+8*6+5], *p;
+  struct syslinux_memmap *tmap, *tp;
+  addr_t regstub;
 
-  nd = syslinux_prepare_shuffle(fraglist, memmap);
-  if (nd < 0)
+  tmap = syslinux_target_memmap(fraglist, memmap);
+  if (!tmap)
     return -1;
 
-  regbuf = (char *)__com32.cs_bounce + (12*nd);
-  memcpy(regbuf, regs, sizeof(*regs));
+  /* Search for a good place to put the real-mode register stub.
+     We prefer to put it as high as possible in the low 640K. */
+  regstub = 0;
+  for (tp = tmap; tp->type != SMT_END; tp = tp->next) {
+    addr_t xend, xlen;
+    if (tp->start >= 640*1024)
+      continue;
+    if (tp->type != SMT_FREE)
+      continue;
+    xend = tp->next->start;
+    if (xend > 640*1024)
+      xend = 640*1024;
+    xlen = xend - tp->start;
+    if (xlen < sizeof handoff_code)
+      continue;
+    regstub = xend - sizeof handoff_code; /* Best alternative so far */
+  }
 
-  memset(&ireg, 0, sizeof ireg);
+  syslinux_free_memmap(tmap);
 
-  ireg.eax.w[0] = 0x001b;
-  ireg.edx.w[0] = bootflags;
-  ireg.es       = SEG(__com32.cs_bounce);
-  ireg.edi.l    = OFFS(__com32.cs_bounce);
-  ireg.ecx.l    = nd;
-  ireg.ds       = SEG(regbuf);
-  ireg.esi.l    = OFFS(regbuf);
-  __intcall(0x22, &ireg, NULL);
+  /* XXX: it might be possible to do something insane here like
+     putting the stub in the IRQ vectors... */
+  if (!regstub)
+    return -1;			/* No space at all */
 
-  return -1;			/* Too many descriptors? */
+  /* Build register-setting stub */
+  p = handoff_code;
+  rp = (const struct syslinux_rm_regs_alt *)regs;
+  for (i = 0; i < 6; i++) {
+    if (i != 1) {		/* Skip CS */
+      p[0] = 0xb8;		/* MOV AX,imm16 */
+      *(uint16_t *)(p+1) = rp->seg[i];
+      *(uint16_t *)(p+3) = 0xc08e + (i << 11); /* MOV seg,AX */
+      p += 5;
+    }
+  }
+  for (i = 0; i < 8; i++) {
+    p[0] = 0x66;		/* MOV exx,imm32 */
+    p[1] = 0xb8 + i;
+    *(uint32_t *)(p+2) = rp->gpr[i];
+    p += 6;
+  }
+  *p++ = 0xea;			/* JMP FAR */
+  *(uint32_t *)p = rp->csip;
+
+  /* Add register-setting stub to shuffle list */
+  if (syslinux_add_movelist(&fraglist, regstub, (addr_t)handoff_code,
+			    sizeof handoff_code))
+    return -1;
+
+  /* Convert regstub to a CS:IP entrypoint pair */
+  regstub = (SEG((void *)regstub) << 16) + OFFS((void *)regstub);
+
+  return syslinux_do_shuffle(fraglist, memmap, regstub, 0, bootflags);
 }
diff --git a/com32/lib/syslinux/zonelist.c b/com32/lib/syslinux/zonelist.c
index 62b1cf3e..86db51e2 100644
--- a/com32/lib/syslinux/zonelist.c
+++ b/com32/lib/syslinux/zonelist.c
@@ -1,6 +1,6 @@
 /* ----------------------------------------------------------------------- *
  *
- *   Copyright 2007-2008 H. Peter Anvin - All Rights Reserved
+ *   Copyright 2007-2009 H. Peter Anvin - All Rights Reserved
  *
  *   Permission is hereby granted, free of charge, to any person
  *   obtaining a copy of this software and associated documentation
@@ -201,6 +201,33 @@ int syslinux_memmap_largest(struct syslinux_memmap *list,
 }
 
 /*
+ * Find the first (lowest address) zone of a specific type and of
+ * a certain minimum size, with an optional starting address.
+ * The input values of start and len are used as minima.
+ */
+int syslinux_memmap_find(struct syslinux_memmap *list,
+			 enum syslinux_memmap_types type,
+			 addr_t *start, addr_t *len)
+{
+  addr_t min_start = *start;
+  addr_t min_len = *len;
+
+  while (list->type != SMT_END) {
+    if (list->type == type && list->next->start > min_start) {
+      addr_t xstart = min_start > list->start ? min_start : list->start;
+      addr_t xlen = list->next->start - xstart;
+      if (xlen >= min_len) {
+	*start = xstart;
+	*len = xlen;
+	return 0;
+      }
+    }
+  }
+
+  return -1;			/* Not found */
+}
+
+/*
  * Free a zonelist.
  */
 void syslinux_free_memmap(struct syslinux_memmap *list)
diff --git a/core/bcopyxx.inc b/core/bcopyxx.inc
index aa1ce4a1..3eeeaef4 100644
--- a/core/bcopyxx.inc
+++ b/core/bcopyxx.inc
@@ -30,16 +30,6 @@
 		bits 32
 		section .bcopyxx
 		align 16
-bcopyxx_before:
-		; target > source, need reverse copy
-		lea esi,[esi+ecx-4]
-		lea edi,[edx+ecx-4]
-		std
-		rep movsd
-		cld
-		jmp eax
-
-		align 4
 bcopyxx_start	equ $
 ;
 ; pm_bcopy:
@@ -52,6 +42,9 @@ bcopyxx_start	equ $
 ;
 	
 pm_bcopy:
+		push ebx
+		push edx
+
 		cmp esi,-1
 		je .bzero
 
@@ -91,6 +84,9 @@ pm_bcopy:
 		jz .fab1
 		a32 movsb
 .fab1:
+.done:
+		pop edx
+		pop ebx
 		ret
 
 .reverse:
@@ -141,7 +137,7 @@ pm_bcopy:
 		a32 movsb
 .rab1:
 		cld
-		ret
+		jmp short .done
 
 .bzero:
 		xor eax,eax
@@ -177,7 +173,7 @@ pm_bcopy:
 		jz .zab1
 		a32 stosb
 .zab1:
-		ret
+		jmp short .done
 
 ;
 ; shuffle_and_boot:
@@ -191,8 +187,10 @@ pm_bcopy:
 ; stub; *especially* when going to real mode.
 ;
 ; Inputs:
-;	EBX		-> Pointer to list of (dst, src, len) pairs(*)
-;	EDX		-> Pointer to safe memory area
+;	ESI		-> Pointer to list of (dst, src, len) pairs(*)
+;	EDI		-> Pointer to safe area for list + shuffler
+;			   (must not overlap this code nor the RM stack)
+;	ECX		-> Byte count of list area (for initial copy)
 ;
 ;     If src == -1: then the memory pointed to by (dst, len) is bzeroed;
 ;		    this is handled inside the bcopy routine.
@@ -200,32 +198,17 @@ pm_bcopy:
 ;     If len == 0:  this marks the end of the list; dst indicates
 ;		    the entry point and src the mode (0 = pm, 1 = rm)
 ;
-;
-; Note: we're hideously strict with the relocation, so we never touch
-; any memory we don't need to.  This is important for our own internal
-; use of the code.
-;
 pm_shuffle:
+		mov ebx,edi		; EBX <- descriptor list
+		lea edx,[edi+ecx]	; EDX <- shuffler end location
+		push edx
+		call pm_bcopy
+		pop edx
 		mov edi,edx
 		mov esi,bcopyxx_start
 		mov ecx,bcopyxx_dwords
 		lea eax,[edx+.safe-bcopyxx_start]	; Resume point
-		cmp edx,bcopyxx_end
-		jae .no_overlap		; Safe area start >= end
-		lea ebp,[edx+bcopyxx_len]
-		cmp edi,ebp
-		jae .no_overlap		; Safe area end <= start
-		cmp edx,esi
-		je .safe		; OK, this was too easy
-
-		; OK, we have some overlap one way or the other.
-		; We bounce this to one of two routines *outside*
-		; the safe area... one on each side.
-		ja bcopyxx_before	; target > source
-		jmp bcopyxx_after	; source > target
-
-.no_overlap:
-		; No overlap, do the copying inside the safe area
+		; Relocate this code
 		rep movsd
 		jmp eax			; Jump to safe location
 .safe:
@@ -336,12 +319,5 @@ bcopyxx_safe	equ bcopyxx_len + bcopyxx_stack
 ;
 DummyTSS	equ 0x800
 
-		bits 32
-		align 4
-bcopyxx_after:
-		; source > target, forward copy
-		rep movsd
-		jmp eax	
-
 		bits 16
 		section .text
diff --git a/core/bootsect.inc b/core/bootsect.inc
index 515f1f5e..67080907 100644
--- a/core/bootsect.inc
+++ b/core/bootsect.inc
@@ -156,6 +156,7 @@ replace_bootstrap:
 		mov ebx,trackbuf
 		imul di,ax,12
 		add di,bx		; DI <- end of list
+		push di
 
 		; Terminating entry...
 		lea eax,[di+12]
@@ -169,16 +170,20 @@ replace_bootstrap:
 		mov cx,replace_stub.len >> 2
 		rep movsd
 		
+		xor eax,eax
+		pop cx			; ECX <- length of list
+
 		pop word [di+replace_stub.ss]
 		pop word [di+replace_stub.esp]
 		pop dword [di+replace_stub.csip]
 
-		movzx edx,di		; "Safe area"
-
 		cli
 		mov ss,[di+replace_stub.ss]
 		mov esp,[di+replace_stub.esp]
 
+		mov edi,trackbuf
+		mov esi,edi
+
 		jmp shuffle_and_boot_raw
 
 		; This stub gets run after the shuffle, but not in-place.
diff --git a/core/comboot.inc b/core/comboot.inc
index 5e28df34..e4a03efe 100644
--- a/core/comboot.inc
+++ b/core/comboot.inc
@@ -1002,8 +1002,6 @@ comapi_shufsize:
 ;
 comapi_shufraw:
 		call comapi_cleanup
-		mov ebx,P_EDI
-		mov edx,P_EBX
 		jmp shuffle_and_boot_raw
 
 		section .data
diff --git a/core/rllpack.inc b/core/rllpack.inc
index a556e00a..996486c7 100644
--- a/core/rllpack.inc
+++ b/core/rllpack.inc
@@ -31,19 +31,22 @@
 ;
 ; rllpack:
 ;	Pack CX bytes from SI into EDI.
-;	Returns updated SI and EDI.
+;	Returns updated (E)SI and EDI.
 ;
 rllpack:
 		push word .pmentry
 		call simple_pm_call
 		ret
 
+		bits 32
 .pmentry:
-		push cx
+		push ecx
 		push ebx
 		push edx
+		movzx ecx,cx
+		movzx esi,si
 .startseq:
-		xor ax,ax		; Zero byte
+		xor eax,eax		; Zero byte
 		xor ebx,ebx		; Run length zero
 		dec edi
 		mov edx,edi		; Pointer to header byte
@@ -53,14 +56,14 @@ rllpack:
 		lodsb
 		dec edi
 		mov [edi],al
-		dec cx
+		dec ecx
 		cmp ah,al
 		je .same
 .diff:
 		mov ah,al
-		xor bx,bx
+		xor ebx,ebx
 .plainbyte:
-		inc bx
+		inc ebx
 		inc byte [edx]
 		jcxz .startseq
 		jns .stdbyte
@@ -74,7 +77,7 @@ rllpack:
 		inc edi			; We killed a whole stretch,
 					; drop start byte
 .normal:
-		inc bx
+		inc ebx
 		add edi,ebx		; Remove the stored run bytes
 .getrun:
 		jcxz .nomatch
@@ -83,8 +86,8 @@ rllpack:
 		jne .nomatch
 		cmp bx,(256-224)*256-1	; Maximum run size
 		jae .nomatch
-		inc bx
-		dec cx
+		inc ebx
+		dec ecx
 		jmp .getrun
 .nomatch:
 		cmp bx,224-126
@@ -101,26 +104,31 @@ rllpack:
 .storebyte:
 		dec edi
 		mov [edi],ah
-		dec si			; Reload subsequent byte
+		dec esi			; Reload subsequent byte
 		jmp .startseq
 .done:
 		pop edx
 		pop ebx
-		pop cx
+		pop ecx
 		ret
+
+		bits 16
 ;
 ; rllunpack:
-;	Unpack bytes from ESI into DI
-;	On return ESI, DI are updated and CX contains number of bytes output.
+;	Unpack bytes from SI into EDI
+;	On return (E)SI, EDI are updated and
+;	(E)CX contains number of bytes output.
 ;
 rllunpack:
 		push word .pmentry
 		call simple_pm_call
 		ret
 
+		bits 32
 .pmentry:
-		push di
-		xor cx,cx
+		push edi
+		movzx esi,si
+		xor ecx,ecx
 .header:
 		dec esi
 		mov cl,[esi]
@@ -150,7 +158,9 @@ rllunpack:
 		mov cl,[esi]
 		jmp .dorun
 .done:
-		pop cx
-		sub cx,di
-		neg cx
+		pop ecx
+		sub ecx,edi
+		neg ecx
 		ret
+
+		bits 16
diff --git a/doc/comboot.txt b/doc/comboot.txt
index 96678671..3ebadf33 100644
--- a/doc/comboot.txt
+++ b/doc/comboot.txt
@@ -897,8 +897,9 @@ AX=0023h [3.75] Get shuffler parameters
 AX=0024h [3.75] Cleanup, shuffle and boot, raw version
 	Input:	AX	0024h
 		DX	derivative-specific flags (see function 000Ch)
-		EDI	shuffle descriptor list
-		EBX	pointer to a "safe area" in memory
+		EDI	shuffle descriptor list safe area
+		ESI	shuffle descriptor list source
+		ECX	byte count of shuffle descriptor list
 	Output:	Does not return
 
 	This routine performs final cleanup, then performs a sequence
@@ -911,7 +912,7 @@ AX=0024h [3.75] Cleanup, shuffle and boot, raw version
 	address 7C00h.  Either the shuffle descriptor list or the safe
 	area (or both) may be located in high memory.
 
-	EDI points to a list of descriptors each of the form:
+	ESI points to a list of descriptors each of the form:
 
 		Offset	Size	Meaning
 		 0	dword	destination address
@@ -920,6 +921,13 @@ AX=0024h [3.75] Cleanup, shuffle and boot, raw version
 
 	The copies are overlap-safe, like memmove().
 
+	Before actually executing the move list, the list is moved to
+	the address specified in EDI.  The caller is responsibe to
+	ensure that the moved descriptor list plus a "safe area"
+	immediately afterwards (the size of which is specified by
+	function 0023h) is not disturbed by the copy sequence.  It is,
+	however, safe to overwrite descriptors already consumed.
+
 	If the source address is -1 (FFFFFFFFh) then the block
 	specified by the destination address and the length is set to
 	all zero.
@@ -938,7 +946,3 @@ AX=0024h [3.75] Cleanup, shuffle and boot, raw version
 	point reloads all data segment registers at the earliest
 	possible point.
 
-	It is the responsibility of the caller that neither the
-	descriptor list nor the "safe area" is disturbed by the copy
-	sequence.  It is, however, safe to overwrite descriptors
-	already consumed.