checkpoint. booting second processor. stack is messed up, but thanks to cliff

and plan 9 code, at least boots and gets into C code.

13 files changed, 505 insertions, 151 deletions

diff --git a/Makefile b/Makefile
index 936bc40..c3495c0 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
 OBJS = main.o console.o string.o kalloc.o proc.o trapasm.o trap.o vectors.o \
-       syscall.o ide.o picirq.o mp.o
+       syscall.o ide.o picirq.o mp.o spinlock.o
 
 CC = i386-jos-elf-gcc
 LD = i386-jos-elf-ld
@@ -20,8 +20,12 @@ bootblock : bootasm.S bootmain.c
 	$(OBJCOPY) -S -O binary bootblock.o bootblock
 	./sign.pl bootblock
 
-kernel : $(OBJS)
-	$(LD) -Ttext 0x100000 -e main -o kernel $(OBJS)
+kernel : $(OBJS) bootother.S
+	$(CC) -nostdinc -I. -c bootother.S
+	$(LD) -N -e start -Ttext 0x7000 -o bootother.out bootother.o
+	$(OBJCOPY) -S -O binary bootother.out bootother
+	$(OBJDUMP) -S bootother.o > bootother.asm
+	$(LD) -Ttext 0x100000 -e main -o kernel $(OBJS) -b binary bootother
 	$(OBJDUMP) -S kernel > kernel.asm
 
 vectors.S : vectors.pl
diff --git a/bootasm.S b/bootasm.S
index 00cbdc9..c2a3c3e 100644
--- a/bootasm.S
+++ b/bootasm.S
@@ -1,17 +1,4 @@
-#define SEG_NULL						\
-	.word 0, 0;						\
-	.byte 0, 0, 0, 0
-#define SEG(type,base,lim)					\
-	.word (((lim) >> 12) & 0xffff), ((base) & 0xffff);	\
-	.byte (((base) >> 16) & 0xff), (0x90 | (type)),		\
-		(0xC0 | (((lim) >> 28) & 0xf)), (((base) >> 24) & 0xff)
-
-#define STA_X		0x8	    // Executable segment
-#define STA_E		0x4	    // Expand down (non-executable segments)
-#define STA_C		0x4	    // Conforming code segment (executable only)
-#define STA_W		0x2	    // Writeable (non-executable segments)
-#define STA_R		0x2	    // Readable (executable segments)
-#define STA_A		0x1	    // Accessed
+#include "asm.h"
 	
 .set PROT_MODE_CSEG,0x8		# code segment selector
 .set PROT_MODE_DSEG,0x10        # data segment selector
diff --git a/bootother.S b/bootother.S
new file mode 100644
index 0000000..cba4a5f
--- /dev/null
+++ b/bootother.S
@@ -0,0 +1,77 @@
+#include "asm.h"
+	
+/*
+ * Start an Application Processor. This must be placed on a 4KB boundary
+ * somewhere in the 1st MB of conventional memory (APBOOTSTRAP). However,
+ * due to some shortcuts below it's restricted further to within the 1st
+ * 64KB. The AP starts in real-mode, with
+ *   CS selector set to the startup memory address/16;
+ *   CS base set to startup memory address;
+ *   CS limit set to 64KB;
+ *   CPL and IP set to 0.
+ *	
+ * Credit: Cliff Frey
+ */
+
+.set PROT_MODE_CSEG,0x8		# code segment selector
+.set PROT_MODE_DSEG,0x10        # data segment selector
+.set CR0_PE_ON,0x1		# protected mode enable flag
+
+.globl start
+start:		.code16				# This runs in real mode
+		cli				# Disable interrupts
+		cld				# String operations increment
+
+		# Set up the important data segment registers (DS, ES, SS).
+		xorw	%ax,%ax			# Segment number zero
+		movw	%ax,%ds			# -> Data Segment
+		movw	%ax,%es			# -> Extra Segment
+		movw	%ax,%ss			# -> Stack Segment
+
+		# Set up the stack pointer, growing downward from 0x7000.
+		movw	$start,%sp         	# Stack Pointer
+	
+#### Switch from real to protected mode	
+####     The descriptors in our GDT allow all physical memory to be accessed.
+####     Furthermore, the descriptors have base addresses of 0, so that the
+####     segment translation is a NOP, ie. virtual addresses are identical to
+####     their physical addresses.  With this setup, immediately after
+####	 enabling protected mode it will still appear to this code
+####	 that it is running directly on physical memory with no translation.
+####	 This initial NOP-translation setup is required by the processor
+####	 to ensure that the transition to protected mode occurs smoothly.
+	
+		lgdt	gdtdesc			# load GDT -- mandatory in protected mode
+		movl	%cr0, %eax		# turn on protected mode
+		orl	$CR0_PE_ON, %eax	# 
+		movl	%eax, %cr0		# 
+	        ### CPU magic: jump to relocation, flush prefetch queue, and reload %cs
+		### Has the effect of just jmp to the next instruction, but simultaneous
+		### loads CS with $PROT_MODE_CSEG.
+		ljmp	$PROT_MODE_CSEG, $protcseg
+	
+#### we are in 32-bit protected mode (hence the .code32)
+.code32
+protcseg:	
+		# Set up the protected-mode data segment registers
+		movw	$PROT_MODE_DSEG, %ax	# Our data segment selector
+		movw	%ax, %ds		# -> DS: Data Segment
+		movw	%ax, %es		# -> ES: Extra Segment
+		movw	%ax, %fs		# -> FS
+		movw	%ax, %gs		# -> GS
+		movw	%ax, %ss		# -> SS: Stack Segment
+
+	# XXX hack
+		movl	0x10018, %eax		# elfhdr->entry (left over in scratch space)
+		# subl	$KERNBASE, %eax
+		jmp	*%eax                   # this jumps to _start in kern/entry.S
+	
+.p2align 2					# force 4 byte alignment
+gdt:
+	SEG_NULL				# null seg
+	SEG(STA_X|STA_R, 0x0, 0xffffffff)	# code seg
+	SEG(STA_W, 0x0, 0xffffffff)	        # data seg
+	
+gdtdesc:
+	.word	0x17			# sizeof(gdt) - 1
+	.long	gdt			# address gdt
diff --git a/defs.h b/defs.h
index 4e1e8a5..70394bb 100644
--- a/defs.h
+++ b/defs.h
@@ -22,6 +22,7 @@ void tinit(void);
 void * memcpy(void *dst, void *src, unsigned n);
 void * memset(void *dst, int c, unsigned n);
 int memcmp(const void *v1, const void *v2, unsigned n);
+void *memmove(void *dst, const void *src, unsigned n);
 
 // syscall.c
 void syscall(void);
@@ -31,5 +32,13 @@ void irq_setmask_8259A(uint16_t mask);
 void pic_init(void);
 
 // mp.c
-void mpinit(void);
+void mp_init(void);
+int lapic_cpu_number(void);
+int mp_isbcpu(void);
+
+// spinlock.c
+extern uint32_t kernel_lock;
+void acquire_spinlock(uint32_t* lock);
+void release_spinlock(uint32_t* lock);
+void release_grant_spinlock(uint32_t* lock, int cpu);
 
diff --git a/main.c b/main.c
index 07b3862..0a43b85 100644
--- a/main.c
+++ b/main.c
@@ -8,6 +8,7 @@
 #include "syscall.h"
 
 extern char edata[], end[];
+extern int acpu;
 
 char buf[512];
 
@@ -16,13 +17,19 @@ main()
 {
   struct proc *p;
   int i;
-  
+
+  if (acpu) {
+    cprintf("an application processor\n");
+    release_spinlock(&kernel_lock);
+    while (1) ;
+  }
+  acpu = 1;
   // clear BSS
   memset(edata, 0, end - edata);
 
   cprintf("\nxV6\n\n");
 
-  mpinit(); // multiprocessor
+  mp_init(); // multiprocessor
   kinit(); // physical memory allocator
   tinit(); // traps and interrupts
   pic_init();
diff --git a/mp.c b/mp.c
index 9d47e50..d3db697 100644
--- a/mp.c
+++ b/mp.c
@@ -2,11 +2,201 @@
 #include "mp.h"
 #include "defs.h"
 #include "memlayout.h"
+#include "param.h"
+#include "x86.h"
+#include "mmu.h"
 
-static struct _MP_* _mp_;  /* The MP floating point structure */
+/* 
+ * Credit: Plan 9 sources, Intel MP spec, and Cliff Frey
+ */
+
+enum {					/* Local APIC registers */
+  LAPIC_ID  = 0x0020,	/* ID */
+  LAPIC_VER = 0x0030,	/* Version */
+  LAPIC_TPR = 0x0080,	/* Task Priority */
+  LAPIC_APR = 0x0090,	/* Arbitration Priority */
+  LAPIC_PPR = 0x00A0,	/* Processor Priority */
+  LAPIC_EOI = 0x00B0,	/* EOI */
+  LAPIC_LDR = 0x00D0,	/* Logical Destination */
+  LAPIC_DFR = 0x00E0,	/* Destination Format */
+  LAPIC_SVR = 0x00F0,	/* Spurious Interrupt Vector */
+  LAPIC_ISR = 0x0100,	/* Interrupt Status (8 registers) */
+  LAPIC_TMR = 0x0180,	/* Trigger Mode (8 registers) */
+  LAPIC_IRR = 0x0200,	/* Interrupt Request (8 registers) */
+  LAPIC_ESR = 0x0280,	/* Error Status */
+  LAPIC_ICRLO = 0x0300,	/* Interrupt Command */
+  LAPIC_ICRHI = 0x0310,	/* Interrupt Command [63:32] */
+  LAPIC_TIMER = 0x0320,	/* Local Vector Table 0 (TIMER) */
+  LAPIC_PCINT = 0x0340,	/* Performance Counter LVT */
+  LAPIC_LINT0 = 0x0350,	/* Local Vector Table 1 (LINT0) */
+  LAPIC_LINT1 = 0x0360,	/* Local Vector Table 2 (LINT1) */
+  LAPIC_ERROR = 0x0370,	/* Local Vector Table 3 (ERROR) */
+  LAPIC_TICR = 0x0380,	/* Timer Initial Count */
+  LAPIC_TCCR = 0x0390,	/* Timer Current Count */
+  LAPIC_TDCR = 0x03E0,	/* Timer Divide Configuration */
+};
+
+enum {					/* LAPIC_SVR */
+  LAPIC_ENABLE	= 0x00000100,	/* Unit Enable */
+  LAPIC_FOCUS	= 0x00000200,	/* Focus Processor Checking Disable */
+};
+
+enum {					/* LAPIC_ICRLO */
+					/* [14] IPI Trigger Mode Level (RW) */
+  LAPIC_DEASSERT = 0x00000000,	/* Deassert level-sensitive interrupt */
+  LAPIC_ASSERT	= 0x00004000,	/* Assert level-sensitive interrupt */
+
+  /* [17:16] Remote Read Status */
+  LAPIC_INVALID	= 0x00000000,	/* Invalid */
+  LAPIC_WAIT	= 0x00010000,	/* In-Progress */
+  LAPIC_VALID	= 0x00020000,	/* Valid */
+
+  /* [19:18] Destination Shorthand */
+  LAPIC_FIELD	= 0x00000000,	/* No shorthand */
+  LAPIC_SELF	= 0x00040000,	/* Self is single destination */
+  LAPIC_ALLINC	= 0x00080000,	/* All including self */
+  LAPIC_ALLEXC	= 0x000C0000,	/* All Excluding self */
+};
+
+enum {					/* LAPIC_ESR */
+  LAPIC_SENDCS	= 0x00000001,	/* Send CS Error */
+  LAPIC_RCVCS	= 0x00000002,	/* Receive CS Error */
+  LAPIC_SENDACCEPT = 0x00000004,	/* Send Accept Error */
+  LAPIC_RCVACCEPT = 0x00000008,	/* Receive Accept Error */
+  LAPIC_SENDVECTOR = 0x00000020,	/* Send Illegal Vector */
+  LAPIC_RCVVECTOR = 0x00000040,	/* Receive Illegal Vector */
+  LAPIC_REGISTER = 0x00000080,	/* Illegal Register Address */
+};
+
+enum {					/* LAPIC_TIMER */
+					/* [17] Timer Mode (RW) */
+  LAPIC_ONESHOT	= 0x00000000,	/* One-shot */
+  LAPIC_PERIODIC = 0x00020000,	/* Periodic */
+
+  /* [19:18] Timer Base (RW) */
+  LAPIC_CLKIN	= 0x00000000,	/* use CLKIN as input */
+  LAPIC_TMBASE	= 0x00040000,	/* use TMBASE */
+  LAPIC_DIVIDER	= 0x00080000,	/* use output of the divider */
+};
+
+enum {					/* LAPIC_TDCR */
+  LAPIC_X2 = 0x00000000,	/* divide by 2 */
+  LAPIC_X4 = 0x00000001,	/* divide by 4 */
+  LAPIC_X8 = 0x00000002,	/* divide by 8 */
+  LAPIC_X16 = 0x00000003,	/* divide by 16 */
+  LAPIC_X32 = 0x00000008,	/* divide by 32 */
+  LAPIC_X64 = 0x00000009,	/* divide by 64 */
+  LAPIC_X128 = 0x0000000A,	/* divide by 128 */
+  LAPIC_X1 = 0x0000000B,	/* divide by 1 */
+};
+
+#define APBOOTCODE 0x7000 // XXX hack
+
+static struct MP* mp;  /* The MP floating point structure */
+static uint32_t *lapicaddr;
+static struct cpu {
+  uint8_t apicid;       /* Local APIC ID */
+  int lintr[2];		/* Local APIC */
+} cpu[NCPU];
 static int ncpu;
+static struct cpu *bcpu;
+
+static int
+lapic_read(int r)
+{
+  return *(lapicaddr+(r/sizeof(*lapicaddr)));
+}
+
+static void
+lapic_write(int r, int data)
+{
+  *(lapicaddr+(r/sizeof(*lapicaddr))) = data;
+}
+
+static void
+lapic_init(int c)
+{
+  uint32_t r, lvt;
+
+  cprintf("lapic_init %d\n", c);
+  lapic_write(LAPIC_DFR, 0xFFFFFFFF);
+  r = (lapic_read(LAPIC_ID)>>24) & 0xFF;
+  lapic_write(LAPIC_LDR, (1<<r)<<24);
+  lapic_write(LAPIC_TPR, 0xFF);
+  lapic_write(LAPIC_SVR, LAPIC_ENABLE|(IRQ_OFFSET+IRQ_SPURIOUS));
+
+  /*
+   * Set the local interrupts. It's likely these should just be
+   * masked off for SMP mode as some Pentium Pros have problems if
+   * LINT[01] are set to ExtINT.
+   * Acknowledge any outstanding interrupts.
+   */
+  lapic_write(LAPIC_LINT0, cpu[c].lintr[0]);
+  lapic_write(LAPIC_LINT1, cpu[c].lintr[1]);
+  lapic_write(LAPIC_EOI, 0);
+
+  lvt = (lapic_read(LAPIC_VER)>>16) & 0xFF;
+  if(lvt >= 4)
+    lapic_write(LAPIC_PCINT, APIC_IMASK);
+  lapic_write(LAPIC_ERROR, IRQ_OFFSET+IRQ_ERROR);
+  lapic_write(LAPIC_ESR, 0);
+  lapic_read(LAPIC_ESR);
+
+  /*
+   * Issue an INIT Level De-Assert to synchronise arbitration ID's.
+   */
+  lapic_write(LAPIC_ICRHI, 0);
+  lapic_write(LAPIC_ICRLO, LAPIC_ALLINC|APIC_LEVEL|LAPIC_DEASSERT|APIC_INIT);
+  while(lapic_read(LAPIC_ICRLO) & APIC_DELIVS)
+    ;
 
-static struct _MP_*
+  /*
+   * Do not allow acceptance of interrupts until all initialisation
+   * for this processor is done. For the bootstrap processor this can be
+   * early duing initialisation. For the application processors this should
+   * be after the bootstrap processor has lowered priority and is accepting
+   * interrupts.
+   */
+  lapic_write(LAPIC_TPR, 0);
+  cprintf("Done init of an apic\n");
+}
+
+static void
+lapic_online(void) 
+{
+  lapic_write(LAPIC_TPR, 0);
+}
+
+int
+lapic_cpu_number(void)
+{
+  return (lapic_read(LAPIC_ID)>>24) & 0xFF;
+}
+
+static void
+lapic_startap(struct cpu *c, int v)
+{
+  int crhi, i;
+  volatile int j = 0;
+
+  crhi = c->apicid<<24;
+  lapic_write(LAPIC_ICRHI, crhi);
+  lapic_write(LAPIC_ICRLO, LAPIC_FIELD|APIC_LEVEL|LAPIC_ASSERT|APIC_INIT);
+
+  while (j++ < 10000) {;}
+  lapic_write(LAPIC_ICRLO, LAPIC_FIELD|APIC_LEVEL|LAPIC_DEASSERT|APIC_INIT);
+
+  while (j++ < 1000000) {;}
+
+  // in p9 code, this was i < 2, which is what the spec says on page B-3
+  for(i = 0; i < 1; i++){
+    lapic_write(LAPIC_ICRHI, crhi);
+    lapic_write(LAPIC_ICRLO, LAPIC_FIELD|APIC_EDGE|APIC_STARTUP|(v/PGSIZE));
+    while (j++ < 100000) {;}
+  }
+}
+
+static struct MP*
 mp_scan(uint8_t *addr, int len)
 {
   uint8_t *e, *p, sum;
@@ -14,24 +204,24 @@ mp_scan(uint8_t *addr, int len)
 
   cprintf("scanning: 0x%x\n", (uint32_t)addr);
   e = addr+len;
-  for(p = addr; p < e; p += sizeof(struct _MP_)){
+  for(p = addr; p < e; p += sizeof(struct MP)){
     if(memcmp(p, "_MP_", 4))
       continue;
     sum = 0;
-    for(i = 0; i < sizeof(struct _MP_); i++)
+    for(i = 0; i < sizeof(struct MP); i++)
       sum += p[i];
     if(sum == 0)
-      return (struct _MP_ *)p;
+      return (struct MP *)p;
   }
   return 0;
 }
 
-static struct _MP_*
+static struct MP*
 mp_search(void)
 {
   uint8_t *bda;
   uint32_t p;
-  struct _MP_ *mp;
+  struct MP *mp;
 
   /*
    * Search for the MP Floating Pointer Structure, which according to the
@@ -56,7 +246,7 @@ mp_search(void)
 static int 
 mp_detect(void)
 {
-  struct PCMP *pcmp;
+  struct MPCTB *pcmp;
   uint8_t *p, sum;
   uint32_t length;
 
@@ -67,10 +257,10 @@ mp_detect(void)
    * if correct, check the version.
    * To do: check extended table checksum.
    */
-  if((_mp_ = mp_search()) == 0 || _mp_->physaddr == 0)
+  if((mp = mp_search()) == 0 || mp->physaddr == 0)
     return 1;
 
-  pcmp = KADDR(_mp_->physaddr);
+  pcmp = KADDR(mp->physaddr);
   if(memcmp(pcmp, "PCMP", 4))
     return 2;
 
@@ -82,48 +272,65 @@ mp_detect(void)
   if(sum || (pcmp->version != 1 && pcmp->version != 4))
     return 3;
 
-  cprintf("MP spec rev #: %x\n", _mp_->specrev);
+  cprintf("MP spec rev #: %x\n", mp->specrev);
   return 0;
 }
 
+int
+mp_isbcpu()
+{
+  if (bcpu == 0) return 1;
+  else return 0;
+}
+
 void
-mpinit()
+mp_init()
 { 
   int r;
   uint8_t *p, *e;
-  struct PCMP *pcmp;
+  struct MPCTB *mpctb;
+  struct MPPE *proc;
+  struct cpu *c;
 
   ncpu = 0;
   if ((r = mp_detect()) != 0) return;
-  cprintf ("This computer is multiprocessor!\n");
+
+  cprintf ("This computer is a multiprocessor!\n");
 
   /*
    * Run through the table saving information needed for starting
    * application processors and initialising any I/O APICs. The table
    * is guaranteed to be in order such that only one pass is necessary.
    */
-  pcmp = KADDR(_mp_->physaddr);
-  p = ((uint8_t*)pcmp)+sizeof(struct PCMP);
-  e = ((uint8_t*)pcmp)+pcmp->length;
+  mpctb = KADDR(mp->physaddr);
+  lapicaddr = KADDR(mpctb->lapicaddr);
+  cprintf("apicaddr: %x\n", lapicaddr);
+  p = ((uint8_t*)mpctb)+sizeof(struct MPCTB);
+  e = ((uint8_t*)mpctb)+mpctb->length;
 
   while(p < e) {
     switch(*p){
-    case PcmpPROCESSOR:
-      cprintf("a processor\n");
+    case MPPROCESSOR:
+      proc = (struct MPPE *) p;
+      cpu[ncpu].apicid = proc->apicid;
+      cpu[ncpu].lintr[0] = APIC_IMASK;
+      cpu[ncpu].lintr[1] = APIC_IMASK;
+      cprintf("a processor %x\n", cpu[ncpu].apicid);
+      if (proc->flags & MPBP) {
+	bcpu = &cpu[ncpu];
+      }
       ncpu++;
-      p += sizeof(struct PCMPprocessor);
+      p += sizeof(struct MPPE);
       continue;
-    case PcmpBUS:
-      cprintf("a bus\n");
-      p += sizeof(struct PCMPbus);
+    case MPBUS:
+      p += sizeof(struct MPBE);
       continue;
-    case PcmpIOAPIC:
-      cprintf("an IO APIC\n");
-      p += sizeof(struct PCMPioapic);
+    case MPIOAPIC:
+      cprintf("an I/O APIC\n");
+      p += sizeof(struct MPIOAPIC);
       continue;
-    case PcmpIOINTR:
-      cprintf("an IO interrupt assignment\n");
-      p += sizeof(struct PCMPintr);
+    case MPIOINTR:
+      p += sizeof(struct MPIE);
       continue;
     default:
       cprintf("mpinit: unknown PCMP type 0x%x (e-p 0x%x)\n", *p, e-p);
@@ -134,6 +341,23 @@ mpinit()
       break;
     }
   }
+  
+  lapic_init(cpu-bcpu);
+  cprintf("ncpu: %d boot %d\n", ncpu, cpu-bcpu);
 
-  cprintf("ncpu: %d\n", ncpu);
+  lapic_online();
+
+  extern uint8_t _binary_bootother_start[], _binary_bootother_size[];
+  memmove(KADDR(APBOOTCODE),_binary_bootother_start, 
+	  (uint32_t) _binary_bootother_size);
+
+  acquire_spinlock(&kernel_lock);
+  for (c = cpu; c < &cpu[ncpu]; c++) {
+    if (c == bcpu) continue;
+    cprintf ("starting processor %d\n", c - cpu);
+    release_grant_spinlock(&kernel_lock, c - cpu);
+    lapic_startap(c, (uint32_t) KADDR(APBOOTCODE));
+    acquire_spinlock(&kernel_lock);
+    cprintf ("done starting processor %d\n", c - cpu);
+  }
 }
diff --git a/mp.h b/mp.h
index df46574..21d19c5 100644
--- a/mp.h
+++ b/mp.h
@@ -1,7 +1,10 @@
 /*
  * MultiProcessor Specification Version 1.[14].
+ *
+ * Credit: Plan 9 sources
  */
-struct _MP_ {			/* floating pointer */
+
+struct MP {			/* floating pointer */
   uint8_t signature[4];		/* "_MP_" */
   physaddr_t physaddr;	        /* physical address of MP configuration table */
   uint8_t length;		/* 1 */
@@ -12,7 +15,7 @@ struct _MP_ {			/* floating pointer */
   uint8_t reserved[3];
 };
 
-struct PCMP {			/* configuration table header */
+struct MPCTB {			/* configuration table header */
   uint8_t signature[4];		/* "PCMP" */
   uint16_t length;		/* total table length */
   uint8_t version;		/* [14] */
@@ -21,15 +24,15 @@ struct PCMP {			/* configuration table header */
   uintptr_t oemtable;		/* OEM table pointer */
   uint16_t oemlength;		/* OEM table length */
   uint16_t entry;		/* entry count */
-  uintptr_t lapicbase;		/* address of local APIC */
+  uintptr_t lapicaddr;		/* address of local APIC */
   uint16_t xlength;		/* extended table length */
   uint8_t xchecksum;		/* extended table checksum */
   uint8_t reserved;
 };
 
-struct PCMPprocessor {		/* processor table entry */
+struct MPPE {		/* processor table entry */
   uint8_t type;			/* entry type (0) */
-  uint8_t apicno;		/* local APIC id */
+  uint8_t apicid;		/* local APIC id */
   uint8_t version;		/* local APIC verison */
   uint8_t flags;		/* CPU flags */
   uint8_t signature[4];		/* CPU signature */
@@ -37,13 +40,13 @@ struct PCMPprocessor {		/* processor table entry */
   uint8_t reserved[8];
 };
 
-struct PCMPbus {		/* bus table entry */
+struct MPBE {		/* bus table entry */
   uint8_t type;			/* entry type (1) */
   uint8_t busno;		/* bus id */
   char string[6];		/* bus type string */
 };
 
-struct PCMPioapic {		/* I/O APIC table entry */
+struct MPIOAPIC {	/* I/O APIC table entry */
   uint8_t type;			/* entry type (2) */
   uint8_t apicno;		/* I/O APIC id */
   uint8_t version;		/* I/O APIC version */
@@ -51,7 +54,7 @@ struct PCMPioapic {		/* I/O APIC table entry */
   uintptr_t addr;		/* I/O APIC address */
 };
 
-struct PCMPintr {		/* interrupt table entry */
+struct MPIE {		/* interrupt table entry */
   uint8_t type;			/* entry type ([34]) */
   uint8_t intr;			/* interrupt type */
   uint16_t flags;		/* interrupt flag */
@@ -61,71 +64,34 @@ struct PCMPintr {		/* interrupt table entry */
   uint8_t intin;		/* destination APIC [L]INTIN# */
 };
 
-struct PCMPsasm {		/* system address space mapping entry */
-  uint8_t type;			/* entry type (128) */
-  uint8_t length;		/* of this entry (20) */
-  uint8_t busno;		/* bus id */
-  uint8_t addrtype;
-  uintptr_t addrbase[2];
-  uint32_t addrlength[2];
-};
-
-struct PCMPhierarchy {		/* bus hierarchy descriptor entry */
-  uint8_t type;			/* entry type (129) */
-  uint8_t length;		/* of this entry (8) */
-  uint8_t busno;		/* bus id */
-  uint8_t info;			/* bus info */
-  uint8_t parent;		/* parent bus */
-  uint8_t reserved[3];
-};
-
-struct PCMPcbasm {		/* compatibility bus address space modifier entry */
-  uint8_t type;			/* entry type (130) */
-  uint8_t length;		/* of this entry (8) */
-  uint8_t busno;		/* bus id */
-  uint8_t modifier;		/* address modifier */
-  uint32_t range;		/* predefined range list */
-};
-
-enum {				/* table entry types */
-  PcmpPROCESSOR	= 0x00,		/* one entry per processor */
-  PcmpBUS = 0x01,		/* one entry per bus */
-  PcmpIOAPIC = 0x02,		/* one entry per I/O APIC */
-  PcmpIOINTR = 0x03,		/* one entry per bus interrupt source */
-  PcmpLINTR = 0x04,		/* one entry per system interrupt source */
-
-  PcmpSASM = 0x80,
-  PcmpHIERARCHY	= 0x81,
-  PcmpCBASM = 0x82,
-
-  /* PCMPprocessor and PCMPioapic flags */
-  PcmpEN = 0x01,		/* enabled */
-  PcmpBP = 0x02,		/* bootstrap processor */
-
-				/* PCMPiointr and PCMPlintr flags */
-  PcmpPOMASK = 0x03,		/* polarity conforms to specifications of bus */
-  PcmpHIGH = 0x01,		/* active high */
-  PcmpLOW = 0x03,		/* active low */
-  PcmpELMASK = 0x0C,		/* trigger mode of APIC input signals */
-  PcmpEDGE = 0x04,		/* edge-triggered */
-  PcmpLEVEL = 0x0C,		/* level-triggered */
-
-  /* PCMPiointr and PCMPlintr interrupt type */
-  PcmpINT = 0x00,		/* vectored interrupt from APIC Rdt */
-  PcmpNMI = 0x01,		/* non-maskable interrupt */
-  PcmpSMI = 0x02,		/* system management interrupt */
-  PcmpExtINT = 0x03,		/* vectored interrupt from external PIC */
-
-  /* PCMPsasm addrtype */
-  PcmpIOADDR = 0x00,		/* I/O address */
-  PcmpMADDR = 0x01,		/* memory address */
-  PcmpPADDR = 0x02,		/* prefetch address */
-
-  /* PCMPhierarchy info */
-  PcmpSD = 0x01,		/* subtractive decode bus */
-
-				/* PCMPcbasm modifier */
-  PcmpPR = 0x01,		/* predefined range list */
+enum {			/* table entry types */
+  MPPROCESSOR	= 0x00,		/* one entry per processor */
+  MPBUS = 0x01,		        /* one entry per bus */
+  MPIOAPIC = 0x02,		/* one entry per I/O APIC */
+  MPIOINTR = 0x03,		/* one entry per bus interrupt source */
+  MPLINTR = 0x04,		/* one entry per system interrupt source */
+
+  MPSASM = 0x80,
+  MPHIERARCHY	= 0x81,
+  MPCBASM = 0x82,
+
+                        /* PCMPprocessor and PCMPioapic flags */
+  MPEN = 0x01,		        /* enabled */
+  MPBP = 0x02,		        /* bootstrap processor */
+
+			/* PCMPiointr and PCMPlintr flags */
+  MPPOMASK = 0x03,		/* polarity conforms to specifications of bus */
+  MPHIGH = 0x01,		/* active high */
+  MPLOW = 0x03,		/* active low */
+  MPELMASK = 0x0C,		/* trigger mode of APIC input signals */
+  MPEDGE = 0x04,		/* edge-triggered */
+  MPLEVEL = 0x0C,		/* level-triggered */
+
+                        /* PCMPiointr and PCMPlintr interrupt type */
+  MPINT = 0x00,		        /* vectored interrupt from APIC Rdt */
+  MPNMI = 0x01,		        /* non-maskable interrupt */
+  MPSMI = 0x02,		        /* system management interrupt */
+  MPExtINT = 0x03,		/* vectored interrupt from external PIC */
 };
 
 /*
@@ -136,23 +102,23 @@ enum {				/* table entry types */
  *	Local APIC Timer Vector Table.
  */
 enum {
-  ApicFIXED = 0x00000000,	/* [10:8] Delivery Mode */
-  ApicLOWEST = 0x00000100,	/* Lowest priority */
-  ApicSMI = 0x00000200,	        /* System Management Interrupt */
-  ApicRR = 0x00000300,	        /* Remote Read */
-  ApicNMI = 0x00000400,
-  ApicINIT = 0x00000500,	/* INIT/RESET */
-  ApicSTARTUP = 0x00000600,	/* Startup IPI */
-  ApicExtINT = 0x00000700,
-
-  ApicPHYSICAL = 0x00000000,	/* [11] Destination Mode (RW) */
-  ApicLOGICAL = 0x00000800,
-
-  ApicDELIVS = 0x00001000,	/* [12] Delivery Status (RO) */
-  ApicHIGH = 0x00000000,	/* [13] Interrupt Input Pin Polarity (RW) */
-  ApicLOW = 0x00002000,
-  ApicRemoteIRR	= 0x00004000,	/* [14] Remote IRR (RO) */
-  ApicEDGE = 0x00000000,	/* [15] Trigger Mode (RW) */
-  ApicLEVEL = 0x00008000,
-  ApicIMASK = 0x00010000,	/* [16] Interrupt Mask */
+  APIC_FIXED = 0x00000000,	/* [10:8] Delivery Mode */
+  APIC_LOWEST = 0x00000100,	/* Lowest priority */
+  APIC_SMI = 0x00000200,	        /* System Management Interrupt */
+  APIC_RR = 0x00000300,	        /* Remote Read */
+  APIC_NMI = 0x00000400,
+  APIC_INIT = 0x00000500,	/* INIT/RESET */
+  APIC_STARTUP = 0x00000600,	/* Startup IPI */
+  APIC_ExtINT = 0x00000700,
+
+  APIC_PHYSICAL = 0x00000000,	/* [11] Destination Mode (RW) */
+  APIC_LOGICAL = 0x00000800,
+
+  APIC_DELIVS = 0x00001000,	/* [12] Delivery Status (RO) */
+  APIC_HIGH = 0x00000000,	/* [13] Interrupt Input Pin Polarity (RW) */
+  APIC_LOW = 0x00002000,
+  APIC_RemoteIRR	= 0x00004000,	/* [14] Remote IRR (RO) */
+  APIC_EDGE = 0x00000000,	/* [15] Trigger Mode (RW) */
+  APIC_LEVEL = 0x00008000,
+  APIC_IMASK = 0x00010000,	/* [16] Interrupt Mask */
 };
diff --git a/param.h b/param.h
index 798dc5b..0a931d9 100644
--- a/param.h
+++ b/param.h
@@ -1,3 +1,4 @@
 #define NPROC 64
 #define PAGE 4096
 #define KSTACKSIZE PAGE
+#define NCPU 8
diff --git a/picirq.c b/picirq.c
index 5fc90c5..ba131a3 100644
--- a/picirq.c
+++ b/picirq.c
@@ -4,15 +4,6 @@
 #include "x86.h"
 #include "defs.h"
 
-#define MAX_IRQS	16	// Number of IRQs
-
-// I/O Addresses of the two 8259A programmable interrupt controllers
-#define IO_PIC1		0x20	// Master (IRQs 0-7)
-#define IO_PIC2		0xA0	// Slave (IRQs 8-15)
-
-#define IRQ_SLAVE	2	// IRQ at which slave connects to master
-#define IRQ_OFFSET	32	// IRQ 0 corresponds to int IRQ_OFFSET
-
 // Current IRQ mask.
 // Initial IRQ mask has interrupt 2 enabled (for slave 8259A).
 uint16_t irq_mask_8259A = 0xFFFF & ~(1<<IRQ_SLAVE);
diff --git a/spinlock.c b/spinlock.c
new file mode 100644
index 0000000..fa04a27
--- /dev/null
+++ b/spinlock.c
@@ -0,0 +1,39 @@
+#include "types.h"
+#include "defs.h"
+#include "x86.h"
+
+#define LOCK_FREE -1
+
+uint32_t kernel_lock = LOCK_FREE;
+
+// lock = LOCK_FREE if free, else = cpu_id of owner CPU
+void 
+acquire_spinlock(uint32_t* lock)
+{
+  int cpu_id = lapic_cpu_number();
+  cprintf ("acquire: %d\n", cpu_id);
+
+  if (*lock == cpu_id)
+    return;
+  while ( cmpxchg(LOCK_FREE, cpu_id, lock) != cpu_id ) { ; }
+}
+
+void
+release_spinlock(uint32_t* lock)
+{
+  int cpu_id = lapic_cpu_number();
+  cprintf ("release: %d\n", cpu_id);
+  if (*lock != cpu_id)
+    panic("release_spinlock: releasing a lock that i don't own\n");
+  *lock = LOCK_FREE;
+}
+
+void
+release_grant_spinlock(uint32_t* lock, int c)
+{
+  int cpu_id = lapic_cpu_number();
+  cprintf ("release_grant: %d -> %d\n", cpu_id, c);
+  if (*lock != cpu_id)
+    panic("release_spinlock: releasing a lock that i don't own\n");
+  *lock = c;
+}
diff --git a/string.c b/string.c
index 40019d5..54f4ba8 100644
--- a/string.c
+++ b/string.c
@@ -38,3 +38,23 @@ memcmp(const void *v1, const void *v2, unsigned n)
 
   return 0;
 }
+
+void *
+memmove(void *dst, const void *src, unsigned n)
+{
+  const char *s;
+  char *d;
+	
+  s = src;
+  d = dst;
+  if (s < d && s + n > d) {
+    s += n;
+    d += n;
+    while (n-- > 0)
+      *--d = *--s;
+  } else
+    while (n-- > 0)
+      *d++ = *s++;
+
+  return dst;
+}
diff --git a/trapasm.S b/trapasm.S
index c01cec0..2608328 100644
--- a/trapasm.S
+++ b/trapasm.S
@@ -27,3 +27,8 @@ trapret:
         popl %ds
         addl $0x8, %esp /* trapno and errcode */
         iret
+
+		
+.globl 	acpu
+acpu:
+	.long 0
diff --git a/x86.h b/x86.h
index 134c6d2..b905f83 100644
--- a/x86.h
+++ b/x86.h
@@ -261,6 +261,17 @@ cpuid(uint32_t info, uint32_t *eaxp, uint32_t *ebxp, uint32_t *ecxp, uint32_t *e
 		*edxp = edx;
 }
 
+static __inline uint32_t
+cmpxchg(uint32_t oldval, uint32_t newval, volatile uint32_t* lock_addr)
+{
+  uint32_t result;
+  __asm__ __volatile__(
+		       "lock; cmpxchgl %2, %0"
+		       :"+m" (*lock_addr), "=a" (result) : "r"(newval), "1"(oldval) : "cc"
+		       );
+  return result;
+}
+
 static __inline uint64_t
 read_tsc(void)
 {
@@ -299,3 +310,16 @@ struct Trapframe {
     uint16_t tf_ss;
     uint16_t tf_padding4;
 };
+
+
+#define MAX_IRQS	16	// Number of IRQs
+
+// I/O Addresses of the two 8259A programmable interrupt controllers
+#define IO_PIC1		0x20	// Master (IRQs 0-7)
+#define IO_PIC2	        0xA0	// Slave (IRQs 8-15)
+
+#define IRQ_SLAVE       2	// IRQ at which slave connects to master
+#define IRQ_OFFSET      32	// IRQ 0 corresponds to int IRQ_OFFSET
+
+#define IRQ_ERROR       19
+#define IRQ_SPURIOUS    31