start at support for multiple CPUs

21 files changed, 87 insertions, 837 deletions

diff --git a/.gdbinit.tmpl-i386 b/.gdbinit.tmpl-i386
index f4f85d2..a3a274b 100644
--- a/.gdbinit.tmpl-i386
+++ b/.gdbinit.tmpl-i386
@@ -1,3 +1,4 @@
+set confirm off
 python
 gdb.execute("target remote localhost:26000")
 gdb.execute("set architecture i386")
diff --git a/.gdbinit.tmpl-riscv b/.gdbinit.tmpl-riscv
index 6ea36e1..c1616b6 100644
--- a/.gdbinit.tmpl-riscv
+++ b/.gdbinit.tmpl-riscv
@@ -1,3 +1,4 @@
+set confirm off
 set architecture riscv
 target remote 127.0.0.1:1234
 symbol-file kernel
diff --git a/Makefile b/Makefile
index 84355e0..38ec5c7 100644
--- a/Makefile
+++ b/Makefile
@@ -187,13 +187,13 @@ endif
 QEMUOPTS = -machine virt -kernel kernel -m 3G -smp $(CPUS) -nographic
 QEMUOPTS += -initrd fs.img
 
-qemu: kernel
+qemu: kernel fs.img
 	$(QEMU) $(QEMUOPTS)
 
 .gdbinit: .gdbinit.tmpl-riscv
 	sed "s/:1234/:$(GDBPORT)/" < $^ > $@
 
-qemu-gdb: kernel .gdbinit
+qemu-gdb: kernel .gdbinit fs.img
 	@echo "*** Now run 'gdb'." 1>&2
 	$(QEMU) $(QEMUOPTS) -S $(QEMUGDB)
 
diff --git a/asm.h b/asm.h
deleted file mode 100644
index b8a7353..0000000
--- a/asm.h
+++ /dev/null
@@ -1,18 +0,0 @@
-//
-// assembler macros to create x86 segments
-//
-
-#define SEG_NULLASM                                             \
-        .word 0, 0;                                             \
-        .byte 0, 0, 0, 0
-
-// The 0xC0 means the limit is in 4096-byte units
-// and (for executable segments) 32-bit mode.
-#define SEG_ASM(type,base,lim)                                  \
-        .word (((lim) >> 12) & 0xffff), ((base) & 0xffff);      \
-        .byte (((base) >> 16) & 0xff), (0x90 | (type)),         \
-                (0xC0 | (((lim) >> 28) & 0xf)), (((base) >> 24) & 0xff)
-
-#define STA_X     0x8       // Executable segment
-#define STA_W     0x2       // Writeable (non-executable segments)
-#define STA_R     0x2       // Readable (executable segments)
diff --git a/bootmain.c b/bootmain.c
deleted file mode 100644
index 1f20e5b..0000000
--- a/bootmain.c
+++ /dev/null
@@ -1,96 +0,0 @@
-// Boot loader.
-//
-// Part of the boot block, along with bootasm.S, which calls bootmain().
-// bootasm.S has put the processor into protected 32-bit mode.
-// bootmain() loads an ELF kernel image from the disk starting at
-// sector 1 and then jumps to the kernel entry routine.
-
-#include "types.h"
-#include "elf.h"
-#include "x86.h"
-#include "memlayout.h"
-
-#define SECTSIZE  512
-
-void readseg(uchar*, uint, uint);
-
-void
-bootmain(void)
-{
-  struct elfhdr *elf;
-  struct proghdr *ph, *eph;
-  void (*entry)(void);
-  uchar* pa;
-
-  elf = (struct elfhdr*)0x10000;  // scratch space
-
-  // Read 1st page off disk
-  readseg((uchar*)elf, 4096, 0);
-
-  // Is this an ELF executable?
-  if(elf->magic != ELF_MAGIC)
-    return;  // let bootasm.S handle error
-
-  // Load each program segment (ignores ph flags).
-  ph = (struct proghdr*)((uchar*)elf + elf->phoff);
-  eph = ph + elf->phnum;
-  for(; ph < eph; ph++){
-    pa = (uchar*)ph->paddr;
-    readseg(pa, ph->filesz, ph->off);
-    if(ph->memsz > ph->filesz)
-      stosb(pa + ph->filesz, 0, ph->memsz - ph->filesz);
-  }
-
-  // Call the entry point from the ELF header.
-  // Does not return!
-  entry = (void(*)(void))(elf->entry);
-  entry();
-}
-
-void
-waitdisk(void)
-{
-  // Wait for disk ready.
-  while((inb(0x1F7) & 0xC0) != 0x40)
-    ;
-}
-
-// Read a single sector at offset into dst.
-void
-readsect(void *dst, uint offset)
-{
-  // Issue command.
-  waitdisk();
-  outb(0x1F2, 1);   // count = 1
-  outb(0x1F3, offset);
-  outb(0x1F4, offset >> 8);
-  outb(0x1F5, offset >> 16);
-  outb(0x1F6, (offset >> 24) | 0xE0);
-  outb(0x1F7, 0x20);  // cmd 0x20 - read sectors
-
-  // Read data.
-  waitdisk();
-  insl(0x1F0, dst, SECTSIZE/4);
-}
-
-// Read 'count' bytes at 'offset' from kernel into physical address 'pa'.
-// Might copy more than asked.
-void
-readseg(uchar* pa, uint count, uint offset)
-{
-  uchar* epa;
-
-  epa = pa + count;
-
-  // Round down to sector boundary.
-  pa -= offset % SECTSIZE;
-
-  // Translate from bytes to sectors; kernel starts at sector 1.
-  offset = (offset / SECTSIZE) + 1;
-
-  // If this is too slow, we could read lots of sectors at a time.
-  // We'd write more to memory than asked, but it doesn't matter --
-  // we load in increasing order.
-  for(; pa < epa; pa += SECTSIZE, offset++)
-    readsect(pa, offset);
-}
diff --git a/entry.S b/entry.S
index 8b3316c..b3d2c55 100644
--- a/entry.S
+++ b/entry.S
@@ -10,12 +10,15 @@
 .section .text
 .globl _entry
 _entry:
-	# set up a stack for C; stack0 is declared in start.
+	# set up a stack for C.
+        # stack0 is declared in start,
+        # with 4096 bytes per CPU.
         la sp, stack0
-        addi sp, sp, 1024
-        addi sp, sp, 1024
-        addi sp, sp, 1024
-        addi sp, sp, 1024
+        li a0, 1024*4
+	csrr a1, mhartid
+        addi a1, a1, 1
+        mul a0, a0, a1
+        add sp, sp, a0
 	# jump to mstart() in start.c
         call mstart
 junk:
diff --git a/kbd.c b/kbd.c
deleted file mode 100644
index 32c1463..0000000
--- a/kbd.c
+++ /dev/null
@@ -1,50 +0,0 @@
-#include "types.h"
-#include "x86.h"
-#include "defs.h"
-#include "kbd.h"
-
-int
-kbdgetc(void)
-{
-  static uint shift;
-  static uchar *charcode[4] = {
-    normalmap, shiftmap, ctlmap, ctlmap
-  };
-  uint st, data, c;
-
-  st = inb(KBSTATP);
-  if((st & KBS_DIB) == 0)
-    return -1;
-  data = inb(KBDATAP);
-
-  if(data == 0xE0){
-    shift |= E0ESC;
-    return 0;
-  } else if(data & 0x80){
-    // Key released
-    data = (shift & E0ESC ? data : data & 0x7F);
-    shift &= ~(shiftcode[data] | E0ESC);
-    return 0;
-  } else if(shift & E0ESC){
-    // Last character was an E0 escape; or with 0x80
-    data |= 0x80;
-    shift &= ~E0ESC;
-  }
-
-  shift |= shiftcode[data];
-  shift ^= togglecode[data];
-  c = charcode[shift & (CTL | SHIFT)][data];
-  if(shift & CAPSLOCK){
-    if('a' <= c && c <= 'z')
-      c += 'A' - 'a';
-    else if('A' <= c && c <= 'Z')
-      c += 'a' - 'A';
-  }
-  return c;
-}
-
-void
-kbdintr(void)
-{
-  consoleintr(kbdgetc);
-}
diff --git a/kbd.h b/kbd.h
deleted file mode 100644
index babbd6e..0000000
--- a/kbd.h
+++ /dev/null
@@ -1,112 +0,0 @@
-// PC keyboard interface constants
-
-#define KBSTATP         0x64    // kbd controller status port(I)
-#define KBS_DIB         0x01    // kbd data in buffer
-#define KBDATAP         0x60    // kbd data port(I)
-
-#define NO              0
-
-#define SHIFT           (1<<0)
-#define CTL             (1<<1)
-#define ALT             (1<<2)
-
-#define CAPSLOCK        (1<<3)
-#define NUMLOCK         (1<<4)
-#define SCROLLLOCK      (1<<5)
-
-#define E0ESC           (1<<6)
-
-// Special keycodes
-#define KEY_HOME        0xE0
-#define KEY_END         0xE1
-#define KEY_UP          0xE2
-#define KEY_DN          0xE3
-#define KEY_LF          0xE4
-#define KEY_RT          0xE5
-#define KEY_PGUP        0xE6
-#define KEY_PGDN        0xE7
-#define KEY_INS         0xE8
-#define KEY_DEL         0xE9
-
-// C('A') == Control-A
-#define C(x) (x - '@')
-
-static uchar shiftcode[256] =
-{
-  [0x1D] CTL,
-  [0x2A] SHIFT,
-  [0x36] SHIFT,
-  [0x38] ALT,
-  [0x9D] CTL,
-  [0xB8] ALT
-};
-
-static uchar togglecode[256] =
-{
-  [0x3A] CAPSLOCK,
-  [0x45] NUMLOCK,
-  [0x46] SCROLLLOCK
-};
-
-static uchar normalmap[256] =
-{
-  NO,   0x1B, '1',  '2',  '3',  '4',  '5',  '6',  // 0x00
-  '7',  '8',  '9',  '0',  '-',  '=',  '\b', '\t',
-  'q',  'w',  'e',  'r',  't',  'y',  'u',  'i',  // 0x10
-  'o',  'p',  '[',  ']',  '\n', NO,   'a',  's',
-  'd',  'f',  'g',  'h',  'j',  'k',  'l',  ';',  // 0x20
-  '\'', '`',  NO,   '\\', 'z',  'x',  'c',  'v',
-  'b',  'n',  'm',  ',',  '.',  '/',  NO,   '*',  // 0x30
-  NO,   ' ',  NO,   NO,   NO,   NO,   NO,   NO,
-  NO,   NO,   NO,   NO,   NO,   NO,   NO,   '7',  // 0x40
-  '8',  '9',  '-',  '4',  '5',  '6',  '+',  '1',
-  '2',  '3',  '0',  '.',  NO,   NO,   NO,   NO,   // 0x50
-  [0x9C] '\n',      // KP_Enter
-  [0xB5] '/',       // KP_Div
-  [0xC8] KEY_UP,    [0xD0] KEY_DN,
-  [0xC9] KEY_PGUP,  [0xD1] KEY_PGDN,
-  [0xCB] KEY_LF,    [0xCD] KEY_RT,
-  [0x97] KEY_HOME,  [0xCF] KEY_END,
-  [0xD2] KEY_INS,   [0xD3] KEY_DEL
-};
-
-static uchar shiftmap[256] =
-{
-  NO,   033,  '!',  '@',  '#',  '$',  '%',  '^',  // 0x00
-  '&',  '*',  '(',  ')',  '_',  '+',  '\b', '\t',
-  'Q',  'W',  'E',  'R',  'T',  'Y',  'U',  'I',  // 0x10
-  'O',  'P',  '{',  '}',  '\n', NO,   'A',  'S',
-  'D',  'F',  'G',  'H',  'J',  'K',  'L',  ':',  // 0x20
-  '"',  '~',  NO,   '|',  'Z',  'X',  'C',  'V',
-  'B',  'N',  'M',  '<',  '>',  '?',  NO,   '*',  // 0x30
-  NO,   ' ',  NO,   NO,   NO,   NO,   NO,   NO,
-  NO,   NO,   NO,   NO,   NO,   NO,   NO,   '7',  // 0x40
-  '8',  '9',  '-',  '4',  '5',  '6',  '+',  '1',
-  '2',  '3',  '0',  '.',  NO,   NO,   NO,   NO,   // 0x50
-  [0x9C] '\n',      // KP_Enter
-  [0xB5] '/',       // KP_Div
-  [0xC8] KEY_UP,    [0xD0] KEY_DN,
-  [0xC9] KEY_PGUP,  [0xD1] KEY_PGDN,
-  [0xCB] KEY_LF,    [0xCD] KEY_RT,
-  [0x97] KEY_HOME,  [0xCF] KEY_END,
-  [0xD2] KEY_INS,   [0xD3] KEY_DEL
-};
-
-static uchar ctlmap[256] =
-{
-  NO,      NO,      NO,      NO,      NO,      NO,      NO,      NO,
-  NO,      NO,      NO,      NO,      NO,      NO,      NO,      NO,
-  C('Q'),  C('W'),  C('E'),  C('R'),  C('T'),  C('Y'),  C('U'),  C('I'),
-  C('O'),  C('P'),  NO,      NO,      '\r',    NO,      C('A'),  C('S'),
-  C('D'),  C('F'),  C('G'),  C('H'),  C('J'),  C('K'),  C('L'),  NO,
-  NO,      NO,      NO,      C('\\'), C('Z'),  C('X'),  C('C'),  C('V'),
-  C('B'),  C('N'),  C('M'),  NO,      NO,      C('/'),  NO,      NO,
-  [0x9C] '\r',      // KP_Enter
-  [0xB5] C('/'),    // KP_Div
-  [0xC8] KEY_UP,    [0xD0] KEY_DN,
-  [0xC9] KEY_PGUP,  [0xD1] KEY_PGDN,
-  [0xCB] KEY_LF,    [0xCD] KEY_RT,
-  [0x97] KEY_HOME,  [0xCF] KEY_END,
-  [0xD2] KEY_INS,   [0xD3] KEY_DEL
-};
-
diff --git a/lapic.c b/lapic.c
deleted file mode 100644
index b22bbd7..0000000
--- a/lapic.c
+++ /dev/null
@@ -1,229 +0,0 @@
-// The local APIC manages internal (non-I/O) interrupts.
-// See Chapter 8 & Appendix C of Intel processor manual volume 3.
-
-#include "param.h"
-#include "types.h"
-#include "defs.h"
-#include "date.h"
-#include "memlayout.h"
-#include "traps.h"
-#include "mmu.h"
-#include "x86.h"
-
-// Local APIC registers, divided by 4 for use as uint[] indices.
-#define ID      (0x0020/4)   // ID
-#define VER     (0x0030/4)   // Version
-#define TPR     (0x0080/4)   // Task Priority
-#define EOI     (0x00B0/4)   // EOI
-#define SVR     (0x00F0/4)   // Spurious Interrupt Vector
-  #define ENABLE     0x00000100   // Unit Enable
-#define ESR     (0x0280/4)   // Error Status
-#define ICRLO   (0x0300/4)   // Interrupt Command
-  #define INIT       0x00000500   // INIT/RESET
-  #define STARTUP    0x00000600   // Startup IPI
-  #define DELIVS     0x00001000   // Delivery status
-  #define ASSERT     0x00004000   // Assert interrupt (vs deassert)
-  #define DEASSERT   0x00000000
-  #define LEVEL      0x00008000   // Level triggered
-  #define BCAST      0x00080000   // Send to all APICs, including self.
-  #define BUSY       0x00001000
-  #define FIXED      0x00000000
-#define ICRHI   (0x0310/4)   // Interrupt Command [63:32]
-#define TIMER   (0x0320/4)   // Local Vector Table 0 (TIMER)
-  #define X1         0x0000000B   // divide counts by 1
-  #define PERIODIC   0x00020000   // Periodic
-#define PCINT   (0x0340/4)   // Performance Counter LVT
-#define LINT0   (0x0350/4)   // Local Vector Table 1 (LINT0)
-#define LINT1   (0x0360/4)   // Local Vector Table 2 (LINT1)
-#define ERROR   (0x0370/4)   // Local Vector Table 3 (ERROR)
-  #define MASKED     0x00010000   // Interrupt masked
-#define TICR    (0x0380/4)   // Timer Initial Count
-#define TCCR    (0x0390/4)   // Timer Current Count
-#define TDCR    (0x03E0/4)   // Timer Divide Configuration
-
-volatile uint *lapic;  // Initialized in mp.c
-
-//PAGEBREAK!
-static void
-lapicw(int index, int value)
-{
-  lapic[index] = value;
-  lapic[ID];  // wait for write to finish, by reading
-}
-
-void
-lapicinit(void)
-{
-  if(!lapic)
-    return;
-
-  // Enable local APIC; set spurious interrupt vector.
-  lapicw(SVR, ENABLE | (T_IRQ0 + IRQ_SPURIOUS));
-
-  // The timer repeatedly counts down at bus frequency
-  // from lapic[TICR] and then issues an interrupt.
-  // If xv6 cared more about precise timekeeping,
-  // TICR would be calibrated using an external time source.
-  lapicw(TDCR, X1);
-  lapicw(TIMER, PERIODIC | (T_IRQ0 + IRQ_TIMER));
-  lapicw(TICR, 10000000);
-
-  // Disable logical interrupt lines.
-  lapicw(LINT0, MASKED);
-  lapicw(LINT1, MASKED);
-
-  // Disable performance counter overflow interrupts
-  // on machines that provide that interrupt entry.
-  if(((lapic[VER]>>16) & 0xFF) >= 4)
-    lapicw(PCINT, MASKED);
-
-  // Map error interrupt to IRQ_ERROR.
-  lapicw(ERROR, T_IRQ0 + IRQ_ERROR);
-
-  // Clear error status register (requires back-to-back writes).
-  lapicw(ESR, 0);
-  lapicw(ESR, 0);
-
-  // Ack any outstanding interrupts.
-  lapicw(EOI, 0);
-
-  // Send an Init Level De-Assert to synchronise arbitration ID's.
-  lapicw(ICRHI, 0);
-  lapicw(ICRLO, BCAST | INIT | LEVEL);
-  while(lapic[ICRLO] & DELIVS)
-    ;
-
-  // Enable interrupts on the APIC (but not on the processor).
-  lapicw(TPR, 0);
-}
-
-int
-lapicid(void)
-{
-  if (!lapic)
-    return 0;
-  return lapic[ID] >> 24;
-}
-
-// Acknowledge interrupt.
-void
-lapiceoi(void)
-{
-  if(lapic)
-    lapicw(EOI, 0);
-}
-
-// Spin for a given number of microseconds.
-// On real hardware would want to tune this dynamically.
-void
-microdelay(int us)
-{
-}
-
-#define CMOS_PORT    0x70
-#define CMOS_RETURN  0x71
-
-// Start additional processor running entry code at addr.
-// See Appendix B of MultiProcessor Specification.
-void
-lapicstartap(uchar apicid, uint addr)
-{
-  int i;
-  ushort *wrv;
-
-  // "The BSP must initialize CMOS shutdown code to 0AH
-  // and the warm reset vector (DWORD based at 40:67) to point at
-  // the AP startup code prior to the [universal startup algorithm]."
-  outb(CMOS_PORT, 0xF);  // offset 0xF is shutdown code
-  outb(CMOS_PORT+1, 0x0A);
-  wrv = (ushort*)P2V((0x40<<4 | 0x67));  // Warm reset vector
-  wrv[0] = 0;
-  wrv[1] = addr >> 4;
-
-  // "Universal startup algorithm."
-  // Send INIT (level-triggered) interrupt to reset other CPU.
-  lapicw(ICRHI, apicid<<24);
-  lapicw(ICRLO, INIT | LEVEL | ASSERT);
-  microdelay(200);
-  lapicw(ICRLO, INIT | LEVEL);
-  microdelay(100);    // should be 10ms, but too slow in Bochs!
-
-  // Send startup IPI (twice!) to enter code.
-  // Regular hardware is supposed to only accept a STARTUP
-  // when it is in the halted state due to an INIT.  So the second
-  // should be ignored, but it is part of the official Intel algorithm.
-  // Bochs complains about the second one.  Too bad for Bochs.
-  for(i = 0; i < 2; i++){
-    lapicw(ICRHI, apicid<<24);
-    lapicw(ICRLO, STARTUP | (addr>>12));
-    microdelay(200);
-  }
-}
-
-#define CMOS_STATA   0x0a
-#define CMOS_STATB   0x0b
-#define CMOS_UIP    (1 << 7)        // RTC update in progress
-
-#define SECS    0x00
-#define MINS    0x02
-#define HOURS   0x04
-#define DAY     0x07
-#define MONTH   0x08
-#define YEAR    0x09
-
-static uint
-cmos_read(uint reg)
-{
-  outb(CMOS_PORT,  reg);
-  microdelay(200);
-
-  return inb(CMOS_RETURN);
-}
-
-static void
-fill_rtcdate(struct rtcdate *r)
-{
-  r->second = cmos_read(SECS);
-  r->minute = cmos_read(MINS);
-  r->hour   = cmos_read(HOURS);
-  r->day    = cmos_read(DAY);
-  r->month  = cmos_read(MONTH);
-  r->year   = cmos_read(YEAR);
-}
-
-// qemu seems to use 24-hour GWT and the values are BCD encoded
-void
-cmostime(struct rtcdate *r)
-{
-  struct rtcdate t1, t2;
-  int sb, bcd;
-
-  sb = cmos_read(CMOS_STATB);
-
-  bcd = (sb & (1 << 2)) == 0;
-
-  // make sure CMOS doesn't modify time while we read it
-  for(;;) {
-    fill_rtcdate(&t1);
-    if(cmos_read(CMOS_STATA) & CMOS_UIP)
-        continue;
-    fill_rtcdate(&t2);
-    if(memcmp(&t1, &t2, sizeof(t1)) == 0)
-      break;
-  }
-
-  // convert
-  if(bcd) {
-#define    CONV(x)     (t1.x = ((t1.x >> 4) * 10) + (t1.x & 0xf))
-    CONV(second);
-    CONV(minute);
-    CONV(hour  );
-    CONV(day   );
-    CONV(month );
-    CONV(year  );
-#undef     CONV
-  }
-
-  *r = t1;
-  r->year += 2000;
-}
diff --git a/main.c b/main.c
index db9a6b9..24793cd 100644
--- a/main.c
+++ b/main.c
@@ -8,11 +8,12 @@
 // Allocate a real stack and switch to it, first
 // doing some setup required for memory allocator to work.
 void
-main()
+main(int hartid)
 {
+  w_tp(hartid);    // save hartid where cpuid() can find it
   uartinit();      // serial port
   consoleinit();
-  printf("entering main()\n");
+  printf("entering main() on hart %d\n", hartid);
   kinit();         // physical page allocator
   kvminit();       // kernel page table
   procinit();      // process table
diff --git a/memide.c b/memide.c
deleted file mode 100644
index ba267ac..0000000
--- a/memide.c
+++ /dev/null
@@ -1,60 +0,0 @@
-// Fake IDE disk; stores blocks in memory.
-// Useful for running kernel without scratch disk.
-
-#include "types.h"
-#include "defs.h"
-#include "param.h"
-#include "mmu.h"
-#include "proc.h"
-#include "x86.h"
-#include "traps.h"
-#include "spinlock.h"
-#include "sleeplock.h"
-#include "fs.h"
-#include "buf.h"
-
-extern uchar _binary_fs_img_start[], _binary_fs_img_size[];
-
-static int disksize;
-static uchar *memdisk;
-
-void
-ideinit(void)
-{
-  memdisk = _binary_fs_img_start;
-  disksize = (uint)_binary_fs_img_size/BSIZE;
-}
-
-// Interrupt handler.
-void
-ideintr(void)
-{
-  // no-op
-}
-
-// Sync buf with disk.
-// If B_DIRTY is set, write buf to disk, clear B_DIRTY, set B_VALID.
-// Else if B_VALID is not set, read buf from disk, set B_VALID.
-void
-iderw(struct buf *b)
-{
-  uchar *p;
-
-  if(!holdingsleep(&b->lock))
-    panic("iderw: buf not locked");
-  if((b->flags & (B_VALID|B_DIRTY)) == B_VALID)
-    panic("iderw: nothing to do");
-  if(b->dev != 1)
-    panic("iderw: request not for disk 1");
-  if(b->blockno >= disksize)
-    panic("iderw: block out of range");
-
-  p = memdisk + b->blockno*BSIZE;
-
-  if(b->flags & B_DIRTY){
-    b->flags &= ~B_DIRTY;
-    memmove(p, b->data, BSIZE);
-  } else
-    memmove(b->data, p, BSIZE);
-  b->flags |= B_VALID;
-}
diff --git a/memlayout.h b/memlayout.h
index db7c076..9c9cfdb 100644
--- a/memlayout.h
+++ b/memlayout.h
@@ -23,8 +23,7 @@
 
 // local interrupt controller, which contains the timer.
 #define CLINT 0x2000000L
-#define CLINT_MSIP0 (CLINT + 0x0)
-#define CLINT_MTIMECMP0 (CLINT + 0x4000)
+#define CLINT_MTIMECMP(hartid) (CLINT + 0x4000 + 8*(hartid))
 #define CLINT_MTIME (CLINT + 0xBFF8)
 
 // qemu puts programmable interrupt controller here.
diff --git a/mp.c b/mp.c
deleted file mode 100644
index e36e45c..0000000
--- a/mp.c
+++ /dev/null
@@ -1,139 +0,0 @@
-// Multiprocessor support
-// Search memory for MP description structures.
-// http://developer.intel.com/design/pentium/datashts/24201606.pdf
-
-#include "types.h"
-#include "defs.h"
-#include "param.h"
-#include "memlayout.h"
-#include "mp.h"
-#include "x86.h"
-#include "mmu.h"
-#include "proc.h"
-
-struct cpu cpus[NCPU];
-int ncpu;
-uchar ioapicid;
-
-static uchar
-sum(uchar *addr, int len)
-{
-  int i, sum;
-
-  sum = 0;
-  for(i=0; i<len; i++)
-    sum += addr[i];
-  return sum;
-}
-
-// Look for an MP structure in the len bytes at addr.
-static struct mp*
-mpsearch1(uint64 a, int len)
-{
-  uchar *e, *p, *addr;
-
-  addr = P2V(a);
-  e = addr+len;
-  for(p = addr; p < e; p += sizeof(struct mp))
-    if(memcmp(p, "_MP_", 4) == 0 && sum(p, sizeof(struct mp)) == 0)
-      return (struct mp*)p;
-  return 0;
-}
-
-// Search for the MP Floating Pointer Structure, which according to the
-// spec is in one of the following three locations:
-// 1) in the first KB of the EBDA;
-// 2) in the last KB of system base memory;
-// 3) in the BIOS ROM between 0xE0000 and 0xFFFFF.
-static struct mp*
-mpsearch(void)
-{
-  uchar *bda;
-  uint p;
-  struct mp *mp;
-
-  bda = (uchar *) P2V(0x400);
-  if((p = ((bda[0x0F]<<8)| bda[0x0E]) << 4)){
-    if((mp = mpsearch1(p, 1024)))
-      return mp;
-  } else {
-    p = ((bda[0x14]<<8)|bda[0x13])*1024;
-    if((mp = mpsearch1(p-1024, 1024)))
-      return mp;
-  }
-  return mpsearch1(0xF0000, 0x10000);
-}
-
-// Search for an MP configuration table.  For now,
-// don't accept the default configurations (physaddr == 0).
-// Check for correct signature, calculate the checksum and,
-// if correct, check the version.
-// To do: check extended table checksum.
-static struct mpconf*
-mpconfig(struct mp **pmp)
-{
-  struct mpconf *conf;
-  struct mp *mp;
-
-  if((mp = mpsearch()) == 0 || mp->physaddr == 0)
-    return 0;
-  conf = (struct mpconf*) P2V((uint64) mp->physaddr);
-  if(memcmp(conf, "PCMP", 4) != 0)
-    return 0;
-  if(conf->version != 1 && conf->version != 4)
-    return 0;
-  if(sum((uchar*)conf, conf->length) != 0)
-    return 0;
-  *pmp = mp;
-  return conf;
-}
-
-void
-mpinit(void)
-{
-  uchar *p, *e;
-  int ismp;
-  struct mp *mp;
-  struct mpconf *conf;
-  struct mpproc *proc;
-  struct mpioapic *ioapic;
-
-  if((conf = mpconfig(&mp)) == 0)
-    panic("Expect to run on an SMP");
-  ismp = 1;
-  lapic = P2V((uint64)conf->lapicaddr_p);
-  for(p=(uchar*)(conf+1), e=(uchar*)conf+conf->length; p<e; ){
-    switch(*p){
-    case MPPROC:
-      proc = (struct mpproc*)p;
-      if(ncpu < NCPU) {
-        cpus[ncpu].apicid = proc->apicid;  // apicid may differ from ncpu
-        ncpu++;
-      }
-      p += sizeof(struct mpproc);
-      continue;
-    case MPIOAPIC:
-      ioapic = (struct mpioapic*)p;
-      ioapicid = ioapic->apicno;
-      p += sizeof(struct mpioapic);
-      continue;
-    case MPBUS:
-    case MPIOINTR:
-    case MPLINTR:
-      p += 8;
-      continue;
-    default:
-      ismp = 0;
-      break;
-    }
-  }
-  if(!ismp)
-    panic("Didn't find a suitable machine");
-
-  if(mp->imcrp){
-    // Bochs doesn't support IMCR, so this doesn't run on Bochs.
-    // But it would on real hardware.
-    outb(0x22, 0x70);   // Select IMCR
-    outb(0x23, inb(0x23) | 1);  // Mask external interrupts.
-  }
-}
diff --git a/mp.h b/mp.h
deleted file mode 100644
index 5964b63..0000000
--- a/mp.h
+++ /dev/null
@@ -1,56 +0,0 @@
-// See MultiProcessor Specification Version 1.[14]
-
-struct mp {             // floating pointer
-  uchar signature[4];           // "_MP_"
-  uint32 physaddr;               // phys addr of MP config table
-  uchar length;                 // 1
-  uchar specrev;                // [14]
-  uchar checksum;               // all bytes must add up to 0
-  uchar type;                   // MP system config type
-  uchar imcrp;
-  uchar reserved[3];
-};
-
-struct mpconf {         // configuration table header
-  uchar signature[4];           // "PCMP"
-  ushort length;                // total table length
-  uchar version;                // [14]
-  uchar checksum;               // all bytes must add up to 0
-  uchar product[20];            // product id
-  uint32 oemtable;               // OEM table pointer
-  ushort oemlength;             // OEM table length
-  ushort entry;                 // entry count
-  uint32 lapicaddr_p;              // address of local APIC
-  ushort xlength;               // extended table length
-  uchar xchecksum;              // extended table checksum
-  uchar reserved;
-};
-
-struct mpproc {         // processor table entry
-  uchar type;                   // entry type (0)
-  uchar apicid;                 // local APIC id
-  uchar version;                // local APIC verison
-  uchar flags;                  // CPU flags
-    #define MPBOOT 0x02           // This proc is the bootstrap processor.
-  uchar signature[4];           // CPU signature
-  uint feature;                 // feature flags from CPUID instruction
-  uchar reserved[8];
-};
-
-struct mpioapic {       // I/O APIC table entry
-  uchar type;                   // entry type (2)
-  uchar apicno;                 // I/O APIC id
-  uchar version;                // I/O APIC version
-  uchar flags;                  // I/O APIC flags
-  uint32 addr_p;                  // I/O APIC address
-};
-
-// Table entry types
-#define MPPROC    0x00  // One per processor
-#define MPBUS     0x01  // One per bus
-#define MPIOAPIC  0x02  // One per I/O APIC
-#define MPIOINTR  0x03  // One per bus interrupt source
-#define MPLINTR   0x04  // One per system interrupt source
-
-//PAGEBREAK!
-// Blank page.
diff --git a/picirq.c b/picirq.c
deleted file mode 100644
index e26957f..0000000
--- a/picirq.c
+++ /dev/null
@@ -1,19 +0,0 @@
-#include "types.h"
-#include "x86.h"
-#include "traps.h"
-
-// I/O Addresses of the two programmable interrupt controllers
-#define IO_PIC1         0x20    // Master (IRQs 0-7)
-#define IO_PIC2         0xA0    // Slave (IRQs 8-15)
-
-// Don't use the 8259A interrupt controllers.  Xv6 assumes SMP hardware.
-void
-picinit(void)
-{
-  // mask all interrupts
-  outb(IO_PIC1+1, 0xFF);
-  outb(IO_PIC2+1, 0xFF);
-}
-
-//PAGEBREAK!
-// Blank page.
diff --git a/proc.c b/proc.c
index 768aa6d..766fd93 100644
--- a/proc.c
+++ b/proc.c
@@ -32,28 +32,33 @@ procinit(void)
   initlock(&ptable.lock, "ptable");
 }
 
-// Must be called with interrupts disabled.
-// XXX riscv
+// Must be called with interrupts disabled,
+// to prevent race with process being moved
+// to a different CPU.
 int
-cpuid() {
-  return 0;
+cpuid()
+{
+  int id = r_tp();
+  return id;
 }
 
 // Return this core's cpu struct.
-// XXX riscv
+// Interrupts must be disabled.
 struct cpu*
 mycpu(void) {
-  struct cpu *c;
-  c = &cpus[0];
+  int id = cpuid();
+  struct cpu *c = &cpus[id];
   return c;
 }
 
-// Disable interrupts so that we are not rescheduled
-// while reading proc from the cpu structure
-// XXX riscv
+// Return the current struct proc *.
 struct proc*
 myproc(void) {
-  return cpus[0].proc;
+  // XXX push intr off
+  struct cpu *c = mycpu();
+  struct proc *p = c->proc;
+  // XXX pop intr
+  return p;
 }
 
 //PAGEBREAK: 32
diff --git a/proc.h b/proc.h
index 7e1149f..00a1cb9 100644
--- a/proc.h
+++ b/proc.h
@@ -20,10 +20,7 @@ struct context {
 
 // Per-CPU state
 struct cpu {
-  uint64 syscallno;            // Temporary used by sysentry
-  uint64 usp;                  // Temporary used by sysentry
   struct proc *proc;           // The process running on this cpu or null
-  struct cpu *cpu;             // XXX
   struct context scheduler;   // swtch() here to enter scheduler
   volatile uint started;       // Has the CPU started?
   int ncli;                    // Depth of pushcli nesting.
@@ -31,7 +28,6 @@ struct cpu {
 };
 
 extern struct cpu cpus[NCPU];
-extern int ncpu;
 
 //PAGEBREAK: 17
 
@@ -47,9 +43,9 @@ extern int ncpu;
 struct trapframe {
   /*   0 */ uint64 kernel_satp;
   /*   8 */ uint64 kernel_sp;
-  /*  16 */ uint64 kernel_trap; // address of trap()
+  /*  16 */ uint64 kernel_trap; // usertrap()
   /*  24 */ uint64 epc; // saved user program counter
-  /*  32 */ uint64 unused;
+  /*  32 */ uint64 hartid;
   /*  40 */ uint64 ra;
   /*  48 */ uint64 sp;
   /*  56 */ uint64 gp;
diff --git a/riscv.h b/riscv.h
index 14c8738..b12e5d3 100644
--- a/riscv.h
+++ b/riscv.h
@@ -1,3 +1,12 @@
+// which hart (core) is this?
+static inline uint64
+r_mhartid()
+{
+  uint64 x;
+  asm("csrr %0, mhartid" : "=r" (x) );
+  return x;
+}
+
 // Machine Status Register, mstatus
 
 #define MSTATUS_MPP_MASK (3L << 11)
@@ -279,6 +288,22 @@ r_sp()
   return x;
 }
 
+// read and write tp, the thread pointer, which holds
+// this core's hartid (core number), the index into cpus[].
+static inline uint64
+r_tp()
+{
+  uint64 x;
+  asm("mv %0, tp" : "=r" (x) );
+  return x;
+}
+
+static inline void 
+w_tp(uint64 x)
+{
+  asm("mv tp, %0" : : "r" (x));
+}
+
 #define PGSIZE 4096 // bytes per page
 #define PGSHIFT 12  // bits of offset within a page
 
diff --git a/start.c b/start.c
index 8743955..53edb8e 100644
--- a/start.c
+++ b/start.c
@@ -1,25 +1,19 @@
 #include "types.h"
+#include "param.h"
 #include "memlayout.h"
 #include "riscv.h"
 #include "defs.h"
 
 void main();
 
-// entry.S uses this as the initial stack.
-__attribute__ ((aligned (16))) char stack0[4096];
+// entry.S needs one stack per CPU.
+__attribute__ ((aligned (16))) char stack0[4096 * NCPU];
 
 // assembly code in kernelvec for machine-mode timer interrupt.
 extern void machinevec();
 
-// scratch area for timer interrupt.
-uint64 mscratch0[8];
-
-__attribute__ ((aligned (16)))
-void
-xyzzy()
-{
-  uartputc('I');
-}
+// scratch area for timer interrupt, one per CPU.
+uint64 mscratch0[NCPU * 32];
 
 // entry.S jumps here in machine mode on stack0.
 void
@@ -42,15 +36,19 @@ mstart()
   w_medeleg(0xffff);
   w_mideleg(0xffff);
 
-  // set up to receive timer interrupts in machine mode.
-  *(uint64*)CLINT_MTIMECMP0 = *(uint64*)CLINT_MTIME + 10000;
-  mscratch0[4] = CLINT_MTIMECMP0;
-  mscratch0[5] = 10000000;
-  w_mscratch((uint64)mscratch0);
+  // set up to receive timer interrupts in machine mode,
+  // for pre-emptive switching and (on hart 0) to drive time.
+  int id = r_mhartid();
+  uint64 *scratch = &mscratch0[32 * id];
+  *(uint64*)CLINT_MTIMECMP(id) = *(uint64*)CLINT_MTIME + 10000;
+  scratch[4] = CLINT_MTIMECMP(id);
+  scratch[5] = 10000000;
+  w_mscratch((uint64)scratch);
   w_mtvec((uint64)machinevec);
   w_mstatus(r_mstatus() | MSTATUS_MIE);
   w_mie(r_mie() |  MIE_MTIE);
 
-  // jump to main in supervisor mode.
-  asm("mret");
+  // call main(hartid) in supervisor mode.
+  asm("csrr a0, mhartid ; \
+       mret");
 }
diff --git a/trampoline.S b/trampoline.S
index 5886942..dd4eb02 100644
--- a/trampoline.S
+++ b/trampoline.S
@@ -120,6 +120,9 @@ trampin:
         # restore kernel stack pointer from p->tf->kernel_sp
         ld sp, 8(a0)
 
+        # make tp hold the current hartid, from p->tf->hartid
+        ld tp, 32(a0)
+
         # remember the address of usertrap(), p->tf->kernel_trap
         ld t0, 16(a0)
 
diff --git a/trap.c b/trap.c
index 5f5d4a0..693c596 100644
--- a/trap.c
+++ b/trap.c
@@ -24,8 +24,6 @@ trapinit(void)
   // set up to take exceptions and traps while in the kernel.
   w_stvec((uint64)kernelvec);
 
-  // time, cycle, instret CSRs
-
   initlock(&tickslock, "time");
 }
 
@@ -45,10 +43,6 @@ usertrap(void)
   // since we're now in the kernel.
   w_stvec((uint64)kernelvec);
 
-  //printf("mtimecmp %x mtime %x\n", *(uint64*)CLINT_MTIMECMP0, *(uint64*)CLINT_MTIME);
-
-  *(uint64*)CLINT_MTIMECMP0 = *(uint64*)CLINT_MTIME + 10000;
-
   struct proc *p = myproc();
   
   // save user program counter.
@@ -102,6 +96,7 @@ usertrapret(void)
   p->tf->kernel_satp = r_satp();
   p->tf->kernel_sp = (uint64)p->kstack + PGSIZE;
   p->tf->kernel_trap = (uint64)usertrap;
+  p->tf->hartid = r_tp();
 
   // set up the registers that trampoline.S's sret will use
   // to get to user space.
@@ -132,9 +127,12 @@ kerneltrap()
 {
   uint64 sstatus = r_sstatus();
   uint64 scause = r_scause();
+  uint64 sepc = r_sepc(); // XXX needed only for check at end?
   
   if((sstatus & SSTATUS_SPP) == 0)
     panic("kerneltrap: not from supervisor mode");
+  if(intr_get() != 0)
+    panic("kerneltrap: interrupts enabled");
 
   if(devintr() == 0){
     printf("scause 0x%p\n", scause);
@@ -142,12 +140,11 @@ kerneltrap()
     panic("kerneltrap");
   }
 
-  // turn off interrupts to ensure we
-  // return with the correct sstatus.
-  intr_off();
-
-  // restore previous interrupt status.
-  w_sstatus(sstatus);
+  // XXX assert that we don't have to save/restore sstatus or sepc.
+  if(r_sstatus() != sstatus)
+    panic("kerneltrap sstatus");
+  if(r_sepc() != sepc)
+    panic("kerneltrap sepc");
 }
 
 // check if it's an external interrupt or software interrupt,