14 files changed, 158 insertions, 79 deletions
diff --git a/kernel/exec.c b/kernel/exec.c
index c9af395..b21afbb 100644
--- a/kernel/exec.c
+++ b/kernel/exec.c
@@ -2,6 +2,7 @@
 #include "param.h"
 #include "memlayout.h"
 #include "riscv.h"
+#include "spinlock.h"
 #include "proc.h"
 #include "defs.h"
 #include "elf.h"
@@ -19,7 +20,6 @@ exec(char *path, char **argv)
   struct proghdr ph;
   pagetable_t pagetable = 0, oldpagetable;
   struct proc *p = myproc();
-  uint64 oldsz = p->sz;
 
   begin_op();
 
@@ -60,6 +60,9 @@ exec(char *path, char **argv)
   end_op();
   ip = 0;
 
+  p = myproc();
+  uint64 oldsz = p->sz;
+
   // Allocate two pages at the next page boundary.
   // Use the second as the user stack.
   sz = PGROUNDUP(sz);
diff --git a/kernel/fs.c b/kernel/fs.c
index ebd8f7a..c241b3c 100644
--- a/kernel/fs.c
+++ b/kernel/fs.c
@@ -14,8 +14,8 @@
 #include "defs.h"
 #include "param.h"
 #include "stat.h"
-#include "proc.h"
 #include "spinlock.h"
+#include "proc.h"
 #include "sleeplock.h"
 #include "fs.h"
 #include "buf.h"
diff --git a/kernel/pipe.c b/kernel/pipe.c
index 31bf0cc..eca5959 100644
--- a/kernel/pipe.c
+++ b/kernel/pipe.c
@@ -2,9 +2,9 @@
 #include "riscv.h"
 #include "defs.h"
 #include "param.h"
+#include "spinlock.h"
 #include "proc.h"
 #include "fs.h"
-#include "spinlock.h"
 #include "sleeplock.h"
 #include "file.h"
 
diff --git a/kernel/proc.c b/kernel/proc.c
index 20d5085..8ea09b5 100644
--- a/kernel/proc.c
+++ b/kernel/proc.c
@@ -2,8 +2,8 @@
 #include "param.h"
 #include "memlayout.h"
 #include "riscv.h"
-#include "proc.h"
 #include "spinlock.h"
+#include "proc.h"
 #include "defs.h"
 
 struct {
@@ -21,7 +21,7 @@ extern void forkret(void);
 // for returning  out of the kernel
 extern void sysexit(void);
 
-static void wakeup1(void *chan);
+static void wakeup1(struct proc *chan);
 
 extern char trampout[]; // trampoline.S
 
@@ -80,11 +80,13 @@ allocproc(void)
   return 0;
 
 found:
+  initlock(&p->lock, "proc");
+  acquire(&p->lock);
+  release(&ptable.lock);
+  
   p->state = EMBRYO;
   p->pid = nextpid++;
 
-  release(&ptable.lock);
-
   // Allocate a page for the kernel stack.
   if((p->kstack = kalloc()) == 0){
     p->state = UNUSED;
@@ -202,15 +204,9 @@ userinit(void)
   safestrcpy(p->name, "initcode", sizeof(p->name));
   p->cwd = namei("/");
 
-  // this assignment to p->state lets other cores
-  // run this process. the acquire forces the above
-  // writes to be visible, and the lock is also needed
-  // because the assignment might not be atomic.
-  acquire(&ptable.lock);
-
   p->state = RUNNABLE;
 
-  release(&ptable.lock);
+  release(&p->lock);
 }
 
 // Grow current process's memory by n bytes.
@@ -221,15 +217,22 @@ growproc(int n)
   uint sz;
   struct proc *p = myproc();
 
+  acquire(&p->lock);
+
   sz = p->sz;
   if(n > 0){
-    if((sz = uvmalloc(p->pagetable, sz, sz + n)) == 0)
+    if((sz = uvmalloc(p->pagetable, sz, sz + n)) == 0) {
+      release(&p->lock);
       return -1;
+    }
   } else if(n < 0){
-    if((sz = uvmdealloc(p->pagetable, sz, sz + n)) == 0)
+    if((sz = uvmdealloc(p->pagetable, sz, sz + n)) == 0) {
+      release(&p->lock);
       return -1;
+    }
   }
   p->sz = sz;
+  release(&p->lock);
   return 0;
 }
 
@@ -272,11 +275,9 @@ fork(void)
 
   pid = np->pid;
 
-  acquire(&ptable.lock);
-
   np->state = RUNNABLE;
 
-  release(&ptable.lock);
+  release(&np->lock);
 
   return pid;
 }
@@ -288,43 +289,64 @@ void
 exit(void)
 {
   struct proc *p = myproc();
-  struct proc *pp;
   int fd;
 
   if(p == initproc)
     panic("init exiting");
 
+  acquire(&p->lock);
+  
   // Close all open files.
   for(fd = 0; fd < NOFILE; fd++){
     if(p->ofile[fd]){
-      fileclose(p->ofile[fd]);
+      struct file *f = p->ofile[fd];
+      release(&p->lock);
+      
+      fileclose(f);
+      
+      acquire(&p->lock);
       p->ofile[fd] = 0;
     }
   }
 
+  struct inode *cwd = p->cwd;
+  release(&p->lock);
+  
   begin_op();
-  iput(p->cwd);
+  iput(cwd);
   end_op();
+
+  acquire(&p->lock);
   p->cwd = 0;
 
-  acquire(&ptable.lock);
+  // Jump into the scheduler, never to return.
+  p->state = ZOMBIE;
+  sched();
+  panic("zombie exit");
+}
 
+void reparent(struct proc *p) {
+  struct proc *pp;
+  struct proc *parent = p->parent;
+
+  acquire(&parent->lock);
+  
   // Parent might be sleeping in wait().
-  wakeup1(p->parent);
+  wakeup1(parent);
 
-  // Pass abandoned children to init.
+  // Pass p's abandoned children to init.
   for(pp = ptable.proc; pp < &ptable.proc[NPROC]; pp++){
     if(pp->parent == p){
       pp->parent = initproc;
-      if(pp->state == ZOMBIE)
-        wakeup1(initproc);
+      if(pp->state == ZOMBIE) {
+          acquire(&initproc->lock);
+          wakeup1(initproc);
+          acquire(&initproc->lock);
+      }
     }
   }
 
-  // Jump into the scheduler, never to return.
-  p->state = ZOMBIE;
-  sched();
-  panic("zombie exit");
+  release(&parent->lock);
 }
 
 // Wait for a child process to exit and return its pid.
@@ -335,35 +357,56 @@ wait(void)
   struct proc *np;
   int havekids, pid;
   struct proc *p = myproc();
-  
-  acquire(&ptable.lock);
+
+  acquire(&p->lock);
   for(;;){
     // Scan through table looking for exited children.
     havekids = 0;
     for(np = ptable.proc; np < &ptable.proc[NPROC]; np++){
       if(np->parent != p)
         continue;
+      acquire(&np->lock);
       havekids = 1;
       if(np->state == ZOMBIE){
         // Found one.
         pid = np->pid;
         freeproc(np);
-        release(&ptable.lock);
+        release(&np->lock);
+        release(&p->lock);
         return pid;
       }
+      release(&np->lock);
     }
 
     // No point waiting if we don't have any children.
     if(!havekids || p->killed){
-      release(&ptable.lock);
+      release(&p->lock);
       return -1;
     }
-
-    // Wait for children to exit.  (See wakeup1 call in proc_exit.)
-    sleep(p, &ptable.lock);  //DOC: wait-sleep
+    
+    // Wait for children to exit.  (See wakeup1 call in reparent.)
+    sleep(p, &p->lock);  //DOC: wait-sleep
   }
 }
 
+// Loop over process table looking for process to run.
+struct proc *find_runnable(int start) {
+  struct proc *p;
+  acquire(&ptable.lock);
+  for(int i = start; i < start+NPROC; i++) {
+    p = &ptable.proc[i % NPROC];
+    acquire(&p->lock);
+    if(p->state == RUNNABLE) {
+      p->state = RUNNING;
+      release(&ptable.lock);
+      return p;
+    }
+    release(&p->lock);
+  }
+  release(&ptable.lock);
+  return 0;
+}
+  
 //PAGEBREAK: 42
 // Per-CPU process scheduler.
 // Each CPU calls scheduler() after setting itself up.
@@ -377,35 +420,33 @@ scheduler(void)
 {
   struct proc *p;
   struct cpu *c = mycpu();
-
+  int next = 0;
+  
   c->proc = 0;
   for(;;){
     // Enable interrupts on this processor.
     intr_on();
 
-    // Loop over process table looking for process to run.
-    acquire(&ptable.lock);
-    for(p = ptable.proc; p < &ptable.proc[NPROC]; p++){
-      if(p->state != RUNNABLE)
-        continue;
-
+    if((p = find_runnable(next)) != 0) {
+      next = (next + 1) & NPROC;
       // Switch to chosen process.  It is the process's job
-      // to release ptable.lock and then reacquire it
+      // to release its lock and then reacquire it
       // before jumping back to us.
       c->proc = p;
-      p->state = RUNNING;
-
       swtch(&c->scheduler, &p->context);
 
       // Process is done running for now.
       // It should have changed its p->state before coming back.
       c->proc = 0;
+      release(&p->lock);
+      if(p->state == ZOMBIE) {
+        reparent(p);
+      }
     }
-    release(&ptable.lock);
   }
 }
 
-// Enter scheduler.  Must hold only ptable.lock
+// Enter scheduler.  Must hold only p->lock
 // and have changed proc->state. Saves and restores
 // intena because intena is a property of this
 // kernel thread, not this CPU. It should
@@ -418,8 +459,8 @@ sched(void)
   int intena;
   struct proc *p = myproc();
 
-  if(!holding(&ptable.lock))
-    panic("sched ptable.lock");
+  if(!holding(&p->lock))
+    panic("sched p->lock");
   if(mycpu()->noff != 1)
     panic("sched locks");
   if(p->state == RUNNING)
@@ -436,10 +477,10 @@ sched(void)
 void
 yield(void)
 {
-  acquire(&ptable.lock);  //DOC: yieldlock
+  acquire(&myproc()->lock);  //DOC: yieldlock
   myproc()->state = RUNNABLE;
   sched();
-  release(&ptable.lock);
+  release(&myproc()->lock);
 }
 
 // A fork child's very first scheduling by scheduler()
@@ -449,8 +490,8 @@ forkret(void)
 {
   static int first = 1;
 
-  // Still holding ptable.lock from scheduler.
-  release(&ptable.lock);
+  // Still holding p->lock from scheduler.
+  release(&myproc()->lock);
 
   if (first) {
     // Some initialization functions must be run in the context
@@ -477,14 +518,14 @@ sleep(void *chan, struct spinlock *lk)
   if(lk == 0)
     panic("sleep without lk");
 
-  // Must acquire ptable.lock in order to
+  // Must acquire p->lock in order to
   // change p->state and then call sched.
-  // Once we hold ptable.lock, we can be
+  // Once we hold p->lock, we can be
   // guaranteed that we won't miss any wakeup
-  // (wakeup runs with ptable.lock locked),
+  // (wakeup runs with p->lock locked),
   // so it's okay to release lk.
-  if(lk != &ptable.lock){  //DOC: sleeplock0
-    acquire(&ptable.lock);  //DOC: sleeplock1
+  if(lk != &p->lock){  //DOC: sleeplock0
+    acquire(&p->lock);  //DOC: sleeplock1
     release(lk);
   }
   // Go to sleep.
@@ -497,32 +538,43 @@ sleep(void *chan, struct spinlock *lk)
   p->chan = 0;
 
   // Reacquire original lock.
-  if(lk != &ptable.lock){  //DOC: sleeplock2
-    release(&ptable.lock);
+  if(lk != &p->lock){  //DOC: sleeplock2
+    release(&p->lock);
     acquire(lk);
   }
 }
 
 //PAGEBREAK!
-// Wake up all processes sleeping on chan.
-// The ptable lock must be held.
+// Wake up all processes sleeping on chan,
+// where chan is a proc, which is locked.
 static void
-wakeup1(void *chan)
+wakeup1(struct proc *chan)
 {
   struct proc *p;
 
   for(p = ptable.proc; p < &ptable.proc[NPROC]; p++)
-    if(p->state == SLEEPING && p->chan == chan)
+    if(p == chan && p->state == SLEEPING && p->chan == chan) {
+      if(p->state != SLEEPING || p->chan != chan)
+        panic("wakeup1");
       p->state = RUNNABLE;
+    }
 }
 
-// Wake up all processes sleeping on chan.
+// Wake up all processes sleeping on chan. Never
+// called when holding a p->lock
 void
 wakeup(void *chan)
 {
-  acquire(&ptable.lock);
-  wakeup1(chan);
-  release(&ptable.lock);
+  struct proc *p;
+
+  for(p = ptable.proc; p < &ptable.proc[NPROC]; p++)
+    if(p->state == SLEEPING && p->chan == chan) {
+      acquire(&p->lock);
+      if(p->state != SLEEPING || p->chan != chan)
+        panic("wakeup");
+      p->state = RUNNABLE;
+      release(&p->lock);
+    }
 }
 
 // Kill the process with the given pid.
@@ -533,18 +585,19 @@ kill(int pid)
 {
   struct proc *p;
 
-  acquire(&ptable.lock);
   for(p = ptable.proc; p < &ptable.proc[NPROC]; p++){
     if(p->pid == pid){
+      acquire(&p->lock);
+      if(p->pid != pid)
+        panic("kill");
       p->killed = 1;
       // Wake process from sleep if necessary.
       if(p->state == SLEEPING)
         p->state = RUNNABLE;
-      release(&ptable.lock);
+      release(&p->lock);
       return 0;
     }
   }
-  release(&ptable.lock);
   return -1;
 }
 
diff --git a/kernel/proc.h b/kernel/proc.h
index 278e4cd..0fa61ff 100644
--- a/kernel/proc.h
+++ b/kernel/proc.h
@@ -82,6 +82,7 @@ enum procstate { UNUSED, EMBRYO, SLEEPING, RUNNABLE, RUNNING, ZOMBIE };
 
 // Per-process state
 struct proc {
+  struct spinlock lock;
   char *kstack;                // Bottom of kernel stack for this process
   uint64 sz;                   // Size of process memory (bytes)
   pagetable_t pagetable;       // Page table
diff --git a/kernel/riscv.h b/kernel/riscv.h
index c3371a4..e5c0f64 100644
--- a/kernel/riscv.h
+++ b/kernel/riscv.h
@@ -304,6 +304,15 @@ w_tp(uint64 x)
   asm volatile("mv tp, %0" : : "r" (x));
 }
 
+static inline uint64
+r_ra()
+{
+  uint64 x;
+  asm volatile("mv %0, ra" : "=r" (x) );
+  return x;
+}
+
+
 #define PGSIZE 4096 // bytes per page
 #define PGSHIFT 12  // bits of offset within a page
 
diff --git a/kernel/sleeplock.c b/kernel/sleeplock.c
index b490370..81de585 100644
--- a/kernel/sleeplock.c
+++ b/kernel/sleeplock.c
@@ -5,8 +5,8 @@
 #include "defs.h"
 #include "param.h"
 #include "memlayout.h"
-#include "proc.h"
 #include "spinlock.h"
+#include "proc.h"
 #include "sleeplock.h"
 
 void
diff --git a/kernel/spinlock.c b/kernel/spinlock.c
index 5a44a46..1d0b16a 100644
--- a/kernel/spinlock.c
+++ b/kernel/spinlock.c
@@ -47,9 +47,16 @@ acquire(struct spinlock *lk)
 void
 release(struct spinlock *lk)
 {
-  if(!holding(lk))
+  uint64 x;
+  asm volatile("mv %0, ra" : "=r" (x) );
+  if(!holding(lk)) {
+    printf("%p: !holding %d %s %p %p %p %p\n", mycpu(), lk->locked, lk->name, lk->cpu,
+           lk->last_release, lk->last_pc, x);
     panic("release");
+  }
 
+  lk->last_release = lk->cpu;
+  lk->last_pc  = x;
   lk->cpu = 0;
 
   // Tell the C compiler and the CPU to not move loads or stores
diff --git a/kernel/spinlock.h b/kernel/spinlock.h
index 4392820..5f244c9 100644
--- a/kernel/spinlock.h
+++ b/kernel/spinlock.h
@@ -5,5 +5,7 @@ struct spinlock {
   // For debugging:
   char *name;        // Name of lock.
   struct cpu *cpu;   // The cpu holding the lock.
+  struct cpu *last_release;
+  uint64 last_pc;
 };
 
diff --git a/kernel/syscall.c b/kernel/syscall.c
index ff10f9c..a054da2 100644
--- a/kernel/syscall.c
+++ b/kernel/syscall.c
@@ -2,6 +2,7 @@
 #include "param.h"
 #include "memlayout.h"
 #include "riscv.h"
+#include "spinlock.h"
 #include "proc.h"
 #include "syscall.h"
 #include "defs.h"
@@ -170,7 +171,9 @@ dosyscall(void)
 
   num = p->tf->a7;
   if(num > 0 && num < NELEM(syscalls) && syscalls[num]) {
+    //printf("%d: syscall %d\n", p->pid, num);
     p->tf->a0 = syscalls[num]();
+    //printf("%d: syscall %d -> %d\n", p->pid, num, p->tf->a0);
   } else {
     printf("%d %s: unknown sys call %d\n",
             p->pid, p->name, num);
diff --git a/kernel/sysfile.c b/kernel/sysfile.c
index 77e273f..533e097 100644
--- a/kernel/sysfile.c
+++ b/kernel/sysfile.c
@@ -9,9 +9,9 @@
 #include "defs.h"
 #include "param.h"
 #include "stat.h"
+#include "spinlock.h"
 #include "proc.h"
 #include "fs.h"
-#include "spinlock.h"
 #include "sleeplock.h"
 #include "file.h"
 #include "fcntl.h"
diff --git a/kernel/sysproc.c b/kernel/sysproc.c
index 93ea9bc..face81a 100644
--- a/kernel/sysproc.c
+++ b/kernel/sysproc.c
@@ -4,6 +4,7 @@
 #include "date.h"
 #include "param.h"
 #include "memlayout.h"
+#include "spinlock.h"
 #include "proc.h"
 
 uint64
diff --git a/kernel/trap.c b/kernel/trap.c
index 835a3b0..018b7db 100644
--- a/kernel/trap.c
+++ b/kernel/trap.c
@@ -2,8 +2,8 @@
 #include "param.h"
 #include "memlayout.h"
 #include "riscv.h"
-#include "proc.h"
 #include "spinlock.h"
+#include "proc.h"
 #include "defs.h"
 
 struct spinlock tickslock;
diff --git a/kernel/uart.c b/kernel/uart.c
index 35fac1b..b8dd664 100644
--- a/kernel/uart.c
+++ b/kernel/uart.c
@@ -2,8 +2,8 @@
 #include "param.h"
 #include "memlayout.h"
 #include "riscv.h"
-#include "proc.h"
 #include "spinlock.h"
+#include "proc.h"
 #include "defs.h"
 
 //