diff -Nur 2.4.7/CREDITS linux/CREDITS --- 2.4.7/CREDITS Mon Jul 23 11:09:13 2001 +++ linux/CREDITS Mon Jul 23 12:20:37 2001 @@ -926,8 +926,8 @@ N: Nigel Gamble E: nigel@nrg.org -E: nigel@sgi.com D: Interrupt-driven printer driver +D: Preemptible kernel S: 120 Alley Way S: Mountain View, California 94040 S: USA diff -Nur 2.4.7/Documentation/Configure.help linux/Documentation/Configure.help --- 2.4.7/Documentation/Configure.help Mon Jul 23 11:09:47 2001 +++ linux/Documentation/Configure.help Mon Jul 23 12:20:38 2001 @@ -132,6 +132,23 @@ If you have system with several CPU's, you do not need to say Y here: APIC will be used automatically. +Preemptible Kernel +CONFIG_PREEMPT + This option reduces the latency of the kernel when reacting to + real-time or interactive events by allowing a low priority process to + be preempted even if it is in kernel mode executing a system call. + This allows applications that need real-time response, such as audio + and other multimedia applications, to run more reliably even when the + system is under load due to other, lower priority, processes. + + This option is currently experimental if used in conjuction with SMP + support. + + Say Y here if you are building a kernel for a desktop system, embedded + system or real-time system. Say N if you are building a kernel for a + system where throughput is more important than interactive response, + such as a server system. Say N if you are unsure. + Kernel math emulation CONFIG_MATH_EMULATION Linux can emulate a math coprocessor (used for floating point diff -Nur 2.4.7/arch/i386/config.in linux/arch/i386/config.in --- 2.4.7/arch/i386/config.in Mon Jul 23 11:09:17 2001 +++ linux/arch/i386/config.in Mon Jul 23 12:20:43 2001 @@ -176,6 +176,9 @@ define_bool CONFIG_X86_LOCAL_APIC y fi fi +if [ "$CONFIG_SMP" != "y" -o "$CONFIG_EXPERIMENTAL" = "y" ]; then + bool 'Preemptible Kernel' CONFIG_PREEMPT +fi if [ "$CONFIG_SMP" = "y" -a "$CONFIG_X86_CMPXCHG" = "y" ]; then define_bool CONFIG_HAVE_DEC_LOCK y diff -Nur 2.4.7/arch/i386/kernel/entry.S linux/arch/i386/kernel/entry.S --- 2.4.7/arch/i386/kernel/entry.S Thu Jul 5 13:52:11 2001 +++ linux/arch/i386/kernel/entry.S Fri Jul 27 17:31:51 2001 @@ -72,7 +72,7 @@ * these are offsets into the task-struct. */ state = 0 -flags = 4 +preempt_count = 4 sigpending = 8 addr_limit = 12 exec_domain = 16 @@ -80,8 +80,28 @@ tsk_ptrace = 24 processor = 52 + /* These are offsets into the irq_stat structure + * There is one per cpu and it is aligned to 32 + * byte boundry (we put that here as a shift count) + */ +irq_array_shift = CONFIG_X86_L1_CACHE_SHIFT + +irq_stat_local_irq_count = 4 +irq_stat_local_bh_count = 8 + ENOSYS = 38 +#ifdef CONFIG_SMP +#define GET_CPU_INDX movl processor(%ebx),%eax; \ + shll $irq_array_shift,%eax +#define GET_CURRENT_CPU_INDX GET_CURRENT(%ebx); \ + GET_CPU_INDX +#define CPU_INDX (,%eax) +#else +#define GET_CPU_INDX +#define GET_CURRENT_CPU_INDX GET_CURRENT(%ebx) +#define CPU_INDX +#endif #define SAVE_ALL \ cld; \ @@ -248,12 +268,30 @@ ALIGN ENTRY(ret_from_intr) GET_CURRENT(%ebx) +#ifdef CONFIG_PREEMPT + cli + decl preempt_count(%ebx) +#endif ret_from_exception: movl EFLAGS(%esp),%eax # mix EFLAGS and CS movb CS(%esp),%al testl $(VM_MASK | 3),%eax # return to VM86 mode or non-supervisor? jne ret_from_sys_call +#ifdef CONFIG_PREEMPT + cmpl $0,preempt_count(%ebx) + jnz restore_all + cmpl $0,need_resched(%ebx) + jz restore_all + movl SYMBOL_NAME(irq_stat)+irq_stat_local_bh_count CPU_INDX,%ecx + addl SYMBOL_NAME(irq_stat)+irq_stat_local_irq_count CPU_INDX,%ecx + jnz restore_all + incl preempt_count(%ebx) + sti + call SYMBOL_NAME(preempt_schedule) + jmp ret_from_intr +#else jmp restore_all +#endif ALIGN reschedule: @@ -290,6 +328,9 @@ GET_CURRENT(%ebx) call *%edi addl $8,%esp +#ifdef CONFIG_PREEMPT + cli +#endif jmp ret_from_exception ENTRY(coprocessor_error) @@ -309,12 +350,18 @@ movl %cr0,%eax testl $0x4,%eax # EM (math emulation bit) jne device_not_available_emulate +#ifdef CONFIG_PREEMPT + cli +#endif call SYMBOL_NAME(math_state_restore) jmp ret_from_exception device_not_available_emulate: pushl $0 # temporary storage for ORIG_EIP call SYMBOL_NAME(math_emulate) addl $4,%esp +#ifdef CONFIG_PREEMPT + cli +#endif jmp ret_from_exception ENTRY(debug) diff -Nur 2.4.7/arch/i386/kernel/traps.c linux/arch/i386/kernel/traps.c --- 2.4.7/arch/i386/kernel/traps.c Thu Jul 5 13:52:11 2001 +++ linux/arch/i386/kernel/traps.c Fri Jul 27 17:31:51 2001 @@ -742,6 +742,11 @@ */ asmlinkage void math_state_restore(struct pt_regs regs) { + /* + * CONFIG_PREEMPT + * Must be called with preemption disabled + */ + __asm__ __volatile__("clts"); /* Allow maths ops (or we recurse) */ if (current->used_math) { diff -Nur 2.4.7/arch/i386/lib/dec_and_lock.c linux/arch/i386/lib/dec_and_lock.c --- 2.4.7/arch/i386/lib/dec_and_lock.c Mon Mar 26 18:41:06 2001 +++ linux/arch/i386/lib/dec_and_lock.c Tue Mar 27 15:13:33 2001 @@ -8,6 +8,7 @@ */ #include +#include #include int atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock) diff -Nur 2.4.7/fs/exec.c linux/fs/exec.c --- 2.4.7/fs/exec.c Mon Jul 23 11:10:55 2001 +++ linux/fs/exec.c Mon Jul 23 12:21:20 2001 @@ -415,8 +415,8 @@ active_mm = current->active_mm; current->mm = mm; current->active_mm = mm; - task_unlock(current); activate_mm(active_mm, mm); + task_unlock(current); mm_release(); if (old_mm) { if (active_mm != old_mm) BUG(); diff -Nur 2.4.7/fs/fat/cache.c linux/fs/fat/cache.c --- 2.4.7/fs/fat/cache.c Wed Jun 13 15:13:55 2001 +++ linux/fs/fat/cache.c Wed Jun 13 16:14:52 2001 @@ -14,6 +14,7 @@ #include #include #include +#include #include "msbuffer.h" diff -Nur 2.4.7/include/asm-i386/hardirq.h linux/include/asm-i386/hardirq.h --- 2.4.7/include/asm-i386/hardirq.h Mon Jul 23 11:11:08 2001 +++ linux/include/asm-i386/hardirq.h Mon Jul 23 12:21:25 2001 @@ -36,6 +36,8 @@ #define synchronize_irq() barrier() +#define release_irqlock(cpu) do { } while (0) + #else #include diff -Nur 2.4.7/include/asm-i386/hw_irq.h linux/include/asm-i386/hw_irq.h --- 2.4.7/include/asm-i386/hw_irq.h Thu Jul 5 13:55:30 2001 +++ linux/include/asm-i386/hw_irq.h Thu Jul 5 16:36:48 2001 @@ -95,6 +95,18 @@ #define __STR(x) #x #define STR(x) __STR(x) +#define GET_CURRENT \ + "movl %esp, %ebx\n\t" \ + "andl $-8192, %ebx\n\t" + +#ifdef CONFIG_PREEMPT +#define BUMP_CONTEX_SWITCH_LOCK \ + GET_CURRENT \ + "incl 4(%ebx)\n\t" +#else +#define BUMP_CONTEX_SWITCH_LOCK +#endif + #define SAVE_ALL \ "cld\n\t" \ "pushl %es\n\t" \ @@ -108,14 +120,11 @@ "pushl %ebx\n\t" \ "movl $" STR(__KERNEL_DS) ",%edx\n\t" \ "movl %edx,%ds\n\t" \ - "movl %edx,%es\n\t" + "movl %edx,%es\n\t" \ + BUMP_CONTEX_SWITCH_LOCK #define IRQ_NAME2(nr) nr##_interrupt(void) #define IRQ_NAME(nr) IRQ_NAME2(IRQ##nr) - -#define GET_CURRENT \ - "movl %esp, %ebx\n\t" \ - "andl $-8192, %ebx\n\t" /* * SMP has a few special interrupts for IPI messages diff -Nur 2.4.7/include/asm-i386/mmu_context.h linux/include/asm-i386/mmu_context.h --- 2.4.7/include/asm-i386/mmu_context.h Mon Mar 26 18:39:41 2001 +++ linux/include/asm-i386/mmu_context.h Tue Mar 27 15:13:38 2001 @@ -27,6 +27,10 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, struct task_struct *tsk, unsigned cpu) { +#ifdef CONFIG_PREEMPT + if (in_ctx_sw_off() == 0) + BUG(); +#endif if (prev != next) { /* stop flush ipis for the previous mm */ clear_bit(cpu, &prev->cpu_vm_mask); diff -Nur 2.4.7/include/asm-i386/pgalloc.h linux/include/asm-i386/pgalloc.h --- 2.4.7/include/asm-i386/pgalloc.h Mon Apr 30 12:45:32 2001 +++ linux/include/asm-i386/pgalloc.h Mon Apr 30 13:21:24 2001 @@ -65,20 +65,26 @@ { unsigned long *ret; + ctx_sw_off(); if ((ret = pgd_quicklist) != NULL) { pgd_quicklist = (unsigned long *)(*ret); ret[0] = 0; pgtable_cache_size--; - } else + ctx_sw_on(); + } else { + ctx_sw_on(); ret = (unsigned long *)get_pgd_slow(); + } return (pgd_t *)ret; } extern __inline__ void free_pgd_fast(pgd_t *pgd) { + ctx_sw_off(); *(unsigned long *)pgd = (unsigned long) pgd_quicklist; pgd_quicklist = (unsigned long *) pgd; pgtable_cache_size++; + ctx_sw_on(); } extern __inline__ void free_pgd_slow(pgd_t *pgd) @@ -108,19 +114,23 @@ { unsigned long *ret; + ctx_sw_off(); if ((ret = (unsigned long *)pte_quicklist) != NULL) { pte_quicklist = (unsigned long *)(*ret); ret[0] = ret[1]; pgtable_cache_size--; } + ctx_sw_on(); return (pte_t *)ret; } extern __inline__ void pte_free_fast(pte_t *pte) { + ctx_sw_off(); *(unsigned long *)pte = (unsigned long) pte_quicklist; pte_quicklist = (unsigned long *) pte; pgtable_cache_size++; + ctx_sw_on(); } extern __inline__ void pte_free_slow(pte_t *pte) diff -Nur 2.4.7/include/asm-i386/smplock.h linux/include/asm-i386/smplock.h --- 2.4.7/include/asm-i386/smplock.h Mon Mar 26 18:39:41 2001 +++ linux/include/asm-i386/smplock.h Tue Mar 27 15:13:38 2001 @@ -10,7 +10,15 @@ extern spinlock_t kernel_flag; +#ifdef CONFIG_SMP #define kernel_locked() spin_is_locked(&kernel_flag) +#else +#ifdef CONFIG_PREEMPT +#define kernel_locked() in_ctx_sw_off() +#else +#define kernel_locked() 1 +#endif +#endif /* * Release global kernel lock and global interrupt lock @@ -42,6 +50,11 @@ */ extern __inline__ void lock_kernel(void) { +#ifdef CONFIG_PREEMPT + if (current->lock_depth == -1) + spin_lock(&kernel_flag); + ++current->lock_depth; +#else #if 1 if (!++current->lock_depth) spin_lock(&kernel_flag); @@ -53,6 +66,7 @@ "\n9:" :"=m" (__dummy_lock(&kernel_flag)), "=m" (current->lock_depth)); +#endif #endif } diff -Nur 2.4.7/include/asm-i386/softirq.h linux/include/asm-i386/softirq.h --- 2.4.7/include/asm-i386/softirq.h Mon Jul 23 11:11:09 2001 +++ linux/include/asm-i386/softirq.h Mon Jul 23 12:21:25 2001 @@ -5,9 +5,9 @@ #include #define __cpu_bh_enable(cpu) \ - do { barrier(); local_bh_count(cpu)--; } while (0) + do { barrier(); local_bh_count(cpu)--; ctx_sw_on(); } while (0) #define cpu_bh_disable(cpu) \ - do { local_bh_count(cpu)++; barrier(); } while (0) + do { ctx_sw_off(); local_bh_count(cpu)++; barrier(); } while (0) #define local_bh_disable() cpu_bh_disable(smp_processor_id()) #define __local_bh_enable() __cpu_bh_enable(smp_processor_id()) @@ -22,7 +22,7 @@ * If you change the offsets in irq_stat then you have to * update this code as well. */ -#define local_bh_enable() \ +#define _raw_local_bh_enable() \ do { \ unsigned int *ptr = &local_bh_count(smp_processor_id()); \ \ @@ -44,6 +44,13 @@ : "r" (ptr), "i" (do_softirq) \ /* no registers clobbered */ ); \ } while (0) + +#ifdef CONFIG_PREEMPT +#define local_bh_enable() \ + do { _raw_local_bh_enable(); ctx_sw_on(); } while (0) +#else +#define local_bh_enable() _raw_local_bh_enable() +#endif #define __cpu_raise_softirq(cpu, nr) __set_bit(nr, &softirq_pending(cpu)) diff -Nur 2.4.7/include/asm-i386/spinlock.h linux/include/asm-i386/spinlock.h --- 2.4.7/include/asm-i386/spinlock.h Mon Mar 26 18:39:42 2001 +++ linux/include/asm-i386/spinlock.h Tue Mar 27 15:13:38 2001 @@ -65,7 +65,7 @@ #define spin_unlock_string \ "movb $1,%0" -static inline int spin_trylock(spinlock_t *lock) +static inline int _raw_spin_trylock(spinlock_t *lock) { char oldval; __asm__ __volatile__( @@ -75,7 +75,7 @@ return oldval > 0; } -static inline void spin_lock(spinlock_t *lock) +static inline void _raw_spin_lock(spinlock_t *lock) { #if SPINLOCK_DEBUG __label__ here; @@ -90,7 +90,7 @@ :"=m" (lock->lock) : : "memory"); } -static inline void spin_unlock(spinlock_t *lock) +static inline void _raw_spin_unlock(spinlock_t *lock) { #if SPINLOCK_DEBUG if (lock->magic != SPINLOCK_MAGIC) @@ -143,7 +143,7 @@ */ /* the spinlock helpers are in arch/i386/kernel/semaphore.c */ -static inline void read_lock(rwlock_t *rw) +static inline void _raw_read_lock(rwlock_t *rw) { #if SPINLOCK_DEBUG if (rw->magic != RWLOCK_MAGIC) @@ -152,7 +152,7 @@ __build_read_lock(rw, "__read_lock_failed"); } -static inline void write_lock(rwlock_t *rw) +static inline void _raw_write_lock(rwlock_t *rw) { #if SPINLOCK_DEBUG if (rw->magic != RWLOCK_MAGIC) @@ -161,10 +161,10 @@ __build_write_lock(rw, "__write_lock_failed"); } -#define read_unlock(rw) asm volatile("lock ; incl %0" :"=m" ((rw)->lock) : : "memory") -#define write_unlock(rw) asm volatile("lock ; addl $" RW_LOCK_BIAS_STR ",%0":"=m" ((rw)->lock) : : "memory") +#define _raw_read_unlock(rw) asm volatile("lock ; incl %0" :"=m" ((rw)->lock) : : "memory") +#define _raw_write_unlock(rw) asm volatile("lock ; addl $" RW_LOCK_BIAS_STR ",%0":"=m" ((rw)->lock) : : "memory") -static inline int write_trylock(rwlock_t *lock) +static inline int _raw_write_trylock(rwlock_t *lock) { atomic_t *count = (atomic_t *)lock; if (atomic_sub_and_test(RW_LOCK_BIAS, count)) diff -Nur 2.4.7/include/linux/brlock.h linux/include/linux/brlock.h --- 2.4.7/include/linux/brlock.h Mon Mar 26 18:39:14 2001 +++ linux/include/linux/brlock.h Tue Mar 27 15:13:38 2001 @@ -170,12 +170,19 @@ __br_write_unlock(idx); } +#else /* CONFIG_SMP */ +#ifdef CONFIG_PREEMPT +# define br_read_lock(idx) ({ (void)(idx); ctx_sw_off(); }) +# define br_read_unlock(idx) ({ (void)(idx); ctx_sw_on(); }) +# define br_write_lock(idx) ({ (void)(idx); ctx_sw_off(); }) +# define br_write_unlock(idx) ({ (void)(idx); ctx_sw_on(); }) #else # define br_read_lock(idx) ((void)(idx)) # define br_read_unlock(idx) ((void)(idx)) # define br_write_lock(idx) ((void)(idx)) # define br_write_unlock(idx) ((void)(idx)) #endif +#endif /* CONFIG_SMP */ /* * Now enumerate all of the possible sw/hw IRQ protected diff -Nur 2.4.7/include/linux/dcache.h linux/include/linux/dcache.h --- 2.4.7/include/linux/dcache.h Thu Jul 5 13:55:55 2001 +++ linux/include/linux/dcache.h Thu Jul 5 16:36:59 2001 @@ -126,31 +126,6 @@ extern spinlock_t dcache_lock; -/** - * d_drop - drop a dentry - * @dentry: dentry to drop - * - * d_drop() unhashes the entry from the parent - * dentry hashes, so that it won't be found through - * a VFS lookup any more. Note that this is different - * from deleting the dentry - d_delete will try to - * mark the dentry negative if possible, giving a - * successful _negative_ lookup, while d_drop will - * just make the cache lookup fail. - * - * d_drop() is used mainly for stuff that wants - * to invalidate a dentry for some reason (NFS - * timeouts or autofs deletes). - */ - -static __inline__ void d_drop(struct dentry * dentry) -{ - spin_lock(&dcache_lock); - list_del(&dentry->d_hash); - INIT_LIST_HEAD(&dentry->d_hash); - spin_unlock(&dcache_lock); -} - static __inline__ int dname_external(struct dentry *d) { return d->d_name.name != d->d_iname; @@ -272,3 +247,34 @@ #endif /* __KERNEL__ */ #endif /* __LINUX_DCACHE_H */ + +#if !defined(__LINUX_DCACHE_H_INLINES) && defined(_TASK_STRUCT_DEFINED) +#define __LINUX_DCACHE_H_INLINES + +#ifdef __KERNEL__ +/** + * d_drop - drop a dentry + * @dentry: dentry to drop + * + * d_drop() unhashes the entry from the parent + * dentry hashes, so that it won't be found through + * a VFS lookup any more. Note that this is different + * from deleting the dentry - d_delete will try to + * mark the dentry negative if possible, giving a + * successful _negative_ lookup, while d_drop will + * just make the cache lookup fail. + * + * d_drop() is used mainly for stuff that wants + * to invalidate a dentry for some reason (NFS + * timeouts or autofs deletes). + */ + +static __inline__ void d_drop(struct dentry * dentry) +{ + spin_lock(&dcache_lock); + list_del(&dentry->d_hash); + INIT_LIST_HEAD(&dentry->d_hash); + spin_unlock(&dcache_lock); +} +#endif +#endif diff -Nur 2.4.7/include/linux/fs_struct.h linux/include/linux/fs_struct.h --- 2.4.7/include/linux/fs_struct.h Mon Jul 23 11:11:25 2001 +++ linux/include/linux/fs_struct.h Mon Jul 23 12:21:26 2001 @@ -20,6 +20,15 @@ extern void exit_fs(struct task_struct *); extern void set_fs_altroot(void); +struct fs_struct *copy_fs_struct(struct fs_struct *old); +void put_fs_struct(struct fs_struct *fs); + +#endif +#endif + +#if !defined(_LINUX_FS_STRUCT_H_INLINES) && defined(_TASK_STRUCT_DEFINED) +#define _LINUX_FS_STRUCT_H_INLINES +#ifdef __KERNEL__ /* * Replace the fs->{rootmnt,root} with {mnt,dentry}. Put the old values. * It can block. Requires the big lock held. @@ -65,9 +74,5 @@ mntput(old_pwdmnt); } } - -struct fs_struct *copy_fs_struct(struct fs_struct *old); -void put_fs_struct(struct fs_struct *fs); - #endif #endif diff -Nur 2.4.7/include/linux/sched.h linux/include/linux/sched.h --- 2.4.7/include/linux/sched.h Mon Jul 23 11:11:27 2001 +++ linux/include/linux/sched.h Mon Jul 23 12:21:27 2001 @@ -86,6 +86,7 @@ #define TASK_UNINTERRUPTIBLE 2 #define TASK_ZOMBIE 4 #define TASK_STOPPED 8 +#define TASK_PREEMPTED 64 #define __set_task_state(tsk, state_value) \ do { (tsk)->state = (state_value); } while (0) @@ -152,6 +153,9 @@ #define MAX_SCHEDULE_TIMEOUT LONG_MAX extern signed long FASTCALL(schedule_timeout(signed long timeout)); asmlinkage void schedule(void); +#ifdef CONFIG_PREEMPT +asmlinkage void preempt_schedule(void); +#endif extern int schedule_task(struct tq_struct *task); extern void flush_scheduled_tasks(void); @@ -287,7 +291,17 @@ * offsets of these are hardcoded elsewhere - touch with care */ volatile long state; /* -1 unrunnable, 0 runnable, >0 stopped */ +#ifdef CONFIG_PREEMPT + /* + * We want the preempt_count in this cache line, but we + * a) don't want to mess up the offsets in asm code and + * b) the alignment of the next line below so + * we move "flags" down + */ + atomic_t preempt_count; /* 0=> preemptable, < 0 => BUG */ +#else unsigned long flags; /* per process flags, defined below */ +#endif int sigpending; mm_segment_t addr_limit; /* thread address space: 0-0xBFFFFFFF for user-thead @@ -319,6 +333,9 @@ struct task_struct *next_task, *prev_task; struct mm_struct *active_mm; +#ifdef CONFIG_PREEMPT + unsigned long flags; /* per process flags, defined below */ +#endif /* task state */ struct linux_binfmt *binfmt; @@ -891,6 +908,11 @@ mntput(rootmnt); return res; } + +#define _TASK_STRUCT_DEFINED +#include +#include +#include #endif /* __KERNEL__ */ diff -Nur 2.4.7/include/linux/smp.h linux/include/linux/smp.h --- 2.4.7/include/linux/smp.h Mon Mar 26 18:39:16 2001 +++ linux/include/linux/smp.h Tue Mar 27 15:13:38 2001 @@ -81,7 +81,9 @@ #define smp_processor_id() 0 #define hard_smp_processor_id() 0 #define smp_threads_ready 1 +#ifndef CONFIG_PREEMPT #define kernel_lock() +#endif #define cpu_logical_map(cpu) 0 #define cpu_number_map(cpu) 0 #define smp_call_function(func,info,retry,wait) ({ 0; }) diff -Nur 2.4.7/include/linux/smp_lock.h linux/include/linux/smp_lock.h --- 2.4.7/include/linux/smp_lock.h Mon Mar 26 18:39:16 2001 +++ linux/include/linux/smp_lock.h Tue Mar 27 15:13:38 2001 @@ -3,7 +3,7 @@ #include -#ifndef CONFIG_SMP +#if !defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT) #define lock_kernel() do { } while(0) #define unlock_kernel() do { } while(0) diff -Nur 2.4.7/include/linux/spinlock.h linux/include/linux/spinlock.h --- 2.4.7/include/linux/spinlock.h Mon Jul 23 11:11:28 2001 +++ linux/include/linux/spinlock.h Mon Jul 23 12:21:27 2001 @@ -41,7 +41,9 @@ #if (DEBUG_SPINLOCKS < 1) +#ifndef CONFIG_PREEMPT #define atomic_dec_and_lock(atomic,lock) atomic_dec_and_test(atomic) +#endif /* * Your basic spinlocks, allowing only a single CPU anywhere @@ -57,11 +59,11 @@ #endif #define spin_lock_init(lock) do { } while(0) -#define spin_lock(lock) (void)(lock) /* Not "unused variable". */ +#define _raw_spin_lock(lock) (void)(lock) /* Not "unused variable". */ #define spin_is_locked(lock) (0) -#define spin_trylock(lock) ({1; }) +#define _raw_spin_trylock(lock) ({1; }) #define spin_unlock_wait(lock) do { } while(0) -#define spin_unlock(lock) do { } while(0) +#define _raw_spin_unlock(lock) do { } while(0) #elif (DEBUG_SPINLOCKS < 2) @@ -120,12 +122,74 @@ #endif #define rwlock_init(lock) do { } while(0) -#define read_lock(lock) (void)(lock) /* Not "unused variable". */ -#define read_unlock(lock) do { } while(0) -#define write_lock(lock) (void)(lock) /* Not "unused variable". */ -#define write_unlock(lock) do { } while(0) +#define _raw_read_lock(lock) (void)(lock) /* Not "unused variable". */ +#define _raw_read_unlock(lock) do { } while(0) +#define _raw_write_lock(lock) (void)(lock) /* Not "unused variable". */ +#define _raw_write_unlock(lock) do { } while(0) #endif /* !SMP */ + +#ifdef CONFIG_PREEMPT + +#define switch_lock_count() current->preempt_count + +#define in_ctx_sw_off() (switch_lock_count().counter) +#define atomic_ptr_in_ctx_sw_off() (&switch_lock_count()) + +#define ctx_sw_off() \ +do { \ + atomic_inc(atomic_ptr_in_ctx_sw_off()); \ +} while (0) + +#define ctx_sw_on_no_preempt() \ +do { \ + atomic_dec(atomic_ptr_in_ctx_sw_off()); \ +} while (0) + +#define ctx_sw_on() \ +do { \ + if (atomic_dec_and_test(atomic_ptr_in_ctx_sw_off()) && \ + current->need_resched) \ + preempt_schedule(); \ +} while (0) + +#define spin_lock(lock) \ +do { \ + ctx_sw_off(); \ + _raw_spin_lock(lock); \ +} while(0) +#define spin_trylock(lock) ({ctx_sw_off(); _raw_spin_trylock(lock) ? \ + 1 : ({ctx_sw_on(); 0;});}) +#define spin_unlock(lock) \ +do { \ + _raw_spin_unlock(lock); \ + ctx_sw_on(); \ +} while (0) + +#define read_lock(lock) ({ctx_sw_off(); _raw_read_lock(lock);}) +#define read_unlock(lock) ({_raw_read_unlock(lock); ctx_sw_on();}) +#define write_lock(lock) ({ctx_sw_off(); _raw_write_lock(lock);}) +#define write_unlock(lock) ({_raw_write_unlock(lock); ctx_sw_on();}) +#define write_trylock(lock) ({ctx_sw_off(); _raw_write_trylock(lock) ? \ + 1 : ({ctx_sw_on(); 0;});}) + +#else + +#define in_ctx_sw_off() do { } while (0) +#define ctx_sw_off() do { } while (0) +#define ctx_sw_on_no_preempt() +#define ctx_sw_on() do { } while (0) + +#define spin_lock(lock) _raw_spin_lock(lock) +#define spin_trylock(lock) _raw_spin_trylock(lock) +#define spin_unlock(lock) _raw_spin_unlock(lock) + +#define read_lock(lock) _raw_read_lock(lock) +#define read_unlock(lock) _raw_read_unlock(lock) +#define write_lock(lock) _raw_write_lock(lock) +#define write_unlock(lock) _raw_write_unlock(lock) +#define write_trylock(lock) _raw_write_trylock(lock) +#endif /* "lock on reference count zero" */ #ifndef atomic_dec_and_lock diff -Nur 2.4.7/include/linux/tqueue.h linux/include/linux/tqueue.h --- 2.4.7/include/linux/tqueue.h Thu Apr 19 15:35:41 2001 +++ linux/include/linux/tqueue.h Mon Apr 23 16:39:48 2001 @@ -94,6 +94,22 @@ extern spinlock_t tqueue_lock; /* + * Call all "bottom halfs" on a given list. + */ + +extern void __run_task_queue(task_queue *list); + +static inline void run_task_queue(task_queue *list) +{ + if (TQ_ACTIVE(*list)) + __run_task_queue(list); +} + +#endif /* _LINUX_TQUEUE_H */ + +#if !defined(_LINUX_TQUEUE_H_INLINES) && defined(_TASK_STRUCT_DEFINED) +#define _LINUX_TQUEUE_H_INLINES +/* * Queue a task on a tq. Return non-zero if it was successfully * added. */ @@ -109,17 +125,4 @@ } return ret; } - -/* - * Call all "bottom halfs" on a given list. - */ - -extern void __run_task_queue(task_queue *list); - -static inline void run_task_queue(task_queue *list) -{ - if (TQ_ACTIVE(*list)) - __run_task_queue(list); -} - -#endif /* _LINUX_TQUEUE_H */ +#endif diff -Nur 2.4.7/kernel/exit.c linux/kernel/exit.c --- 2.4.7/kernel/exit.c Wed Jun 13 15:14:48 2001 +++ linux/kernel/exit.c Wed Jun 13 16:15:10 2001 @@ -277,6 +277,10 @@ struct mm_struct * start_lazy_tlb(void) { struct mm_struct *mm = current->mm; +#ifdef CONFIG_PREEMPT + if (in_ctx_sw_off() == 0) + BUG(); +#endif current->mm = NULL; /* active_mm is still 'mm' */ atomic_inc(&mm->mm_count); @@ -288,6 +292,10 @@ { struct mm_struct *active_mm = current->active_mm; +#ifdef CONFIG_PREEMPT + if (in_ctx_sw_off() == 0) + BUG(); +#endif current->mm = mm; if (mm != active_mm) { current->active_mm = mm; @@ -311,8 +319,8 @@ /* more a memory barrier than a real lock */ task_lock(tsk); tsk->mm = NULL; - task_unlock(tsk); enter_lazy_tlb(mm, current, smp_processor_id()); + task_unlock(tsk); mmput(mm); } } diff -Nur 2.4.7/kernel/fork.c linux/kernel/fork.c --- 2.4.7/kernel/fork.c Mon Jul 23 11:11:31 2001 +++ linux/kernel/fork.c Mon Jul 23 12:21:27 2001 @@ -599,6 +599,12 @@ if (p->binfmt && p->binfmt->module) __MOD_INC_USE_COUNT(p->binfmt->module); +#ifdef CONFIG_PREEMPT + /* Since we are keeping the context switch off state as part + * of the context, make sure we start with it off. + */ + p->preempt_count.counter = 1; +#endif p->did_exec = 0; p->swappable = 0; p->state = TASK_UNINTERRUPTIBLE; diff -Nur 2.4.7/kernel/ksyms.c linux/kernel/ksyms.c --- 2.4.7/kernel/ksyms.c Mon Jul 23 11:11:31 2001 +++ linux/kernel/ksyms.c Mon Jul 23 12:21:28 2001 @@ -435,6 +435,9 @@ EXPORT_SYMBOL(interruptible_sleep_on); EXPORT_SYMBOL(interruptible_sleep_on_timeout); EXPORT_SYMBOL(schedule); +#ifdef CONFIG_PREEMPT +EXPORT_SYMBOL(preempt_schedule); +#endif EXPORT_SYMBOL(schedule_timeout); EXPORT_SYMBOL(jiffies); EXPORT_SYMBOL(xtime); diff -Nur 2.4.7/kernel/sched.c linux/kernel/sched.c --- 2.4.7/kernel/sched.c Mon Jul 23 11:11:31 2001 +++ linux/kernel/sched.c Mon Jul 23 12:21:28 2001 @@ -472,7 +472,7 @@ task_lock(prev); prev->has_cpu = 0; mb(); - if (prev->state == TASK_RUNNING) + if (task_on_runqueue(prev)) goto needs_resched; out_unlock: @@ -502,7 +502,7 @@ goto out_unlock; spin_lock_irqsave(&runqueue_lock, flags); - if ((prev->state == TASK_RUNNING) && !prev->has_cpu) + if (task_on_runqueue(prev) && !prev->has_cpu) reschedule_idle(prev); spin_unlock_irqrestore(&runqueue_lock, flags); goto out_unlock; @@ -515,6 +515,9 @@ void schedule_tail(struct task_struct *prev) { __schedule_tail(prev); +#ifdef CONFIG_PREEMPT + ctx_sw_on(); +#endif } /* @@ -534,6 +537,10 @@ struct list_head *tmp; int this_cpu, c; +#ifdef CONFIG_PREEMPT + ctx_sw_off(); +#endif + if (!current->active_mm) BUG(); need_resched_back: prev = current; @@ -564,7 +571,14 @@ break; } default: +#ifdef CONFIG_PREEMPT + if (prev->state & TASK_PREEMPTED) + break; +#endif del_from_runqueue(prev); +#ifdef CONFIG_PREEMPT + case TASK_RUNNING|TASK_PREEMPTED: +#endif case TASK_RUNNING:; } prev->need_resched = 0; @@ -579,7 +593,7 @@ */ next = idle_task(this_cpu); c = -1000; - if (prev->state == TASK_RUNNING) + if (task_on_runqueue(prev)) goto still_running; still_running_back: @@ -673,6 +687,9 @@ if (current->need_resched) goto need_resched_back; +#ifdef CONFIG_PREEMPT + ctx_sw_on_no_preempt(); +#endif return; recalculate: @@ -1288,3 +1305,15 @@ atomic_inc(&init_mm.mm_count); enter_lazy_tlb(&init_mm, current, cpu); } +#ifdef CONFIG_PREEMPT +asmlinkage void preempt_schedule(void) +{ + while (current->need_resched) { + ctx_sw_off(); + current->state |= TASK_PREEMPTED; + schedule(); + current->state &= ~TASK_PREEMPTED; + ctx_sw_on_no_preempt(); + } +} +#endif diff -Nur 2.4.7/lib/dec_and_lock.c linux/lib/dec_and_lock.c --- 2.4.7/lib/dec_and_lock.c Mon Mar 26 18:41:26 2001 +++ linux/lib/dec_and_lock.c Tue Mar 27 15:13:39 2001 @@ -1,4 +1,5 @@ #include +#include #include /* diff -Nur 2.4.7/net/socket.c linux/net/socket.c --- 2.4.7/net/socket.c Mon Jul 23 11:11:32 2001 +++ linux/net/socket.c Mon Jul 23 12:21:28 2001 @@ -135,7 +135,7 @@ static struct net_proto_family *net_families[NPROTO]; -#ifdef CONFIG_SMP +#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT) static atomic_t net_family_lockct = ATOMIC_INIT(0); static spinlock_t net_family_lock = SPIN_LOCK_UNLOCKED;