source: trunk/source/lisp-kernel/thread_manager.c @ 11370

Last change on this file since 11370 was 11370, checked in by gb, 11 years ago

Using i386_set_ldt() to point %fs at the tcr on 32-bit FreeBSD doesn't work
on a 64-bit kernel. Using i386_set_fsbase does work on both 32-bit and
64-bit kernels, but doesn't allow writing to %fs: at least by the time
we've called i386_set_fsbase() on thread startup, %fs contains an appropriate
selector and it never changes throughout the life of the thread.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 55.3 KB
Line 
1/*
2   Copyright (C) 1994-2001 Digitool, Inc
3   This file is part of OpenMCL. 
4
5   OpenMCL is licensed under the terms of the Lisp Lesser GNU Public
6   License , known as the LLGPL and distributed with OpenMCL as the
7   file "LICENSE".  The LLGPL consists of a preamble and the LGPL,
8   which is distributed with OpenMCL as the file "LGPL".  Where these
9   conflict, the preamble takes precedence. 
10
11   OpenMCL is referenced in the preamble as the "LIBRARY."
12
13   The LLGPL is also available online at
14   http://opensource.franz.com/preamble.html
15*/
16
17
18#include "Threads.h"
19
20
21typedef struct {
22  TCR *tcr;
23  natural vsize, tsize;
24  void *created;
25} thread_activation;
26
27#ifdef HAVE_TLS
28__thread TCR __attribute__ ((aligned (16))) current_tcr;
29#endif
30
31extern natural
32store_conditional(natural*, natural, natural);
33
34extern signed_natural
35atomic_swap(signed_natural*, signed_natural);
36
37#ifdef USE_FUTEX
38#define futex_wait(futex,val) syscall(SYS_futex,futex,FUTEX_WAIT,val)
39#define futex_wake(futex,n) syscall(SYS_futex,futex,FUTEX_WAKE,n)
40#define FUTEX_AVAIL (0)
41#define FUTEX_LOCKED (1)
42#define FUTEX_CONTENDED (2)
43#endif
44
45#ifdef WINDOWS
46extern pc spentry_start, spentry_end,subprims_start,subprims_end;
47extern pc restore_windows_context_start, restore_windows_context_end,
48  restore_windows_context_load_rcx, restore_windows_context_iret;
49
50extern void interrupt_handler(int, siginfo_t *, ExceptionInformation *);
51
52void CALLBACK
53nullAPC(ULONG_PTR arg) 
54{
55}
56 
57BOOL (*pCancelIoEx)(HANDLE, OVERLAPPED*) = NULL;
58
59
60extern void *windows_find_symbol(void*, char*);
61
62int
63raise_thread_interrupt(TCR *target)
64{
65  /* GCC doesn't align CONTEXT corrcectly */
66  char _contextbuf[sizeof(CONTEXT)+__alignof(CONTEXT)];
67  CONTEXT  *pcontext;
68  HANDLE hthread = (HANDLE)(target->osid);
69  pc where;
70  area *cs = target->cs_area, *ts = target->cs_area;
71  DWORD rc;
72  BOOL io_pending;
73
74  pcontext = (CONTEXT *)((((natural)&_contextbuf)+15)&~15);
75  rc = SuspendThread(hthread);
76  if (rc == -1) {
77    return -1;
78  }
79  /* What if the suspend count is > 1 at this point ?  I don't think
80     that that matters, but I'm not sure */
81  pcontext->ContextFlags = CONTEXT_ALL;
82  rc = GetThreadContext(hthread, pcontext);
83  if (rc == 0) {
84    return ESRCH;
85  }
86
87  where = (pc)(xpPC(pcontext));
88 
89  if ((target->valence != TCR_STATE_LISP) ||
90      (TCR_INTERRUPT_LEVEL(target) < 0) ||
91      (target->unwinding != 0) ||
92      (!((where < (pc)lisp_global(HEAP_END)) &&
93         (where >= (pc)lisp_global(HEAP_START))) &&
94       !((where < spentry_end) && (where >= spentry_start)) &&
95       !((where < subprims_end) && (where >= subprims_start)) &&
96       !((where < (pc) 0x16000) &&
97         (where >= (pc) 0x15000)) &&
98       !((where < (pc) (ts->high)) &&
99         (where >= (pc) (ts->low))))) {
100    /* If the thread's in a blocking syscall, it'd be nice to
101       get it out of that state here. */
102    GetThreadIOPendingFlag(hthread,&io_pending);
103    target->interrupt_pending = (1LL << (nbits_in_word - 1LL));
104    if (io_pending) {
105      pending_io * pending = (pending_io *) (target->pending_io_info);
106      if (pending) {
107        if (pCancelIoEx) {
108          pCancelIoEx(pending->h, pending->o);
109        } else {
110          CancelIo(pending->h);
111        }
112      }
113    }
114    QueueUserAPC(nullAPC, hthread, 0);
115    ResumeThread(hthread);
116    return 0;
117  } else {
118    /* Thread is running lisp code with interupts enabled.  Set it
119       so that it calls out and then returns to the context,
120       handling any necessary pc-lusering. */
121    LispObj foreign_rsp = (((LispObj)(target->foreign_sp))-0x200)&~15;
122    CONTEXT *icontext = ((CONTEXT *) foreign_rsp) -1;
123    icontext = (CONTEXT *)(((LispObj)icontext)&~15);
124   
125    *icontext = *pcontext;
126
127#ifdef WIN_64   
128    xpGPR(pcontext,REG_RCX) = SIGNAL_FOR_PROCESS_INTERRUPT;
129    xpGPR(pcontext,REG_RDX) = 0;
130    xpGPR(pcontext,REG_R8) = (LispObj) icontext;
131    xpGPR(pcontext,REG_RSP) = (LispObj)(((LispObj *)icontext)-1);
132    *(((LispObj *)icontext)-1) = (LispObj)raise_thread_interrupt;
133#else
134    {
135      LispObj *p = (LispObj *)icontext;
136      p -= 4;
137      p[0] = SIGNAL_FOR_PROCESS_INTERRUPT;
138      p[1] = 0;
139      p[2] = (DWORD)icontext;
140      *(--p) = (LispObj)raise_thread_interrupt;;
141      xpGPR(pcontext,Isp) = (DWORD)p;
142#ifdef WIN32_ES_HACK
143      pcontext->SegEs = pcontext->SegDs;
144#endif
145    }
146#endif
147    pcontext->EFlags &= ~0x400;  /* clear direction flag */
148    xpPC(pcontext) = (LispObj)interrupt_handler;
149    SetThreadContext(hthread,pcontext);
150    ResumeThread(hthread);
151    return 0;
152  }
153}
154#else
155int
156raise_thread_interrupt(TCR *target)
157{
158  pthread_t thread = (pthread_t)target->osid;
159#ifdef DARWIN_not_yet
160  if (use_mach_exception_handling) {
161    return mach_raise_thread_interrupt(target);
162  }
163#endif
164  if (thread != (pthread_t) 0) {
165    return pthread_kill(thread, SIGNAL_FOR_PROCESS_INTERRUPT);
166  }
167  return ESRCH;
168}
169#endif
170
171signed_natural
172atomic_incf_by(signed_natural *ptr, signed_natural by)
173{
174  signed_natural old, new;
175  do {
176    old = *ptr;
177    new = old+by;
178  } while (store_conditional((natural *)ptr, (natural) old, (natural) new) !=
179           (natural) old);
180  return new;
181}
182
183signed_natural
184atomic_incf(signed_natural *ptr)
185{
186  return atomic_incf_by(ptr, 1);
187}
188
189signed_natural
190atomic_decf(signed_natural *ptr)
191{
192  signed_natural old, new;
193  do {
194    old = *ptr;
195    new = old == 0 ? old : old-1;
196  } while (store_conditional((natural *)ptr, (natural) old, (natural) new) !=
197           (natural) old);
198  return old-1;
199}
200
201
202#ifndef USE_FUTEX
203int spin_lock_tries = 1;
204
205void
206get_spin_lock(signed_natural *p, TCR *tcr)
207{
208  int i, n = spin_lock_tries;
209 
210  while (1) {
211    for (i = 0; i < n; i++) {
212      if (atomic_swap(p,(signed_natural)tcr) == 0) {
213        return;
214      }
215    }
216#ifndef WINDOWS
217    sched_yield();
218#endif
219  }
220}
221#endif
222
223#ifndef USE_FUTEX
224int
225lock_recursive_lock(RECURSIVE_LOCK m, TCR *tcr)
226{
227
228  if (tcr == NULL) {
229    tcr = get_tcr(true);
230  }
231  if (m->owner == tcr) {
232    m->count++;
233    return 0;
234  }
235  while (1) {
236    LOCK_SPINLOCK(m->spinlock,tcr);
237    ++m->avail;
238    if (m->avail == 1) {
239      m->owner = tcr;
240      m->count = 1;
241      RELEASE_SPINLOCK(m->spinlock);
242      break;
243    }
244    RELEASE_SPINLOCK(m->spinlock);
245    SEM_WAIT_FOREVER(m->signal);
246  }
247  return 0;
248}
249
250#else /* USE_FUTEX */
251
252static void inline
253lock_futex(signed_natural *p)
254{
255 
256  while (1) {
257    if (store_conditional(p,FUTEX_AVAIL,FUTEX_LOCKED) == FUTEX_AVAIL) {
258      return;
259    }
260    while (1) {
261      if (atomic_swap(p,FUTEX_CONTENDED) == FUTEX_AVAIL) {
262        return;
263      }
264      futex_wait(p,FUTEX_CONTENDED);
265    }
266  }
267}
268
269static void inline
270unlock_futex(signed_natural *p)
271{
272  if (atomic_decf(p) != FUTEX_AVAIL) {
273    *p = FUTEX_AVAIL;
274    futex_wake(p,INT_MAX);
275  }
276}
277   
278int
279lock_recursive_lock(RECURSIVE_LOCK m, TCR *tcr)
280{
281  if (tcr == NULL) {
282    tcr = get_tcr(true);
283  }
284  if (m->owner == tcr) {
285    m->count++;
286    return 0;
287  }
288  lock_futex(&m->avail);
289  m->owner = tcr;
290  m->count = 1;
291  return 0;
292}
293#endif /* USE_FUTEX */
294
295
296#ifndef USE_FUTEX 
297int
298unlock_recursive_lock(RECURSIVE_LOCK m, TCR *tcr)
299{
300  int ret = EPERM, pending;
301
302  if (tcr == NULL) {
303    tcr = get_tcr(true);
304  }
305
306  if (m->owner == tcr) {
307    --m->count;
308    if (m->count == 0) {
309      LOCK_SPINLOCK(m->spinlock,tcr);
310      m->owner = NULL;
311      pending = m->avail-1 + m->waiting;     /* Don't count us */
312      m->avail = 0;
313      --pending;
314      if (pending > 0) {
315        m->waiting = pending;
316      } else {
317        m->waiting = 0;
318      }
319      RELEASE_SPINLOCK(m->spinlock);
320      if (pending >= 0) {
321        SEM_RAISE(m->signal);
322      }
323    }
324    ret = 0;
325  }
326  return ret;
327}
328#else /* USE_FUTEX */
329int
330unlock_recursive_lock(RECURSIVE_LOCK m, TCR *tcr)
331{
332  int ret = EPERM;
333
334   if (tcr == NULL) {
335    tcr = get_tcr(true);
336  }
337
338  if (m->owner == tcr) {
339    --m->count;
340    if (m->count == 0) {
341      m->owner = NULL;
342      unlock_futex(&m->avail);
343    }
344    ret = 0;
345  }
346  return ret;
347}
348#endif /* USE_FUTEX */
349
350void
351destroy_recursive_lock(RECURSIVE_LOCK m)
352{
353#ifndef USE_FUTEX
354  destroy_semaphore((void **)&m->signal);
355#endif
356  postGCfree((void *)(m->malloced_ptr));
357}
358
359/*
360  If we're already the owner (or if the lock is free), lock it
361  and increment the lock count; otherwise, return EBUSY without
362  waiting.
363*/
364
365#ifndef USE_FUTEX
366int
367recursive_lock_trylock(RECURSIVE_LOCK m, TCR *tcr, int *was_free)
368{
369  TCR *owner = m->owner;
370
371  LOCK_SPINLOCK(m->spinlock,tcr);
372  if (owner == tcr) {
373    m->count++;
374    if (was_free) {
375      *was_free = 0;
376      RELEASE_SPINLOCK(m->spinlock);
377      return 0;
378    }
379  }
380  if (store_conditional((natural*)&(m->avail), 0, 1) == 0) {
381    m->owner = tcr;
382    m->count = 1;
383    if (was_free) {
384      *was_free = 1;
385    }
386    RELEASE_SPINLOCK(m->spinlock);
387    return 0;
388  }
389
390  RELEASE_SPINLOCK(m->spinlock);
391  return EBUSY;
392}
393#else
394int
395recursive_lock_trylock(RECURSIVE_LOCK m, TCR *tcr, int *was_free)
396{
397  TCR *owner = m->owner;
398
399  if (owner == tcr) {
400    m->count++;
401    if (was_free) {
402      *was_free = 0;
403      return 0;
404    }
405  }
406  if (store_conditional((natural*)&(m->avail), 0, 1) == 0) {
407    m->owner = tcr;
408    m->count = 1;
409    if (was_free) {
410      *was_free = 1;
411    }
412    return 0;
413  }
414
415  return EBUSY;
416}
417#endif
418
419void
420sem_wait_forever(SEMAPHORE s)
421{
422  int status;
423
424  do {
425#ifdef USE_MACH_SEMAPHORES
426    mach_timespec_t q = {1,0};
427    status = SEM_TIMEDWAIT(s,q);
428#endif
429#ifdef USE_POSIX_SEMAPHORES
430    struct timespec q;
431    gettimeofday((struct timeval *)&q, NULL);
432    q.tv_sec += 1;
433    status = SEM_TIMEDWAIT(s,&q);
434#endif
435#ifdef USE_WINDOWS_SEMAPHORES
436    status = (WaitForSingleObject(s,1000L) == WAIT_TIMEOUT) ? 1 : 0;
437#endif
438  } while (status != 0);
439}
440
441int
442wait_on_semaphore(void *s, int seconds, int millis)
443{
444#ifdef USE_POSIX_SEMAPHORES
445  int nanos = (millis % 1000) * 1000000;
446  int status;
447
448  struct timespec q;
449  gettimeofday((struct timeval *)&q, NULL);
450  q.tv_nsec *= 1000L;  /* microseconds -> nanoseconds */
451   
452  q.tv_nsec += nanos;
453  if (q.tv_nsec >= 1000000000L) {
454    q.tv_nsec -= 1000000000L;
455    seconds += 1;
456  }
457  q.tv_sec += seconds;
458  status = SEM_TIMEDWAIT(s, &q);
459  if (status < 0) {
460    return errno;
461  }
462  return status;
463#endif
464#ifdef USE_MACH_SEMAPHORES
465  int nanos = (millis % 1000) * 1000000;
466  mach_timespec_t q = {seconds, nanos};
467  int status = SEM_TIMEDWAIT(s, q);
468
469 
470  switch (status) {
471  case 0: return 0;
472  case KERN_OPERATION_TIMED_OUT: return ETIMEDOUT;
473  case KERN_ABORTED: return EINTR;
474  default: return EINVAL;
475  }
476#endif
477#ifdef USE_WINDOWS_SEMAPHORES
478  switch (WaitForSingleObjectEx(s, seconds*1000L+(DWORD)millis,true)) {
479  case WAIT_OBJECT_0:
480    return 0;
481  case WAIT_TIMEOUT:
482    return /* ETIMEDOUT */ WAIT_TIMEOUT;
483  case WAIT_IO_COMPLETION:
484    return EINTR;
485  default:
486    break;
487  }
488  return EINVAL;
489
490#endif
491}
492
493
494int
495semaphore_maybe_timedwait(void *s, struct timespec *t)
496{
497  if (t) {
498    return wait_on_semaphore(s, t->tv_sec, t->tv_nsec/1000000L);
499  }
500  SEM_WAIT_FOREVER(s);
501  return 0;
502}
503
504void
505signal_semaphore(SEMAPHORE s)
506{
507  SEM_RAISE(s);
508}
509
510 
511#ifdef WINDOWS
512LispObj
513current_thread_osid()
514{
515  TCR *tcr = get_tcr(false);
516  LispObj current = 0;
517
518  if (tcr) {
519    current = tcr->osid;
520  }
521  if (current == 0) {
522    DuplicateHandle(GetCurrentProcess(),
523                    GetCurrentThread(),
524                    GetCurrentProcess(),
525                    (LPHANDLE)(&current),
526                    0,
527                    FALSE,
528                    DUPLICATE_SAME_ACCESS);
529    if (tcr) {
530      tcr->osid = current;
531    }
532  }
533  return current;
534}
535#else
536LispObj
537current_thread_osid()
538{
539  return (LispObj)ptr_to_lispobj(pthread_self());
540}
541#endif
542
543
544int thread_suspend_signal = 0, thread_resume_signal = 0;
545
546
547
548void
549linux_exception_init(TCR *tcr)
550{
551}
552
553
554TCR *
555get_interrupt_tcr(Boolean create)
556{
557  return get_tcr(create);
558}
559 
560void
561suspend_resume_handler(int signo, siginfo_t *info, ExceptionInformation *context)
562{
563#ifdef DARWIN_GS_HACK
564  Boolean gs_was_tcr = ensure_gs_pthread();
565#endif
566  TCR *tcr = get_interrupt_tcr(false);
567
568  if (TCR_INTERRUPT_LEVEL(tcr) <= (-2<<fixnumshift)) {
569    SET_TCR_FLAG(tcr,TCR_FLAG_BIT_PENDING_SUSPEND);
570  } else {
571    tcr->suspend_context = context;
572    SEM_RAISE(tcr->suspend);
573    SEM_WAIT_FOREVER(tcr->resume);
574    tcr->suspend_context = NULL;
575  }
576#ifdef DARWIN_GS_HACK
577  if (gs_was_tcr) {
578    set_gs_address(tcr);
579  }
580#endif
581  SIGRETURN(context);
582}
583
584 
585
586/*
587  'base' should be set to the bottom (origin) of the stack, e.g., the
588  end from which it grows.
589*/
590 
591#ifdef WINDOWS
592void
593os_get_current_thread_stack_bounds(void **base, natural *size)
594{
595  natural natbase;
596  MEMORY_BASIC_INFORMATION info;
597  void *addr = (void *)current_stack_pointer();
598 
599  VirtualQuery(addr, &info, sizeof(info));
600  natbase = (natural)info.BaseAddress+info.RegionSize;
601  *size = natbase - (natural)(info.AllocationBase);
602  *base = (void *)natbase;
603}
604#else
605void
606os_get_current_thread_stack_bounds(void **base, natural *size)
607{
608  pthread_t p = pthread_self();
609#ifdef DARWIN
610  *base = pthread_get_stackaddr_np(p);
611  *size = pthread_get_stacksize_np(p);
612#endif
613#ifdef LINUX
614  pthread_attr_t attr;
615
616  pthread_getattr_np(p,&attr);
617  pthread_attr_getstack(&attr, base, size);
618  pthread_attr_destroy(&attr);
619  *(natural *)base += *size;
620#endif
621#ifdef FREEBSD
622  pthread_attr_t attr;
623  void * temp_base;
624  size_t temp_size;
625 
626
627  pthread_attr_init(&attr); 
628  pthread_attr_get_np(p, &attr);
629  pthread_attr_getstackaddr(&attr,&temp_base);
630  pthread_attr_getstacksize(&attr,&temp_size);
631  *base = (void *)((natural)temp_base + temp_size);
632  *size = temp_size;
633  pthread_attr_destroy(&attr);
634#endif
635#ifdef SOLARIS
636  stack_t st;
637 
638  thr_stksegment(&st);
639  *size = st.ss_size;
640  *base = st.ss_sp;
641 
642#endif
643}
644#endif
645
646void *
647new_semaphore(int count)
648{
649#ifdef USE_POSIX_SEMAPHORES
650  sem_t *s = malloc(sizeof(sem_t));
651  sem_init(s, 0, count);
652  return s;
653#endif
654#ifdef USE_MACH_SEMAPHORES
655  semaphore_t s = (semaphore_t)0;
656  semaphore_create(mach_task_self(),&s, SYNC_POLICY_FIFO, count);
657  return (void *)(natural)s;
658#endif
659#ifdef USE_WINDOWS_SEMAPHORES
660  return CreateSemaphore(NULL, count, 0x7fffL, NULL);
661#endif
662}
663
664RECURSIVE_LOCK
665new_recursive_lock()
666{
667  extern int cache_block_size;
668  void *p = calloc(1,sizeof(_recursive_lock)+cache_block_size-1);
669  RECURSIVE_LOCK m = NULL;
670#ifndef USE_FUTEX
671  void *signal = new_semaphore(0);
672#endif
673  if (p) {
674    m = (RECURSIVE_LOCK) ((((natural)p)+cache_block_size-1) & (~(cache_block_size-1)));
675    m->malloced_ptr = p;
676  }
677
678#ifdef USE_FUTEX
679  if (m) {
680    return m;
681  }
682#else
683  if (m && signal) {
684    m->signal = signal;
685    return m;
686  }
687  if (m) {
688    free(p);
689  }
690  if (signal) {
691    destroy_semaphore(&signal);
692  }
693#endif
694  return NULL;
695}
696
697void
698destroy_semaphore(void **s)
699{
700  if (*s) {
701#ifdef USE_POSIX_SEMAPHORES
702    sem_destroy((sem_t *)*s);
703    free(*s);
704#endif
705#ifdef USE_MACH_SEMAPHORES
706    semaphore_destroy(mach_task_self(),((semaphore_t)(natural) *s));
707#endif
708#ifdef USE_WINDOWS_SEMAPHORES
709    CloseHandle(*s);
710#endif
711    *s=NULL;
712  }
713}
714
715#ifdef WINDOWS
716void
717tsd_set(LispObj key, void *datum)
718{
719  TlsSetValue((DWORD)key, datum);
720}
721
722void *
723tsd_get(LispObj key)
724{
725  return TlsGetValue((DWORD)key);
726}
727#else
728void
729tsd_set(LispObj key, void *datum)
730{
731  pthread_setspecific((pthread_key_t)key, datum);
732}
733
734void *
735tsd_get(LispObj key)
736{
737  return pthread_getspecific((pthread_key_t)key);
738}
739#endif
740
741void
742dequeue_tcr(TCR *tcr)
743{
744  TCR *next, *prev;
745
746  next = tcr->next;
747  prev = tcr->prev;
748
749  prev->next = next;
750  next->prev = prev;
751  tcr->prev = tcr->next = NULL;
752#ifdef X8664
753  tcr->linear = NULL;
754#endif
755}
756 
757void
758enqueue_tcr(TCR *new)
759{
760  TCR *head, *tail;
761 
762  LOCK(lisp_global(TCR_AREA_LOCK),new);
763  head = (TCR *)ptr_from_lispobj(lisp_global(INITIAL_TCR));
764  tail = head->prev;
765  tail->next = new;
766  head->prev = new;
767  new->prev = tail;
768  new->next = head;
769  UNLOCK(lisp_global(TCR_AREA_LOCK),new);
770}
771
772#ifdef WIN_32
773TCR *
774allocate_tcr()
775{
776  void *p = calloc(1,sizeof(TCR)+15);
777  TCR *tcr = (TCR *)((((natural)p)+15)&~15);
778
779  tcr->allocated = p;
780  return tcr;
781}
782#else
783TCR *
784allocate_tcr()
785{
786  TCR *tcr, *chain = NULL, *next;
787#ifdef DARWIN
788  extern Boolean use_mach_exception_handling;
789  kern_return_t kret;
790  mach_port_t
791    thread_exception_port,
792    task_self = mach_task_self();
793#endif
794  for (;;) {
795    tcr = calloc(1, sizeof(TCR));
796#ifdef DARWIN
797#if WORD_SIZE == 64
798    if (((unsigned)((natural)tcr)) != ((natural)tcr)) {
799      tcr->next = chain;
800      chain = tcr;
801      continue;
802    }
803#endif
804    if (use_mach_exception_handling) {
805      thread_exception_port = (mach_port_t)((natural)tcr);
806      kret = mach_port_allocate_name(task_self,
807                                     MACH_PORT_RIGHT_RECEIVE,
808                                     thread_exception_port);
809    } else {
810      kret = KERN_SUCCESS;
811    }
812
813    if (kret != KERN_SUCCESS) {
814      tcr->next = chain;
815      chain = tcr;
816      continue;
817    }
818#endif
819    for (next = chain; next;) {
820      next = next->next;
821      free(chain);
822    }
823    return tcr;
824  }
825}
826#endif
827
828#ifdef X8664
829#ifdef LINUX
830#include <asm/prctl.h>
831#include <sys/prctl.h>
832#endif
833#ifdef FREEBSD
834#include <machine/sysarch.h>
835#endif
836
837void
838setup_tcr_extra_segment(TCR *tcr)
839{
840#ifdef FREEBSD
841  amd64_set_gsbase(tcr);
842#endif
843#ifdef LINUX
844  arch_prctl(ARCH_SET_GS, (natural)tcr);
845#endif
846#ifdef DARWIN
847  /* There's no way to do this yet.  See DARWIN_GS_HACK */
848  /* darwin_set_x8664_fs_reg(tcr); */
849#endif
850#ifdef SOLARIS
851  /* Chris Curtis found this and suggested the use of syscall here */
852  syscall(SYS_lwp_private,_LWP_SETPRIVATE, _LWP_GSBASE, tcr);
853#endif
854}
855
856#endif
857
858#ifdef X8632
859
860#ifdef DARWIN
861#include <architecture/i386/table.h>
862#include <architecture/i386/sel.h>
863#include <i386/user_ldt.h>
864
865void setup_tcr_extra_segment(TCR *tcr)
866{
867    uintptr_t addr = (uintptr_t)tcr;
868    unsigned int size = sizeof(*tcr);
869    ldt_entry_t desc;
870    sel_t sel;
871    int i;
872
873    desc.data.limit00 = (size - 1) & 0xffff;
874    desc.data.limit16 = ((size - 1) >> 16) & 0xf;
875    desc.data.base00 = addr & 0xffff;
876    desc.data.base16 = (addr >> 16) & 0xff;
877    desc.data.base24 = (addr >> 24) & 0xff;
878    desc.data.type = DESC_DATA_WRITE;
879    desc.data.dpl = USER_PRIV;
880    desc.data.present = 1;
881    desc.data.stksz = DESC_CODE_32B;
882    desc.data.granular = DESC_GRAN_BYTE;
883   
884    i = i386_set_ldt(LDT_AUTO_ALLOC, &desc, 1);
885
886    if (i < 0) {
887        perror("i386_set_ldt");
888    } else {
889        sel.index = i;
890        sel.rpl = USER_PRIV;
891        sel.ti = SEL_LDT;
892        tcr->ldt_selector = sel;
893    }
894}
895
896void free_tcr_extra_segment(TCR *tcr)
897{
898  /* load %fs with null segement selector */
899  __asm__ volatile ("mov %0,%%fs" : : "r"(0));
900  if (i386_set_ldt(tcr->ldt_selector.index, NULL, 1) < 0)
901    perror("i386_set_ldt");
902  tcr->ldt_selector = NULL_SEL;
903}
904#endif
905
906#ifdef LINUX
907
908#include <asm/ldt.h>
909#include <sys/syscall.h>
910
911/* see desc_struct in kernel/include/asm-i386/processor.h */
912typedef struct {
913  uint32_t a;
914  uint32_t b;
915} linux_desc_struct;
916
917
918#define desc_avail(d) (((d)->a) == 0)
919
920linux_desc_struct linux_ldt_entries[LDT_ENTRIES];
921
922/* We have to ask the Linux kernel for a copy of the ldt table
923   and manage it ourselves.  It's not clear that this is
924   thread-safe in general, but we can at least ensure that
925   it's thread-safe wrt lisp threads. */
926
927pthread_mutex_t ldt_lock = PTHREAD_MUTEX_INITIALIZER;  /* simple, non-recursive mutex */
928
929int
930modify_ldt(int func, void *ptr, unsigned long bytecount)
931{
932  return syscall(__NR_modify_ldt, func, ptr, bytecount);
933}
934
935
936void
937setup_tcr_extra_segment(TCR *tcr)
938{
939  int i, n;
940  short sel;
941  struct user_desc u = {1, 0, 0, 1, MODIFY_LDT_CONTENTS_DATA, 0, 0, 0, 1};
942  linux_desc_struct *d = linux_ldt_entries;
943
944  pthread_mutex_lock(&ldt_lock);
945  n = modify_ldt(0,d,LDT_ENTRIES*LDT_ENTRY_SIZE)/LDT_ENTRY_SIZE;
946  for (i = 0; i < n; i++,d++) {
947    if (desc_avail(d)) {
948      break;
949    }
950  }
951  if (i == LDT_ENTRIES) {
952    pthread_mutex_unlock(&ldt_lock);
953    fprintf(stderr, "All 8192 ldt entries in use ?\n");
954    _exit(1);
955  }
956  u.entry_number = i;
957  u.base_addr = (uint32_t)tcr;
958  u.limit = sizeof(TCR);
959  u.limit_in_pages = 0;
960  if (modify_ldt(1,&u,sizeof(struct user_desc)) != 0) {
961    pthread_mutex_unlock(&ldt_lock);
962    fprintf(stderr,"Can't assign LDT entry\n");
963    _exit(1);
964  }
965  sel = (i << 3) | 7;
966  tcr->ldt_selector = sel;
967  pthread_mutex_unlock(&ldt_lock);
968}
969
970void
971free_tcr_extra_segment(TCR *tcr)
972{
973  struct user_desc u = {0, 0, 0, 0, MODIFY_LDT_CONTENTS_DATA, 0, 0, 0, 0};
974  short sel = tcr->ldt_selector;
975
976  pthread_mutex_lock(&ldt_lock);
977  /* load %fs with null segment selector */
978  __asm__ volatile ("mov %0,%%fs" : : "r"(0));
979  tcr->ldt_selector = 0;
980  u.entry_number = (sel>>3);
981  modify_ldt(1,&u,sizeof(struct user_desc));
982  pthread_mutex_unlock(&ldt_lock);
983 
984}
985
986#endif
987
988#ifdef WINDOWS
989bitvector ldt_entries_in_use = NULL;
990HANDLE ldt_lock;
991
992typedef struct {
993  DWORD offset;
994  DWORD size;
995  LDT_ENTRY entry;
996} win32_ldt_info;
997
998
999int WINAPI (*NtQueryInformationProcess)(HANDLE,DWORD,VOID*,DWORD,DWORD*);
1000int WINAPI (*NtSetInformationProcess)(HANDLE,DWORD,VOID*,DWORD);
1001
1002void
1003init_win32_ldt()
1004{
1005  HANDLE hNtdll;
1006  int status = 0xc0000002;
1007  win32_ldt_info info;
1008  DWORD nret;
1009 
1010
1011  ldt_entries_in_use=malloc(8192/8);
1012  zero_bits(ldt_entries_in_use,8192);
1013  ldt_lock = CreateMutex(NULL,0,NULL);
1014
1015  hNtdll = LoadLibrary("ntdll.dll");
1016  NtQueryInformationProcess = (void*)GetProcAddress(hNtdll, "NtQueryInformationProcess");
1017  NtSetInformationProcess = (void*)GetProcAddress(hNtdll, "NtSetInformationProcess");
1018  if (NtQueryInformationProcess != NULL) {
1019    info.offset = 0;
1020    info.size = sizeof(LDT_ENTRY);
1021    status = NtQueryInformationProcess(GetCurrentProcess(),10,&info,sizeof(info),&nret);
1022  }
1023
1024  if (status) {
1025    fprintf(stderr, "This application can't run under this OS version\n");
1026    _exit(1);
1027  }
1028}
1029
1030void
1031setup_tcr_extra_segment(TCR *tcr)
1032{
1033  int i, status;
1034  DWORD nret;
1035  win32_ldt_info info;
1036  LDT_ENTRY *entry = &(info.entry);
1037  DWORD *words = (DWORD *)entry, tcraddr = (DWORD)tcr;
1038
1039
1040  WaitForSingleObject(ldt_lock,INFINITE);
1041
1042  for (i = 0; i < 8192; i++) {
1043    if (!ref_bit(ldt_entries_in_use,i)) {
1044      info.offset = i << 3;
1045      info.size = sizeof(LDT_ENTRY);
1046      words[0] = 0;
1047      words[1] = 0;
1048      status = NtQueryInformationProcess(GetCurrentProcess(),10,&info,sizeof(info),&nret);
1049      if (status == 0) {
1050        if ((info.size == 0) ||
1051            ((words[0] == 0) && (words[1] == 0))) {
1052          break;
1053        }
1054      }
1055    }
1056  }
1057  if (i == 8192) {
1058    ReleaseMutex(ldt_lock);
1059    fprintf(stderr, "All 8192 ldt entries in use ?\n");
1060    _exit(1);
1061  }
1062  set_bit(ldt_entries_in_use,i);
1063  words[0] = 0;
1064  words[1] = 0;
1065  entry->LimitLow = sizeof(TCR);
1066  entry->BaseLow = tcraddr & 0xffff;
1067  entry->HighWord.Bits.BaseMid = (tcraddr >> 16) & 0xff;
1068  entry->HighWord.Bits.BaseHi = (tcraddr >> 24);
1069  entry->HighWord.Bits.Pres = 1;
1070  entry->HighWord.Bits.Default_Big = 1;
1071  entry->HighWord.Bits.Type = 16 | 2; /* read-write data */
1072  entry->HighWord.Bits.Dpl = 3; /* for use by the great unwashed */
1073  info.size = sizeof(LDT_ENTRY);
1074  status = NtSetInformationProcess(GetCurrentProcess(),10,&info,sizeof(info));
1075  if (status != 0) {
1076    ReleaseMutex(ldt_lock);
1077    FBug(NULL, "can't set LDT entry %d, status = 0x%x", i, status);
1078  }
1079#if 1
1080  /* Sanity check */
1081  info.offset = i << 3;
1082  info.size = sizeof(LDT_ENTRY);
1083  words[0] = 0;
1084  words[0] = 0;
1085  NtQueryInformationProcess(GetCurrentProcess(),10,&info,sizeof(info),&nret);
1086  if (((entry->BaseLow)|((entry->HighWord.Bits.BaseMid)<<16)|((entry->HighWord.Bits.BaseHi)<<24)) != tcraddr) {
1087    Bug(NULL, "you blew it: bad address in ldt entry\n");
1088  }
1089#endif
1090  tcr->ldt_selector = (i << 3) | 7;
1091  ReleaseMutex(ldt_lock);
1092}
1093
1094void 
1095free_tcr_extra_segment(TCR *tcr)
1096{
1097}
1098
1099#endif
1100#ifdef FREEBSD
1101#include <machine/segments.h>
1102#include <machine/sysarch.h>
1103
1104/* It'd be tempting to use i386_set_fsbase() here, but there doesn't
1105   seem to be any way to free the GDT entry it creates.  Actually,
1106   it's not clear that that really sets a GDT entry; let's see */
1107
1108#define FREEBSD_USE_SET_FSBASE 1
1109void
1110setup_tcr_extra_segment(TCR *tcr)
1111{
1112#if !FREEBSD_USE_SET_FSBASE
1113  struct segment_descriptor sd;
1114  uintptr_t addr = (uintptr_t)tcr;
1115  unsigned int size = sizeof(*tcr);
1116  int i;
1117
1118  sd.sd_lolimit = (size - 1) & 0xffff;
1119  sd.sd_hilimit = ((size - 1) >> 16) & 0xf;
1120  sd.sd_lobase = addr & ((1<<24)-1);
1121  sd.sd_hibase = (addr>>24)&0xff;
1122
1123
1124
1125  sd.sd_type = 18;
1126  sd.sd_dpl = SEL_UPL;
1127  sd.sd_p = 1;
1128  sd.sd_def32 = 1;
1129  sd.sd_gran = 0;
1130
1131  i = i386_set_ldt(LDT_AUTO_ALLOC, (union descriptor *)&sd, 1);
1132
1133  if (i < 0) {
1134    perror("i386_set_ldt");
1135    exit(1);
1136  } else {
1137    tcr->ldt_selector = LSEL(i,SEL_UPL);
1138  }
1139#else
1140  if (i386_set_fsbase((void*)tcr)) {
1141    perror("i386_set_fsbase");
1142    exit(1);
1143  }
1144  /* Once we've called i386_set_fsbase, we can't write to %fs. */
1145  tcr->ldt_selector = GSEL(GUFS_SEL, SEL_UPL);
1146#endif
1147}
1148
1149void 
1150free_tcr_extra_segment(TCR *tcr)
1151{
1152#if FREEBSD_USE_SET_FSBASE
1153  /* On a 32-bit kernel, this allocates a GDT entry.  It's not clear
1154     what it would mean to deallocate that entry. */
1155  /* If we're running on a 64-bit kernel, we can't write to %fs */
1156#else
1157  int idx = tcr->ldt_selector >> 3;
1158  /* load %fs with null segment selector */
1159  __asm__ volatile ("mov %0,%%fs" : : "r"(0));
1160  if (i386_set_ldt(idx, NULL, 1) < 0)
1161    perror("i386_set_ldt");
1162#endif
1163  tcr->ldt_selector = 0;
1164}
1165#endif
1166
1167#ifdef SOLARIS
1168#include <sys/sysi86.h>
1169
1170bitvector ldt_entries_in_use = NULL;
1171pthread_mutex_t ldt_lock = PTHREAD_MUTEX_INITIALIZER;  /* simple, non-recursive mutex */
1172
1173void
1174solaris_ldt_init()
1175{
1176  int fd;
1177  struct ssd s;
1178
1179  ldt_entries_in_use=malloc(8192/8);
1180  zero_bits(ldt_entries_in_use,8192);
1181 
1182  fd = open("/proc/self/ldt", O_RDONLY);
1183
1184  while(read(fd,&s,sizeof(s)) == sizeof(s)) {
1185    set_bit(ldt_entries_in_use,s.sel>>3);
1186  }
1187  close(fd);
1188}
1189   
1190
1191void
1192setup_tcr_extra_segment(TCR *tcr)
1193{
1194  struct ssd s;
1195  int i;
1196
1197  pthread_mutex_lock(&ldt_lock);
1198
1199  for (i = 0; i < 8192; i++) {
1200    if (!ref_bit(ldt_entries_in_use,i)) {
1201      s.sel = (i<<3)|7;
1202      s.bo = (unsigned int)tcr;
1203      s.ls = sizeof(TCR);
1204      s.acc1 = 0xf2;
1205      s.acc2 = 4;
1206
1207      if (sysi86(SI86DSCR, &s) >= 0) {
1208        set_bit(ldt_entries_in_use,i);
1209        tcr->ldt_selector = (i<<3)|7;
1210        pthread_mutex_unlock(&ldt_lock);
1211        return;
1212      }
1213      set_bit(ldt_entries_in_use,i);
1214    }
1215  }
1216  pthread_mutex_unlock(&ldt_lock);
1217  fprintf(stderr, "All 8192 LDT descriptors in use\n");
1218  _exit(1);
1219
1220
1221 
1222}
1223
1224void 
1225free_tcr_extra_segment(TCR *tcr)
1226{
1227  struct ssd s;
1228  int i;
1229
1230  pthread_mutex_lock(&ldt_lock);
1231  __asm__ volatile ("mov %0,%%fs" : : "r"(0));
1232  s.sel = tcr->ldt_selector;
1233  i = s.sel>>3;
1234  tcr->ldt_selector = 0;
1235  s.bo = 0;
1236  s.ls = 0;
1237  s.acc1 = 0;
1238  s.acc2 = 0;
1239  sysi86(SI86DSCR, &s);
1240  clr_bit(ldt_entries_in_use,i);
1241  pthread_mutex_unlock(&ldt_lock);
1242}
1243
1244#endif
1245#endif
1246
1247/*
1248  Caller must hold the area_lock.
1249*/
1250TCR *
1251new_tcr(natural vstack_size, natural tstack_size)
1252{
1253  extern area
1254    *allocate_vstack_holding_area_lock(natural),
1255    *allocate_tstack_holding_area_lock(natural);
1256  area *a;
1257  int i;
1258#ifndef WINDOWS
1259  sigset_t sigmask;
1260
1261  sigemptyset(&sigmask);
1262  pthread_sigmask(SIG_SETMASK,&sigmask, NULL);
1263#endif
1264
1265#ifdef HAVE_TLS
1266  TCR *tcr = &current_tcr;
1267#else /* no TLS */
1268  TCR *tcr = allocate_tcr();
1269#endif
1270
1271#ifdef X86
1272  setup_tcr_extra_segment(tcr);
1273  tcr->linear = tcr;
1274#ifdef X8632
1275  tcr->node_regs_mask = X8632_DEFAULT_NODE_REGS_MASK;
1276#endif
1277#endif
1278
1279#if (WORD_SIZE == 64)
1280  tcr->single_float_convert.tag = subtag_single_float;
1281#endif
1282  lisp_global(TCR_COUNT) += (1<<fixnumshift);
1283  tcr->suspend = new_semaphore(0);
1284  tcr->resume = new_semaphore(0);
1285  tcr->reset_completion = new_semaphore(0);
1286  tcr->activate = new_semaphore(0);
1287  LOCK(lisp_global(TCR_AREA_LOCK),tcr);
1288  a = allocate_vstack_holding_area_lock(vstack_size);
1289  tcr->vs_area = a;
1290  a->owner = tcr;
1291  tcr->save_vsp = (LispObj *) a->active; 
1292  a = allocate_tstack_holding_area_lock(tstack_size);
1293  UNLOCK(lisp_global(TCR_AREA_LOCK),tcr);
1294  tcr->ts_area = a;
1295  a->owner = tcr;
1296  tcr->save_tsp = (LispObj *) a->active;
1297#ifdef X86
1298  tcr->next_tsp = tcr->save_tsp;
1299#endif
1300
1301  tcr->valence = TCR_STATE_FOREIGN;
1302#ifdef PPC
1303  tcr->lisp_fpscr.words.l = 0xd0;
1304#endif
1305#ifdef X86
1306  tcr->lisp_mxcsr = (1 << MXCSR_DM_BIT) | 
1307#if 1                           /* Mask underflow; too hard to
1308                                   deal with denorms if underflow is
1309                                   enabled */
1310    (1 << MXCSR_UM_BIT) | 
1311#endif
1312    (1 << MXCSR_PM_BIT);
1313#endif
1314  tcr->save_allocbase = tcr->save_allocptr = (void *) VOID_ALLOCPTR;
1315  tcr->tlb_limit = 2048<<fixnumshift;
1316  tcr->tlb_pointer = (LispObj *)malloc(tcr->tlb_limit);
1317  for (i = 0; i < 2048; i++) {
1318    tcr->tlb_pointer[i] = (LispObj) no_thread_local_binding_marker;
1319  }
1320  TCR_INTERRUPT_LEVEL(tcr) = (LispObj) (-1<<fixnum_shift);
1321#ifndef WINDOWS
1322  tcr->shutdown_count = PTHREAD_DESTRUCTOR_ITERATIONS;
1323#else
1324  tcr->shutdown_count = 1;
1325#endif
1326  return tcr;
1327}
1328
1329void
1330shutdown_thread_tcr(void *arg)
1331{
1332  TCR *tcr = TCR_FROM_TSD(arg),*current=get_tcr(0);
1333
1334  area *vs, *ts, *cs;
1335  void *termination_semaphore;
1336 
1337  if (current == NULL) {
1338    current = tcr;
1339  }
1340
1341  if (--(tcr->shutdown_count) == 0) {
1342    if (tcr->flags & (1<<TCR_FLAG_BIT_FOREIGN)) {
1343      LispObj callback_macptr = nrs_FOREIGN_THREAD_CONTROL.vcell,
1344        callback_ptr = ((macptr *)ptr_from_lispobj(untag(callback_macptr)))->address;
1345   
1346      tsd_set(lisp_global(TCR_KEY), TCR_TO_TSD(tcr));
1347      ((void (*)())ptr_from_lispobj(callback_ptr))(1);
1348      tsd_set(lisp_global(TCR_KEY), NULL);
1349    }
1350#ifdef DARWIN
1351    darwin_exception_cleanup(tcr);
1352#endif
1353    LOCK(lisp_global(TCR_AREA_LOCK),current);
1354    vs = tcr->vs_area;
1355    tcr->vs_area = NULL;
1356    ts = tcr->ts_area;
1357    tcr->ts_area = NULL;
1358    cs = tcr->cs_area;
1359    tcr->cs_area = NULL;
1360    if (vs) {
1361      condemn_area_holding_area_lock(vs);
1362    }
1363    if (ts) {
1364      condemn_area_holding_area_lock(ts);
1365    }
1366    if (cs) {
1367      condemn_area_holding_area_lock(cs);
1368    }
1369    destroy_semaphore(&tcr->suspend);
1370    destroy_semaphore(&tcr->resume);
1371    destroy_semaphore(&tcr->reset_completion);
1372    destroy_semaphore(&tcr->activate);
1373    tcr->tlb_limit = 0;
1374    free(tcr->tlb_pointer);
1375    tcr->tlb_pointer = NULL;
1376    tcr->osid = 0;
1377    tcr->interrupt_pending = 0;
1378    termination_semaphore = tcr->termination_semaphore;
1379    tcr->termination_semaphore = NULL;
1380#ifdef HAVE_TLS
1381    dequeue_tcr(tcr);
1382#endif
1383#ifdef X8632
1384    free_tcr_extra_segment(tcr);
1385#endif
1386#ifdef WIN32
1387    CloseHandle((HANDLE)tcr->io_datum);
1388    tcr->io_datum = NULL;
1389    free(tcr->native_thread_info);
1390    tcr->native_thread_info = NULL;
1391#endif
1392    UNLOCK(lisp_global(TCR_AREA_LOCK),current);
1393    if (termination_semaphore) {
1394      SEM_RAISE(termination_semaphore);
1395    }
1396  } else {
1397    tsd_set(lisp_global(TCR_KEY), TCR_TO_TSD(tcr));
1398  }
1399}
1400
1401void
1402tcr_cleanup(void *arg)
1403{
1404  TCR *tcr = (TCR *)arg;
1405  area *a;
1406
1407  a = tcr->vs_area;
1408  if (a) {
1409    a->active = a->high;
1410  }
1411  a = tcr->ts_area;
1412  if (a) {
1413    a->active = a->high;
1414  }
1415  a = tcr->cs_area;
1416  if (a) {
1417    a->active = a->high;
1418  }
1419  tcr->valence = TCR_STATE_FOREIGN;
1420  tcr->shutdown_count = 1;
1421  shutdown_thread_tcr(tcr);
1422  tsd_set(lisp_global(TCR_KEY), NULL);
1423}
1424
1425void *
1426current_native_thread_id()
1427{
1428  return ((void *) (natural)
1429#ifdef LINUX
1430#ifdef __NR_gettid
1431          syscall(__NR_gettid)
1432#else
1433          getpid()
1434#endif
1435#endif
1436#ifdef DARWIN
1437          mach_thread_self()
1438#endif
1439#ifdef FREEBSD
1440          pthread_self()
1441#endif
1442#ifdef SOLARIS
1443          pthread_self()
1444#endif
1445#ifdef WINDOWS
1446          GetCurrentThreadId()
1447#endif
1448          );
1449}
1450
1451
1452void
1453thread_init_tcr(TCR *tcr, void *stack_base, natural stack_size)
1454{
1455  area *a, *register_cstack_holding_area_lock(BytePtr, natural);
1456
1457  tcr->osid = current_thread_osid();
1458  tcr->native_thread_id = current_native_thread_id();
1459  LOCK(lisp_global(TCR_AREA_LOCK),tcr);
1460  a = register_cstack_holding_area_lock((BytePtr)stack_base, stack_size);
1461  UNLOCK(lisp_global(TCR_AREA_LOCK),tcr);
1462  tcr->cs_area = a;
1463  a->owner = tcr;
1464  if (!(tcr->flags & (1<<TCR_FLAG_BIT_FOREIGN))) {
1465    tcr->cs_limit = (LispObj)ptr_to_lispobj(a->softlimit);
1466  }
1467#ifdef LINUX
1468#ifdef PPC
1469#ifndef PPC64
1470  tcr->native_thread_info = current_r2;
1471#endif
1472#endif
1473#endif
1474  tcr->errno_loc = &errno;
1475  tsd_set(lisp_global(TCR_KEY), TCR_TO_TSD(tcr));
1476#ifdef DARWIN
1477  extern Boolean use_mach_exception_handling;
1478  if (use_mach_exception_handling) {
1479    darwin_exception_init(tcr);
1480  }
1481#endif
1482#ifdef LINUX
1483  linux_exception_init(tcr);
1484#endif
1485#ifdef WINDOWS
1486  tcr->io_datum = (VOID *)CreateEvent(NULL, true, false, NULL);
1487  tcr->native_thread_info = malloc(sizeof(CONTEXT));
1488#endif
1489  tcr->log2_allocation_quantum = unbox_fixnum(lisp_global(DEFAULT_ALLOCATION_QUANTUM));
1490}
1491
1492/*
1493  Register the specified tcr as "belonging to" the current thread.
1494  Under Darwin, setup Mach exception handling for the thread.
1495  Install cleanup handlers for thread termination.
1496*/
1497void
1498register_thread_tcr(TCR *tcr)
1499{
1500  void *stack_base = NULL;
1501  natural stack_size = 0;
1502
1503  os_get_current_thread_stack_bounds(&stack_base, &stack_size);
1504  thread_init_tcr(tcr, stack_base, stack_size);
1505  enqueue_tcr(tcr);
1506}
1507
1508
1509 
1510 
1511#ifndef MAP_GROWSDOWN
1512#define MAP_GROWSDOWN 0
1513#endif
1514
1515Ptr
1516create_stack(natural size)
1517{
1518  Ptr p;
1519  size=align_to_power_of_2(size, log2_page_size);
1520  p = (Ptr) MapMemoryForStack((size_t)size);
1521  if (p != (Ptr)(-1)) {
1522    *((size_t *)p) = size;
1523    return p;
1524  }
1525  allocation_failure(true, size);
1526
1527}
1528
1529void *
1530allocate_stack(natural size)
1531{
1532  return create_stack(size);
1533}
1534
1535void
1536free_stack(void *s)
1537{
1538  size_t size = *((size_t *)s);
1539  UnMapMemory(s, size);
1540}
1541
1542Boolean threads_initialized = false;
1543
1544#ifndef USE_FUTEX
1545#ifdef WINDOWS
1546void
1547count_cpus()
1548{
1549  SYSTEM_INFO si;
1550
1551  GetSystemInfo(&si);
1552  if (si.dwNumberOfProcessors > 1) {
1553    spin_lock_tries = 1024;
1554  }
1555}
1556#else
1557void
1558count_cpus()
1559{
1560#ifdef DARWIN
1561  /* As of OSX 10.4, Darwin doesn't define _SC_NPROCESSORS_ONLN */
1562#include <mach/host_info.h>
1563
1564  struct host_basic_info info;
1565  mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
1566 
1567  if (KERN_SUCCESS == host_info(mach_host_self(), HOST_BASIC_INFO,(host_info_t)(&info),&count)) {
1568    if (info.max_cpus > 1) {
1569      spin_lock_tries = 1024;
1570    }
1571  }
1572#else
1573  int n = sysconf(_SC_NPROCESSORS_ONLN);
1574 
1575  if (n > 1) {
1576    spin_lock_tries = 1024;
1577  }
1578#endif
1579}
1580#endif
1581#endif
1582
1583void
1584init_threads(void * stack_base, TCR *tcr)
1585{
1586  lisp_global(INITIAL_TCR) = (LispObj)ptr_to_lispobj(tcr);
1587#ifdef WINDOWS
1588  lisp_global(TCR_KEY) = TlsAlloc();
1589  pCancelIoEx = windows_find_symbol(NULL, "CancelIoEx");
1590#else
1591  pthread_key_create((pthread_key_t *)&(lisp_global(TCR_KEY)), shutdown_thread_tcr);
1592  thread_signal_setup();
1593#endif
1594 
1595#ifndef USE_FUTEX
1596  count_cpus();
1597#endif
1598  threads_initialized = true;
1599}
1600
1601
1602#ifdef WINDOWS
1603unsigned CALLBACK
1604#else
1605void *
1606#endif
1607lisp_thread_entry(void *param)
1608{
1609  thread_activation *activation = (thread_activation *)param;
1610  TCR *tcr = new_tcr(activation->vsize, activation->tsize);
1611#ifndef WINDOWS
1612  sigset_t mask, old_mask;
1613
1614  sigemptyset(&mask);
1615  pthread_sigmask(SIG_SETMASK, &mask, &old_mask);
1616#endif
1617
1618  register_thread_tcr(tcr);
1619
1620#ifndef WINDOWS
1621  pthread_cleanup_push(tcr_cleanup,(void *)tcr);
1622#endif
1623  tcr->vs_area->active -= node_size;
1624  *(--tcr->save_vsp) = lisp_nil;
1625  enable_fp_exceptions();
1626  SET_TCR_FLAG(tcr,TCR_FLAG_BIT_AWAITING_PRESET);
1627  activation->tcr = tcr;
1628  SEM_RAISE(activation->created);
1629  do {
1630    SEM_RAISE(tcr->reset_completion);
1631    SEM_WAIT_FOREVER(tcr->activate);
1632    /* Now go run some lisp code */
1633    start_lisp(TCR_TO_TSD(tcr),0);
1634  } while (tcr->flags & (1<<TCR_FLAG_BIT_AWAITING_PRESET));
1635#ifndef WINDOWS
1636  pthread_cleanup_pop(true);
1637#else
1638  tcr_cleanup(tcr);
1639#endif
1640#ifdef WINDOWS
1641  return 0;
1642#else
1643  return NULL;
1644#endif
1645}
1646
1647void *
1648xNewThread(natural control_stack_size,
1649           natural value_stack_size,
1650           natural temp_stack_size)
1651
1652{
1653  thread_activation activation;
1654
1655
1656  activation.tsize = temp_stack_size;
1657  activation.vsize = value_stack_size;
1658  activation.tcr = 0;
1659  activation.created = new_semaphore(0);
1660  if (create_system_thread(control_stack_size +(CSTACK_HARDPROT+CSTACK_SOFTPROT), 
1661                           NULL, 
1662                           lisp_thread_entry,
1663                           (void *) &activation)) {
1664   
1665    SEM_WAIT_FOREVER(activation.created);       /* Wait until thread's entered its initial function */
1666  }
1667  destroy_semaphore(&activation.created); 
1668  return TCR_TO_TSD(activation.tcr);
1669}
1670
1671Boolean
1672active_tcr_p(TCR *q)
1673{
1674  TCR *head = (TCR *)ptr_from_lispobj(lisp_global(INITIAL_TCR)), *p = head;
1675 
1676  do {
1677    if (p == q) {
1678      return true;
1679    }
1680    p = p->next;
1681  } while (p != head);
1682  return false;
1683}
1684
1685#ifdef WINDOWS
1686OSErr
1687xDisposeThread(TCR *tcr)
1688{
1689  return 0;                     /* I don't think that this is ever called. */
1690}
1691#else
1692OSErr
1693xDisposeThread(TCR *tcr)
1694{
1695  if (tcr != (TCR *)ptr_from_lispobj(lisp_global(INITIAL_TCR))) {
1696    if (active_tcr_p(tcr) && (tcr != get_tcr(false))) {
1697      pthread_cancel((pthread_t)(tcr->osid));
1698      return 0;
1699    }
1700  }
1701  return -50;
1702}
1703#endif
1704
1705OSErr
1706xYieldToThread(TCR *target)
1707{
1708  Bug(NULL, "xYieldToThread ?");
1709  return 0;
1710}
1711 
1712OSErr
1713xThreadCurrentStackSpace(TCR *tcr, unsigned *resultP)
1714{
1715  Bug(NULL, "xThreadCurrentStackSpace ?");
1716  return 0;
1717}
1718
1719
1720#ifdef WINDOWS
1721LispObj
1722create_system_thread(size_t stack_size,
1723                     void* stackaddr,
1724                     unsigned CALLBACK (*start_routine)(void *),
1725                     void* param)
1726{
1727  HANDLE thread_handle;
1728
1729  stack_size = ((stack_size+(((1<<16)-1)))&~((1<<16)-1));
1730
1731  thread_handle = (HANDLE)_beginthreadex(NULL, 
1732                                         0/*stack_size*/,
1733                                         start_routine,
1734                                         param,
1735                                         0, 
1736                                         NULL);
1737
1738  if (thread_handle == NULL) {
1739    wperror("CreateThread");
1740  }
1741  return (LispObj) ptr_to_lispobj(thread_handle);
1742}
1743#else
1744LispObj
1745create_system_thread(size_t stack_size,
1746                     void* stackaddr,
1747                     void* (*start_routine)(void *),
1748                     void* param)
1749{
1750  pthread_attr_t attr;
1751  pthread_t returned_thread = (pthread_t) 0;
1752
1753  pthread_attr_init(&attr);
1754  pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); 
1755
1756  if (stack_size == MINIMAL_THREAD_STACK_SIZE) {
1757    stack_size = PTHREAD_STACK_MIN;
1758  }
1759
1760  stack_size = ensure_stack_limit(stack_size);
1761  if (stackaddr != NULL) {
1762    /* Size must have been specified.  Sort of makes sense ... */
1763#ifdef DARWIN
1764    Fatal("no pthread_attr_setsetstack. "," Which end of stack does address refer to?");
1765#else
1766    pthread_attr_setstack(&attr, stackaddr, stack_size);
1767#endif
1768  } else if (stack_size != DEFAULT_THREAD_STACK_SIZE) {
1769    pthread_attr_setstacksize(&attr,stack_size);
1770  }
1771
1772  /*
1773     I think that's just about enough ... create the thread.
1774  */
1775  pthread_create(&returned_thread, &attr, start_routine, param);
1776  pthread_attr_destroy(&attr);
1777  return (LispObj) ptr_to_lispobj(returned_thread);
1778}
1779#endif
1780
1781TCR *
1782get_tcr(Boolean create)
1783{
1784#ifdef HAVE_TLS
1785  TCR *current = current_tcr.linear;
1786#else
1787  void *tsd = (void *)tsd_get(lisp_global(TCR_KEY));
1788  TCR *current = (tsd == NULL) ? NULL : TCR_FROM_TSD(tsd);
1789#endif
1790
1791  if ((current == NULL) && create) {
1792    LispObj callback_macptr = nrs_FOREIGN_THREAD_CONTROL.vcell,
1793      callback_ptr = ((macptr *)ptr_from_lispobj(untag(callback_macptr)))->address;
1794    int i, nbindwords = 0;
1795    extern unsigned initial_stack_size;
1796   
1797    /* Make one. */
1798    current = new_tcr(initial_stack_size, MIN_TSTACK_SIZE);
1799    SET_TCR_FLAG(current,TCR_FLAG_BIT_FOREIGN);
1800    register_thread_tcr(current);
1801#ifdef DEBUG_TCR_CREATION
1802#ifndef WINDOWS
1803    fprintf(stderr, "\ncreating TCR for pthread 0x%x", pthread_self());
1804#endif
1805#endif
1806    current->vs_area->active -= node_size;
1807    *(--current->save_vsp) = lisp_nil;
1808#ifdef PPC
1809#define NSAVEREGS 8
1810#endif
1811#ifdef X8664
1812#define NSAVEREGS 4
1813#endif
1814#ifdef X8632
1815#define NSAVEREGS 0
1816#endif
1817    for (i = 0; i < NSAVEREGS; i++) {
1818      *(--current->save_vsp) = 0;
1819      current->vs_area->active -= node_size;
1820    }
1821    nbindwords = ((int (*)())ptr_from_lispobj(callback_ptr))(-1);
1822    for (i = 0; i < nbindwords; i++) {
1823      *(--current->save_vsp) = 0;
1824      current->vs_area->active -= node_size;
1825    }
1826    current->shutdown_count = 1;
1827    ((void (*)())ptr_from_lispobj(callback_ptr))(0);
1828
1829  }
1830 
1831  return current;
1832}
1833
1834#ifdef WINDOWS
1835
1836Boolean
1837suspend_tcr(TCR *tcr)
1838{
1839  int suspend_count = atomic_incf(&(tcr->suspend_count));
1840  DWORD rc;
1841  if (suspend_count == 1) {
1842    CONTEXT  *pcontext = (CONTEXT *)tcr->native_thread_info;
1843    HANDLE hthread = (HANDLE)(tcr->osid);
1844    pc where;
1845    area *cs = tcr->cs_area;
1846    LispObj foreign_rsp;
1847
1848    if (hthread == NULL) {
1849      return false;
1850    }
1851    rc = SuspendThread(hthread);
1852    if (rc == -1) {
1853      /* If the thread's simply dead, we should handle that here */
1854      return false;
1855    }
1856    pcontext->ContextFlags = CONTEXT_ALL;
1857    rc = GetThreadContext(hthread, pcontext);
1858    if (rc == 0) {
1859      return false;
1860    }
1861    where = (pc)(xpPC(pcontext));
1862
1863    if (tcr->valence == TCR_STATE_LISP) {
1864      if ((where >= restore_windows_context_start) &&
1865          (where < restore_windows_context_end)) {
1866        /* Thread has started to return from an exception. */
1867        if (where < restore_windows_context_load_rcx) {
1868          /* In the process of restoring registers; context still in
1869             %rcx.  Just make our suspend_context be the context
1870             we're trying to restore, so that we'll resume from
1871             the suspend in the same context that we're trying to
1872             restore */
1873#ifdef WIN_64
1874          *pcontext = * (CONTEXT *)(pcontext->Rcx);
1875#else
1876          fprintf(stderr, "missing win32 suspend code, case (1)\n");
1877#endif
1878        } else {
1879          /* Most of the context has already been restored; fix %rcx
1880             if need be, then restore ss:rsp, cs:rip, and flags. */
1881#ifdef WIN64
1882          x64_iret_frame *iret_frame = (x64_iret_frame *) (pcontext->Rsp);
1883          if (where == restore_windows_context_load_rcx) {
1884            pcontext->Rcx = ((CONTEXT*)(pcontext->Rcx))->Rcx;
1885          }
1886          pcontext->Rip = iret_frame->Rip;
1887          pcontext->SegCs = (WORD) iret_frame->Cs;
1888          pcontext->EFlags = (DWORD) iret_frame->Rflags;
1889          pcontext->Rsp = iret_frame->Rsp;
1890          pcontext->SegSs = (WORD) iret_frame->Ss;
1891#else
1892#warning need context setup for win32
1893          fprintf(stderr, "missing win32 suspend code, case (2)\n");
1894#endif
1895        }
1896        tcr->suspend_context = NULL;
1897      } else {
1898        area *ts = tcr->ts_area;
1899        /* If we're in the lisp heap, or in x86-spentry??.o, or in
1900           x86-subprims??.o, or in the subprims jump table at #x15000,
1901           or on the tstack ... we're just executing lisp code.  Otherwise,
1902           we got an exception while executing lisp code, but haven't
1903           entered the handler yet (still in Windows exception glue
1904           or switching stacks or something.)  In the latter case, we
1905           basically want to get to he handler and have it notice
1906           the pending exception request, and suspend the thread at that
1907           point. */
1908        if (!((where < (pc)lisp_global(HEAP_END)) &&
1909              (where >= (pc)lisp_global(HEAP_START))) &&
1910            !((where < spentry_end) && (where >= spentry_start)) &&
1911            !((where < subprims_end) && (where >= subprims_start)) &&
1912            !((where < (pc) 0x16000) &&
1913              (where >= (pc) 0x15000)) &&
1914            !((where < (pc) (ts->high)) &&
1915              (where >= (pc) (ts->low)))) {
1916          /* The thread has lisp valence, but is not executing code
1917             where we expect lisp code to be and is not exiting from
1918             an exception handler.  That pretty much means that it's
1919             on its way into an exception handler; we have to handshake
1920             until it enters an exception-wait state. */
1921          /* There are likely race conditions here */
1922          SET_TCR_FLAG(tcr,TCR_FLAG_BIT_PENDING_SUSPEND);
1923          ResumeThread(hthread);
1924          SEM_WAIT_FOREVER(tcr->suspend);
1925          SuspendThread(hthread);
1926          /* The thread is either waiting for its resume semaphore to
1927             be signaled or is about to wait.  Signal it now, while
1928             the thread's suspended. */
1929          SEM_RAISE(tcr->resume);
1930          pcontext->ContextFlags = CONTEXT_ALL;
1931          GetThreadContext(hthread, pcontext);
1932        }
1933      }
1934    } else {
1935      if (tcr->valence == TCR_STATE_EXCEPTION_RETURN) {
1936        if (!tcr->pending_exception_context) {
1937          FBug(pcontext, "we're confused here.");
1938        }
1939        *pcontext = *tcr->pending_exception_context;
1940        tcr->pending_exception_context = NULL;
1941        tcr->valence = TCR_STATE_LISP;
1942      }
1943    }
1944    tcr->suspend_context = pcontext;
1945    return true;
1946  }
1947  return false;
1948}
1949#else
1950Boolean
1951suspend_tcr(TCR *tcr)
1952{
1953  int suspend_count = atomic_incf(&(tcr->suspend_count));
1954  pthread_t thread;
1955  if (suspend_count == 1) {
1956    thread = (pthread_t)(tcr->osid);
1957    if ((thread != (pthread_t) 0) &&
1958        (pthread_kill(thread, thread_suspend_signal) == 0)) {
1959      SET_TCR_FLAG(tcr,TCR_FLAG_BIT_SUSPEND_ACK_PENDING);
1960    } else {
1961      /* A problem using pthread_kill.  On Darwin, this can happen
1962         if the thread has had its signal mask surgically removed
1963         by pthread_exit.  If the native (Mach) thread can be suspended,
1964         do that and return true; otherwise, flag the tcr as belonging
1965         to a dead thread by setting tcr->osid to 0.
1966      */
1967      tcr->osid = 0;
1968      return false;
1969    }
1970    return true;
1971  }
1972  return false;
1973}
1974#endif
1975
1976#ifdef WINDOWS
1977Boolean
1978tcr_suspend_ack(TCR *tcr)
1979{
1980  return true;
1981}
1982#else
1983Boolean
1984tcr_suspend_ack(TCR *tcr)
1985{
1986  if (tcr->flags & (1<<TCR_FLAG_BIT_SUSPEND_ACK_PENDING)) {
1987    SEM_WAIT_FOREVER(tcr->suspend);
1988    tcr->flags &= ~(1<<TCR_FLAG_BIT_SUSPEND_ACK_PENDING);
1989  }
1990  return true;
1991}
1992#endif
1993     
1994
1995Boolean
1996kill_tcr(TCR *tcr)
1997{
1998  TCR *current = get_tcr(true);
1999  Boolean result = false;
2000
2001  LOCK(lisp_global(TCR_AREA_LOCK),current);
2002  {
2003    LispObj osid = tcr->osid;
2004   
2005    if (osid) {
2006      result = true;
2007#ifdef WINDOWS
2008      /* What we really want to de hear is (something like)
2009         forcing the thread to run quit_handler().  For now,
2010         mark the TCR as dead and kill thw Windows thread. */
2011      tcr->osid = 0;
2012      if (!TerminateThread((HANDLE)osid, 0)) {
2013        result = false;
2014      } else {
2015        shutdown_thread_tcr(tcr);
2016      }
2017#else
2018      if (pthread_kill((pthread_t)osid,SIGQUIT)) {
2019        result = false;
2020      }
2021#endif
2022    }
2023  }
2024  UNLOCK(lisp_global(TCR_AREA_LOCK), current);
2025  return result;
2026}
2027
2028Boolean
2029lisp_suspend_tcr(TCR *tcr)
2030{
2031  Boolean suspended;
2032  TCR *current = get_tcr(true);
2033 
2034  LOCK(lisp_global(TCR_AREA_LOCK),current);
2035  suspended = suspend_tcr(tcr);
2036  if (suspended) {
2037    while (!tcr_suspend_ack(tcr));
2038  }
2039  UNLOCK(lisp_global(TCR_AREA_LOCK),current);
2040  return suspended;
2041}
2042         
2043#ifdef WINDOWS
2044Boolean
2045resume_tcr(TCR *tcr)
2046{
2047  int suspend_count = atomic_decf(&(tcr->suspend_count)), err;
2048  DWORD rc;
2049  if (suspend_count == 0) {
2050    CONTEXT *context = tcr->suspend_context;
2051    HANDLE hthread = (HANDLE)(tcr->osid);
2052
2053    if (context) {
2054      context->ContextFlags = CONTEXT_ALL;
2055      tcr->suspend_context = NULL;
2056      SetThreadContext(hthread,context);
2057      rc = ResumeThread(hthread);
2058      if (rc == -1) {
2059        wperror("ResumeThread");
2060        return false;
2061      }
2062      return true;
2063    }
2064  }
2065  return false;
2066}   
2067#else
2068Boolean
2069resume_tcr(TCR *tcr)
2070{
2071  int suspend_count = atomic_decf(&(tcr->suspend_count));
2072  if (suspend_count == 0) {
2073    void *s = (tcr->resume);
2074    if (s != NULL) {
2075      SEM_RAISE(s);
2076      return true;
2077    }
2078  }
2079  return false;
2080}
2081#endif
2082
2083   
2084
2085
2086Boolean
2087lisp_resume_tcr(TCR *tcr)
2088{
2089  Boolean resumed;
2090  TCR *current = get_tcr(true);
2091 
2092  LOCK(lisp_global(TCR_AREA_LOCK),current);
2093  resumed = resume_tcr(tcr);
2094  UNLOCK(lisp_global(TCR_AREA_LOCK), current);
2095  return resumed;
2096}
2097
2098
2099TCR *freed_tcrs = NULL;
2100
2101void
2102enqueue_freed_tcr (TCR *tcr)
2103{
2104#ifndef HAVE_TLS
2105  tcr->next = freed_tcrs;
2106  freed_tcrs = tcr;
2107#endif
2108}
2109
2110/* It's not clear that we can safely condemn a dead tcr's areas, since
2111   we may not be able to call free() if a suspended thread owns a
2112   malloc lock. At least make the areas appear to be empty.
2113*/
2114   
2115
2116void
2117normalize_dead_tcr_areas(TCR *tcr)
2118{
2119  area *a;
2120
2121  a = tcr->vs_area;
2122  if (a) {
2123    a->active = a->high;
2124  }
2125
2126  a = tcr->ts_area;
2127  if (a) {
2128    a->active = a->high;
2129  }
2130
2131  a = tcr->cs_area;
2132  if (a) {
2133    a->active = a->high;
2134  }
2135}
2136   
2137void
2138free_freed_tcrs ()
2139{
2140  TCR *current, *next;
2141
2142  for (current = freed_tcrs; current; current = next) {
2143    next = current->next;
2144#ifndef HAVE_TLS
2145#ifdef WIN32
2146    free(current->allocated);
2147#else
2148    free(current);
2149#endif
2150#endif
2151  }
2152  freed_tcrs = NULL;
2153}
2154
2155void
2156suspend_other_threads(Boolean for_gc)
2157{
2158  TCR *current = get_tcr(true), *other, *next;
2159  int dead_tcr_count = 0;
2160  Boolean all_acked;
2161
2162  LOCK(lisp_global(TCR_AREA_LOCK), current);
2163  for (other = current->next; other != current; other = other->next) {
2164    if ((other->osid != 0)) {
2165      suspend_tcr(other);
2166      if (other->osid == 0) {
2167        dead_tcr_count++;
2168      }
2169    } else {
2170      dead_tcr_count++;
2171    }
2172  }
2173
2174  do {
2175    all_acked = true;
2176    for (other = current->next; other != current; other = other->next) {
2177      if ((other->osid != 0)) {
2178        if (!tcr_suspend_ack(other)) {
2179          all_acked = false;
2180        }
2181      }
2182    }
2183  } while(! all_acked);
2184
2185     
2186
2187  /* All other threads are suspended; can safely delete dead tcrs now */
2188  if (dead_tcr_count) {
2189    for (other = current->next; other != current; other = next) {
2190      next = other->next;
2191      if ((other->osid == 0))  {
2192        normalize_dead_tcr_areas(other);
2193        dequeue_tcr(other);
2194        enqueue_freed_tcr(other);
2195      }
2196    }
2197  }
2198}
2199
2200void
2201lisp_suspend_other_threads()
2202{
2203  suspend_other_threads(false);
2204}
2205
2206void
2207resume_other_threads(Boolean for_gc)
2208{
2209  TCR *current = get_tcr(true), *other;
2210  for (other = current->next; other != current; other = other->next) {
2211    if ((other->osid != 0)) {
2212      resume_tcr(other);
2213    }
2214  }
2215  free_freed_tcrs();
2216  UNLOCK(lisp_global(TCR_AREA_LOCK), current);
2217}
2218
2219void
2220lisp_resume_other_threads()
2221{
2222  resume_other_threads(false);
2223}
2224
2225
2226
2227rwlock *
2228rwlock_new()
2229{
2230  extern int cache_block_size;
2231
2232  void *p = calloc(1,sizeof(rwlock)+cache_block_size-1);
2233  rwlock *rw = NULL;;
2234 
2235  if (p) {
2236    rw = (rwlock *) ((((natural)p)+cache_block_size-1) & (~(cache_block_size-1)));
2237    rw->malloced_ptr = p;
2238#ifndef USE_FUTEX
2239    rw->reader_signal = new_semaphore(0);
2240    rw->writer_signal = new_semaphore(0);
2241    if ((rw->reader_signal == NULL) || (rw->writer_signal == NULL)) {
2242      if (rw->reader_signal) {
2243        destroy_semaphore(&(rw->reader_signal));
2244      } else {
2245        destroy_semaphore(&(rw->writer_signal));
2246      }
2247      free(rw);
2248      rw = NULL;
2249    }
2250#endif
2251  }
2252  return rw;
2253}
2254
2255     
2256/*
2257  Try to get read access to a multiple-readers/single-writer lock.  If
2258  we already have read access, return success (indicating that the
2259  lock is held another time.  If we already have write access to the
2260  lock ... that won't work; return EDEADLK.  Wait until no other
2261  thread has or is waiting for write access, then indicate that we
2262  hold read access once.
2263*/
2264#ifndef USE_FUTEX
2265int
2266rwlock_rlock(rwlock *rw, TCR *tcr, struct timespec *waitfor)
2267{
2268  int err = 0;
2269 
2270  LOCK_SPINLOCK(rw->spin, tcr);
2271
2272  if (rw->writer == tcr) {
2273    RELEASE_SPINLOCK(rw->spin);
2274    return EDEADLK;
2275  }
2276
2277  while (rw->blocked_writers || (rw->state > 0)) {
2278    rw->blocked_readers++;
2279    RELEASE_SPINLOCK(rw->spin);
2280    err = semaphore_maybe_timedwait(rw->reader_signal,waitfor);
2281    LOCK_SPINLOCK(rw->spin,tcr);
2282    rw->blocked_readers--;
2283    if (err == EINTR) {
2284      err = 0;
2285    }
2286    if (err) {
2287      RELEASE_SPINLOCK(rw->spin);
2288      return err;
2289    }
2290  }
2291  rw->state--;
2292  RELEASE_SPINLOCK(rw->spin);
2293  return err;
2294}
2295#else
2296int
2297rwlock_rlock(rwlock *rw, TCR *tcr, struct timespec *waitfor)
2298{
2299  natural waitval;
2300
2301  lock_futex(&rw->spin);
2302
2303  if (rw->writer == tcr) {
2304    unlock_futex(&rw->spin);
2305    return EDEADLOCK;
2306  }
2307  while (1) {
2308    if (rw->writer == NULL) {
2309      --rw->state;
2310      unlock_futex(&rw->spin);
2311      return 0;
2312    }
2313    rw->blocked_readers++;
2314    waitval = rw->reader_signal;
2315    unlock_futex(&rw->spin);
2316    futex_wait(&rw->reader_signal,waitval);
2317    lock_futex(&rw->spin);
2318    rw->blocked_readers--;
2319  }
2320  return 0;
2321}
2322#endif   
2323
2324
2325/*
2326  Try to obtain write access to the lock.
2327  It is an error if we already have read access, but it's hard to
2328  detect that.
2329  If we already have write access, increment the count that indicates
2330  that.
2331  Otherwise, wait until the lock is not held for reading or writing,
2332  then assert write access.
2333*/
2334
2335#ifndef USE_FUTEX
2336int
2337rwlock_wlock(rwlock *rw, TCR *tcr, struct timespec *waitfor)
2338{
2339  int err = 0;
2340
2341  LOCK_SPINLOCK(rw->spin,tcr);
2342  if (rw->writer == tcr) {
2343    rw->state++;
2344    RELEASE_SPINLOCK(rw->spin);
2345    return 0;
2346  }
2347
2348  while (rw->state != 0) {
2349    rw->blocked_writers++;
2350    RELEASE_SPINLOCK(rw->spin);
2351    err = semaphore_maybe_timedwait(rw->writer_signal, waitfor);
2352    LOCK_SPINLOCK(rw->spin,tcr);
2353    rw->blocked_writers--;
2354    if (err == EINTR) {
2355      err = 0;
2356    }
2357    if (err) {
2358      RELEASE_SPINLOCK(rw->spin);
2359      return err;
2360    }
2361  }
2362  rw->state = 1;
2363  rw->writer = tcr;
2364  RELEASE_SPINLOCK(rw->spin);
2365  return err;
2366}
2367
2368#else
2369int
2370rwlock_wlock(rwlock *rw, TCR *tcr, struct timespec *waitfor)
2371{
2372  int err = 0;
2373  natural waitval;
2374
2375  lock_futex(&rw->spin);
2376  if (rw->writer == tcr) {
2377    rw->state++;
2378    unlock_futex(&rw->spin);
2379    return 0;
2380  }
2381
2382  while (rw->state != 0) {
2383    rw->blocked_writers++;
2384    waitval = rw->writer_signal;
2385    unlock_futex(&rw->spin);
2386    futex_wait(&rw->writer_signal,waitval);
2387    lock_futex(&rw->spin);
2388    rw->blocked_writers--;
2389  }
2390  rw->state = 1;
2391  rw->writer = tcr;
2392  unlock_futex(&rw->spin);
2393  return err;
2394}
2395#endif
2396
2397/*
2398  Sort of the same as above, only return EBUSY if we'd have to wait.
2399*/
2400#ifndef USE_FUTEX
2401int
2402rwlock_try_wlock(rwlock *rw, TCR *tcr)
2403{
2404  int ret = EBUSY;
2405
2406  LOCK_SPINLOCK(rw->spin,tcr);
2407  if (rw->writer == tcr) {
2408    rw->state++;
2409    ret = 0;
2410  } else {
2411    if (rw->state == 0) {
2412      rw->writer = tcr;
2413      rw->state = 1;
2414      ret = 0;
2415    }
2416  }
2417  RELEASE_SPINLOCK(rw->spin);
2418  return ret;
2419}
2420#else
2421int
2422rwlock_try_wlock(rwlock *rw, TCR *tcr)
2423{
2424  int ret = EBUSY;
2425
2426  lock_futex(&rw->spin);
2427  if (rw->writer == tcr) {
2428    rw->state++;
2429    ret = 0;
2430  } else {
2431    if (rw->state == 0) {
2432      rw->writer = tcr;
2433      rw->state = 1;
2434      ret = 0;
2435    }
2436  }
2437  unlock_futex(&rw->spin);
2438  return ret;
2439}
2440#endif
2441
2442#ifndef USE_FUTEX
2443int
2444rwlock_try_rlock(rwlock *rw, TCR *tcr)
2445{
2446  int ret = EBUSY;
2447
2448  LOCK_SPINLOCK(rw->spin,tcr);
2449  if (rw->state <= 0) {
2450    --rw->state;
2451    ret = 0;
2452  }
2453  RELEASE_SPINLOCK(rw->spin);
2454  return ret;
2455}
2456#else
2457int
2458rwlock_try_rlock(rwlock *rw, TCR *tcr)
2459{
2460  int ret = EBUSY;
2461
2462  lock_futex(&rw->spin);
2463  if (rw->state <= 0) {
2464    --rw->state;
2465    ret = 0;
2466  }
2467  unlock_futex(&rw->spin);
2468  return ret;
2469}
2470#endif
2471
2472
2473
2474#ifndef USE_FUTEX
2475int
2476rwlock_unlock(rwlock *rw, TCR *tcr)
2477{
2478
2479  int err = 0;
2480  natural blocked_readers = 0;
2481
2482  LOCK_SPINLOCK(rw->spin,tcr);
2483  if (rw->state > 0) {
2484    if (rw->writer != tcr) {
2485      err = EINVAL;
2486    } else {
2487      --rw->state;
2488      if (rw->state == 0) {
2489        rw->writer = NULL;
2490      }
2491    }
2492  } else {
2493    if (rw->state < 0) {
2494      ++rw->state;
2495    } else {
2496      err = EINVAL;
2497    }
2498  }
2499  if (err) {
2500    RELEASE_SPINLOCK(rw->spin);
2501    return err;
2502  }
2503 
2504  if (rw->state == 0) {
2505    if (rw->blocked_writers) {
2506      SEM_RAISE(rw->writer_signal);
2507    } else {
2508      blocked_readers = rw->blocked_readers;
2509      if (blocked_readers) {
2510        SEM_BROADCAST(rw->reader_signal, blocked_readers);
2511      }
2512    }
2513  }
2514  RELEASE_SPINLOCK(rw->spin);
2515  return 0;
2516}
2517#else
2518int
2519rwlock_unlock(rwlock *rw, TCR *tcr)
2520{
2521
2522  int err = 0;
2523
2524  lock_futex(&rw->spin);
2525  if (rw->state > 0) {
2526    if (rw->writer != tcr) {
2527      err = EINVAL;
2528    } else {
2529      --rw->state;
2530      if (rw->state == 0) {
2531        rw->writer = NULL;
2532      }
2533    }
2534  } else {
2535    if (rw->state < 0) {
2536      ++rw->state;
2537    } else {
2538      err = EINVAL;
2539    }
2540  }
2541  if (err) {
2542    unlock_futex(&rw->spin);
2543    return err;
2544  }
2545 
2546  if (rw->state == 0) {
2547    if (rw->blocked_writers) {
2548      ++rw->writer_signal;
2549      unlock_futex(&rw->spin);
2550      futex_wake(&rw->writer_signal,1);
2551      return 0;
2552    }
2553    if (rw->blocked_readers) {
2554      ++rw->reader_signal;
2555      unlock_futex(&rw->spin);
2556      futex_wake(&rw->reader_signal, INT_MAX);
2557      return 0;
2558    }
2559  }
2560  unlock_futex(&rw->spin);
2561  return 0;
2562}
2563#endif
2564
2565       
2566void
2567rwlock_destroy(rwlock *rw)
2568{
2569#ifndef USE_FUTEX
2570  destroy_semaphore((void **)&rw->reader_signal);
2571  destroy_semaphore((void **)&rw->writer_signal);
2572#endif
2573  postGCfree((void *)(rw->malloced_ptr));
2574}
2575
2576
2577
Note: See TracBrowser for help on using the repository browser.