Context Navigation

x86-spentry64.s @ 15572

Visit:

Last change on this file since 15572 was 15572, checked in by gb, 7 years ago

If we get a SIGSEGV or SIGBUS while executing foreign code when
tcr->safe_ref_address is non-nil, try to recover by making it look
like the current ffcall has returned a null pointer.

Try to use this mechanism to recognize tagged ObjC instances. (Part
of that process involves sending -[NSObject class] to something that
may or may not be an ObjC instance, and if the instance isn't valid
that message will likely generate a memory fault.)

Property svn:eol-style set to native
Property svn:keywords set to Author Date Id Revision

File size: 167.0 KB

Line
1	/* Copyright (C) 2005-2009 Clozure Associates and contributors */
2	/* This file is part of Clozure CL. */
3
4	/* Clozure CL is licensed under the terms of the Lisp Lesser GNU Public */
5	/* License , known as the LLGPL and distributed with Clozure CL as the */
6	/* file "LICENSE". The LLGPL consists of a preamble and the LGPL, */
7	/* which is distributed with Clozure CL as the file "LGPL". Where these */
8	/* conflict, the preamble takes precedence. */
9
10	/* Clozure CL is referenced in the preamble as the "LIBRARY." */
11
12	/* The LLGPL is also available online at */
13	/* http://opensource.franz.com/preamble.html */
14
15
16
17	include(lisp.s)
18	_beginfile
19
20	.align 2
21	define(`_spentry',`ifdef(`__func_name',`_endfn',`')
22	.p2align 3
23	_exportfn(_SP$1)
24	.line __line__
25	')
26
27
28	define(`_endsubp',`
29	_endfn(_SP$1)
30	# __line__
31	')
32
33	define(`jump_builtin',`
34	ref_nrs_value(builtin_functions,%fname)
35	set_nargs($2)
36	vrefr(%fname,%fname,$1)
37	jump_fname()
38	')
39
40
41
42	_spentry(bad_funcall)
43	Xspentry_start:
44	.globl C(bad_funcall)
45	__(tra(C(bad_funcall)))
46	__(uuo_error_not_callable)
47	_endsubp(bad_funcall)
48
49	/* %arg_z has overflowed by one bit. Make a bignum with 2 (32-bit) digits. */
50
51	_spentry(fix_overflow)
52	C(fix_one_bit_overflow):
53	__(movq $two_digit_bignum_header,%imm0)
54	__(Misc_Alloc_Fixed(`',aligned_bignum_size(2)))
55	__(unbox_fixnum(%arg_z,%imm0))
56	__(movq $0xe000000000000000,%imm1)
57	__(mov %temp0,%arg_z)
58	__(xorq %imm1,%imm0)
59	__(movq %imm0,misc_data_offset(%arg_z))
60	__(ret)
61	_endsubp(fix_overflow)
62
63
64	/* Make a lisp integer (fixnum or two-digit bignum) from the signed */
65	/* 64-bit value in %imm0. */
66
67	_spentry(makes64)
68	__(movq %imm0,%imm1)
69	__(shlq $fixnumshift,%imm1)
70	__(movq %imm1,%arg_z)
71	__(sarq $fixnumshift,%imm1)
72	__(cmpq %imm1,%imm0)
73	__(jz 0f)
74	__(movd %imm0,%mm0)
75	__(movq $two_digit_bignum_header,%imm0)
76	__(Misc_Alloc_Fixed(%arg_z,aligned_bignum_size(2)))
77	__(movq %mm0,misc_data_offset(%arg_z))
78	0: __(repret)
79	_endsubp(makes64)
80
81
82
83	/* %imm1:%imm0 constitute a signed integer, almost certainly a bignum. */
84	/* Make a lisp integer out of those 128 bits .. */
85
86	_startfn(C(makes128))
87
88	/* We're likely to have to make a bignum out of the integer in %imm1 and */
89	/* %imm0. We'll need to use %imm0 and %imm1 to cons the bignum, and */
90	/* will need to do some arithmetic (determining significant bigits) */
91	/* on %imm0 and %imm1 in order to know how large that bignum needs to be. */
92	/* Cache %imm0 and %imm1 in %mm0 and %mm1. */
93
94	__(movd %imm0,%mm0)
95	__(movd %imm1,%mm1)
96
97	/* If %imm1 is just a sign extension of %imm0, make a 64-bit signed integer. */
98
99	__(sarq $63,%imm0)
100	__(cmpq %imm0,%imm1)
101	__(movd %mm0,%imm0)
102	__(je _SPmakes64)
103
104	/* Otherwise, if the high 32 bits of %imm1 are a sign-extension of the */
105	/* low 32 bits of %imm1, make a 3-digit bignum. If the upper 32 bits */
106	/* of %imm1 are significant, make a 4 digit bignum */
107
108	__(movq %imm1,%imm0)
109	__(shlq $32,%imm0)
110	__(sarq $32,%imm0)
111	__(cmpq %imm0,%imm1)
112	__(jz 3f)
113	__(mov $four_digit_bignum_header,%imm0)
114	__(Misc_Alloc_Fixed(%arg_z,aligned_bignum_size(4)))
115	__(movq %mm0,misc_data_offset(%arg_z))
116	__(movq %mm1,misc_data_offset+8(%arg_z))
117	__(ret)
118	3: __(mov $three_digit_bignum_header,%imm0)
119	__(Misc_Alloc_Fixed(%arg_z,aligned_bignum_size(3)))
120	__(movq %mm0,misc_data_offset(%arg_z))
121	__(movd %mm1,misc_data_offset+8(%arg_z))
122	__(ret)
123	_endfn
124
125
126	/* %imm1:%imm0 constitute an unsigned integer, almost certainly a bignum. */
127	/* Make a lisp integer out of those 128 bits .. */
128
129	_startfn(C(makeu128))
130
131	/* We're likely to have to make a bignum out of the integer in %imm1 and */
132	/* %imm0. We'll need to use %imm0 and %imm1 to cons the bignum, and */
133	/* will need to do some arithmetic (determining significant bigits) */
134	/* on %imm0 and %imm1 in order to know how large that bignum needs to be. */
135	/* Cache %imm0 and %imm1 in %mm0 and %mm1. */
136
137	/* If the high word is 0, make an unsigned-byte 64 ... */
138
139	__(testq %imm1,%imm1)
140	__(jz _SPmakeu64)
141
142	__(movd %imm0,%mm0)
143	__(movd %imm1,%mm1)
144
145	__(js 5f) /* Sign bit set in %imm1. Need 5 digits */
146	__(bsrq %imm1,%imm0)
147	__(rcmpb(%imm0_b,$31))
148	__(jae 4f) /* Some high bits in %imm1. Need 4 digits */
149	__(testl %imm1_l,%imm1_l)
150	__(movd %mm0,%imm0)
151	__(jz _SPmakeu64)
152
153	/* Need 3 digits */
154
155	__(movq $three_digit_bignum_header,%imm0)
156	__(Misc_Alloc_Fixed(%arg_z,aligned_bignum_size(3)))
157	__(movq %mm0,misc_data_offset(%arg_z))
158	__(movd %mm1,misc_data_offset+8(%arg_z))
159	__(ret)
160	4: __(movq $four_digit_bignum_header,%imm0)
161	__(Misc_Alloc_Fixed(%arg_z,aligned_bignum_size(4)))
162	__(jmp 6f)
163	5: __(movq $five_digit_bignum_header,%imm0)
164	__(Misc_Alloc_Fixed(%arg_z,aligned_bignum_size(5)))
165	6: __(movq %mm0,misc_data_offset(%arg_z))
166	__(movq %mm0,misc_data_offset+8(%arg_z))
167	__(ret)
168	_endfn
169
170	_spentry(misc_ref)
171	__(movb $tagmask,%imm0_b)
172	__(andb %arg_y_b,%imm0_b)
173	__(cmpb $tag_misc,%imm0_b)
174	__(jne 0f)
175	__(testb $fixnummask,%arg_z_b)
176	__(jne 1f)
177	__(movq misc_header_offset(%arg_y),%imm0)
178	__(xorb %imm0_b,%imm0_b)
179	__(shrq $num_subtag_bits-fixnumshift,%imm0)
180	__(cmpq %imm0,%arg_z)
181	__(jae 2f)
182	__(movb misc_subtag_offset(%arg_y),%imm1_b)
183	__(jmp C(misc_ref_common))
184
185	0: __(uuo_error_reg_not_tag(Rarg_y,tag_misc))
186	1: __(uuo_error_reg_not_fixnum(Rarg_z))
187	2: __(uuo_error_vector_bounds(Rarg_z,Rarg_y))
188	_endsubp(misc_ref)
189
190	/* %imm1.b = subtag, %arg_y = uvector, %arg_z = index. */
191	/* Bounds/type-checking done in caller */
192
193	_startfn(C(misc_ref_common))
194	__(movzbl %imm1_b,%imm1_l)
195	__(lea local_label(misc_ref_jmp)(%rip),%imm2)
196	__(jmp *(%imm2,%imm1,8))
197	.p2align 3
198	local_label(misc_ref_jmp):
199	/* 00-0f */
200	.quad local_label(misc_ref_invalid) /* 00 even_fixnum */
201	.quad local_label(misc_ref_invalid) /* 01 imm_1 */
202	.quad local_label(misc_ref_invalid) /* 02 imm_2 */
203	.quad local_label(misc_ref_invalid) /* 03 cons */
204	.quad local_label(misc_ref_invalid) /* 04 tra_0 */
205	.quad local_label(misc_ref_invalid) /* 05 nodeheader_0 */
206	.quad local_label(misc_ref_invalid) /* 06 nodeheader_1 */
207	.quad local_label(misc_ref_invalid) /* 07 immheader_0 */
208	.quad local_label(misc_ref_invalid) /* 08 odd_fixnum */
209	.quad local_label(misc_ref_invalid) /* 09 immheader_1 */
210	.quad local_label(misc_ref_invalid) /* 0a immheader_2 */
211	.quad local_label(misc_ref_invalid) /* 0b nil */
212	.quad local_label(misc_ref_invalid) /* 0c tra_1 */
213	.quad local_label(misc_ref_invalid) /* 0d misc */
214	.quad local_label(misc_ref_invalid) /* 0e symbol */
215	.quad local_label(misc_ref_invalid) /* 0f function */
216	/* 10-1f */
217	.quad local_label(misc_ref_invalid) /* 10 even_fixnum */
218	.quad local_label(misc_ref_invalid) /* 11 imm_1 */
219	.quad local_label(misc_ref_invalid) /* 12 imm_2 */
220	.quad local_label(misc_ref_invalid) /* 13 cons */
221	.quad local_label(misc_ref_invalid) /* 14 tra_0 */
222	.quad local_label(misc_ref_node) /* 15 symbol_vector */
223	.quad local_label(misc_ref_node) /* 16 ratio */
224	.quad local_label(misc_ref_invalid) /* 17 immheader_0 */
225	.quad local_label(misc_ref_invalid) /* 18 odd_fixnum */
226	.quad local_label(misc_ref_u32) /* 19 bignum */
227	.quad local_label(misc_ref_u64) /* 1a macptr */
228	.quad local_label(misc_ref_invalid) /* 1b nil */
229	.quad local_label(misc_ref_invalid) /* 1c tra_1 */
230	.quad local_label(misc_ref_invalid) /* 1d misc */
231	.quad local_label(misc_ref_invalid) /* 1e symbol */
232	.quad local_label(misc_ref_invalid) /* 1f function */
233	/* 20-2f */
234	.quad local_label(misc_ref_invalid) /* 20 even_fixnum */
235	.quad local_label(misc_ref_invalid) /* 21 imm_1 */
236	.quad local_label(misc_ref_invalid) /* 22 imm_2 */
237	.quad local_label(misc_ref_invalid) /* 23 cons */
238	.quad local_label(misc_ref_invalid) /* 24 tra_0 */
239	.quad local_label(misc_ref_node) /* 25 catch_frame */
240	.quad local_label(misc_ref_node) /* 26 complex */
241	.quad local_label(misc_ref_invalid) /* 27 immheader_0 */
242	.quad local_label(misc_ref_invalid) /* 28 odd_fixnum */
243	.quad local_label(misc_ref_u32) /* 29 double_float */
244	.quad local_label(misc_ref_u64) /* 2a dead_macptr */
245	.quad local_label(misc_ref_invalid) /* 2b nil */
246	.quad local_label(misc_ref_invalid) /* 2c tra_1 */
247	.quad local_label(misc_ref_invalid) /* 2d misc */
248	.quad local_label(misc_ref_invalid) /* 2e symbol */
249	.quad local_label(misc_ref_invalid) /* 2f function */
250	/* 30-3f */
251	.quad local_label(misc_ref_invalid) /* 30 even_fixnum */
252	.quad local_label(misc_ref_invalid) /* 31 imm_1 */
253	.quad local_label(misc_ref_invalid) /* 32 imm_2 */
254	.quad local_label(misc_ref_invalid) /* 33 cons */
255	.quad local_label(misc_ref_invalid) /* 34 tra_0 */
256	.quad local_label(misc_ref_node) /* 35 hash_vector */
257	.quad local_label(misc_ref_node) /* 36 struct */
258	.quad local_label(misc_ref_invalid) /* 37 immheader_0 */
259	.quad local_label(misc_ref_invalid) /* 38 odd_fixnum */
260	.quad local_label(misc_ref_u32) /* 39 xcode_vector */
261	.quad local_label(misc_ref_invalid) /* 3a immheader_2 */
262	.quad local_label(misc_ref_invalid) /* 3b nil */
263	.quad local_label(misc_ref_invalid) /* 3c tra_1 */
264	.quad local_label(misc_ref_invalid) /* 3d misc */
265	.quad local_label(misc_ref_invalid) /* 3e symbol */
266	.quad local_label(misc_ref_invalid) /* 3f function */
267	/* 40-4f */
268	.quad local_label(misc_ref_invalid) /* 40 even_fixnum */
269	.quad local_label(misc_ref_invalid) /* 41 imm_1 */
270	.quad local_label(misc_ref_invalid) /* 42 imm_2 */
271	.quad local_label(misc_ref_invalid) /* 43 cons */
272	.quad local_label(misc_ref_invalid) /* 44 tra_0 */
273	.quad local_label(misc_ref_node) /* 45 pool */
274	.quad local_label(misc_ref_node) /* 46 istruct */
275	.quad local_label(misc_ref_invalid) /* 47 immheader_0 */
276	.quad local_label(misc_ref_invalid) /* 48 odd_fixnum */
277	.quad local_label(misc_ref_invalid) /* 49 immheader_1 */
278	.quad local_label(misc_ref_invalid) /* 4a immheader_2 */
279	.quad local_label(misc_ref_invalid) /* 4b nil */
280	.quad local_label(misc_ref_invalid) /* 4c tra_1 */
281	.quad local_label(misc_ref_invalid) /* 4d misc */
282	.quad local_label(misc_ref_invalid) /* 4e symbol */
283	.quad local_label(misc_ref_invalid) /* 4f function */
284	/* 50-5f */
285	.quad local_label(misc_ref_invalid) /* 50 even_fixnum */
286	.quad local_label(misc_ref_invalid) /* 51 imm_1 */
287	.quad local_label(misc_ref_invalid) /* 52 imm_2 */
288	.quad local_label(misc_ref_invalid) /* 53 cons */
289	.quad local_label(misc_ref_invalid) /* 54 tra_0 */
290	.quad local_label(misc_ref_node) /* 55 weak */
291	.quad local_label(misc_ref_node) /* 56 value_cell */
292	.quad local_label(misc_ref_invalid) /* 57 immheader_0 */
293	.quad local_label(misc_ref_invalid) /* 58 odd_fixnum */
294	.quad local_label(misc_ref_invalid) /* 59 immheader_1 */
295	.quad local_label(misc_ref_invalid) /* 5a immheader_2 */
296	.quad local_label(misc_ref_invalid) /* 5b nil */
297	.quad local_label(misc_ref_invalid) /* 5c tra_1 */
298	.quad local_label(misc_ref_invalid) /* 5d misc */
299	.quad local_label(misc_ref_invalid) /* 5e symbol */
300	.quad local_label(misc_ref_invalid) /* 5f function */
301	/* 60-6f */
302	.quad local_label(misc_ref_invalid) /* 60 even_fixnum */
303	.quad local_label(misc_ref_invalid) /* 61 imm_1 */
304	.quad local_label(misc_ref_invalid) /* 62 imm_2 */
305	.quad local_label(misc_ref_invalid) /* 63 cons */
306	.quad local_label(misc_ref_invalid) /* 64 tra_0 */
307	.quad local_label(misc_ref_node) /* 65 package */
308	.quad local_label(misc_ref_node) /* 66 xfunction */
309	.quad local_label(misc_ref_invalid) /* 67 immheader_0 */
310	.quad local_label(misc_ref_invalid) /* 68 odd_fixnum */
311	.quad local_label(misc_ref_invalid) /* 69 immheader_1 */
312	.quad local_label(misc_ref_invalid) /* 6a immheader_2 */
313	.quad local_label(misc_ref_invalid) /* 6b nil */
314	.quad local_label(misc_ref_invalid) /* 6c tra_1 */
315	.quad local_label(misc_ref_invalid) /* 6d misc */
316	.quad local_label(misc_ref_invalid) /* 6e symbol */
317	.quad local_label(misc_ref_invalid) /* 6f function */
318	/* 70-7f */
319	.quad local_label(misc_ref_invalid) /* 70 even_fixnum */
320	.quad local_label(misc_ref_invalid) /* 71 imm_1 */
321	.quad local_label(misc_ref_invalid) /* 72 imm_2 */
322	.quad local_label(misc_ref_invalid) /* 73 cons */
323	.quad local_label(misc_ref_invalid) /* 74 tra_0 */
324	.quad local_label(misc_ref_node) /* 75 slot_vector */
325	.quad local_label(misc_ref_node) /* 76 lock */
326	.quad local_label(misc_ref_invalid) /* 77 immheader_0 */
327	.quad local_label(misc_ref_invalid) /* 78 odd_fixnum */
328	.quad local_label(misc_ref_invalid) /* 79 immheader_1 */
329	.quad local_label(misc_ref_invalid) /* 7a immheader_2 */
330	.quad local_label(misc_ref_invalid) /* 7b nil */
331	.quad local_label(misc_ref_invalid) /* 7c tra_1 */
332	.quad local_label(misc_ref_invalid) /* 7d misc */
333	.quad local_label(misc_ref_invalid) /* 7e symbol */
334	.quad local_label(misc_ref_invalid) /* 7f function */
335	/* 80-8f */
336	.quad local_label(misc_ref_invalid) /* 80 even_fixnum */
337	.quad local_label(misc_ref_invalid) /* 81 imm_1 */
338	.quad local_label(misc_ref_invalid) /* 82 imm_2 */
339	.quad local_label(misc_ref_invalid) /* 83 cons */
340	.quad local_label(misc_ref_invalid) /* 84 tra_0 */
341	.quad local_label(misc_ref_node) /* 85 lisp_thread */
342	.quad local_label(misc_ref_node) /* 86 instance */
343	.quad local_label(misc_ref_invalid) /* 87 immheader_0 */
344	.quad local_label(misc_ref_invalid) /* 88 odd_fixnum */
345	.quad local_label(misc_ref_invalid) /* 89 immheader_1 */
346	.quad local_label(misc_ref_invalid) /* 8a immheader_2 */
347	.quad local_label(misc_ref_invalid) /* 8b nil */
348	.quad local_label(misc_ref_invalid) /* 8c tra_1 */
349	.quad local_label(misc_ref_invalid) /* 8d misc */
350	.quad local_label(misc_ref_invalid) /* 8e symbol */
351	.quad local_label(misc_ref_invalid) /* 8f function */
352	/* 90-9f */
353	.quad local_label(misc_ref_invalid) /* 90 even_fixnum */
354	.quad local_label(misc_ref_invalid) /* 91 imm_1 */
355	.quad local_label(misc_ref_invalid) /* 92 imm_2 */
356	.quad local_label(misc_ref_invalid) /* 93 cons */
357	.quad local_label(misc_ref_invalid) /* 94 tra_0 */
358	.quad local_label(misc_ref_function) /* 95 function_vector */
359	.quad local_label(misc_ref_invalid) /* 96 nodeheader_1 */
360	.quad local_label(misc_ref_invalid) /* 97 immheader_0 */
361	.quad local_label(misc_ref_invalid) /* 98 odd_fixnum */
362	.quad local_label(misc_ref_invalid) /* 99 immheader_1 */
363	.quad local_label(misc_ref_invalid) /* 9a immheader_2 */
364	.quad local_label(misc_ref_invalid) /* 9b nil */
365	.quad local_label(misc_ref_invalid) /* 9c tra_1 */
366	.quad local_label(misc_ref_invalid) /* 9d misc */
367	.quad local_label(misc_ref_invalid) /* 9e symbol */
368	.quad local_label(misc_ref_invalid) /* 9f function */
369	/* a0-af */
370	.quad local_label(misc_ref_invalid) /* a0 even_fixnum */
371	.quad local_label(misc_ref_invalid) /* a1 imm_1 */
372	.quad local_label(misc_ref_invalid) /* a2 imm_2 */
373	.quad local_label(misc_ref_invalid) /* a3 cons */
374	.quad local_label(misc_ref_invalid) /* a4 tra_0 */
375	.quad local_label(misc_ref_node) /* a5 arrayH */
376	.quad local_label(misc_ref_node) /* a6 vectorH */
377	.quad local_label(misc_ref_s16) /* a7 s16 */
378	.quad local_label(misc_ref_invalid) /* a8 odd_fixnum */
379	.quad local_label(misc_ref_invalid) /* a9 immheader_1 */
380	.quad local_label(misc_ref_invalid) /* aa immheader_2 */
381	.quad local_label(misc_ref_invalid) /* ab nil */
382	.quad local_label(misc_ref_invalid) /* ac tra_1 */
383	.quad local_label(misc_ref_invalid) /* ad misc */
384	.quad local_label(misc_ref_invalid) /* ae symbol */
385	.quad local_label(misc_ref_invalid) /* af function */
386	/* b0-bf */
387	.quad local_label(misc_ref_invalid) /* b0 even_fixnum */
388	.quad local_label(misc_ref_invalid) /* b1 imm_1 */
389	.quad local_label(misc_ref_invalid) /* b2 imm_2 */
390	.quad local_label(misc_ref_invalid) /* b3 cons */
391	.quad local_label(misc_ref_invalid) /* b4 tra_0 */
392	.quad local_label(misc_ref_invalid) /* b5 nodeheader_0 */
393	.quad local_label(misc_ref_node) /* b6 simple_vector */
394	.quad local_label(misc_ref_u16) /* b7 immheader_0 */
395	.quad local_label(misc_ref_invalid) /* b8 odd_fixnum */
396	.quad local_label(misc_ref_invalid) /* b9 immheader_1 */
397	.quad local_label(misc_ref_invalid) /* ba immheader_2 */
398	.quad local_label(misc_ref_invalid) /* bb nil */
399	.quad local_label(misc_ref_invalid) /* bc tra_1 */
400	.quad local_label(misc_ref_invalid) /* bd misc */
401	.quad local_label(misc_ref_invalid) /* be symbol */
402	.quad local_label(misc_ref_invalid) /* bf function */
403	/* c0-cf */
404	.quad local_label(misc_ref_invalid) /* c0 even_fixnum */
405	.quad local_label(misc_ref_invalid) /* c1 imm_1 */
406	.quad local_label(misc_ref_invalid) /* c2 imm_2 */
407	.quad local_label(misc_ref_invalid) /* c3 cons */
408	.quad local_label(misc_ref_invalid) /* c4 tra_0 */
409	.quad local_label(misc_ref_invalid) /* c5 nodeheader_0 */
410	.quad local_label(misc_ref_invalid) /* c6 nodeheader_1 */
411	.quad local_label(misc_ref_string) /* c7 simple_base_string */
412	.quad local_label(misc_ref_invalid) /* c8 odd_fixnum */
413	.quad local_label(misc_ref_new_string) /* c9 new_string_1 */
414	.quad local_label(misc_ref_fixnum_vector) /* ca fixnum_vector */
415	.quad local_label(misc_ref_invalid) /* cb nil */
416	.quad local_label(misc_ref_invalid) /* cc tra_1 */
417	.quad local_label(misc_ref_invalid) /* cd misc */
418	.quad local_label(misc_ref_invalid) /* ce symbol */
419	.quad local_label(misc_ref_invalid) /* cf function */
420	/* d0-df */
421	.quad local_label(misc_ref_invalid) /* d0 even_fixnum */
422	.quad local_label(misc_ref_invalid) /* d1 imm_1 */
423	.quad local_label(misc_ref_invalid) /* d2 imm_2 */
424	.quad local_label(misc_ref_invalid) /* d3 cons */
425	.quad local_label(misc_ref_invalid) /* d4 tra_0 */
426	.quad local_label(misc_ref_invalid) /* d5 nodeheader_0 */
427	.quad local_label(misc_ref_invalid) /* d6 nodeheader_1 */
428	.quad local_label(misc_ref_s8) /* d7 s8 */
429	.quad local_label(misc_ref_invalid) /* d8 odd_fixnum */
430	.quad local_label(misc_ref_s32) /* d9 s32 */
431	.quad local_label(misc_ref_s64) /* da s64 */
432	.quad local_label(misc_ref_invalid) /* db nil */
433	.quad local_label(misc_ref_invalid) /* dc tra_1 */
434	.quad local_label(misc_ref_invalid) /* dd misc */
435	.quad local_label(misc_ref_invalid) /* de symbol */
436	.quad local_label(misc_ref_invalid) /* df function */
437	/* e0-ef */
438	.quad local_label(misc_ref_invalid) /* e0 even_fixnum */
439	.quad local_label(misc_ref_invalid) /* e1 imm_1 */
440	.quad local_label(misc_ref_invalid) /* e2 imm_2 */
441	.quad local_label(misc_ref_invalid) /* e3 cons */
442	.quad local_label(misc_ref_invalid) /* e4 tra_0 */
443	.quad local_label(misc_ref_invalid) /* e5 nodeheader_0 */
444	.quad local_label(misc_ref_invalid) /* e6 nodeheader_1 */
445	.quad local_label(misc_ref_u8) /* e7 u8 */
446	.quad local_label(misc_ref_invalid) /* e8 odd_fixnum */
447	.quad local_label(misc_ref_u32) /* e9 u32 */
448	.quad local_label(misc_ref_u64) /* ea u64 */
449	.quad local_label(misc_ref_invalid) /* eb nil */
450	.quad local_label(misc_ref_invalid) /* ec tra_1 */
451	.quad local_label(misc_ref_invalid) /* ed misc */
452	.quad local_label(misc_ref_invalid) /* ee symbol */
453	.quad local_label(misc_ref_invalid) /* ef function */
454	/* f0-ff */
455	.quad local_label(misc_ref_invalid) /* f0 even_fixnum */
456	.quad local_label(misc_ref_invalid) /* f1 imm_1 */
457	.quad local_label(misc_ref_invalid) /* f2 imm_2 */
458	.quad local_label(misc_ref_invalid) /* f3 cons */
459	.quad local_label(misc_ref_invalid) /* f4 tra_0 */
460	.quad local_label(misc_ref_invalid) /* f5 nodeheader_0 */
461	.quad local_label(misc_ref_invalid) /* f6 nodeheader_1 */
462	.quad local_label(misc_ref_bit_vector) /* f7 bitvector */
463	.quad local_label(misc_ref_invalid) /* f8 odd_fixnum */
464	.quad local_label(misc_ref_single_float_vector) /* f9 single_float */
465	.quad local_label(misc_ref_double_float_vector) /* fa double_float */
466	.quad local_label(misc_ref_invalid) /* fb nil */
467	.quad local_label(misc_ref_invalid) /* fc tra_1 */
468	.quad local_label(misc_ref_invalid) /* fd misc */
469	.quad local_label(misc_ref_invalid) /* fe symbol */
470	.quad local_label(misc_ref_invalid) /* ff function */
471
472
473	/* Node vector. Functions are funny: the first N words */
474	/* are treated as (UNSIGNED-BYTE 64), where N is the low */
475	/* 32 bits of the first word. */
476
477	local_label(misc_ref_function):
478	__(movl misc_data_offset(%arg_y),%imm0_l)
479	__(shl $fixnumshift,%imm0)
480	__(rcmpq(%arg_z,%imm0))
481	__(jb local_label(misc_ref_u64))
482	local_label(misc_ref_node):
483	__(movq misc_data_offset(%arg_y,%arg_z),%arg_z)
484	__(ret)
485	local_label(misc_ref_u64):
486	__(movq misc_data_offset(%arg_y,%arg_z),%imm0)
487	__(jmp _SPmakeu64)
488	local_label(misc_ref_double_float_vector):
489	__(movsd misc_data_offset(%arg_y,%arg_z),%fp1)
490	__(movq $double_float_header,%imm0)
491	__(Misc_Alloc_Fixed(%arg_z,double_float.size))
492	__(movsd %fp1,double_float.value(%arg_z))
493	__(ret)
494	local_label(misc_ref_fixnum_vector):
495	__(movq misc_data_offset(%arg_y,%arg_z),%imm0)
496	__(box_fixnum(%imm0,%arg_z))
497	__(ret)
498	local_label(misc_ref_s64):
499	__(movq misc_data_offset(%arg_y,%arg_z),%imm0)
500	__(jmp _SPmakes64)
501	local_label(misc_ref_u32):
502	__(movq %arg_z,%imm0)
503	__(shr $1,%imm0)
504	__(movl misc_data_offset(%arg_y,%imm0),%imm0_l)
505	__(box_fixnum(%imm0,%arg_z))
506	__(ret)
507	local_label(misc_ref_s32):
508	__(movq %arg_z,%imm0)
509	__(shr $1,%imm0)
510	__(movslq misc_data_offset(%arg_y,%imm0),%imm0)
511	__(box_fixnum(%imm0,%arg_z))
512	__(ret)
513	local_label(misc_ref_single_float_vector):
514	__(movq %arg_z,%imm0)
515	__(shr $1,%imm0)
516	__(movsd misc_data_offset(%arg_y,%imm0),%fp1)
517	__(movd %fp1,%imm0_l)
518	__(shl $32,%imm0)
519	__(lea subtag_single_float(%imm0),%arg_z)
520	__(ret)
521	local_label(misc_ref_u8):
522	__(movq %arg_z,%imm0)
523	__(shr $3,%imm0)
524	__(movzbl misc_data_offset(%arg_y,%imm0),%imm0_l)
525	__(box_fixnum(%imm0,%arg_z))
526	__(ret)
527	local_label(misc_ref_s8):
528	__(movq %arg_z,%imm0)
529	__(shr $3,%imm0)
530	__(movsbq misc_data_offset(%arg_y,%imm0),%imm0)
531	__(box_fixnum(%imm0,%arg_z))
532	__(ret)
533	local_label(misc_ref_string):
534	__(movq %arg_z,%imm0)
535	__(shr $3,%imm0)
536	__(movzbl misc_data_offset(%arg_y,%imm0),%imm0_l)
537	__(shlq $charcode_shift,%imm0)
538	__(leaq subtag_character(%imm0),%arg_z)
539	__(ret)
540	local_label(misc_ref_new_string):
541	__(movq %arg_z,%imm0)
542	__(shr $1,%imm0)
543	__(movl misc_data_offset(%arg_y,%imm0),%imm0_l)
544	__(shlq $charcode_shift,%imm0)
545	__(leaq subtag_character(%imm0),%arg_z)
546	__(ret)
547	local_label(misc_ref_u16):
548	__(movq %arg_z,%imm0)
549	__(shrq $2,%imm0)
550	__(movzwl misc_data_offset(%arg_y,%imm0),%imm0_l)
551	__(box_fixnum(%imm0,%arg_z))
552	__(ret)
553	local_label(misc_ref_s16):
554	__(movq %arg_z,%imm0)
555	__(shrq $2,%imm0)
556	__(movswq misc_data_offset(%arg_y,%imm0),%imm0)
557	__(box_fixnum(%imm0,%arg_z))
558	__(ret)
559	local_label(misc_ref_bit_vector):
560	__(unbox_fixnum(%arg_z,%imm0))
561	__(btq %imm0,misc_data_offset(%arg_y))
562	__(setc %imm0_b)
563	__(movzbl %imm0_b,%imm0_l)
564	__(imull $fixnumone,%imm0_l,%arg_z_l)
565	__(ret)
566	local_label(misc_ref_invalid):
567	__(movq $XBADVEC,%arg_x)
568	__(set_nargs(3))
569	__(jmp _SPksignalerr)
570	_endfn(C(misc_ref_common))
571
572	/* like misc_ref, only the boxed subtag is in arg_x. */
573
574	_spentry(subtag_misc_ref)
575	__(movb $tagmask,%imm0_b)
576	__(andb %arg_y_b,%imm0_b)
577	__(cmpb $tag_misc,%imm0_b)
578	__(jne 0f)
579	__(testb $fixnummask,%arg_z_b)
580	__(jne 1f)
581	__(movq misc_header_offset(%arg_y),%imm0)
582	__(xorb %imm0_b,%imm0_b)
583	__(shrq $num_subtag_bits-fixnumshift,%imm0)
584	__(cmpq %imm0,%arg_z)
585	__(jae 2f)
586	__(unbox_fixnum(%arg_x,%imm1))
587	__(jmp C(misc_ref_common))
588	0: __(uuo_error_reg_not_tag(Rarg_y,tag_misc))
589	1: __(uuo_error_reg_not_fixnum(Rarg_z))
590	2: __(uuo_error_vector_bounds(Rarg_z,Rarg_y))
591
592	_endsubp(subtag_misc_ref)
593
594	_spentry(subtag_misc_set)
595	__(movb $tagmask,%imm0_b)
596	__(andb %arg_x_b,%imm0_b)
597	__(cmpb $tag_misc,%imm0_b)
598	__(jne 0f)
599	__(testb $fixnummask,%arg_y_b)
600	__(jne 1f)
601	__(movq misc_header_offset(%arg_x),%imm0)
602	__(xorb %imm0_b,%imm0_b)
603	__(shrq $num_subtag_bits-fixnumshift,%imm0)
604	__(cmpq %imm0,%arg_y)
605	__(jae 2f)
606	__(unbox_fixnum(%temp0,%imm1))
607	__(jmp C(misc_set_common))
608	0: __(uuo_error_reg_not_tag(Rarg_x,tag_misc))
609	1: __(uuo_error_reg_not_fixnum(Rarg_y))
610	2: __(uuo_error_vector_bounds(Rarg_y,Rarg_x))
611	_endsubp(subtag_misc_set)
612
613	_spentry(misc_set)
614	__(movb $tagmask,%imm0_b)
615	__(andb %arg_x_b,%imm0_b)
616	__(cmpb $tag_misc,%imm0_b)
617	__(jne 0f)
618	__(testb $fixnummask,%arg_y_b)
619	__(jne 1f)
620	__(movq misc_header_offset(%arg_x),%imm0)
621	__(xorb %imm0_b,%imm0_b)
622	__(shrq $num_subtag_bits-fixnumshift,%imm0)
623	__(cmpq %imm0,%arg_y)
624	__(jae 2f)
625	__(movb misc_subtag_offset(%arg_x),%imm1_b)
626	__(jmp C(misc_set_common))
627
628	0: __(uuo_error_reg_not_tag(Rarg_x,tag_misc))
629	1: __(uuo_error_reg_not_fixnum(Rarg_y))
630	2: __(uuo_error_vector_bounds(Rarg_y,Rarg_x))
631	_endsubp(misc_set)
632
633	_startfn(C(misc_set_common))
634	__(movzbl %imm1_b,%imm1_l)
635	__(lea local_label(misc_set_jmp)(%rip),%imm2)
636	__(jmp *(%imm2,%imm1,8))
637	.p2align 3
638	local_label(misc_set_jmp):
639	/* 00-0f */
640	.quad local_label(misc_set_invalid) /* 00 even_fixnum */
641	.quad local_label(misc_set_invalid) /* 01 imm_1 */
642	.quad local_label(misc_set_invalid) /* 02 imm_2 */
643	.quad local_label(misc_set_invalid) /* 03 cons */
644	.quad local_label(misc_set_invalid) /* 04 tra_0 */
645	.quad local_label(misc_set_invalid) /* 05 nodeheader_0 */
646	.quad local_label(misc_set_invalid) /* 06 nodeheader_1 */
647	.quad local_label(misc_set_invalid) /* 07 immheader_0 */
648	.quad local_label(misc_set_invalid) /* 08 odd_fixnum */
649	.quad local_label(misc_set_invalid) /* 09 immheader_1 */
650	.quad local_label(misc_set_invalid) /* 0a immheader_2 */
651	.quad local_label(misc_set_invalid) /* 0b nil */
652	.quad local_label(misc_set_invalid) /* 0c tra_1 */
653	.quad local_label(misc_set_invalid) /* 0d misc */
654	.quad local_label(misc_set_invalid) /* 0e symbol */
655	.quad local_label(misc_set_invalid) /* 0f function */
656	/* 10-1f */
657	.quad local_label(misc_set_invalid) /* 10 even_fixnum */
658	.quad local_label(misc_set_invalid) /* 11 imm_1 */
659	.quad local_label(misc_set_invalid) /* 12 imm_2 */
660	.quad local_label(misc_set_invalid) /* 13 cons */
661	.quad local_label(misc_set_invalid) /* 14 tra_0 */
662	.quad _SPgvset /* 15 symbol_vector */
663	.quad _SPgvset /* 16 ratio */
664	.quad local_label(misc_set_invalid) /* 17 immheader_0 */
665	.quad local_label(misc_set_invalid) /* 18 odd_fixnum */
666	.quad local_label(misc_set_u32) /* 19 bignum */
667	.quad local_label(misc_set_u64) /* 1a macptr */
668	.quad local_label(misc_set_invalid) /* 1b nil */
669	.quad local_label(misc_set_invalid) /* 1c tra_1 */
670	.quad local_label(misc_set_invalid) /* 1d misc */
671	.quad local_label(misc_set_invalid) /* 1e symbol */
672	.quad local_label(misc_set_invalid) /* 1f function */
673	/* 20-2f */
674	.quad local_label(misc_set_invalid) /* 20 even_fixnum */
675	.quad local_label(misc_set_invalid) /* 21 imm_1 */
676	.quad local_label(misc_set_invalid) /* 22 imm_2 */
677	.quad local_label(misc_set_invalid) /* 23 cons */
678	.quad local_label(misc_set_invalid) /* 24 tra_0 */
679	.quad _SPgvset /* 25 catch_frame */
680	.quad _SPgvset /* 26 complex */
681	.quad local_label(misc_set_invalid) /* 27 immheader_0 */
682	.quad local_label(misc_set_invalid) /* 28 odd_fixnum */
683	.quad local_label(misc_set_u32) /* 29 double_float */
684	.quad local_label(misc_set_u64) /* 2a dead_macptr */
685	.quad local_label(misc_set_invalid) /* 2b nil */
686	.quad local_label(misc_set_invalid) /* 2c tra_1 */
687	.quad local_label(misc_set_invalid) /* 2d misc */
688	.quad local_label(misc_set_invalid) /* 2e symbol */
689	.quad local_label(misc_set_invalid) /* 2f function */
690	/* 30-3f */
691	.quad local_label(misc_set_invalid) /* 30 even_fixnum */
692	.quad local_label(misc_set_invalid) /* 31 imm_1 */
693	.quad local_label(misc_set_invalid) /* 32 imm_2 */
694	.quad local_label(misc_set_invalid) /* 33 cons */
695	.quad local_label(misc_set_invalid) /* 34 tra_0 */
696	.quad _SPgvset /* 35 hash_vector */
697	.quad _SPgvset /* 36 struct */
698	.quad local_label(misc_set_invalid) /* 37 immheader_0 */
699	.quad local_label(misc_set_invalid) /* 38 odd_fixnum */
700	.quad local_label(misc_set_u32) /* 39 xcode_vector */
701	.quad local_label(misc_set_invalid) /* 3a immheader_2 */
702	.quad local_label(misc_set_invalid) /* 3b nil */
703	.quad local_label(misc_set_invalid) /* 3c tra_1 */
704	.quad local_label(misc_set_invalid) /* 3d misc */
705	.quad local_label(misc_set_invalid) /* 3e symbol */
706	.quad local_label(misc_set_invalid) /* 3f function */
707	/* 40-4f */
708	.quad local_label(misc_set_invalid) /* 40 even_fixnum */
709	.quad local_label(misc_set_invalid) /* 41 imm_1 */
710	.quad local_label(misc_set_invalid) /* 42 imm_2 */
711	.quad local_label(misc_set_invalid) /* 43 cons */
712	.quad local_label(misc_set_invalid) /* 44 tra_0 */
713	.quad _SPgvset /* 45 pool */
714	.quad _SPgvset /* 46 istruct */
715	.quad local_label(misc_set_invalid) /* 47 immheader_0 */
716	.quad local_label(misc_set_invalid) /* 48 odd_fixnum */
717	.quad local_label(misc_set_invalid) /* 49 immheader_1 */
718	.quad local_label(misc_set_invalid) /* 4a immheader_2 */
719	.quad local_label(misc_set_invalid) /* 4b nil */
720	.quad local_label(misc_set_invalid) /* 4c tra_1 */
721	.quad local_label(misc_set_invalid) /* 4d misc */
722	.quad local_label(misc_set_invalid) /* 4e symbol */
723	.quad local_label(misc_set_invalid) /* 4f function */
724	/* 50-5f */
725	.quad local_label(misc_set_invalid) /* 50 even_fixnum */
726	.quad local_label(misc_set_invalid) /* 51 imm_1 */
727	.quad local_label(misc_set_invalid) /* 52 imm_2 */
728	.quad local_label(misc_set_invalid) /* 53 cons */
729	.quad local_label(misc_set_invalid) /* 54 tra_0 */
730	.quad _SPgvset /* 55 weak */
731	.quad _SPgvset /* 56 value_cell */
732	.quad local_label(misc_set_invalid) /* 57 immheader_0 */
733	.quad local_label(misc_set_invalid) /* 58 odd_fixnum */
734	.quad local_label(misc_set_invalid) /* 59 immheader_1 */
735	.quad local_label(misc_set_invalid) /* 5a immheader_2 */
736	.quad local_label(misc_set_invalid) /* 5b nil */
737	.quad local_label(misc_set_invalid) /* 5c tra_1 */
738	.quad local_label(misc_set_invalid) /* 5d misc */
739	.quad local_label(misc_set_invalid) /* 5e symbol */
740	.quad local_label(misc_set_invalid) /* 5f function */
741	/* 60-6f */
742	.quad local_label(misc_set_invalid) /* 60 even_fixnum */
743	.quad local_label(misc_set_invalid) /* 61 imm_1 */
744	.quad local_label(misc_set_invalid) /* 62 imm_2 */
745	.quad local_label(misc_set_invalid) /* 63 cons */
746	.quad local_label(misc_set_invalid) /* 64 tra_0 */
747	.quad _SPgvset /* 65 package */
748	.quad _SPgvset /* 66 xfunction */
749	.quad local_label(misc_set_invalid) /* 67 immheader_0 */
750	.quad local_label(misc_set_invalid) /* 68 odd_fixnum */
751	.quad local_label(misc_set_invalid) /* 69 immheader_1 */
752	.quad local_label(misc_set_invalid) /* 6a immheader_2 */
753	.quad local_label(misc_set_invalid) /* 6b nil */
754	.quad local_label(misc_set_invalid) /* 6c tra_1 */
755	.quad local_label(misc_set_invalid) /* 6d misc */
756	.quad local_label(misc_set_invalid) /* 6e symbol */
757	.quad local_label(misc_set_invalid) /* 6f function */
758	/* 70-7f */
759	.quad local_label(misc_set_invalid) /* 70 even_fixnum */
760	.quad local_label(misc_set_invalid) /* 71 imm_1 */
761	.quad local_label(misc_set_invalid) /* 72 imm_2 */
762	.quad local_label(misc_set_invalid) /* 73 cons */
763	.quad local_label(misc_set_invalid) /* 74 tra_0 */
764	.quad _SPgvset /* 75 slot_vector */
765	.quad _SPgvset /* 76 lock */
766	.quad local_label(misc_set_invalid) /* 77 immheader_0 */
767	.quad local_label(misc_set_invalid) /* 78 odd_fixnum */
768	.quad local_label(misc_set_invalid) /* 79 immheader_1 */
769	.quad local_label(misc_set_invalid) /* 7a immheader_2 */
770	.quad local_label(misc_set_invalid) /* 7b nil */
771	.quad local_label(misc_set_invalid) /* 7c tra_1 */
772	.quad local_label(misc_set_invalid) /* 7d misc */
773	.quad local_label(misc_set_invalid) /* 7e symbol */
774	.quad local_label(misc_set_invalid) /* 7f function */
775	/* 80-8f */
776	.quad local_label(misc_set_invalid) /* 80 even_fixnum */
777	.quad local_label(misc_set_invalid) /* 81 imm_1 */
778	.quad local_label(misc_set_invalid) /* 82 imm_2 */
779	.quad local_label(misc_set_invalid) /* 83 cons */
780	.quad local_label(misc_set_invalid) /* 84 tra_0 */
781	.quad _SPgvset /* 85 lisp_thread */
782	.quad _SPgvset /* 86 instance */
783	.quad local_label(misc_set_invalid) /* 87 immheader_0 */
784	.quad local_label(misc_set_invalid) /* 88 odd_fixnum */
785	.quad local_label(misc_set_invalid) /* 89 immheader_1 */
786	.quad local_label(misc_set_invalid) /* 8a immheader_2 */
787	.quad local_label(misc_set_invalid) /* 8b nil */
788	.quad local_label(misc_set_invalid) /* 8c tra_1 */
789	.quad local_label(misc_set_invalid) /* 8d misc */
790	.quad local_label(misc_set_invalid) /* 8e symbol */
791	.quad local_label(misc_set_invalid) /* 8f function */
792	/* 90-9f */
793	.quad local_label(misc_set_invalid) /* 90 even_fixnum */
794	.quad local_label(misc_set_invalid) /* 91 imm_1 */
795	.quad local_label(misc_set_invalid) /* 92 imm_2 */
796	.quad local_label(misc_set_invalid) /* 93 cons */
797	.quad local_label(misc_set_invalid) /* 94 tra_0 */
798	.quad local_label(misc_set_function) /* 95 function_vector */
799	.quad local_label(misc_set_invalid) /* 96 nodeheader_1 */
800	.quad local_label(misc_set_invalid) /* 97 immheader_0 */
801	.quad local_label(misc_set_invalid) /* 98 odd_fixnum */
802	.quad local_label(misc_set_invalid) /* 99 immheader_1 */
803	.quad local_label(misc_set_invalid) /* 9a immheader_2 */
804	.quad local_label(misc_set_invalid) /* 9b nil */
805	.quad local_label(misc_set_invalid) /* 9c tra_1 */
806	.quad local_label(misc_set_invalid) /* 9d misc */
807	.quad local_label(misc_set_invalid) /* 9e symbol */
808	.quad local_label(misc_set_invalid) /* 9f function */
809	/* a0-af */
810	.quad local_label(misc_set_invalid) /* a0 even_fixnum */
811	.quad local_label(misc_set_invalid) /* a1 imm_1 */
812	.quad local_label(misc_set_invalid) /* a2 imm_2 */
813	.quad local_label(misc_set_invalid) /* a3 cons */
814	.quad local_label(misc_set_invalid) /* a4 tra_0 */
815	.quad _SPgvset /* a5 arrayH */
816	.quad _SPgvset /* a6 vectorH */
817	.quad local_label(misc_set_s16) /* a7 s16 */
818	.quad local_label(misc_set_invalid) /* a8 odd_fixnum */
819	.quad local_label(misc_set_invalid) /* a9 immheader_1 */
820	.quad local_label(misc_set_invalid) /* aa immheader_2 */
821	.quad local_label(misc_set_invalid) /* ab nil */
822	.quad local_label(misc_set_invalid) /* ac tra_1 */
823	.quad local_label(misc_set_invalid) /* ad misc */
824	.quad local_label(misc_set_invalid) /* ae symbol */
825	.quad local_label(misc_set_invalid) /* af function */
826	/* b0-bf */
827	.quad local_label(misc_set_invalid) /* b0 even_fixnum */
828	.quad local_label(misc_set_invalid) /* b1 imm_1 */
829	.quad local_label(misc_set_invalid) /* b2 imm_2 */
830	.quad local_label(misc_set_invalid) /* b3 cons */
831	.quad local_label(misc_set_invalid) /* b4 tra_0 */
832	.quad local_label(misc_set_invalid) /* b5 nodeheader_0 */
833	.quad _SPgvset /* b6 simple_vector */
834	.quad local_label(misc_set_u16) /* b7 immheader_0 */
835	.quad local_label(misc_set_invalid) /* b8 odd_fixnum */
836	.quad local_label(misc_set_invalid) /* b9 immheader_1 */
837	.quad local_label(misc_set_invalid) /* ba immheader_2 */
838	.quad local_label(misc_set_invalid) /* bb nil */
839	.quad local_label(misc_set_invalid) /* bc tra_1 */
840	.quad local_label(misc_set_invalid) /* bd misc */
841	.quad local_label(misc_set_invalid) /* be symbol */
842	.quad local_label(misc_set_invalid) /* bf function */
843	/* c0-cf */
844	.quad local_label(misc_set_invalid) /* c0 even_fixnum */
845	.quad local_label(misc_set_invalid) /* c1 imm_1 */
846	.quad local_label(misc_set_invalid) /* c2 imm_2 */
847	.quad local_label(misc_set_invalid) /* c3 cons */
848	.quad local_label(misc_set_invalid) /* c4 tra_0 */
849	.quad local_label(misc_set_invalid) /* c5 nodeheader_0 */
850	.quad local_label(misc_set_invalid) /* c6 nodeheader_1 */
851	.quad local_label(misc_set_string) /* c7 simple_base_string */
852	.quad local_label(misc_set_invalid) /* c8 odd_fixnum */
853	.quad local_label(misc_set_new_string) /* c9 new_strin */
854	.quad local_label(misc_set_fixnum_vector) /* ca fixnum_vector */
855	.quad local_label(misc_set_invalid) /* cb nil */
856	.quad local_label(misc_set_invalid) /* cc tra_1 */
857	.quad local_label(misc_set_invalid) /* cd misc */
858	.quad local_label(misc_set_invalid) /* ce symbol */
859	.quad local_label(misc_set_invalid) /* cf function */
860	/* d0-df */
861	.quad local_label(misc_set_invalid) /* d0 even_fixnum */
862	.quad local_label(misc_set_invalid) /* d1 imm_1 */
863	.quad local_label(misc_set_invalid) /* d2 imm_2 */
864	.quad local_label(misc_set_invalid) /* d3 cons */
865	.quad local_label(misc_set_invalid) /* d4 tra_0 */
866	.quad local_label(misc_set_invalid) /* d5 nodeheader_0 */
867	.quad local_label(misc_set_invalid) /* d6 nodeheader_1 */
868	.quad local_label(misc_set_s8) /* d7 s8 */
869	.quad local_label(misc_set_invalid) /* d8 odd_fixnum */
870	.quad local_label(misc_set_s32) /* d9 s32 */
871	.quad local_label(misc_set_s64) /* da s64 */
872	.quad local_label(misc_set_invalid) /* db nil */
873	.quad local_label(misc_set_invalid) /* dc tra_1 */
874	.quad local_label(misc_set_invalid) /* dd misc */
875	.quad local_label(misc_set_invalid) /* de symbol */
876	.quad local_label(misc_set_invalid) /* df function */
877	/* e0-ef */
878	.quad local_label(misc_set_invalid) /* e0 even_fixnum */
879	.quad local_label(misc_set_invalid) /* e1 imm_1 */
880	.quad local_label(misc_set_invalid) /* e2 imm_2 */
881	.quad local_label(misc_set_invalid) /* e3 cons */
882	.quad local_label(misc_set_invalid) /* e4 tra_0 */
883	.quad local_label(misc_set_invalid) /* e5 nodeheader_0 */
884	.quad local_label(misc_set_invalid) /* e6 nodeheader_1 */
885	.quad local_label(misc_set_u8) /* e7 u8 */
886	.quad local_label(misc_set_invalid) /* e8 odd_fixnum */
887	.quad local_label(misc_set_u32) /* e9 u32 */
888	.quad local_label(misc_set_u64) /* ea u64 */
889	.quad local_label(misc_set_invalid) /* eb nil */
890	.quad local_label(misc_set_invalid) /* ec tra_1 */
891	.quad local_label(misc_set_invalid) /* ed misc */
892	.quad local_label(misc_set_invalid) /* ee symbol */
893	.quad local_label(misc_set_invalid) /* ef function */
894	/* f0-ff */
895	.quad local_label(misc_set_invalid) /* f0 even_fixnum */
896	.quad local_label(misc_set_invalid) /* f1 imm_1 */
897	.quad local_label(misc_set_invalid) /* f2 imm_2 */
898	.quad local_label(misc_set_invalid) /* f3 cons */
899	.quad local_label(misc_set_invalid) /* f4 tra_0 */
900	.quad local_label(misc_set_invalid) /* f5 nodeheader_0 */
901	.quad local_label(misc_set_invalid) /* f6 nodeheader_1 */
902	.quad local_label(misc_set_bit_vector) /* f7 bitvector */
903	.quad local_label(misc_set_invalid) /* f8 odd_fixnum */
904	.quad local_label(misc_set_single_float_vector) /* f9 single_float */
905	.quad local_label(misc_set_double_float_vector) /* fa double_float */
906	.quad local_label(misc_set_invalid) /* fb nil */
907	.quad local_label(misc_set_invalid) /* fc tra_1 */
908	.quad local_label(misc_set_invalid) /* fd misc */
909	.quad local_label(misc_set_invalid) /* fe symbol */
910	.quad local_label(misc_set_invalid) /* ff function */
911
912	local_label(misc_set_function):
913	/* Functions are funny: the first N words */
914	/* are treated as (UNSIGNED-BYTE 64), where N is the low */
915	/* 32 bits of the first word. */
916	__(movl misc_data_offset(%arg_x),%imm0_l)
917	__(shl $fixnumshift,%imm0)
918	__(rcmpq(%arg_y,%imm0))
919	__(jae _SPgvset)
920	local_label(misc_set_u64):
921	__(movq $~(target_most_positive_fixnum << fixnumshift),%imm0)
922	__(testq %arg_z,%imm0)
923	__(movq %arg_z,%imm0)
924	__(jne 1f)
925	__(sarq $fixnumshift,%imm0)
926	__(jmp 9f)
927	1: __(andb $tagmask,%imm0_b)
928	__(cmpb $tag_misc,%imm0_b)
929	__(jne local_label(misc_set_bad))
930	__(movb misc_subtag_offset(%arg_z),%imm0_b)
931	__(cmpb $subtag_bignum,%imm0_b)
932	__(jne local_label(misc_set_bad))
933	__(movq misc_header_offset(%arg_z),%imm0)
934	__(cmpq $three_digit_bignum_header,%imm0)
935	__(je 3f)
936	__(cmpq $two_digit_bignum_header,%imm0)
937	__(jne local_label(misc_set_bad))
938	__(movq misc_data_offset(%arg_z),%imm0)
939	__(testq %imm0,%imm0)
940	__(js local_label(misc_set_bad))
941	__(jmp 9f)
942	3: __(movq misc_data_offset(%arg_z),%imm0)
943	__(cmpl $0,misc_data_offset+8(%arg_z))
944	__(jne local_label(misc_set_bad))
945	9: __(movq %imm0,misc_data_offset(%arg_x,%arg_y))
946	__(ret)
947	local_label(misc_set_fixnum_vector):
948	__(movq %arg_z,%imm0)
949	__(sarq $fixnumshift,%imm0)
950	__(testb $fixnummask,%arg_z_b)
951	__(jne local_label(misc_set_bad))
952	__(movq %imm0,misc_data_offset(%arg_x,%arg_y))
953	__(ret)
954	local_label(misc_set_s64):
955	__(movq %arg_z,%imm0)
956	__(sarq $fixnumshift,%imm0)
957	__(testb $fixnummask,%arg_z_b)
958	__(je 9f)
959	1: __(movb %arg_z_b,%imm0_b)
960	__(andb $tagmask,%imm0_b)
961	__(cmpb $tag_misc,%imm0_b)
962	__(jne local_label(misc_set_bad))
963	__(movb misc_subtag_offset(%arg_z),%imm0_b)
964	__(cmpb $subtag_bignum,%imm0_b)
965	__(jne local_label(misc_set_bad))
966	__(movq misc_header_offset(%arg_z),%imm0)
967	__(cmpq $two_digit_bignum_header,%imm0)
968	__(movq misc_data_offset(%arg_z),%imm0)
969	__(jne local_label(misc_set_bad))
970	9: __(movq %imm0,misc_data_offset(%arg_x,%arg_y))
971	__(ret)
972	local_label(misc_set_bad):
973	__(movq %arg_z,%arg_y)
974	__(movq %arg_x,%arg_z)
975	__(movq $XNOTELT,%arg_x)
976	__(set_nargs(3))
977	__(jmp _SPksignalerr)
978	local_label(misc_set_double_float_vector):
979	__(extract_lisptag(%arg_z,%imm0))
980	__(cmpb $tag_misc,%imm0_b)
981	__(jne local_label(misc_set_bad))
982	__(movb misc_subtag_offset(%arg_z),%imm0_b)
983	__(cmpb $subtag_double_float,%imm0_b)
984	__(jne local_label(misc_set_bad))
985	__(movq double_float.value(%arg_z),%imm0)
986	__(movq %imm0,misc_dfloat_offset(%arg_x,%arg_y))
987	__(ret)
988	local_label(misc_set_s32):
989	__(movq %arg_z,%imm0)
990	__(movq %arg_y,%imm1)
991	__(shlq $64-(32+fixnumshift),%imm0)
992	__(shrq $1,%imm1)
993	__(sarq $64-(32+fixnumshift),%imm0)
994	__(cmpq %imm0,%arg_z)
995	__(jne local_label(misc_set_bad))
996	__(testb $fixnummask,%arg_z_b)
997	__(jne local_label(misc_set_bad))
998	__(shr $fixnumshift,%imm0)
999	__(movl %imm0_l,misc_data_offset(%arg_x,%imm1))
1000	__(ret)
1001	local_label(misc_set_single_float_vector):
1002	__(cmpb $tag_single_float,%arg_z_b)
1003	__(movq %arg_z,%imm0)
1004	__(movq %arg_y,%imm1)
1005	__(jne local_label(misc_set_bad))
1006	__(shrq $1,%imm1)
1007	__(shr $32,%imm0)
1008	__(movl %imm0_l,misc_data_offset(%arg_x,%imm1))
1009	__(ret)
1010	local_label(misc_set_u32):
1011	__(movq %arg_y,%imm1)
1012	__(movq $~(0xffffffff<<fixnumshift),%imm0)
1013	__(shrq $1,%imm1)
1014	__(testq %imm0,%arg_z)
1015	__(jne local_label(misc_set_bad))
1016	__(unbox_fixnum(%arg_z,%imm0))
1017	__(movl %imm0_l,misc_data_offset(%arg_x,%imm1))
1018	__(ret)
1019	local_label(misc_set_bit_vector):
1020	__(testq $~fixnumone,%arg_z)
1021	__(jne local_label(misc_set_bad))
1022	__(unbox_fixnum(%arg_y,%imm0))
1023	__(testb %arg_z_b,%arg_z_b)
1024	__(je local_label(misc_set_clr_bit))
1025	local_label(misc_set_set_bit):
1026	__(btsq %imm0,misc_data_offset(%arg_x))
1027	__(ret)
1028	local_label(misc_set_clr_bit):
1029	__(btrq %imm0,misc_data_offset(%arg_x))
1030	__(ret)
1031	local_label(misc_set_u8):
1032	__(testq $~(0xff<<fixnumshift),%arg_z)
1033	__(jne local_label(misc_set_bad))
1034	__(movq %arg_y,%imm1)
1035	__(unbox_fixnum(%arg_z,%imm0))
1036	__(shrq $3,%imm1)
1037	__(movb %imm0_b,misc_data_offset(%arg_x,%imm1))
1038	__(ret)
1039	local_label(misc_set_s8):
1040	__(movq %arg_z,%imm0)
1041	__(shlq $64-(8+fixnumshift),%imm0)
1042	__(sarq $64-(8+fixnumshift),%imm0)
1043	__(cmpq %arg_z,%imm0)
1044	__(jne local_label(misc_set_bad))
1045	__(testb $fixnummask,%arg_z_b)
1046	__(jne local_label(misc_set_bad))
1047	__(movq %arg_y,%imm1)
1048	__(shrq $fixnumshift,%imm0)
1049	__(shrq $3,%imm1)
1050	__(movb %imm0_b,misc_data_offset(%arg_x,%imm1))
1051	__(ret)
1052	local_label(misc_set_string):
1053	__(cmpb $subtag_character,%arg_z_b)
1054	__(movq %arg_z,%imm0)
1055	__(jne local_label(misc_set_bad))
1056	__(movq %arg_y,%imm1)
1057	__(shrq $charcode_shift,%imm0)
1058	__(shrq $3,%imm1)
1059	__(movb %imm0_b,misc_data_offset(%arg_x,%imm1))
1060	__(ret)
1061	local_label(misc_set_new_string):
1062	__(cmpb $subtag_character,%arg_z_b)
1063	__(movq %arg_z,%imm0)
1064	__(jne local_label(misc_set_bad))
1065	__(movq %arg_y,%imm1)
1066	__(shrq $charcode_shift,%imm0)
1067	__(shrq $1,%imm1)
1068	__(movl %imm0_l,misc_data_offset(%arg_x,%imm1))
1069	__(ret)
1070	local_label(misc_set_s16):
1071	__(movq %arg_z,%imm0)
1072	__(movq %arg_y,%imm1)
1073	__(shlq $64-(16+fixnumshift),%imm0)
1074	__(shrq $2,%imm1)
1075	__(sarq $64-(16+fixnumshift),%imm0)
1076	__(cmpq %arg_z,%imm0)
1077	__(jne local_label(misc_set_bad))
1078	__(testb $fixnummask,%arg_z_b)
1079	__(jne local_label(misc_set_bad))
1080	__(shrq $fixnumshift,%imm0)
1081	__(movw %imm0_w,misc_data_offset(%arg_x,%imm1))
1082	__(ret)
1083	local_label(misc_set_u16):
1084	__(movq %arg_y,%imm1)
1085	__(testq $~(0xffff<<fixnumshift),%arg_z)
1086	__(jne local_label(misc_set_bad))
1087	__(shrq $2,%imm1)
1088	__(unbox_fixnum(%arg_z,%imm0))
1089	__(movw %imm0_w,misc_data_offset(%arg_x,%imm1))
1090	__(ret)
1091	local_label(misc_set_invalid):
1092	__(push $XSETBADVEC)
1093	__(set_nargs(4))
1094	__(jmp _SPksignalerr)
1095	_endfn(C(misc_set_common))
1096
1097	/* ret1valn returns "1 multiple value" when a called function does not */
1098	/* return multiple values. Its presence on the stack (as a return address) */
1099	/* identifies the stack frame to code which returns multiple values. */
1100
1101	_spentry(Fret1valn)
1102	.globl C(ret1valn)
1103	__(tra(C(ret1valn)))
1104	__(movq (%rsp),%ra0)
1105	__(movq %arg_z,(%rsp))
1106	__(set_nargs(1))
1107	__(jmpq *%ra0)
1108	_endsubp(Fret1valn)
1109
1110
1111	_spentry(nvalret)
1112	.globl C(nvalret)
1113	C(nvalret):
1114	__(ref_global(ret1val_addr,%temp1))
1115	__(cmpq lisp_frame.savera0(%rbp),%temp1)
1116	__(je 1f)
1117	__(testl %nargs,%nargs)
1118	__(movl $nil_value,%arg_z_l)
1119	__(cmovneq -node_size(%rsp,%nargs_q),%arg_z)
1120	__(leaveq)
1121	__(ret)
1122
1123
1124	/* actually need to return values ; always need to copy */
1125	1: __(leaq 2*node_size(%rbp),%imm1)
1126	__(movq (%imm1),%ra0)
1127	__(addq $node_size,%imm1)
1128	__(movq 0(%rbp),%rbp)
1129	__(leaq (%rsp,%nargs_q),%temp0)
1130	__(xorl %imm0_l,%imm0_l)
1131	__(jmp 3f)
1132	2: __(movq -node_size(%temp0),%temp1)
1133	__(subq $node_size,%temp0)
1134	__(addl $node_size,%imm0_l)
1135	__(movq %temp1,-node_size(%imm1))
1136	__(subq $node_size,%imm1)
1137	3: __(cmpl %imm0_l,%nargs) ;
1138	__(jne 2b)
1139	__(movq %imm1,%rsp)
1140	__(jmp *%ra0)
1141	_endsubp(nvalret)
1142
1143	_spentry(jmpsym)
1144	__(jump_fname())
1145	_endsubp(jmpsym)
1146
1147	_spentry(jmpnfn)
1148	__(movq %temp0,%fn)
1149	__(jmp *%fn)
1150	_endsubp(jmpnfn)
1151
1152	_spentry(funcall)
1153	__(do_funcall())
1154	_endsubp(funcall)
1155
1156	_spentry(mkcatch1v)
1157	__(nMake_Catch(0))
1158	__(ret)
1159	_endsubp(mkcatch1v)
1160
1161	_spentry(mkunwind)
1162	__(movq $undefined,%arg_z)
1163	__(Make_Catch(fixnumone))
1164	__(jmp *%ra0)
1165	_endsubp(mkunwind)
1166
1167	/* this takes a return address in %ra0; it's "new" in that it does the
1168	double binding of interrupt-level out-of-line */
1169	_spentry(nmkunwind)
1170	__(movq rcontext(tcr.tlb_pointer),%arg_x)
1171	__(movq INTERRUPT_LEVEL_BINDING_INDEX(%arg_x),%arg_y)
1172	__(push %arg_y)
1173	__(push $INTERRUPT_LEVEL_BINDING_INDEX)
1174	__(push rcontext(tcr.db_link))
1175	__(movq %rsp,rcontext(tcr.db_link))
1176	__(movq $-1<<fixnumshift,INTERRUPT_LEVEL_BINDING_INDEX(%arg_x))
1177	__(movq $undefined,%arg_z)
1178	__(Make_Catch(fixnumone))
1179	__(movq %arg_y,%arg_z)
1180	__(jmp _SPbind_interrupt_level)
1181	_endsubp(nmkunwind)
1182
1183	_spentry(mkcatchmv)
1184	__(nMake_Catch(fixnumone))
1185	__(ret)
1186	_endsubp(mkcatchmv)
1187
1188	_spentry(throw)
1189	__(movq rcontext(tcr.catch_top),%imm1)
1190	__(xorl %imm0_l,%imm0_l)
1191	__(movq (%rsp,%nargs_q),%temp0) /* temp0 = tag */
1192	__(jmp local_label(_throw_test))
1193	local_label(_throw_loop):
1194	__(cmpq %temp0,catch_frame.catch_tag(%imm1))
1195	__(je local_label(_throw_found))
1196	__(movq catch_frame.link(%imm1),%imm1)
1197	__(addq $fixnum_one,%imm0)
1198	local_label(_throw_test):
1199	__(testq %imm1,%imm1)
1200	__(jne local_label(_throw_loop))
1201	__(push %ra0)
1202	__(uuo_error_reg_not_tag(Rtemp0,subtag_catch_frame))
1203	__(pop %ra0)
1204	__(jmp _SPthrow)
1205	local_label(_throw_found):
1206	__(testb $fulltagmask,catch_frame.mvflag(%imm1))
1207	__(jne local_label(_throw_multiple))
1208	__(testl %nargs,%nargs)
1209	__(movl $nil_value,%arg_z_l)
1210	__(je local_label(_throw_one_value))
1211	__(movq -node_size(%rsp,%nargs_q),%arg_z)
1212	__(add %nargs_q,%rsp)
1213	local_label(_throw_one_value):
1214	__(lea local_label(_threw_one_value)(%rip),%ra0)
1215	__(jmp _SPnthrow1value)
1216	__(tra(local_label(_threw_one_value)))
1217	__(movq rcontext(tcr.catch_top),%temp0)
1218	__(movq catch_frame.db_link(%temp0),%imm0)
1219	__(movq rcontext(tcr.db_link),%imm1)
1220	__(cmpq %imm0,%imm1)
1221	__(jz local_label(_threw_one_value_dont_unbind))
1222	__(lea local_label(_threw_one_value_dont_unbind)(%rip),%ra0)
1223	__(push %ra0)
1224	__(jmp _SPunbind_to)
1225	__(tra(local_label(_threw_one_value_dont_unbind)))
1226	__(movq catch_frame.rbp(%temp0),%rbp)
1227	__(movq catch_frame.foreign_sp(%temp0),%imm0)
1228	__(movq catch_frame.xframe(%temp0),%imm1)
1229	__(movq %imm0,rcontext(tcr.foreign_sp))
1230	__(movq %imm1,rcontext(tcr.xframe))
1231	__(movq catch_frame.rsp(%temp0),%rsp)
1232	__(movq catch_frame.link(%temp0),%imm1)
1233	__(movq catch_frame._save0(%temp0),%save0)
1234	__(movq catch_frame._save1(%temp0),%save1)
1235	__(movq catch_frame._save2(%temp0),%save2)
1236	__ifndef(`TCR_IN_GPR')
1237	__(movq catch_frame._save3(%temp0),%save3)
1238	__endif
1239	__(movq %imm1,rcontext(tcr.catch_top))
1240	__(movq catch_frame.pc(%temp0),%ra0)
1241	__(lea -(tsp_frame.fixed_overhead+fulltag_misc)(%temp0),%imm1)
1242	__(movq (%imm1),%imm1)
1243	__(movq %imm1,rcontext(tcr.save_tsp))
1244	__(movq %imm1,rcontext(tcr.next_tsp))
1245	__(jmp *%ra0)
1246	local_label(_throw_multiple):
1247	__(lea local_label(_threw_multiple)(%rip),%ra0)
1248	__(jmp _SPnthrowvalues)
1249	__(tra(local_label(_threw_multiple)))
1250	__(movq rcontext(tcr.catch_top),%temp0)
1251	__(movq catch_frame.db_link(%temp0),%imm0)
1252	__(movq rcontext(tcr.db_link),%imm1)
1253	__(cmpq %imm0,%imm1)
1254	__(je local_label(_threw_multiple_dont_unbind))
1255	__(leaq local_label(_threw_multiple_dont_unbind)(%rip),%ra0)
1256	__(push %ra0)
1257	__(jmp _SPunbind_to)
1258	__(tra(local_label(_threw_multiple_dont_unbind)))
1259	/* Copy multiple values from the current %rsp to the target %rsp */
1260	__(lea (%rsp,%nargs_q),%imm0)
1261	__(movq catch_frame.rsp(%temp0),%imm1)
1262	__(jmp local_label(_threw_multiple_push_test))
1263	local_label(_threw_multiple_push_loop):
1264	__(subq $node_size,%imm0)
1265	__(subq $node_size,%imm1)
1266	__(movq (%imm0),%arg_z)
1267	__(movq %arg_z,(%imm1))
1268	local_label(_threw_multiple_push_test):
1269	__(cmpq %imm0,%rsp)
1270	__(jne local_label(_threw_multiple_push_loop))
1271	/* target %rsp is now in %imm1 */
1272	__(movq catch_frame.rbp(%temp0),%rbp)
1273	__(movq catch_frame.foreign_sp(%temp0),%imm0)
1274	__(movq %imm0,rcontext(tcr.foreign_sp))
1275	__(movq catch_frame.xframe(%temp0),%imm0)
1276	__(movq %imm0,rcontext(tcr.xframe))
1277	__(movq %imm1,%rsp)
1278	__(movq catch_frame.link(%temp0),%imm1)
1279	__(movq catch_frame._save0(%temp0),%save0)
1280	__(movq catch_frame._save1(%temp0),%save1)
1281	__(movq catch_frame._save2(%temp0),%save2)
1282	__ifndef(`TCR_IN_GPR')
1283	__(movq catch_frame._save3(%temp0),%save3)
1284	__endif
1285	__(movq %imm1,rcontext(tcr.catch_top))
1286	__(movq catch_frame.pc(%temp0),%ra0)
1287	__(lea -(tsp_frame.fixed_overhead+fulltag_misc)(%temp0),%imm1)
1288	__(movq (%imm1),%imm1)
1289	__(movq %imm1,rcontext(tcr.save_tsp))
1290	__(movq %imm1,rcontext(tcr.next_tsp))
1291	__(jmp *%ra0)
1292	_endsubp(throw)
1293
1294	/* This takes N multiple values atop the vstack. */
1295	_spentry(nthrowvalues)
1296	__(movb $1,rcontext(tcr.unwinding))
1297	local_label(_nthrowv_nextframe):
1298	__(subq $fixnumone,%imm0)
1299	__(js local_label(_nthrowv_done))
1300	__(movd %imm0,%mm1)
1301	__(movq rcontext(tcr.catch_top),%temp0)
1302	__(movq catch_frame.link(%temp0),%imm1)
1303	__(movq catch_frame.db_link(%temp0),%imm0)
1304	__(movq %imm1,rcontext(tcr.catch_top))
1305	__(cmpq %imm0,rcontext(tcr.db_link))
1306	__(jz local_label(_nthrowv_dont_unbind))
1307	__(push %ra0)
1308	__(leaq local_label(_nthrowv_back_from_unbind)(%rip),%ra0)
1309	__(push %ra0)
1310	__(jmp _SPunbind_to)
1311	__(tra(local_label(_nthrowv_back_from_unbind)))
1312
1313	__(pop %ra0)
1314	local_label(_nthrowv_dont_unbind):
1315	__(cmpb $unbound_marker,catch_frame.catch_tag(%temp0))
1316	__(je local_label(_nthrowv_do_unwind))
1317	/* A catch frame. If the last one, restore context from there. */
1318	__(movd %mm1,%imm0)
1319	__(testq %imm0,%imm0) /* last catch frame ? */
1320	__(jne local_label(_nthrowv_skip))
1321	__(movq catch_frame.xframe(%temp0),%save0)
1322	__(movq %save0,rcontext(tcr.xframe))
1323	__(leaq (%rsp,%nargs_q),%save1)
1324	__(movq catch_frame.rsp(%temp0),%save2)
1325	__(movq %nargs_q,%save0)
1326	__(jmp local_label(_nthrowv_push_test))
1327	local_label(_nthrowv_push_loop):
1328	__(subq $node_size,%save1)
1329	__(subq $node_size,%save2)
1330	__(movq (%save1),%temp1)
1331	__(movq %temp1,(%save2))
1332	local_label(_nthrowv_push_test):
1333	__(subq $node_size,%save0)
1334	__(jns local_label(_nthrowv_push_loop))
1335	__(movq catch_frame.xframe(%temp0),%save0)
1336	__(movq %save0,rcontext(tcr.xframe))
1337	__(movq %save2,%rsp)
1338	__(movq catch_frame.rbp(%temp0),%rbp)
1339	__ifndef(`TCR_IN_GPR')
1340	__(movq catch_frame._save3(%temp0),%save3)
1341	__endif
1342	__(movq catch_frame._save2(%temp0),%save2)
1343	__(movq catch_frame._save1(%temp0),%save1)
1344	__(movq catch_frame._save0(%temp0),%save0)
1345	__(movq catch_frame.foreign_sp(%temp0),%stack_temp)
1346	__(movq %stack_temp,rcontext(tcr.foreign_sp))
1347	local_label(_nthrowv_skip):
1348	__(movq -(tsp_frame.fixed_overhead+fulltag_misc)(%temp0),%imm1)
1349	__(movq %imm1,rcontext(tcr.save_tsp))
1350	__(movq %imm1,rcontext(tcr.next_tsp))
1351	__(movd %mm1,%imm0)
1352	__(jmp local_label(_nthrowv_nextframe))
1353	local_label(_nthrowv_do_unwind):
1354	/* This is harder. Call the cleanup code with the multiple values and */
1355	/* nargs, the throw count, and the caller's return address in a temp */
1356	/* stack frame. */
1357	__(leaq (%rsp,%nargs_q),%save1)
1358	__(push catch_frame._save0(%temp0))
1359	__(push catch_frame._save1(%temp0))
1360	__(push catch_frame._save2(%temp0))
1361	__ifndef(`TCR_IN_GPR')
1362	__(push catch_frame._save3(%temp0))
1363	__endif
1364	__(push catch_frame.pc(%temp0))
1365	__(movq catch_frame.rbp(%temp0),%rbp)
1366	__(movq catch_frame.xframe(%temp0),%stack_temp)
1367	__(movq catch_frame.rsp(%temp0),%arg_x)
1368	__(movq %stack_temp,rcontext(tcr.xframe))
1369	__(movq catch_frame.foreign_sp(%temp0),%stack_temp)
1370	__(movq %stack_temp,rcontext(tcr.foreign_sp))
1371	/* Discard the catch frame, so we can build a temp frame */
1372	__(movq -(tsp_frame.fixed_overhead+fulltag_misc)(%temp0),%imm1)
1373	__(movq %imm1,rcontext(tcr.save_tsp))
1374	__(movq %imm1,rcontext(tcr.next_tsp))
1375	/* tsp overhead, nargs, throw count, ra0 */
1376	__(dnode_align(%nargs_q,(tsp_frame.fixed_overhead+(3*node_size)),%imm0))
1377	__(TSP_Alloc_Var(%imm0,%imm1))
1378
1379	__(movq %nargs_q,(%imm1))
1380	__(movq %ra0,node_size(%imm1))
1381	__(movq %mm1,node_size*2(%imm1))
1382	__(leaq node_size*3(%imm1),%imm1)
1383	__(jmp local_label(_nthrowv_tpushtest))
1384	local_label(_nthrowv_tpushloop):
1385	__(movq -node_size(%save1),%temp0)
1386	__(subq $node_size,%save1)
1387	__(movq %temp0,(%imm1))
1388	__(addq $node_size,%imm1)
1389	local_label(_nthrowv_tpushtest):
1390	__(subl $node_size,%nargs)
1391	__(jns local_label(_nthrowv_tpushloop))
1392	__(pop %xfn)
1393	__ifndef(`TCR_IN_GPR')
1394	__(pop %save3)
1395	__endif
1396	__(pop %save2)
1397	__(pop %save1)
1398	__(pop %save0)
1399	__(movq %arg_x,%rsp)
1400	/* Ready to call cleanup code. set up tra, jmp to %xfn */
1401	__(leaq local_label(_nthrowv_called_cleanup)(%rip),%ra0)
1402	__(push %ra0)
1403	__(movb $0,rcontext(tcr.unwinding))
1404	__(jmp *%xfn)
1405	__(tra(local_label(_nthrowv_called_cleanup)))
1406
1407	__(movb $1,rcontext(tcr.unwinding))
1408	__(movq rcontext(tcr.save_tsp),%imm1)
1409	__(movq tsp_frame.data_offset+(0*node_size)(%imm1),%nargs_q)
1410	__(movq tsp_frame.data_offset+(1*node_size)(%imm1),%ra0)
1411	__(movq tsp_frame.data_offset+(2*node_size)(%imm1),%mm1)
1412	__(movq %nargs_q,%imm0)
1413	__(addq $tsp_frame.fixed_overhead+(node_size*3),%imm1)
1414	__(jmp local_label(_nthrowv_tpoptest))
1415	local_label(_nthrowv_tpoploop):
1416	__(push (%imm1))
1417	__(addq $node_size,%imm1)
1418	local_label(_nthrowv_tpoptest):
1419	__(subq $node_size,%imm0)
1420	__(jns local_label(_nthrowv_tpoploop))
1421	__(movq rcontext(tcr.save_tsp),%imm1)
1422	__(movq (%imm1),%imm1)
1423	__(movq %imm1,rcontext(tcr.save_tsp))
1424	__(movq %imm1,rcontext(tcr.next_tsp))
1425	__(movd %mm1,%imm0)
1426	__(jmp local_label(_nthrowv_nextframe))
1427	local_label(_nthrowv_done):
1428	__(movb $0,rcontext(tcr.unwinding))
1429	__(check_pending_interrupt(%imm0))
1430	local_label(_nthrowv_return):
1431	__(jmp *%ra0)
1432	_endsubp(nthrowvalues)
1433
1434	/* This is a (slight) optimization. When running an unwind-protect, */
1435	/* save the single value and the throw count in the tstack frame. */
1436	/* Note that this takes a single value in arg_z. */
1437
1438	_spentry(nthrow1value)
1439	__(movb $1,rcontext(tcr.unwinding))
1440	local_label(_nthrow1v_nextframe):
1441	__(subq $fixnumone,%imm0)
1442	__(js local_label(_nthrow1v_done))
1443	__(movd %imm0,%mm1)
1444	__(movq rcontext(tcr.catch_top),%temp0)
1445	__(movq catch_frame.link(%temp0),%imm1)
1446	__(movq catch_frame.db_link(%temp0),%imm0)
1447	__(movq %imm1,rcontext(tcr.catch_top))
1448	__(cmpq %imm0,rcontext(tcr.db_link))
1449	__(jz local_label(_nthrow1v_dont_unbind))
1450	__(push %ra0)
1451	__(leaq local_label(_nthrow1v_back_from_unbind)(%rip),%ra0)
1452	__(push %ra0)
1453	__(jmp _SPunbind_to)
1454	__(tra(local_label(_nthrow1v_back_from_unbind)))
1455
1456	__(pop %ra0)
1457	local_label(_nthrow1v_dont_unbind):
1458	__(cmpb $unbound_marker,catch_frame.catch_tag(%temp0))
1459	__(je local_label(_nthrow1v_do_unwind))
1460	/* A catch frame. If the last one, restore context from there. */
1461	__(movd %mm1,%imm0)
1462	__(testq %imm0,%imm0) /* last catch frame ? */
1463	__(jne local_label(_nthrow1v_skip))
1464	__(movq catch_frame.xframe(%temp0),%save0)
1465	__(movq %save0,rcontext(tcr.xframe))
1466	__(leaq (%rsp,%nargs_q),%save1)
1467	__(movq catch_frame.xframe(%temp0),%save0)
1468	__(movq %save0,rcontext(tcr.xframe))
1469	__(movq catch_frame.rsp(%temp0),%rsp)
1470	__(movq catch_frame.rbp(%temp0),%rbp)
1471	__ifndef(`TCR_IN_GPR')
1472	__(movq catch_frame._save3(%temp0),%save3)
1473	__endif
1474	__(movq catch_frame._save2(%temp0),%save2)
1475	__(movq catch_frame._save1(%temp0),%save1)
1476	__(movq catch_frame._save0(%temp0),%save0)
1477	__(movq catch_frame.foreign_sp(%temp0),%stack_temp)
1478	__(movq %stack_temp,rcontext(tcr.foreign_sp))
1479	local_label(_nthrow1v_skip):
1480	__(movq -(tsp_frame.fixed_overhead+fulltag_misc)(%temp0),%imm1)
1481	__(movq %imm1,rcontext(tcr.save_tsp))
1482	__(movq %imm1,rcontext(tcr.next_tsp))
1483	__(movd %mm1,%imm0)
1484	__(jmp local_label(_nthrow1v_nextframe))
1485	local_label(_nthrow1v_do_unwind):
1486
1487	/* This is harder, but not as hard (not as much BLTing) as the */
1488	/* multiple-value case. */
1489
1490	__(movq catch_frame.xframe(%temp0),%save0)
1491	__(movq %save0,rcontext(tcr.xframe))
1492	__(movq catch_frame._save0(%temp0),%save0)
1493	__(movq catch_frame._save1(%temp0),%save1)
1494	__(movq catch_frame._save2(%temp0),%save2)
1495	__ifndef(`TCR_IN_GPR')
1496	__(movq catch_frame._save3(%temp0),%save3)
1497	__endif
1498	__(movq catch_frame.pc(%temp0),%xfn)
1499	__(movq catch_frame.rbp(%temp0),%rbp)
1500	__(movq catch_frame.rsp(%temp0),%rsp)
1501	__(movq catch_frame.foreign_sp(%temp0),%stack_temp)
1502	__(movq %stack_temp,rcontext(tcr.foreign_sp))
1503	/* Discard the catch frame, so we can build a temp frame */
1504	__(movq -(tsp_frame.fixed_overhead+fulltag_misc)(%temp0),%imm1)
1505	__(movq %imm1,rcontext(tcr.save_tsp))
1506	__(movq %imm1,rcontext(tcr.next_tsp))
1507	__(TSP_Alloc_Fixed((3*node_size),%imm1))
1508	__(addq $tsp_frame.fixed_overhead,%imm1)
1509	__(movq %ra0,(%imm1))
1510	__(movq %mm1,node_size*1(%imm1))
1511	__(movq %arg_z,node_size*2(%imm1))
1512	/* Ready to call cleanup code. set up tra, jmp to %xfn */
1513	__(leaq local_label(_nthrow1v_called_cleanup)(%rip),%ra0)
1514	__(movb $0,rcontext(tcr.unwinding))
1515	__(push %ra0)
1516	__(jmp *%xfn)
1517	__(tra(local_label(_nthrow1v_called_cleanup)))
1518
1519	__(movb $1,rcontext(tcr.unwinding))
1520	__(movq rcontext(tcr.save_tsp),%imm1)
1521	__(movq tsp_frame.data_offset+(0*node_size)(%imm1),%ra0)
1522	__(movq tsp_frame.data_offset+(1*node_size)(%imm1),%mm1)
1523	__(movq tsp_frame.data_offset+(2*node_size)(%imm1),%arg_z)
1524
1525	__(movq (%imm1),%imm1)
1526	__(movq %imm1,rcontext(tcr.save_tsp))
1527	__(movq %imm1,rcontext(tcr.next_tsp))
1528	__(movd %mm1,%imm0)
1529	__(jmp local_label(_nthrow1v_nextframe))
1530	local_label(_nthrow1v_done):
1531	__(movb $0,rcontext(tcr.unwinding))
1532	__(check_pending_interrupt(%imm0))
1533	local_label(_nthrow1v_return):
1534	__(jmp *%ra0)
1535	_endsubp(nthrow1value)
1536
1537	/* This never affects the symbol's vcell */
1538	/* Non-null symbol in arg_y, new value in arg_z */
1539
1540	_spentry(bind)
1541	__(movq symbol.binding_index(%arg_y),%temp0)
1542	__(cmpq rcontext(tcr.tlb_limit),%temp0)
1543	__(jb 0f)
1544	__(push %temp0)
1545	__(tlb_too_small())
1546	0: __(testq %temp0,%temp0)
1547	__(jz 9f)
1548	__(movq rcontext(tcr.tlb_pointer),%temp1)
1549	__(push (%temp1,%temp0))
1550	__(push %temp0)
1551	__(push rcontext(tcr.db_link))
1552	__(movq %rsp,rcontext(tcr.db_link))
1553	__(movq %arg_z,(%temp1,%temp0))
1554	__(jmp *%ra0)
1555	9:
1556	__(movq %arg_y,%arg_z)
1557	__(movq $XSYMNOBIND,%arg_y)
1558	__(set_nargs(2))
1559	__(push %ra0)
1560	__(jmp _SPksignalerr)
1561	_endsubp(bind)
1562
1563	/* arg_z = symbol: bind it to its current value */
1564
1565	_spentry(bind_self)
1566	__(movq symbol.binding_index(%arg_z),%temp0)
1567	__(cmpq rcontext(tcr.tlb_limit),%temp0)
1568	__(jb 0f)
1569	__(push %temp0)
1570	__(tlb_too_small())
1571	0: __(testq %temp0,%temp0)
1572	__(jz 9f)
1573	__(movq rcontext(tcr.tlb_pointer),%temp1)
1574	__(cmpb $no_thread_local_binding_marker,(%temp0,%temp1))
1575	__(jz 2f)
1576	__(push (%temp1,%temp0))
1577	__(push %temp0)
1578	__(push rcontext(tcr.db_link))
1579	__(movq %rsp,rcontext(tcr.db_link))
1580	__(jmp *%ra0)
1581	2: __(movq symbol.vcell(%arg_z),%arg_y)
1582	__(push (%temp1,%temp0))
1583	__(push %temp0)
1584	__(push rcontext(tcr.db_link))
1585	__(movq %arg_y,(%temp1,%temp0))
1586	__(movq %rsp,rcontext(tcr.db_link))
1587	__(jmp *%ra0)
1588	9: __(movq $XSYMNOBIND,%arg_y)
1589	__(set_nargs(2))
1590	__(push %ra0)
1591	__(jmp _SPksignalerr)
1592	_endsubp(bind_self)
1593
1594	_spentry(bind_nil)
1595	__(movq symbol.binding_index(%arg_z),%temp0)
1596	__(cmpq rcontext(tcr.tlb_limit),%temp0)
1597	__(jb 0f)
1598	__(push %temp0)
1599	__(tlb_too_small())
1600	0: __(testq %temp0,%temp0)
1601	__(jz 9f)
1602	__(movq rcontext(tcr.tlb_pointer),%temp1)
1603	__(push (%temp1,%temp0))
1604	__(push %temp0)
1605	__(push rcontext(tcr.db_link))
1606	__(movq %rsp,rcontext(tcr.db_link))
1607	__(movq $nil_value,(%temp1,%temp0))
1608	__(jmp *%ra0)
1609	9: __(movq $XSYMNOBIND,%arg_y)
1610	__(set_nargs(2))
1611	__(push %ra0)
1612	__(jmp _SPksignalerr)
1613	_endsubp(bind_nil)
1614
1615	_spentry(bind_self_boundp_check)
1616	__(movq symbol.binding_index(%arg_z),%temp0)
1617	__(cmpq rcontext(tcr.tlb_limit),%temp0)
1618	__(jb 0f)
1619	__(push %temp0)
1620	__(tlb_too_small())
1621	0: __(testq %temp0,%temp0)
1622	__(jz 9f)
1623	__(movq rcontext(tcr.tlb_pointer),%temp1)
1624	__(cmpb $no_thread_local_binding_marker,(%temp1,%temp0))
1625	__(je 2f)
1626	__(cmpb $unbound_marker,(%temp1,%temp0))
1627	__(je 8f)
1628	__(push (%temp1,%temp0))
1629	__(push %temp0)
1630	__(push rcontext(tcr.db_link))
1631	__(movq %rsp,rcontext(tcr.db_link))
1632	__(jmp *%ra0)
1633	2: __(movq symbol.vcell(%arg_z),%arg_y)
1634	__(cmpb $unbound_marker,%arg_y_b)
1635	__(jz 8f)
1636	__(push (%temp1,%temp0))
1637	__(push %temp0)
1638	__(push rcontext(tcr.db_link))
1639	__(movq %rsp,rcontext(tcr.db_link))
1640	__(movq %arg_y,(%temp1,%temp0))
1641	__(jmp *%ra0)
1642	8: __(push %ra0)
1643	__(uuo_error_reg_unbound(Rarg_z))
1644
1645	9: __(movq $XSYMNOBIND,%arg_y)
1646	__(set_nargs(2))
1647	__(push %ra0)
1648	__(jmp _SPksignalerr)
1649	_endsubp(bind_self_boundp_check)
1650
1651	_spentry(conslist)
1652	__(movl $nil_value,%arg_z_l)
1653	__(testl %nargs,%nargs)
1654	__(jmp 2f)
1655	1: __(pop %arg_y)
1656	__(Cons(%arg_y,%arg_z,%arg_z))
1657	__(subl $node_size,%nargs)
1658	2: __(jnz 1b)
1659	__(jmp *%ra0)
1660	_endsubp(conslist)
1661
1662	/* do list: last arg in arg_z, all others pushed, nargs set to #args pushed. /
1663	/* Cons, one cons cell at at time. Maybe optimize this later. */
1664
1665	_spentry(conslist_star)
1666	__(testl %nargs,%nargs)
1667	__(jmp 2f)
1668	1: __(pop %arg_y)
1669	__(Cons(%arg_y,%arg_z,%arg_z))
1670	__(subl $node_size,%nargs)
1671	2: __(jnz 1b)
1672	__(jmp *%ra0)
1673	_endsubp(conslist_star)
1674
1675	/* We always have to create a tsp frame (even if nargs is 0), so the compiler */
1676	/* doesn't get confused. */
1677	_spentry(stkconslist)
1678	__(movq %nargs_q,%imm1)
1679	__(addq %imm1,%imm1)
1680	__(movl $nil_value,%arg_z_l)
1681	__(dnode_align(%imm1,tsp_frame.fixed_overhead,%imm1))
1682	__(TSP_Alloc_Var(%imm1,%imm0))
1683	__(addq $fulltag_cons,%imm0)
1684	__(testl %nargs,%nargs)
1685	__(jmp 2f)
1686	1: __(pop %temp0)
1687	__(_rplaca(%imm0,%temp0))
1688	__(_rplacd(%imm0,%arg_z))
1689	__(movq %imm0,%arg_z)
1690	__(add $cons.size,%imm0)
1691	__(subl $node_size,%nargs)
1692	2: __(jne 1b)
1693	__(jmp *%ra0)
1694	_endsubp(stkconslist)
1695
1696	/* do list: last arg in arg_z, all others vpushed, /
1697	/* nargs set to #args vpushed. */
1698
1699	_spentry(stkconslist_star)
1700	__(movq %nargs_q,%imm1)
1701	__(addq %imm1,%imm1)
1702	__(dnode_align(%imm1,tsp_frame.fixed_overhead,%imm1))
1703	__(TSP_Alloc_Var(%imm1,%imm0))
1704	__(addq $fulltag_cons,%imm0)
1705	__(testl %nargs,%nargs)
1706	__(jmp 2f)
1707	1: __(pop %temp0)
1708	__(_rplaca(%imm0,%temp0))
1709	__(_rplacd(%imm0,%arg_z))
1710	__(movq %imm0,%arg_z)
1711	__(addq $cons.size,%imm0)
1712	__(subl $node_size,%nargs)
1713	2: __(jne 1b)
1714	__(jmp *%ra0)
1715	_endsubp(stkconslist_star)
1716
1717	/* Make a stack-consed simple-vector out of the NARGS objects */
1718	/* on top of the vstack; return it in arg_z. */
1719
1720	_spentry(mkstackv)
1721	__(dnode_align(%nargs_q,tsp_frame.fixed_overhead+node_size,%imm1))
1722	__(TSP_Alloc_Var(%imm1,%temp0))
1723	__(movl %nargs,%imm0_l)
1724	__(shlq $(num_subtag_bits-fixnumshift),%imm0)
1725	__(movb $subtag_simple_vector,%imm0_b)
1726	__(movq %imm0,(%temp0))
1727	__(leaq fulltag_misc(%temp0),%arg_z)
1728	__(testl %nargs,%nargs)
1729	__(leaq misc_data_offset(%arg_z,%nargs_q),%imm1)
1730	__(jmp 2f)
1731	1: __(pop -node_size(%imm1))
1732	__(subl $node_size,%nargs)
1733	__(leaq -node_size(%imm1),%imm1)
1734	2: __(jne 1b)
1735	__(jmp *%ra0)
1736	_endsubp(mkstackv)
1737
1738
1739	.globl C(egc_write_barrier_start)
1740	C(egc_write_barrier_start):
1741	/* */
1742	/* The function pc_luser_xp() - which is used to ensure that suspended threads */
1743	/* are suspended in a GC-safe way - has to treat these subprims (which implement */
1744	/* the EGC write-barrier) specially. Specifically, a store that might introduce */
1745	/* an intergenerational reference (a young pointer stored in an old object) has */
1746	/* to "memoize" that reference by setting a bit in the global "refbits" bitmap. */
1747	/* This has to happen atomically, and has to happen atomically wrt GC. */
1748
1749	/* Note that updating a word in a bitmap is itself not atomic, unless we use */
1750	/* interlocked loads and stores. */
1751
1752
1753
1754	/* For RPLACA and RPLACD, things are fairly simple: regardless of where we are */
1755	/* in the function, we can do the store (even if it's already been done) and */
1756	/* calculate whether or not we need to set the bit out-of-line. (Actually */
1757	/* setting the bit needs to be done atomically, unless we're sure that other */
1758	/* threads are suspended.) */
1759	/* We can unconditionally set the suspended thread's RIP to the return address. */
1760
1761
1762	_spentry(rplaca)
1763	.globl C(egc_rplaca)
1764	C(egc_rplaca):
1765	__(rcmpq(%arg_z,%arg_y))
1766	__(_rplaca(%arg_y,%arg_z))
1767	__(ja 1f)
1768	0: __(repret)
1769	1: __(movq %arg_y,%imm0)
1770	__(subq lisp_global(ref_base),%imm0)
1771	__(shrq $dnode_shift,%imm0)
1772	__(movq %imm0,%imm1)
1773	__(cmpq lisp_global(oldspace_dnode_count),%imm0)
1774	__(jae 2f)
1775	__(ref_global(refbits,%temp0))
1776	__(xorb $63,%imm0_b)
1777	__(lock)
1778	__(btsq %imm0,(%temp0))
1779	2: __(cmpq lisp_global(managed_static_dnodes),%imm1)
1780	__(jae 0b)
1781	__(ref_global(managed_static_refbits,%temp0))
1782	__(xorb $63,%imm1_b)
1783	__(lock)
1784	__(btsq %imm1,(%temp0))
1785	__(ret)
1786	_endsubp(rplaca)
1787
1788	_spentry(rplacd)
1789	.globl C(egc_rplacd)
1790	C(egc_rplacd):
1791	__(rcmpq(%arg_z,%arg_y))
1792	__(_rplacd(%arg_y,%arg_z))
1793	__(ja 1f)
1794	0: __(repret)
1795	1: __(movq %arg_y,%imm0)
1796	__(subq lisp_global(ref_base),%imm0)
1797	__(shrq $dnode_shift,%imm0)
1798	__(movq %imm0,%imm1)
1799	__(cmpq lisp_global(oldspace_dnode_count),%imm0)
1800	__(jae 2f)
1801	__(ref_global(refbits,%temp0))
1802	__(xorb $63,%imm0_b)
1803	__(lock)
1804	__(btsq %imm0,(%temp0))
1805	__(ret)
1806	2: __(cmpq lisp_global(managed_static_dnodes),%imm1)
1807	__(jae 0b)
1808	__(ref_global(managed_static_refbits,%temp0))
1809	__(xorb $63,%imm1_b)
1810	__(lock)
1811	__(btsq %imm1,(%temp0))
1812	__(ret)
1813	_endsubp(rplacd)
1814
1815	/* Storing into a gvector can be handled the same way as storing into a CONS. */
1816
1817
1818	_spentry(gvset)
1819	.globl C(egc_gvset)
1820	C(egc_gvset):
1821	__(rcmpq(%arg_z,%arg_x))
1822	__(movq %arg_z,misc_data_offset(%arg_x,%arg_y))
1823	__(ja 1f)
1824	0: __(repret)
1825	1: __(lea misc_data_offset(%arg_x,%arg_y),%imm0)
1826	__(subq lisp_global(ref_base),%imm0)
1827	__(shrq $dnode_shift,%imm0)
1828	__(movq %imm0,%imm1)
1829	__(cmpq lisp_global(oldspace_dnode_count),%imm0)
1830	__(jae 2f)
1831	__(ref_global(refbits,%temp0))
1832	__(xorb $63,%imm0_b)
1833	__(lock)
1834	__(btsq %imm0,(%temp0))
1835	2: __(cmpq lisp_global(managed_static_dnodes),%imm1)
1836	__(jae 0b)
1837	__(ref_global(managed_static_refbits,%temp0))
1838	__(xorb $63,%imm1_b)
1839	__(lock)
1840	__(btsq %imm1,(%temp0))
1841	__(ret)
1842	_endsubp(gvset)
1843
1844	/* This is a special case of storing into a gvector: if we need to */
1845	/* memoize the store, record the address of the hash-table vector */
1846	/* in the refmap, as well. */
1847
1848
1849	_spentry(set_hash_key)
1850	.globl C(egc_set_hash_key)
1851	C(egc_set_hash_key):
1852	__(rcmpq(%arg_z,%arg_x))
1853	__(movq %arg_z,misc_data_offset(%arg_x,%arg_y))
1854	__(ja 1f)
1855	0: __(repret)
1856	1: __(lea misc_data_offset(%arg_x,%arg_y),%imm0)
1857	__(subq lisp_global(ref_base),%imm0)
1858	__(shrq $dnode_shift,%imm0)
1859	__(movq %imm0,%imm1)
1860	__(cmpq lisp_global(oldspace_dnode_count),%imm0)
1861	__(jae 2f)
1862	__(ref_global(refbits,%temp0))
1863	__(xorb $63,%imm0_b)
1864	__(lock)
1865	__(btsq %imm0,(%temp0))
1866	/* Now memoize the address of the hash vector */
1867	__(movq %arg_x,%imm0)
1868	__(subq lisp_global(ref_base),%imm0)
1869	__(shrq $dnode_shift,%imm0)
1870	__(xorb $63,%imm0_b)
1871	__(lock)
1872	__(btsq %imm0,(%temp0))
1873	2: __(cmpq lisp_global(managed_static_dnodes),%imm1)
1874	__(jae 0b)
1875	__(ref_global(managed_static_refbits,%temp0))
1876	__(xorb $63,%imm1_b)
1877	__(lock)
1878	__(btsq %imm1,(%temp0))
1879	/* Now memoize the address of the hash vector */
1880	__(movq %arg_x,%imm0)
1881	__(subq lisp_global(ref_base),%imm0)
1882	__(shrq $dnode_shift,%imm0)
1883	__(xorb $63,%imm0_b)
1884	__(lock)
1885	__(btsq %imm0,(%temp0))
1886	__(ret)
1887	_endsubp(set_hash_key)
1888
1889	/* This is a little trickier: if this is interrupted, we need to know */
1890	/* whether or not the STORE-CONDITIONAL (cmpxchgq) has won or not. */
1891	/* If we're interrupted before the PC has reached the "success_test" label, */
1892	/* repeat (luser the PC back to store_node_conditional_retry.) If we're at that */
1893	/* label with the Z flag set, we won and (may) need to memoize. */
1894
1895	_spentry(store_node_conditional)
1896	.globl C(egc_store_node_conditional)
1897	C(egc_store_node_conditional):
1898	__(unbox_fixnum(%temp0,%imm1))
1899	.globl C(egc_store_node_conditional_retry)
1900	C(egc_store_node_conditional_retry):
1901	0: __(movq %arg_y,%imm0)
1902	__(lock)
1903	__(cmpxchgq %arg_z,(%arg_x,%imm1))
1904	.globl C(egc_store_node_conditional_success_test)
1905	C(egc_store_node_conditional_success_test):
1906	__(jne 9f)
1907	__(lea (%arg_x,%imm1),%imm0)
1908	__(subq lisp_global(ref_base),%imm0)
1909	__(shrq $dnode_shift,%imm0)
1910	__(movq %imm0,%imm1)
1911	__(cmpq lisp_global(oldspace_dnode_count),%imm0)
1912	__(ref_global(refbits,%temp1))
1913	__(jae 2f)
1914	__(xorb $63,%imm0_b)
1915	__(lock)
1916	__(btsq %imm0,(%temp1))
1917	2: __(cmpq lisp_global(managed_static_dnodes),%imm1)
1918	__(jae 8f)
1919	__(ref_global(managed_static_refbits,%temp1))
1920	__(xorb $63,%imm1_b)
1921	__(lock)
1922	__(btsq %imm1,(%temp1))
1923	.globl C(egc_store_node_conditional_success_end)
1924	C(egc_store_node_conditional_success_end):
1925	8: __(movl $t_value,%arg_z_l)
1926	__(ret)
1927	9: __(movl $nil_value,%arg_z_l)
1928	__(ret)
1929	_endsubp(store_node_conditional)
1930
1931	_spentry(set_hash_key_conditional)
1932	.globl C(egc_set_hash_key_conditional)
1933	C(egc_set_hash_key_conditional):
1934	.globl C(egc_set_hash_key_conditional_retry)
1935	C(egc_set_hash_key_conditional_retry):
1936	__(unbox_fixnum(%temp0,%imm1))
1937	0: __(movq %arg_y,%imm0)
1938	__(lock)
1939	__(cmpxchgq %arg_z,(%arg_x,%imm1))
1940	.globl C(egc_set_hash_key_conditional_success_test)
1941	C(egc_set_hash_key_conditional_success_test):
1942	__(jne 9f)
1943	__(lea (%arg_x,%imm1),%imm0)
1944	__(subq lisp_global(ref_base),%imm0)
1945	__(shrq $dnode_shift,%imm0)
1946	__(movq %imm0,%imm1)
1947	__(cmpq lisp_global(oldspace_dnode_count),%imm0)
1948	__(ref_global(refbits,%temp1))
1949	__(jae 2f)
1950	__(xorb $63,%imm0_b)
1951	__(lock)
1952	__(btsq %imm0,(%temp1))
1953	/* Now memoize the address of the hash vector */
1954	__(movq %arg_x,%imm0)
1955	__(subq lisp_global(ref_base),%imm0)
1956	__(shrq $dnode_shift,%imm0)
1957	__(xorb $63,%imm0_b)
1958	__(lock)
1959	__(btsq %imm0,(%temp1))
1960	2: __(cmpq lisp_global(managed_static_dnodes),%imm1)
1961	__(jae 8f)
1962	__(ref_global(managed_static_refbits,%temp1))
1963	__(xorb $63,%imm1_b)
1964	__(lock)
1965	__(btsq %imm1,(%temp1))
1966	/* Now memoize the address of the hash vector */
1967	__(movq %arg_x,%imm0)
1968	__(subq lisp_global(ref_base),%imm0)
1969	__(shrq $dnode_shift,%imm0)
1970	__(xorb $63,%imm0_b)
1971	__(lock)
1972	__(btsq %imm0,(%temp1))
1973	.globl C(egc_write_barrier_end)
1974	C(egc_write_barrier_end):
1975	8: __(movl $t_value,%arg_z_l)
1976	__(ret)
1977	9: __(movl $nil_value,%arg_z_l)
1978	__(ret)
1979	_endsubp(set_hash_key_conditional)
1980
1981
1982
1983
1984	_spentry(setqsym)
1985	__(btq $sym_vbit_const,symbol.flags(%arg_y))
1986	__(jae _SPspecset)
1987	__(movq %arg_y,%arg_z)
1988	__(movq $XCONST,%arg_y)
1989	__(set_nargs(2))
1990	__(jmp _SPksignalerr)
1991	_endsubp(setqsym)
1992
1993	_spentry(progvsave)
1994	/* Error if arg_z isn't a proper list. That's unlikely, */
1995	/* but it's better to check now than to crash later. */
1996
1997	__(compare_reg_to_nil(%arg_z))
1998	__(movq %arg_z,%arg_x) /* fast */
1999	__(movq %arg_z,%temp1) /* slow */
2000	__(je 9f) /* Null list is proper */
2001	0:
2002	__(extract_lisptag(%arg_x,%imm0))
2003	__(cmpb $tag_list,%imm0_b)
2004	__(jne 8f)
2005	__(compare_reg_to_nil(%arg_x))
2006	__(je 9f)
2007	__(_cdr(%arg_x,%temp0)) /* (null (cdr fast)) ? */
2008	__(compare_reg_to_nil(%temp0))
2009	__(je 9f)
2010	__(extract_lisptag(%temp0,%imm0))
2011	__(cmpb $tag_list,%imm0_b)
2012	__(jne 8f)
2013	__(_cdr(%temp0,%arg_x))
2014	__(_cdr(%temp1,%temp1))
2015	__(cmpq %temp1,%arg_x)
2016	__(jne 0b)
2017
2018	8: __(movq $XIMPROPERLIST,%arg_y)
2019	__(set_nargs(2))
2020	__(jmp _SPksignalerr)
2021	9: /* Whew */
2022
2023	/* Next, determine the length of arg_y. We */
2024	/* know that it's a proper list. */
2025	__(movq $-fixnumone,%imm0)
2026	__(movq %arg_y,%arg_x)
2027	1: __(compare_reg_to_nil(%arg_x))
2028	__(_cdr(%arg_x,%arg_x))
2029	__(leaq fixnumone(%imm0),%imm0)
2030	__(jne 1b)
2031
2032	/* imm0 is now (boxed) triplet count. */
2033	/* Determine word count, add 1 (to align), and make room. */
2034	/* if count is 0, make an empty tsp frame and exit */
2035	__(testq %imm0,%imm0)
2036	__(jne 2f)
2037	__(TSP_Alloc_Fixed(2*node_size,%imm0))
2038	__(ret)
2039	2: __(movq %imm0,%imm1)
2040	__(add %imm1,%imm1)
2041	__(add %imm0,%imm1)
2042	__(dnode_align(%imm1,tsp_frame.fixed_overhead+node_size,%imm1))
2043	__(TSP_Alloc_Var(%imm1,%temp0))
2044	__(movq %imm0,(%temp0))
2045	__(movq rcontext(tcr.db_link),%temp1)
2046	3: __(movl $unbound_marker,%temp0_l)
2047	__(compare_reg_to_nil(%arg_z))
2048	__(cmovneq cons.car(%arg_z),%temp0)
2049	__(cmovneq cons.cdr(%arg_z),%arg_z)
2050	__(_car(%arg_y,%arg_x))
2051	__(_cdr(%arg_y,%arg_y))
2052	__(movq symbol.binding_index(%arg_x),%arg_x)
2053	__(cmp rcontext(tcr.tlb_limit),%arg_x)
2054	__(jb 4f)
2055	__(push %arg_x)
2056	__(tlb_too_small())
2057	4: __(movq rcontext(tcr.tlb_pointer),%imm0)
2058	__(subq $binding.size,%imm1)
2059	__(compare_reg_to_nil(%arg_y))
2060	__(movq %arg_x,binding.sym(%imm1))
2061	__(push (%imm0,%arg_x))
2062	__(pop binding.val(%imm1))
2063	__(movq %temp0,(%imm0,%arg_x))
2064	__(movq %temp1,binding.link(%imm1))
2065	__(movq %imm1,%temp1)
2066	__(jne 3b)
2067	__(movq %temp1,rcontext(tcr.db_link))
2068	__(ret)
2069	_endsubp(progvsave)
2070
2071	/* Allocate node objects on the temp stack, immediate objects on the foreign */
2072	/* stack. (The caller has to know which stack to discard a frame from.) */
2073	/* %arg_y = boxed element-count, %arg_z = boxed subtype */
2074
2075	_spentry(stack_misc_alloc)
2076	__(movq $~(((1<<56)-1)<<fixnumshift),%temp0)
2077	__(testq %temp0,%arg_y)
2078	__(jne local_label(stack_misc_alloc_not_u56))
2079	__(unbox_fixnum(%arg_z,%imm0))
2080	__(movq %arg_y,%temp0)
2081	__(shl $num_subtag_bits-fixnumshift,%temp0)
2082	__(orq %temp0,%imm0) /* %imm0 now = header */
2083	__(movb $fulltagmask,%imm1_b)
2084	__(andb %imm0_b,%imm1_b)
2085	__(cmpb $fulltag_nodeheader_0,%imm1_b)
2086	__(je local_label(stack_misc_alloc_node))
2087	__(cmpb $fulltag_nodeheader_1,%imm1_b)
2088	__(je local_label(stack_misc_alloc_node))
2089	__(cmpb $ivector_class_64_bit,%imm1_b)
2090	__(jz local_label(stack_misc_alloc_64))
2091	__(cmpb $ivector_class_32_bit,%imm1_b)
2092	__(jz local_label(stack_misc_alloc_32))
2093	__(unbox_fixnum(%arg_y,%imm1))
2094	/* ivector_class_other_bit: 16, 8, or 1 ... */
2095	__(cmpb $subtag_bit_vector,%imm0_b)
2096	__(jne local_label(stack_misc_alloc_8))
2097	__(addq $7,%imm1)
2098	__(shrq $3,%imm1)
2099	__(jmp local_label(stack_misc_alloc_alloc_ivector))
2100	local_label(stack_misc_alloc_8):
2101	__(cmpb $subtag_simple_base_string,%imm0_b)
2102	__(jb local_label(stack_misc_alloc_16))
2103	__(unbox_fixnum(%arg_y,%imm1))
2104	__(jmp local_label(stack_misc_alloc_alloc_ivector))
2105	local_label(stack_misc_alloc_16):
2106	__(unbox_fixnum(%arg_y,%imm1))
2107	__(shlq %imm1)
2108	__(jmp local_label(stack_misc_alloc_alloc_ivector))
2109	local_label(stack_misc_alloc_32):
2110	/* 32-bit ivector */
2111	__(unbox_fixnum(%arg_y,%imm1))
2112	__(shlq $2,%imm1)
2113	__(jmp local_label(stack_misc_alloc_alloc_ivector))
2114	local_label(stack_misc_alloc_64):
2115	/* 64-bit ivector */
2116	__(movq %arg_y,%imm1)
2117	local_label(stack_misc_alloc_alloc_ivector):
2118	__(dnode_align(%imm1,tsp_frame.fixed_overhead+node_size,%imm1))
2119	__(cmpq $tstack_alloc_limit,%imm1)
2120	__(ja local_label(stack_misc_alloc_heap_alloc_ivector))
2121	__ifdef(`WINDOWS')
2122	__(windows_cstack_probe(%imm1,%temp0))
2123	__endif
2124	__(movq rcontext(tcr.foreign_sp),%stack_temp)
2125	__(movd %stack_temp,%temp1)
2126	__(subq %imm1,rcontext(tcr.foreign_sp))
2127	__(movq rcontext(tcr.foreign_sp),%temp0)
2128	0: __(movapd %fpzero,-dnode_size(%temp1))
2129	__(subq $dnode_size,%temp1)
2130	__(cmpq %temp1,%temp0)
2131	__(jnz 0b)
2132	__(movq %stack_temp,(%temp0))
2133	__(movq %rbp,csp_frame.save_rbp(%temp0))
2134	__(movq %imm0,csp_frame.fixed_overhead(%temp0))
2135	__(leaq csp_frame.fixed_overhead+fulltag_misc(%temp0),%arg_z)
2136	__(ret)
2137	local_label(stack_misc_alloc_heap_alloc_ivector):
2138	__(movq rcontext(tcr.foreign_sp),%imm1)
2139	__(subq $dnode_size,rcontext(tcr.foreign_sp))
2140	__(movq rcontext(tcr.foreign_sp),%imm0)
2141	__(movq %imm1,(%imm0))
2142	__(jmp _SPmisc_alloc)
2143	local_label(stack_misc_alloc_node):
2144	__(movq %arg_y,%imm1)
2145	__(dnode_align(%imm1,tsp_frame.fixed_overhead+node_size,%imm1))
2146	__(cmpq $tstack_alloc_limit,%imm1)
2147	__(ja local_label(stack_misc_alloc_heap_alloc_gvector))
2148	__(TSP_Alloc_Var(%imm1,%temp0))
2149	__(movq %imm0,(%temp0))
2150	__(leaq fulltag_misc(%temp0),%arg_z)
2151	__(ret)
2152	local_label(stack_misc_alloc_heap_alloc_gvector):
2153	__(TSP_Alloc_Fixed(0,%imm0))
2154	__(jmp _SPmisc_alloc)
2155
2156	local_label(stack_misc_alloc_not_u56):
2157	__(uuo_error_reg_not_type(Rarg_y,error_object_not_unsigned_byte_56))
2158	_endsubp(stack_misc_alloc)
2159
2160	/* subtype (boxed, of course) is pushed, followed by nargs bytes worth of */
2161	/* initial-contents. Note that this can be used to cons any type of initialized */
2162	/* node-header'ed misc object (symbols, closures, ...) as well as vector-like */
2163	/* objects. */
2164	_spentry(gvector)
2165	__(subl $node_size,%nargs)
2166	__(movq (%rsp,%nargs_q),%imm0) /* boxed subtype */
2167	__(sarq $fixnumshift,%imm0)
2168	__(movq %nargs_q,%imm1)
2169	__(shlq $num_subtag_bits-word_shift,%imm1)
2170	__(orq %imm1,%imm0)
2171	__(dnode_align(%nargs_q,node_size,%imm1))
2172	__(Misc_Alloc(%arg_z))
2173	__(movq %nargs_q,%imm1)
2174	__(jmp 2f)
2175	1: __(movq %temp0,misc_data_offset(%arg_z,%imm1))
2176	2: __(subq $node_size,%imm1)
2177	__(pop %temp0) /* Note the intentional fencepost: */
2178	/* discard the subtype as well. */
2179	__(jge 1b)
2180	__(jmp *%ra0)
2181	_endsubp(gvector)
2182
2183	_spentry(mvpass)
2184	__(hlt)
2185	_endsubp(mvpass)
2186
2187
2188
2189	_spentry(nthvalue)
2190	__(hlt)
2191	_endsubp(nthvalue)
2192
2193	_spentry(values)
2194	__(movq (%temp0),%ra0)
2195	__(ref_global(ret1val_addr,%imm1))
2196	__(cmpq %imm1,%ra0)
2197	__(movl $nil_value,%arg_z_l)
2198	__(je 0f)
2199	__(testl %nargs,%nargs)
2200	__(cmovneq -node_size(%rsp,%nargs_q),%arg_z)
2201	__(movq %temp0,%rsp)
2202	__(ret)
2203	0: __(movq 8(%temp0),%ra0)
2204	__(addq $2*node_size,%temp0)
2205	__(lea (%rsp,%nargs_q),%imm0)
2206	__(jmp 2f)
2207	1: __(subq $node_size,%imm0)
2208	__(movq (%imm0),%temp1)
2209	__(subq $node_size,%temp0)
2210	__(movq %temp1,(%temp0))
2211	2: __(cmpq %imm0,%rsp)
2212	__(jne 1b)
2213	__(movq %temp0,%rsp)
2214	__(jmp *%ra0)
2215	_endsubp(values)
2216
2217	_spentry(default_optional_args)
2218	__(hlt)
2219	_endsubp(default_optional_args)
2220
2221	_spentry(opt_supplied_p)
2222	__(hlt)
2223	_endsubp(opt_supplied_p)
2224
2225	_spentry(lexpr_entry)
2226	__(hlt)
2227	_endsubp(lexpr_entry)
2228
2229	_spentry(heap_rest_arg)
2230	__(push_argregs())
2231	__(movq %next_method_context,%arg_y)
2232	__(movl %nargs,%imm1_l)
2233	__(testl %imm1_l,%imm1_l)
2234	__(movl $nil_value,%arg_z_l)
2235	__(jmp 2f)
2236	.p2align 4
2237	1: __(pop %temp1)
2238	__(Cons(%temp1,%arg_z,%arg_z))
2239	__(subl $node_size,%imm1_l)
2240	2: __(jg 1b)
2241	__(push %arg_z)
2242	__(movq %arg_y,%next_method_context)
2243	__(jmp *%ra0)
2244	_endsubp(heap_rest_arg)
2245
2246	/* %imm0 contains the number of fixed args ; make an &rest arg out of the others */
2247	_spentry(req_heap_rest_arg)
2248	__(push_argregs())
2249	__(movq %next_method_context,%arg_y)
2250	__(movl %nargs,%imm1_l)
2251	__(subl %imm0_l,%imm1_l)
2252	__(movl $nil_value,%arg_z_l)
2253	__(jmp 2f)
2254	.p2align 4
2255	1: __(pop %temp1)
2256	__(Cons(%temp1,%arg_z,%arg_z))
2257	__(subl $node_size,%imm1_l)
2258	2: __(jg 1b)
2259	__(push %arg_z)
2260	__(movq %arg_y,%next_method_context)
2261	__(jmp *%ra0)
2262	_endsubp(req_heap_rest_arg)
2263
2264	/* %imm0 bytes of stuff has already been pushed */
2265	/* make an &rest arg out of any others */
2266	_spentry(heap_cons_rest_arg)
2267	__(movl %nargs,%imm1_l)
2268	__(subl %imm0_l,%imm1_l)
2269	__(movq %next_method_context,%arg_y)
2270	__(movl $nil_value,%arg_z_l)
2271	__(jmp 2f)
2272	.p2align 4
2273	1: __(pop %temp1)
2274	__(Cons(%temp1,%arg_z,%arg_z))
2275	__(subl $node_size,%imm1_l)
2276	2: __(jg 1b)
2277	__(push %arg_z)
2278	__(movq %arg_y,%next_method_context)
2279	__(jmp *%ra0)
2280	_endsubp(heap_cons_rest_arg)
2281
2282	_spentry(simple_keywords)
2283	__(xorl %imm0_l,%imm0_l)
2284	__(push_argregs())
2285	__(jmp _SPkeyword_bind)
2286	_endsubp(simple_keywords)
2287
2288	_spentry(keyword_args)
2289	__(push_argregs())
2290	__(jmp _SPkeyword_bind)
2291	_endsubp(keyword_args)
2292
2293	/* There are %nargs words of arguments on the stack; %imm0 contains the number */
2294	/* of non-keyword args pushed. It's possible that we never actually got */
2295	/* any keyword args, which would make things much simpler. */
2296
2297	/* On entry, temp1 contains a fixnum with bits indicating whether */
2298	/* &allow-other-keys and/or &rest was present in the lambda list. */
2299	/* Once we get here, we can use the arg registers. */
2300
2301	define(`keyword_flags_aok_bit',`fixnumshift')
2302	define(`keyword_flags_unknown_keys_bit',`fixnumshift+1')
2303	define(`keyword_flags_rest_bit',`fixnumshift+2')
2304	define(`keyword_flags_seen_aok_bit',`fixnumshift+3')
2305
2306	_spentry(keyword_bind)
2307	__(movl %nargs,%imm1_l)
2308	__(subq %imm0,%imm1)
2309	__(jbe local_label(no_keyword_values))
2310	__(btq $word_shift,%imm1)
2311	__(jnc local_label(even))
2312	__(movl $nil_value,%arg_z_l)
2313	__(movq %imm1,%nargs_q)
2314	__(testl %nargs,%nargs)
2315	__(jmp 1f)
2316	0: __(pop %arg_y)
2317	__(Cons(%arg_y,%arg_z,%arg_z))
2318	__(subl $node_size,%nargs)
2319	1: __(jnz 0b)
2320	__(movl $XBADKEYS,%arg_y_l)
2321	__(set_nargs(2))
2322	__(jmp _SPksignalerr)
2323	/* Now that we're sure that we have an even number of keywords and values */
2324	/* (in %imm1), copy all pairs to the temp stack */
2325	local_label(even):
2326	/* Get the keyword vector into arg_x, and its length into arg_y. */
2327	__(movl function_data_offset(%fn),%imm0_l)
2328	__(movq function_data_offset(%fn,%imm0,node_size),%arg_x)
2329	__(vector_length(%arg_x,%arg_y))
2330	__(testq %arg_y,%arg_y)
2331	__(jne 1f)
2332	__(btq $keyword_flags_aok_bit,%temp1)
2333	__(jnc 1f)
2334
2335	__(btq $keyword_flags_rest_bit,%temp1)
2336	__(jc 0f)
2337	__(addq %imm1,%rsp)
2338	0:
2339	__(jmp *%ra0)
2340	1:
2341	__(lea tsp_frame.fixed_overhead(%imm1),%arg_z)
2342	__(TSP_Alloc_Var(%arg_z,%imm0))
2343	2: __(subq $node_size,%arg_z)
2344	__(pop (%arg_z))
2345	__(cmpq %arg_z,%imm0)
2346	__(jne 2b)
2347	/* Push arg_y pairs of NILs. */
2348	__(movq %arg_y,%imm0)
2349	__(jmp 4f)
2350	3: __(push $nil_value)
2351	__(push $nil_value)
2352	4: __(subq $fixnumone,%arg_y)
2353	__(jge 3b)
2354	/* Push the %saveN registers, so that we can use them in this loop */
2355	/* Also, borrow %arg_y for a bit */
2356	__(push %arg_y)
2357	__(push %save2)
2358	__(push %save1)
2359	__(push %save0)
2360	__(leaq 4*node_size(%rsp,%imm0,2),%save0)
2361	/* %save0 points to the 0th value/supplied-p pair */
2362	__(leaq (%arg_z,%imm1),%save1)
2363	/* %save1 is the end of the provided keyword/value pairs (the old %tsp). */
2364	__(movq %imm0,%save2)
2365	/* %save2 is the length of the keyword vector */
2366	5: __(movq (%arg_z),%arg_y) /* %arg_y is current keyword */
2367	__(xorl %imm0_l,%imm0_l)
2368	__(cmpq $nrs.kallowotherkeys,%arg_y)
2369	__(jne local_label(next_keyvect_entry))
2370	__(btsq $keyword_flags_seen_aok_bit,%temp1)
2371	__(jc local_label(next_keyvect_entry))
2372	__(cmpb $fulltag_nil,node_size(%arg_z))
2373	__(je local_label(next_keyvect_entry))
2374	__(btsq $keyword_flags_aok_bit,%temp1)
2375	__(jmp local_label(next_keyvect_entry))
2376	6: __(cmpq misc_data_offset(%arg_x,%imm0),%arg_y)
2377	__(jne 7f)
2378	/* Got a match; have we already seen this keyword ? */
2379	__(negq %imm0)
2380	__(cmpb $fulltag_nil,-node_size*2(%save0,%imm0,2))
2381	__(jne 9f) /* already seen keyword, ignore this value */
2382	__(movq node_size(%arg_z),%arg_y)
2383	__(movq %arg_y,-node_size(%save0,%imm0,2))
2384	__(movl $t_value,-node_size*2(%save0,%imm0,2))
2385	__(jmp 9f)
2386	7: __(addq $node_size,%imm0)
2387	local_label(next_keyvect_entry):
2388	__(cmpq %imm0,%save2)
2389	__(jne 6b)
2390	/* Didn't match anything in the keyword vector. Is the keyword */
2391	/* :allow-other-keys ? */
2392	__(cmpq $nrs.kallowotherkeys,%arg_y)
2393	__(je 9f) /* :allow-other-keys is never "unknown" */
2394	8: __(btsq $keyword_flags_unknown_keys_bit,%temp1)
2395	9: __(addq $dnode_size,%arg_z)
2396	__(cmpq %arg_z,%save1)
2397	__(jne 5b)
2398	__(pop %save0)
2399	__(pop %save1)
2400	__(pop %save2)
2401	__(pop %arg_y)
2402	/* If the function takes an &rest arg, or if we got an unrecognized */
2403	/* keyword and don't allow that, copy the incoming keyword/value */
2404	/* pairs from the temp stack back to the value stack */
2405	__(btq $keyword_flags_rest_bit,%temp1)
2406	__(jc 1f)
2407	__(btq $keyword_flags_unknown_keys_bit,%temp1)
2408	__(jnc 0f)
2409	__(btq $keyword_flags_aok_bit,%temp1)
2410	__(jnc 1f)
2411	/* pop the temp frame */
2412	0: __(discard_temp_frame(%imm1))
2413	__(jmp *%ra0)
2414	/* Copy the keyword/value pairs from the tsp back to sp, either because */
2415	/* the function takes an &rest arg or because we need to signal an */
2416	/* "unknown keywords" error */
2417	1: __(movq rcontext(tcr.save_tsp),%arg_z)
2418	__(mov (%arg_z),%arg_y)
2419	__(jmp 3f)
2420	2: __(push (%arg_z))
2421	__(push node_size(%arg_z))
2422	3: __(addq $dnode_size,%arg_z)
2423	__(cmpq %arg_z,%arg_y)
2424	__(jne 2b)
2425	__(discard_temp_frame(%imm0))
2426	__(btq $keyword_flags_unknown_keys_bit,%temp1)
2427	__(jnc 9f)
2428	__(btq $keyword_flags_aok_bit,%temp1)
2429	__(jc 9f)
2430	/* Signal an "unknown keywords" error */
2431	__(movq %imm1,%nargs_q)
2432	__(testl %nargs,%nargs)
2433	__(movl $nil_value,%arg_z_l)
2434	__(jmp 5f)
2435	4: __(pop %arg_y)
2436	__(Cons(%arg_y,%arg_z,%arg_z))
2437	__(subl $node_size,%nargs)
2438	5: __(jnz 4b)
2439	__(movl $XBADKEYS,%arg_y_l)
2440	__(set_nargs(2))
2441	__(push %ra0)
2442	__(jmp _SPksignalerr)
2443	9: __(jmp *%ra0)
2444
2445	/* No keyword values were provided. Access the keyword vector (which is the 0th */
2446	/* constant in %fn), determine its length N, and push N pairs of NILs. */
2447	/* N could be 0 ... */
2448
2449	local_label(no_keyword_values):
2450	__(movl function_data_offset(%fn),%imm0_l)
2451	__(movq function_data_offset(%fn,%imm0,node_size),%arg_x)
2452	__(movl $nil_value,%arg_z_l)
2453	__(vector_length(%arg_x,%arg_y))
2454	__(jmp 1f)
2455	0: __(push %arg_z)
2456	__(push %arg_z)
2457	1: __(subq $fixnumone,%arg_y)
2458	__(jge 0b)
2459	__(jmp *%ra0)
2460	_endsubp(keyword_bind)
2461
2462
2463
2464	_spentry(ksignalerr)
2465	__(movq $nrs.errdisp,%fname)
2466	__(jump_fname)
2467	_endsubp(ksignalerr)
2468
2469	_spentry(stack_rest_arg)
2470	__(xorl %imm0_l,%imm0_l)
2471	__(push_argregs())
2472	__(jmp _SPstack_cons_rest_arg)
2473	_endsubp(stack_rest_arg)
2474
2475	_spentry(req_stack_rest_arg)
2476	__(push_argregs())
2477	__(jmp _SPstack_cons_rest_arg)
2478	_endsubp(req_stack_rest_arg)
2479
2480	_spentry(stack_cons_rest_arg)
2481	__(movl %nargs,%imm1_l)
2482	__(subl %imm0_l,%imm1_l)
2483	__(movl $nil_value,%arg_z_l)
2484	__(jle 2f) /* empty list ; make an empty TSP frame */
2485	__(addq %imm1,%imm1)
2486	__(cmpq $(tstack_alloc_limit-dnode_size),%imm1)
2487	__(ja 3f) /* make empty frame, then heap-cons */
2488	__(dnode_align(%imm1,tsp_frame.fixed_overhead,%imm0))
2489	__(TSP_Alloc_Var(%imm0,%temp1))
2490	__(addq $fulltag_cons,%temp1)
2491	1: __(pop %arg_x)
2492	__(_rplacd(%temp1,%arg_z))
2493	__(_rplaca(%temp1,%arg_x))
2494	__(movq %temp1,%arg_z)
2495	__(addq $cons.size,%temp1)
2496	__(subq $dnode_size,%imm1)
2497	__(jne 1b)
2498	__(push %arg_z)
2499	__(jmp *%ra0)
2500
2501	/* Length 0, make empty frame */
2502
2503	2:
2504	__(TSP_Alloc_Fixed(0,%temp1))
2505	__(push %arg_z)
2506	__(jmp *%ra0)
2507
2508	/* Too big to stack-cons, but make an empty frame before heap-consing */
2509
2510	3:
2511	__(TSP_Alloc_Fixed(0,%temp1))
2512	__(jmp _SPheap_cons_rest_arg)
2513	_endsubp(stack_cons_rest_arg)
2514
2515
2516
2517	_spentry(getxlong)
2518	_endsubp(getxlong)
2519
2520	/* Have to be a little careful here: the caller may or may not have pushed */
2521	/* an empty frame, and we may or may not have needed one. We can't easily */
2522	/* tell whether or not a frame will be needed (if the caller didn't reserve */
2523	/* a frame, whether or not we need one depends on the length of the list */
2524	/* in arg_z. So, if the caller didn't push a frame, we do so ; once everything's */
2525	/* been spread, we discard the reserved frame (regardless of who pushed it) */
2526	/* if all args fit in registers. */
2527	_spentry(spreadargz)
2528	__(testl %nargs,%nargs)
2529	__(jne 0f)
2530	__(push $reserved_frame_marker)
2531	__(push $reserved_frame_marker)
2532	0: __(movq %arg_z,%arg_y) /* save in case of error */
2533	__(xorl %imm0_l,%imm0_l)
2534	__(compare_reg_to_nil(%arg_z))
2535	__(je 2f)
2536	1: __(extract_fulltag(%arg_z,%imm1))
2537	__(cmpb $fulltag_cons,%imm1_b)
2538	__(jne 9f)
2539	__(_car(%arg_z,%arg_x))
2540	__(_cdr(%arg_z,%arg_z))
2541	__(addl $node_size,%imm0_l)
2542	__(cmpl $call_arguments_limit<<fixnumshift, %imm0_l)
2543	__(jae 8f)
2544	__(compare_reg_to_nil(%arg_z))
2545	__(push %arg_x)
2546	__(jne 1b)
2547	2: __(addl %imm0_l,%nargs)
2548	__(jne 4f)
2549	3: __(addq $2*node_size,%rsp)
2550	__(jmp *%ra0)
2551	4: __(cmpl $1*node_size,%nargs)
2552	__(pop %arg_z)
2553	__(je 3b)
2554	__(cmpl $2*node_size,%nargs)
2555	__(pop %arg_y)
2556	__(je 3b)
2557	__(cmpl $3*node_size,%nargs)
2558	__(pop %arg_x)
2559	__(je 3b)
2560	__(jmp *%ra0)
2561	/* Discard everything that's been pushed already, complain */
2562
2563	8: __(lea (%rsp,%imm0),%rsp)
2564	__(movq %arg_y,%arg_z) /* recover original */
2565	__(movq $XTMINPS,%arg_y)
2566	__(set_nargs(2))
2567	__(push %ra0)
2568	__(jmp _SPksignalerr)
2569	/* Discard everything that's been pushed already, complain */
2570	9: __(lea (%rsp,%imm0),%rsp)
2571	__(movq %arg_y,%arg_z) /* recover original */
2572	__(movq $XNOSPREAD,%arg_y)
2573	__(set_nargs(2))
2574	__(push %ra0)
2575	__(jmp _SPksignalerr)
2576	_endsubp(spreadargz)
2577
2578	/* Caller built it's own frame when it was entered. If all outgoing args */
2579	/* are in registers, we can discard that frame; otherwise, we copy outgoing */
2580	/* relative to it and restore %rbp/%ra0 */
2581	_spentry(tfuncallgen)
2582	__(cmpl $nargregs*node_size,%nargs)
2583	__(jbe 9f)
2584	__(lea -nargregs*node_size(%rsp,%nargs_q),%imm0)
2585	__(xorl %imm1_l,%imm1_l)
2586	/* We can use %ra0 as a temporary here, since the real return address */
2587	/* is on the stack */
2588	0: __(movq -node_size(%imm0),%ra0)
2589	__(movq %ra0,-node_size(%rbp,%imm1))
2590	__(subq $node_size,%imm0)
2591	__(subq $node_size,%imm1)
2592	__(cmpq %imm0,%rsp)
2593	__(jne 0b)
2594	__(lea (%rbp,%imm1),%rsp)
2595	__(movq 8(%rbp),%ra0)
2596	__(movq (%rbp),%rbp)
2597	__(pushq %ra0)
2598	__(do_funcall())
2599	/* All args in regs; exactly the same as the tfuncallvsp case */
2600	9:
2601	__(leave)
2602	__(do_funcall())
2603	_endsubp(tfuncallgen)
2604
2605	/* Some args were pushed; move them down in the frame */
2606	_spentry(tfuncallslide)
2607	__(lea -nargregs*node_size(%rsp,%nargs_q),%imm0)
2608	__(xorl %imm1_l,%imm1_l)
2609	/* We can use %ra0 as a temporary here, since the real return address */
2610	/* is on the stack */
2611	0: __(movq -node_size(%imm0),%ra0)
2612	__(movq %ra0,-node_size(%rbp,%imm1))
2613	__(subq $node_size,%imm0)
2614	__(subq $node_size,%imm1)
2615	__(cmpq %imm0,%rsp)
2616	__(jne 0b)
2617	__(lea (%rbp,%imm1),%rsp)
2618	__(movq 8(%rbp),%ra0)
2619	__(movq (%rbp),%rbp)
2620	__(push %ra0)
2621	__(do_funcall())
2622	_endsubp(tfuncallslide)
2623
2624	/* No args were pushed; recover saved context & do funcall */
2625	_spentry(tfuncallvsp)
2626	__(leave)
2627	__(do_funcall())
2628	_endsubp(tfuncallvsp)
2629
2630	_spentry(tcallsymgen)
2631	__(cmpl $nargregs*node_size,%nargs)
2632	__(jbe 9f)
2633	__(lea -nargregs*node_size(%rsp,%nargs_q),%imm0)
2634	__(xorl %imm1_l,%imm1_l)
2635	/* We can use %ra0 as a temporary here, since the real return address */
2636	/* is on the stack */
2637	0: __(movq -node_size(%imm0),%ra0)
2638	__(movq %ra0,-node_size(%rbp,%imm1))
2639	__(subq $node_size,%imm0)
2640	__(subq $node_size,%imm1)
2641	__(cmpq %imm0,%rsp)
2642	__(jne 0b)
2643	__(lea (%rbp,%imm1),%rsp)
2644	__(movq 8(%rbp),%ra0)
2645	__(movq (%rbp),%rbp)
2646	__(pushq %ra0)
2647	__(jump_fname())
2648	/* All args in regs; exactly the same as the tcallsymvsp case */
2649	9:
2650	__(leave)
2651	__(jump_fname())
2652	_endsubp(tcallsymgen)
2653
2654	_spentry(tcallsymslide)
2655	__(lea -nargregs*node_size(%rsp,%nargs_q),%imm0)
2656	__(xorl %imm1_l,%imm1_l)
2657	/* We can use %ra0 as a temporary here, since the real return address */
2658	/* is on the stack */
2659	0: __(movq -node_size(%imm0),%ra0)
2660	__(movq %ra0,-node_size(%rbp,%imm1))
2661	__(subq $node_size,%imm0)
2662	__(subq $node_size,%imm1)
2663	__(cmpq %imm0,%rsp)
2664	__(jne 0b)
2665	__(lea (%rbp,%imm1),%rsp)
2666	__(movq 8(%rbp),%ra0)
2667	__(movq 0(%rbp),%rbp)
2668	__(pushq %ra0)
2669	__(jump_fname())
2670	_endsubp(tcallsymslide)
2671
2672	_spentry(tcallsymvsp)
2673	__(leave)
2674	__(jump_fname())
2675	_endsubp(tcallsymvsp)
2676
2677	_spentry(tcallnfngen)
2678	__(cmpl $nargregs*node_size,%nargs)
2679	__(jbe 9f)
2680	__(lea -nargregs*node_size(%rsp,%nargs_q),%imm0)
2681	__(xorl %imm1_l,%imm1_l)
2682	/* We can use %ra0 as a temporary here, since the real return address */
2683	/* is on the stack */
2684	0: __(movq -node_size(%imm0),%ra0)
2685	__(movq %ra0,-node_size(%rbp,%imm1))
2686	__(subq $node_size,%imm0)
2687	__(subq $node_size,%imm1)
2688	__(cmpq %imm0,%rsp)
2689	__(jne 0b)
2690	__(movq %temp0,%fn)
2691	__(lea (%rbp,%imm1),%rsp)
2692	__(movq lisp_frame.savera0(%rbp),%ra0)
2693	__(movq lisp_frame.backlink(%rbp),%rbp)
2694	__(pushq %ra0)
2695	__(jmp *%fn)
2696	/* All args in regs; exactly the same as the tcallnfnvsp case */
2697	9:
2698	__(movq %temp0,%fn)
2699	__(leave)
2700	__(jmp *%fn)
2701	_endsubp(tcallnfngen)
2702
2703	_spentry(tcallnfnslide)
2704	__(lea -nargregs*node_size(%rsp,%nargs_q),%imm0)
2705	__(xorl %imm1_l,%imm1_l)
2706	/* We can use %ra0 as a temporary here, since the real return address */
2707	/* is on the stack */
2708	0: __(movq -node_size(%imm0),%ra0)
2709	__(movq %ra0,-node_size(%rbp,%imm1))
2710	__(subq $node_size,%imm0)
2711	__(subq $node_size,%imm1)
2712	__(cmpq %imm0,%rsp)
2713	__(jne 0b)
2714	__(movq %temp0,%fn)
2715	__(lea (%rbp,%imm1),%rsp)
2716	__(movq lisp_frame.savera0(%rbp),%ra0)
2717	__(movq lisp_frame.backlink(%rbp),%rbp)
2718	__(pushq %ra0)
2719	__(jmp *%fn)
2720	_endsubp(tcallnfnslide)
2721
2722	_spentry(tcallnfnvsp)
2723	__(movq %temp0,%fn)
2724	__(leave)
2725	__(jmp *%fn)
2726	_endsubp(tcallnfnvsp)
2727
2728
2729	/* Make a "raw" area on the foreign stack, stack-cons a macptr to point to it, */
2730	/* and return the macptr. Size (in bytes, boxed) is in arg_z on entry; macptr */
2731	/* in arg_z on exit. */
2732	_spentry(makestackblock)
2733	__(unbox_fixnum(%arg_z,%imm0))
2734	__(dnode_align(%imm0,tsp_frame.fixed_overhead+macptr.size,%imm0))
2735	__(cmpq $tstack_alloc_limit,%imm0)
2736	__(jae 1f)
2737	__ifdef(`WINDOWS')
2738	__(windows_cstack_probe(%imm0,%arg_z))
2739	__endif
2740	__(movq rcontext(tcr.foreign_sp),%imm1)
2741	__(subq %imm0,rcontext(tcr.foreign_sp))
2742	__(movq rcontext(tcr.foreign_sp),%arg_z)
2743	__(movq %imm1,(%arg_z))
2744	__(movq %rbp,csp_frame.save_rbp(%arg_z))
2745	__(lea macptr.size+tsp_frame.fixed_overhead(%arg_z),%imm0)
2746	__(movq $macptr_header,tsp_frame.fixed_overhead(%arg_z))
2747	__(addq $fulltag_misc+tsp_frame.fixed_overhead,%arg_z)
2748	__(movq %imm0,macptr.address(%arg_z))
2749	__(movsd %fpzero,macptr.domain(%arg_z))
2750	__(movsd %fpzero,macptr.type(%arg_z))
2751	__(ret)
2752	1: __(movq rcontext(tcr.foreign_sp),%imm1)
2753	__(subq $dnode_size,rcontext(tcr.foreign_sp))
2754	__(movq rcontext(tcr.foreign_sp),%imm0)
2755	__(movq %imm1,(%imm0))
2756	__(movq %rbp,csp_frame.save_rbp(%imm0))
2757	__(set_nargs(1))
2758	__(movq $nrs.new_gcable_ptr,%fname)
2759	__(jump_fname())
2760	_endsubp(makestackblock)
2761
2762	_spentry(makestackblock0)
2763	__(unbox_fixnum(%arg_z,%imm0))
2764	__(dnode_align(%imm0,tsp_frame.fixed_overhead+macptr.size,%imm0))
2765	__(cmpq $tstack_alloc_limit,%imm0)
2766	__(jae 9f)
2767	__ifdef(`WINDOWS')
2768	__(windows_cstack_probe(%imm0,%arg_z))
2769	__endif
2770	__(movq rcontext(tcr.foreign_sp),%imm1)
2771	__(subq %imm0,rcontext(tcr.foreign_sp))
2772	__(movq rcontext(tcr.foreign_sp),%arg_z)
2773	__(movq %imm1,(%arg_z))
2774	__(movq %rbp,csp_frame.save_rbp(%arg_z))
2775	__(lea macptr.size+tsp_frame.fixed_overhead(%arg_z),%imm0)
2776	__(movq $macptr_header,tsp_frame.fixed_overhead(%arg_z))
2777	__(addq $fulltag_misc+tsp_frame.fixed_overhead,%arg_z)
2778	__(movq %imm0,macptr.address(%arg_z))
2779	__(movsd %fpzero,macptr.domain(%arg_z))
2780	__(movsd %fpzero,macptr.type(%arg_z))
2781	__(jmp 2f)
2782	1: __(movapd %fpzero,(%imm0))
2783	__(addq $dnode_size,%imm0)
2784	2: __(cmpq %imm0,%imm1)
2785	__(jne 1b)
2786	__(repret)
2787	9: __(movq rcontext(tcr.foreign_sp),%imm1)
2788	__(subq $dnode_size,rcontext(tcr.foreign_sp))
2789	__(movq rcontext(tcr.foreign_sp),%imm0)
2790	__(movq %imm1,(%imm0))
2791	__(movq %rbp,csp_frame.save_rbp(%imm0))
2792	__(set_nargs(1))
2793	__(movq $nrs.new_gcable_ptr,%fname)
2794	__(jump_fname())
2795	_endsubp(makestackblock0)
2796
2797	_spentry(makestacklist)
2798	__(movq $((1<<63)\|fixnummask),%imm0)
2799	__(testq %imm0,%arg_y)
2800	__(jne 9f)
2801	__(movq %arg_y,%imm0)
2802	__(addq %imm0,%imm0)
2803	__(rcmpq(%imm0,$tstack_alloc_limit))
2804	__(movl $nil_value,%temp1_l)
2805	__(jae 2f)
2806	__(addq $tsp_frame.fixed_overhead,%imm0)
2807	__(TSP_Alloc_Var(%imm0,%temp0))
2808	__(addq $fulltag_cons,%temp0)
2809	__(jmp 1f)
2810	0: __(_rplaca(%temp0,%arg_z))
2811	__(_rplacd(%temp0,%temp1))
2812	__(movq %temp0,%temp1)
2813	__(addq $cons.size,%temp0)
2814	1: __(subq $fixnumone,%arg_y)
2815	__(jge 0b)
2816	__(movq %temp1,%arg_z)
2817	__(ret)
2818	2: __(TSP_Alloc_Fixed(0,%imm0))
2819	__(jmp 4f)
2820	3: __(Cons(%arg_z,%temp1,%temp1))
2821	4: __(subq $fixnumone,%arg_y)
2822	__(jge 3b)
2823	__(movq %temp1,%arg_z)
2824	__(ret)
2825	9: __(uuo_error_reg_not_type(Rarg_y,error_object_not_unsigned_byte))
2826	_endsubp(makestacklist)
2827
2828	/* subtype (boxed) vpushed before initial values. (Had better be a */
2829	/* node header subtag.) Nargs set to count of things vpushed. */
2830	_spentry(stkgvector)
2831	__(lea -fixnum_one(%nargs_q),%imm0)
2832	__(lea (%rsp,%imm0),%arg_x)
2833	__(movq %imm0,%arg_y)
2834	__(shlq $num_subtag_bits-fixnumshift,%imm0)
2835	__(movq (%arg_x), %imm1)
2836	__(shrq $fixnumshift,%imm1)
2837	__(orq %imm1,%imm0) /* imm0 = header, %arg_y = unaligned size */
2838	__(dnode_align(%arg_y,(tsp_frame.fixed_overhead+node_size),%imm1))
2839	__(TSP_Alloc_Var(%imm1,%arg_z))
2840	__(movq %imm0,(%arg_z))
2841	__(addq $fulltag_misc,%arg_z)
2842	__(lea -node_size(%nargs_q),%imm0)
2843	__(jmp 2f)
2844	1: __(pop misc_data_offset(%arg_z,%imm0))
2845	2: __(subq $node_size,%imm0)
2846	__(jge 1b)
2847	__(addq $node_size,%rsp)
2848	__(jmp *%ra0)
2849	_endsubp(stkgvector)
2850
2851	_spentry(misc_alloc)
2852	__(movq $~(((1<<56)-1)<<fixnumshift),%imm0)
2853	__(testq %imm0,%arg_y)
2854	__(jne local_label(misc_alloc_not_u56))
2855	__(unbox_fixnum(%arg_z,%imm0))
2856	__(movq %arg_y,%temp0)
2857	__(shl $num_subtag_bits-fixnumshift,%temp0)
2858	__(orq %temp0,%imm0) /* %imm0 now = header */
2859	__(movb $fulltagmask,%imm1_b)
2860	__(andb %imm0_b,%imm1_b)
2861	__(cmpb $fulltag_nodeheader_0,%imm1_b)
2862	__(je local_label(misc_alloc_64))
2863	__(cmpb $fulltag_nodeheader_1,%imm1_b)
2864	__(je local_label(misc_alloc_64))
2865	__(cmpb $ivector_class_64_bit,%imm1_b)
2866	__(jz local_label(misc_alloc_64))
2867	__(cmpb $ivector_class_32_bit,%imm1_b)
2868	__(jz local_label(misc_alloc_32))
2869	__(unbox_fixnum(%arg_y,%imm1))
2870	/* ivector_class_other_bit: 16, 8, or 1 ... */
2871	__(cmpb $subtag_bit_vector,%imm0_b)
2872	__(jne local_label(misc_alloc_8))
2873	__(addq $7,%imm1)
2874	__(shrq $3,%imm1)
2875	__(jmp local_label(misc_alloc_alloc_vector))
2876	local_label(misc_alloc_8):
2877	__(cmpb $subtag_simple_base_string,%imm0_b)
2878	__(jae local_label(misc_alloc_alloc_vector))
2879	local_label(misc_alloc_16):
2880	__(shlq %imm1)
2881	__(jmp local_label(misc_alloc_alloc_vector))
2882	local_label(misc_alloc_32):
2883	/* 32-bit ivector */
2884	__(unbox_fixnum(%arg_y,%imm1))
2885	__(shlq $2,%imm1)
2886	__(jmp local_label(misc_alloc_alloc_vector))
2887	local_label(misc_alloc_64):
2888	/* 64-bit ivector or gvector */
2889	__(movq %arg_y,%imm1)
2890	local_label(misc_alloc_alloc_vector):
2891	__(dnode_align(%imm1,node_size,%imm1))
2892	__(ref_global(tenured_area,%arg_x))
2893	__(cmpq area.low(%arg_x),%imm1)
2894	__(ja local_label(misc_alloc_large))
2895	__(Misc_Alloc(%arg_z))
2896	__(ret)
2897	local_label(misc_alloc_not_u56):
2898	__(uuo_error_reg_not_type(Rarg_y,error_object_not_unsigned_byte_56))
2899	local_label(misc_alloc_large):
2900	/* If we tried to subtract %imm1 from tcr.allocptr, it
2901	might become negative ; we treat addresses as being unsigned,
2902	so that negative value would look like a very large unsigned
2903	value and we'd think that the allocation succeeded.
2904	If we reach this point, we're trying to allocate something
2905	very large indeed. Depending on the platform, that's anywhere
2906	from hundreds of GB to hundreds of TB. Someday, it might be
2907	worth trying that (using a special "large allocation" UUO);
2908	for now, it's probably safe to just report that a memory
2909	allocation attempt failed.
2910	*/
2911	__(movq $XMEMFULL,%arg_z)
2912	__(set_nargs(1))
2913	__(jmp _SPksignalerr)
2914	_endsubp(misc_alloc)
2915
2916
2917	_startfn(C(destbind1))
2918	/* Save entry %rsp in case of error */
2919	__(movd %rsp,%mm0)
2920	/* Extract required arg count. */
2921	__(movzbl %nargs_b,%imm0_l)
2922	__(testl %imm0_l,%imm0_l)
2923	__(je local_label(opt)) /* skip if no required args */
2924	local_label(req_loop):
2925	__(compare_reg_to_nil(%arg_reg))
2926	__(je local_label(toofew))
2927	__(extract_lisptag(%arg_reg,%imm1))
2928	__(cmpb $tag_list,%imm1_b)
2929	__(jne local_label(badlist))
2930	__(subb $1,%imm0_b)
2931	__(pushq cons.car(%arg_reg))
2932	__(_cdr(%arg_reg,%arg_reg))
2933	__(jne local_label(req_loop))
2934	local_label(opt):
2935	__(movw %nargs_w,%imm0_w)
2936	__(shrw $8,%imm0_w)
2937	__(je local_label(rest_keys))
2938	__(btl $initopt_bit,%nargs)
2939	__(jc local_label(opt_supp))
2940	/* 'simple' &optionals: no supplied-p, default to nil. */
2941	local_label(simple_opt_loop):
2942	__(compare_reg_to_nil(%arg_reg))
2943	__(je local_label(default_simple_opt))
2944	__(extract_lisptag(%arg_reg,%imm1))
2945	__(cmpb $tag_list,%imm1_b)
2946	__(jne local_label(badlist))
2947	__(subb $1,%imm0_b)
2948	__(pushq cons.car(%arg_reg))
2949	__(_cdr(%arg_reg,%arg_reg))
2950	__(jne local_label(simple_opt_loop))
2951	__(jmp local_label(rest_keys))
2952	local_label(default_simple_opt):
2953	__(subb $1,%imm0_b)
2954	__(pushq $nil_value)
2955	__(jne local_label(default_simple_opt))
2956	__(jmp local_label(rest_keys))
2957	local_label(opt_supp):
2958	__(extract_lisptag(%arg_reg,%imm1))
2959	__(compare_reg_to_nil(%arg_reg))
2960	__(je local_label(default_hard_opt))
2961	__(cmpb $tag_list,%imm1_b)
2962	__(jne local_label(badlist))
2963	__(subb $1,%imm0_b)
2964	__(pushq cons.car(%arg_reg))
2965	__(_cdr(%arg_reg,%arg_reg))
2966	__(push $t_value)
2967	__(jne local_label(opt_supp))
2968	__(jmp local_label(rest_keys))
2969	local_label(default_hard_opt):
2970	__(subb $1,%imm0_b)
2971	__(push $nil_value)
2972	__(push $nil_value)
2973	__(jne local_label(default_hard_opt))
2974	local_label(rest_keys):
2975	__(btl $restp_bit,%nargs)
2976	__(jc local_label(have_rest))
2977	__(btl $keyp_bit,%nargs)
2978	__(jc local_label(have_keys))
2979	__(compare_reg_to_nil(%arg_reg))
2980	__(jne local_label(toomany))
2981	__(jmp *%ra0)
2982	local_label(have_rest):
2983	__(pushq %arg_reg)
2984	__(btl $keyp_bit,%nargs)
2985	__(jc local_label(have_keys))
2986	__(jmp *%ra0)
2987	/* Ensure that arg_reg contains a proper,even-length list. */
2988	/* Insist that its length is <= 512 (as a cheap circularity check.) */
2989	local_label(have_keys):
2990	__(movw $256,%imm0_w)
2991	__(movq %arg_reg,%arg_y)
2992	local_label(count_keys_loop):
2993	__(compare_reg_to_nil(%arg_y))
2994	__(je local_label(counted_keys))
2995	__(subw $1,%imm0_w)
2996	__(jl local_label(toomany))
2997	__(extract_lisptag(%arg_y,%imm1))
2998	__(cmpb $tag_list,%imm1_b)
2999	__(jne local_label(badlist))
3000	__(_cdr(%arg_y,%arg_y))
3001	__(extract_fulltag(%arg_y,%imm1))
3002	__(cmpb $fulltag_cons,%imm1_b)
3003	__(jne local_label(badlist))
3004	__(_cdr(%arg_y,%arg_y))
3005	__(jmp local_label(count_keys_loop))
3006	local_label(counted_keys):
3007	/* We've got a proper, even-length list of key/value pairs in */
3008	/* arg_reg. For each keyword var in the lambda-list, push a pair */
3009	/* of NILs on the vstack. */
3010
3011	__(movl %nargs,%imm1_l)
3012	__(shrl $16,%imm1_l)
3013	__(movzbl %imm1_b,%imm0_l)
3014	__(movq %rsp,%arg_y)
3015	__(jmp local_label(push_pair_test))
3016	local_label(push_pair_loop):
3017	__(push $nil_value)
3018	__(push $nil_value)
3019	local_label(push_pair_test):
3020	__(subb $1,%imm1_b)
3021	__(jge local_label(push_pair_loop))
3022	/* Push the %saveN registers, so that we can use them in this loop */
3023	/* Also, borrow %arg_z */
3024	__(push %save0)
3025	__(push %save1)
3026	__(push %save2)
3027	__(push %arg_z)
3028	/* save0 points to the 0th value/supplied-p pair */
3029	__(movq %arg_y,%save0)
3030	/* save1 is the length of the keyword vector */
3031	__(vector_length(%arg_x,%save1))
3032	/* save2 is the current keyword */
3033	/* arg_z is the value of the current keyword */
3034	__(xorl %imm0_l,%imm0_l) /* count unknown keywords seen */
3035	local_label(match_keys_loop):
3036	__(compare_reg_to_nil(%arg_reg))
3037	__(je local_label(matched_keys))
3038	__(_car(%arg_reg,%save2))
3039	__(_cdr(%arg_reg,%arg_reg))
3040	__(_car(%arg_reg,%arg_z))
3041	__(_cdr(%arg_reg,%arg_reg))
3042	__(xorl %arg_y_l,%arg_y_l)
3043	__(jmp local_label(match_test))
3044	local_label(match_loop):
3045	__(cmpq misc_data_offset(%arg_x,%arg_y),%save2)
3046	__(je local_label(matched))
3047	__(addq $node_size,%arg_y)
3048	local_label(match_test):
3049	__(cmpq %arg_y,%save1)
3050	__(jne local_label(match_loop))
3051	/* No match. Note unknown keyword, check for :allow-other-keys */
3052	__(addl $1,%imm0_l)
3053	__(cmpq $nrs.kallowotherkeys,%save2)
3054	__(jne local_label(match_keys_loop))
3055	__(subl $1,%imm0_l)
3056	__(btsl $seen_aok_bit,%nargs)
3057	__(jc local_label(match_keys_loop))
3058	/* First time we've seen :allow-other-keys. Maybe set aok_bit. */
3059	__(compare_reg_to_nil(%arg_z))
3060	__(je local_label(match_keys_loop))
3061	__(btsl $aok_bit,%nargs)
3062	__(jmp local_label(match_keys_loop))
3063	/* Got a match. Worry about :allow-other-keys here, too. */
3064	local_label(matched):
3065	__(negq %arg_y)
3066	__(cmpb $fulltag_nil,-node_size*2(%save0,%arg_y,2))
3067	__(jne local_label(match_keys_loop))
3068	__(movq %arg_z,-node_size(%save0,%arg_y,2))
3069	__(movl $t_value,-node_size*2(%save0,%arg_y,2))
3070	__(cmpq $nrs.kallowotherkeys,%save2)
3071	__(jne local_label(match_keys_loop))
3072	__(btsl $seen_aok_bit,%nargs)
3073	__(jnc local_label(match_keys_loop))
3074	__(compare_reg_to_nil(%arg_z))
3075	__(je local_label(match_keys_loop))
3076	__(btsl $aok_bit,%nargs)
3077	__(jmp local_label(match_keys_loop))
3078	local_label(matched_keys):
3079	__(pop %arg_z)
3080	__(pop %save2)
3081	__(pop %save1)
3082	__(pop %save0)
3083	__(testl %imm0_l,%imm0_l)
3084	__(je local_label(keys_ok))
3085	__(btl $aok_bit,%nargs)
3086	__(jnc local_label(badkeys))
3087	local_label(keys_ok):
3088	__(jmp *%ra0)
3089	/* Some unrecognized keywords. Complain generically about */
3090	/* invalid keywords. */
3091	local_label(badkeys):
3092	__(movq $XBADKEYS,%arg_y)
3093	__(jmp local_label(destructure_error))
3094	local_label(toomany):
3095	__(movq $XCALLTOOMANY,%arg_y)
3096	__(jmp local_label(destructure_error))
3097	local_label(toofew):
3098	__(movq $XCALLTOOFEW,%arg_y)
3099	__(jmp local_label(destructure_error))
3100	local_label(badlist):
3101	__(movq $XCALLNOMATCH,%arg_y)
3102	/* jmp local_label(destructure_error) */
3103	local_label(destructure_error):
3104	__(movd %mm0,%rsp) /* undo everything done to the stack */
3105	__(movq %whole_reg,%arg_z)
3106	__(set_nargs(2))
3107	__(push %ra0)
3108	__(jmp _SPksignalerr)
3109	_endfn(C(destbind1))
3110
3111	_spentry(macro_bind)
3112	__(movq %arg_reg,%whole_reg)
3113	__(extract_lisptag(%arg_reg,%imm0))
3114	__(cmpb $tag_list,%imm0_b)
3115	__(jne 1f)
3116	__(_cdr(%arg_reg,%arg_reg))
3117	__(jmp C(destbind1))
3118	1: __(movq $XCALLNOMATCH,%arg_y)
3119	__(movq %whole_reg,%arg_z)
3120	__(set_nargs(2))
3121	__(push %ra0)
3122	__(jmp _SPksignalerr)
3123	_endsubp(macro_bind)
3124
3125	_spentry(destructuring_bind)
3126	__(movq %arg_reg,%whole_reg)
3127	__(jmp C(destbind1))
3128	_endsubp(destructuring_bind)
3129
3130	_spentry(destructuring_bind_inner)
3131	__(movq %arg_z,%whole_reg)
3132	__(jmp C(destbind1))
3133	_endsubp(destructuring_bind_inner)
3134
3135
3136
3137
3138	_spentry(vpopargregs)
3139	_endsubp(vpopargregs)
3140
3141	/* If arg_z is an integer, return in imm0 something whose sign */
3142	/* is the same as arg_z's. If not an integer, error. */
3143	_spentry(integer_sign)
3144	__(testb $tagmask,%arg_z_b)
3145	__(movq %arg_z,%imm0)
3146	__(je 8f)
3147	__(extract_typecode(%arg_z,%imm0))
3148	__(cmpb $subtag_bignum,%imm0_b)
3149	__(jne 9f)
3150	__(getvheader(%arg_z,%imm0))
3151	__(shr $num_subtag_bits,%imm0)
3152	__(movslq misc_data_offset-4(%arg_z,%imm0,4),%imm0)
3153	8: __(repret)
3154	9: __(uuo_error_reg_not_type(Rarg_z,error_object_not_integer))
3155	_endsubp(integer_sign)
3156
3157	/* "slide" nargs worth of values up the stack. IMM0 contains */
3158	/* the difference between the current RSP and the target. */
3159	_spentry(mvslide)
3160	__(movl %nargs,%imm1_l)
3161	__(lea (%rsp,%nargs_q),%temp0)
3162	__(testq %imm1,%imm1)
3163	__(lea (%temp0,%imm0),%imm0)
3164	__(je 2f)
3165	1:
3166	__(subq $node_size,%temp0)
3167	__(movq (%temp0),%temp1)
3168	__(subq $node_size,%imm0)
3169	__(movq %temp1,(%imm0))
3170	__(subq $node_size,%imm1)
3171	__(jne 1b)
3172	2: __(movq %imm0,%rsp)
3173	__(jmp *%ra0)
3174	_endsubp(mvslide)
3175
3176	_spentry(save_values)
3177	__(movq rcontext(tcr.save_tsp),%imm1)
3178	/* common exit: nargs = values in this set, imm1 = ptr to tsp before call to save_values */
3179	local_label(save_values_to_tsp):
3180	__(movq rcontext(tcr.save_tsp),%arg_x)
3181	__(dnode_align(%nargs_q,tsp_frame.fixed_overhead+(2node_size),%imm0)) / count, link */
3182	__(TSP_Alloc_Var(%imm0,%arg_z))
3183	__(movq rcontext(tcr.save_tsp),%imm0)
3184	__(movq %imm1,(%imm0))
3185	__(movq %nargs_q,(%arg_z))
3186	__(movq %arg_x,node_size(%arg_z))
3187	__(leaq 2*node_size(%arg_z,%nargs_q),%arg_y)
3188	__(leaq (%rsp,%nargs_q),%imm0)
3189	__(cmpq %imm0,%rsp)
3190	__(jmp 2f)
3191	1: __(subq $node_size,%imm0)
3192	__(movq (%imm0),%arg_z)
3193	__(subq $node_size,%arg_y)
3194	__(cmpq %imm0,%rsp)
3195	__(movq %arg_z,(%arg_y))
3196	2: __(jne 1b)
3197	__(add %nargs_q,%rsp)
3198	__(jmp *%ra0)
3199	_endsubp(save_values)
3200
3201	/* Add the multiple values that are on top of the vstack to the set */
3202	/* saved in the top tsp frame, popping them off of the vstack in the */
3203	/* process. It is an error (a bad one) if the TSP contains something */
3204	/* other than a previously saved set of multiple-values. */
3205	/* Since adding to the TSP may cause a new TSP segment to be allocated, */
3206	/* each add_values call adds another linked element to the list of */
3207	/* values. This makes recover_values harder. */
3208	_spentry(add_values)
3209	__(testl %nargs,%nargs)
3210	__(movq rcontext(tcr.save_tsp),%imm1)
3211	__(movq (%imm1),%imm1)
3212	__(jne local_label(save_values_to_tsp))
3213	__(jmp *%ra0)
3214	_endsubp(add_values)
3215
3216	/* push the values in the value set atop the sp, incrementing nargs. */
3217	/* Discard the tsp frame; leave values atop the sp. */
3218
3219	_spentry(recover_values)
3220	/* First, walk the segments reversing the pointer to previous */
3221	/* segment pointers Can tell the end because that previous */
3222	/* segment pointer is the prev tsp pointer */
3223	__(movq rcontext(tcr.save_tsp),%temp1)
3224	__(movq %temp1,%arg_x) /* current segment */
3225	__(movq %temp1,%arg_y) /* last segment */
3226	__(movq tsp_frame.backlink(%temp1),%arg_z) /* previous tsp */
3227	local_label(walkloop):
3228	__(movq tsp_frame.fixed_overhead+node_size(%arg_x),%temp0)
3229	__(cmpq %temp0,%arg_z) /* last segment ? */
3230	__(movq %arg_y,tsp_frame.fixed_overhead+node_size(%arg_x))
3231	__(movq %arg_x,%arg_y) /* last segment <- current segment */
3232	__(movq %temp0,%arg_x) /* current segment <- next segment */
3233	__(jne local_label(walkloop))
3234
3235	/* the final segment pointer is now in %arg_y */
3236	/* walk backwards, pushing values on the stack and incrementing %nargs */
3237	local_label(pushloop):
3238	__(movq tsp_frame.data_offset(%arg_y),%imm0) /* nargs in segment */
3239	__(testq %imm0,%imm0)
3240	__(leaq tsp_frame.data_offset+(2*node_size)(%arg_y,%imm0),%temp0)
3241	__(leaq (%nargs_q,%imm0),%nargs_q)
3242	__(jmp 2f)
3243	1: __(pushq -node_size(%temp0))
3244	__(subq $node_size,%temp0)
3245	__(subq $fixnum_one,%imm0)
3246	2: __(jne 1b)
3247	__(cmpq %arg_y,%temp1)
3248	__(movq tsp_frame.data_offset+node_size(%arg_y),%arg_y)
3249	__(jne local_label(pushloop))
3250	__(movq (%temp1),%temp1)
3251	__(movq %temp1,rcontext(tcr.save_tsp))
3252	__(movq %temp1,rcontext(tcr.next_tsp))
3253	__(jmp *%ra0)
3254	_endsubp(recover_values)
3255
3256	/* Exactly like recover_values, but it's necessary to reserve an outgoing */
3257	/* frame if any values (which will be used as outgoing arguments) will */
3258	/* wind up on the stack. We can assume that %nargs contains 0 (and */
3259	/* that no other arguments have been pushed) on entry. */
3260
3261	_spentry(recover_values_for_mvcall)
3262	/* First, walk the segments reversing the pointer to previous */
3263	/* segment pointers Can tell the end because that previous */
3264	/* segment pointer is the prev tsp pointer */
3265	__(xorl %nargs,%nargs)
3266	__(movq rcontext(tcr.save_tsp),%temp1)
3267	__(movq %temp1,%arg_x) /* current segment */
3268	__(movq %temp1,%arg_y) /* last segment */
3269	__(movq tsp_frame.backlink(%temp1),%arg_z) /* previous tsp */
3270	local_label(walkloop_mvcall):
3271	__(movq tsp_frame.fixed_overhead+node_size(%arg_x),%temp0)
3272	__(addq tsp_frame.data_offset(%arg_x),%nargs_q)
3273	__(cmpq %temp0,%arg_z) /* last segment ? */
3274	__(movq %arg_y,tsp_frame.fixed_overhead+node_size(%arg_x))
3275	__(movq %arg_x,%arg_y) /* last segment <- current segment */
3276	__(movq %temp0,%arg_x) /* current segment <- next segment */
3277	__(jne local_label(walkloop_mvcall))
3278
3279	__(cmpl $nargregs*node_size,%nargs)
3280	__(jbe local_label(pushloop_mvcall))
3281	__(push $reserved_frame_marker)
3282	__(push $reserved_frame_marker)
3283
3284	/* the final segment pointer is now in %arg_y */
3285	/* walk backwards, pushing values on the stack and incrementing %nargs */
3286	local_label(pushloop_mvcall):
3287	__(movq tsp_frame.data_offset(%arg_y),%imm0) /* nargs in segment */
3288	__(testq %imm0,%imm0)
3289	__(leaq tsp_frame.data_offset+(2*node_size)(%arg_y,%imm0),%temp0)
3290	__(jmp 2f)
3291	1: __(pushq -node_size(%temp0))
3292	__(subq $node_size,%temp0)
3293	__(subq $fixnum_one,%imm0)
3294	2: __(jne 1b)
3295	__(cmpq %arg_y,%temp1)
3296	__(movq tsp_frame.data_offset+node_size(%arg_y),%arg_y)
3297	__(jne local_label(pushloop_mvcall))
3298	__(movq (%temp1),%temp1)
3299	__(movq %temp1,rcontext(tcr.save_tsp))
3300	__(movq %temp1,rcontext(tcr.next_tsp))
3301	__(jmp *%ra0)
3302	_endsubp(recover_values_for_mvcall)
3303
3304	_spentry(reset)
3305	__(hlt)
3306	_endsubp(reset)
3307
3308
3309
3310	_spentry(misc_alloc_init)
3311	__(push %rbp)
3312	__(movq %rsp,%rbp)
3313	__(push %arg_z)
3314	__(movq %arg_y,%arg_z)
3315	__(movq %arg_x,%arg_y)
3316	__(lea local_label(misc_alloc_init_back)(%rip),%ra0)
3317	__(push %ra0)
3318	__(jmp _SPmisc_alloc)
3319	__(tra(local_label(misc_alloc_init_back)))
3320	__(pop %arg_y)
3321	__(leave)
3322	__(movq $nrs.init_misc,%fname)
3323	__(set_nargs(2))
3324	__(jump_fname())
3325	_endsubp(misc_alloc_init)
3326
3327	_spentry(stack_misc_alloc_init)
3328	__(push %rbp)
3329	__(movq %rsp,%rbp)
3330	__(push %arg_z)
3331	__(movq %arg_y,%arg_z)
3332	__(movq %arg_x,%arg_y)
3333	__(lea local_label(stack_misc_alloc_init_back)(%rip),%ra0)
3334	__(push %ra0)
3335	__(jmp _SPstack_misc_alloc)
3336	__(tra(local_label(stack_misc_alloc_init_back)))
3337	__(pop %arg_y)
3338	__(leave)
3339	__(movq $nrs.init_misc,%fname)
3340	__(set_nargs(2))
3341	__(jump_fname())
3342	_endsubp(stack_misc_alloc_init)
3343
3344
3345
3346	.globl C(popj)
3347	_spentry(popj)
3348	C(popj):
3349	__(leave)
3350	__(ret)
3351	_endsubp(popj)
3352
3353
3354
3355	_spentry(getu64)
3356	__(movq $~(target_most_positive_fixnum << fixnumshift),%imm0)
3357	__(testq %arg_z,%imm0)
3358	__(movq %arg_z,%imm0)
3359	__(jne 1f)
3360	__(sarq $fixnumshift,%imm0)
3361	__(ret)
3362	1: __(andb $tagmask,%imm0_b)
3363	__(cmpb $tag_misc,%imm0_b)
3364	__(jne 9f)
3365	__(movb misc_subtag_offset(%arg_z),%imm0_b)
3366	__(cmpb $subtag_bignum,%imm0_b)
3367	__(jne 9f)
3368	__(movq misc_header_offset(%arg_z),%imm0)
3369	__(cmpq $three_digit_bignum_header,%imm0)
3370	__(je 3f)
3371	__(cmpq $two_digit_bignum_header,%imm0)
3372	__(jne 9f)
3373	__(movq misc_data_offset(%arg_z),%imm0)
3374	__(testq %imm0,%imm0)
3375	__(js 9f)
3376	__(repret)
3377	3: __(movq misc_data_offset(%arg_z),%imm0)
3378	__(cmpl $0,misc_data_offset+8(%arg_z))
3379	__(jne 9f)
3380	__(repret)
3381	9: __(uuo_error_reg_not_type(Rarg_z,error_object_not_u64))
3382	_endsubp(getu64)
3383
3384	_spentry(gets64)
3385	__(movq %arg_z,%imm0)
3386	__(sarq $fixnumshift,%imm0)
3387	__(testb $fixnummask,%arg_z_b)
3388	__(je 8f)
3389	1: __(movb %arg_z_b,%imm0_b)
3390	__(andb $tagmask,%imm0_b)
3391	__(cmpb $tag_misc,%imm0_b)
3392	__(jne 9f)
3393	__(movb misc_subtag_offset(%arg_z),%imm0_b)
3394	__(cmpb $subtag_bignum,%imm0_b)
3395	__(jne 9f)
3396	__(movq misc_header_offset(%arg_z),%imm0)
3397	__(cmpq $two_digit_bignum_header,%imm0)
3398	__(movq misc_data_offset(%arg_z),%imm0)
3399	__(jne 9f)
3400	8: __(repret)
3401	9: __(uuo_error_reg_not_type(Rarg_z,error_object_not_s64))
3402	_endsubp(gets64)
3403
3404	_spentry(makeu64)
3405	__(movq %imm0,%imm1)
3406	__(shlq $fixnumshift+1,%imm1)
3407	__(movq %imm1,%arg_z) /* Tagged as a fixnum, 2x */
3408	__(shrq $fixnumshift+1,%imm1)
3409	__(shrq %arg_z)
3410	__(cmpq %imm0,%imm1)
3411	__(je 9f)
3412	__(testq %imm0,%imm0)
3413	__(movd %imm0,%mm0)
3414	__(js 3f)
3415	/* Make a 2-digit bignum. */
3416	__(movl $two_digit_bignum_header,%imm0_l)
3417	__(movl $aligned_bignum_size(2),%imm1_l)
3418	__(Misc_Alloc(%arg_z))
3419	__(movq %mm0,misc_data_offset(%arg_z))
3420	__(ret)
3421	3: __(movl $three_digit_bignum_header,%imm0_l)
3422	__(movl $aligned_bignum_size(3),%imm1_l)
3423	__(Misc_Alloc(%arg_z))
3424	__(movq %mm0,misc_data_offset(%arg_z))
3425	9: __(repret)
3426	_endsubp(makeu64)
3427
3428	/* on entry: arg_z = symbol. On exit, arg_z = value (possibly */
3429	/* unbound_marker), arg_y = symbol */
3430	_spentry(specref)
3431	__(movq symbol.binding_index(%arg_z),%imm0)
3432	__(xorl %imm2_l,%imm2_l)
3433	__(cmp rcontext(tcr.tlb_limit),%imm0)
3434	__(movq rcontext(tcr.tlb_pointer),%imm1)
3435	__(movq %arg_z,%arg_y)
3436	__(cmovael %imm2_l,%imm0_l)
3437	__(movq (%imm1,%imm0),%arg_z)
3438	__(cmpb $no_thread_local_binding_marker,%arg_z_b)
3439	__(cmoveq symbol.vcell(%arg_y),%arg_z)
3440	__(ret)
3441	_endsubp(specref)
3442
3443	/* arg_y = special symbol, arg_z = new value. */
3444	_spentry(specset)
3445	__(movq symbol.binding_index(%arg_y),%imm0)
3446	__(cmp rcontext(tcr.tlb_limit),%imm0)
3447	__(movq rcontext(tcr.tlb_pointer),%imm1)
3448	__(jae 1f)
3449	__(movq (%imm1,%imm0),%arg_x)
3450	__(cmpb $no_thread_local_binding_marker,%arg_x_b)
3451	__(je 1f)
3452	__(movq %arg_z,(%imm1,%imm0))
3453	__(ret)
3454	1: __(lea fulltag_misc-fulltag_symbol(%arg_y),%arg_x)
3455	__(movq $1<<fixnumshift,%arg_y)
3456	__(jmp _SPgvset)
3457	_endsubp(specset)
3458
3459	_spentry(specrefcheck)
3460	__(movq symbol.binding_index(%arg_z),%imm0)
3461	__(xorl %imm2_l,%imm2_l)
3462	__(cmp rcontext(tcr.tlb_limit),%imm0)
3463	__(movq rcontext(tcr.tlb_pointer),%imm1)
3464	__(movq %arg_z,%arg_y)
3465	__(cmovaeq %imm2,%imm0)
3466	__(movq (%imm1,%imm0),%arg_z)
3467	__(cmpb $no_thread_local_binding_marker,%arg_z_b)
3468	__(cmoveq symbol.vcell(%arg_y),%arg_z)
3469	__(cmpb $unbound_marker,%arg_z_b)
3470	__(je 9f)
3471	__(repret)
3472	9: __(uuo_error_reg_unbound(Rarg_y))
3473	_endsubp(specrefcheck)
3474
3475	_spentry(restoreintlevel)
3476	_endsubp(restoreintlevel)
3477
3478	_spentry(makes32)
3479	__(hlt)
3480	_endsubp(makes32)
3481
3482	_spentry(makeu32)
3483	__(hlt)
3484	_endsubp(makeu32)
3485
3486	_spentry(gets32)
3487	__(hlt)
3488	_endsubp(gets32)
3489
3490	_spentry(getu32)
3491	__(hlt)
3492	_endsubp(getu32)
3493
3494
3495	_spentry(mvpasssym)
3496	_endsubp(mvpasssym)
3497
3498
3499	_spentry(unbind)
3500	__(movq rcontext(tcr.db_link),%imm1)
3501	__(movq rcontext(tcr.tlb_pointer),%arg_x)
3502	__(movq binding.sym(%imm1),%temp1)
3503	__(movq binding.val(%imm1),%arg_y)
3504	__(movq binding.link(%imm1),%imm1)
3505	__(movq %arg_y,(%arg_x,%temp1))
3506	__(movq %imm1,rcontext(tcr.db_link))
3507	__(ret)
3508	_endsubp(unbind)
3509
3510	_spentry(unbind_n)
3511	__(movq rcontext(tcr.db_link),%imm1)
3512	__(movq rcontext(tcr.tlb_pointer),%arg_x)
3513	1:
3514	__(movq binding.sym(%imm1),%temp1)
3515	__(movq binding.val(%imm1),%arg_y)
3516	__(movq binding.link(%imm1),%imm1)
3517	__(movq %arg_y,(%arg_x,%temp1))
3518	__(subq $1,%imm0)
3519	__(jne 1b)
3520	__(movq %imm1,rcontext(tcr.db_link))
3521	__(ret)
3522	_endsubp(unbind_n)
3523
3524	_spentry(unbind_to)
3525	__(movq rcontext(tcr.db_link),%imm1)
3526	__(movq rcontext(tcr.tlb_pointer),%arg_x)
3527	1:
3528	__(movq binding.sym(%imm1),%temp1)
3529	__(movq binding.val(%imm1),%arg_y)
3530	__(movq binding.link(%imm1),%imm1)
3531	__(movq %arg_y,(%arg_x,%temp1))
3532	__(cmpq %imm1,%imm0)
3533	__(jne 1b)
3534	__(movq %imm1,rcontext(tcr.db_link))
3535	__(ret)
3536	_endsubp(unbind_to)
3537
3538
3539	/* Bind CCL::INTERRUPT-LEVEL to 0. If its value had been negative, check */
3540	/* for pending interrupts after doing so. */
3541
3542	_spentry(bind_interrupt_level_0)
3543	__(movq rcontext(tcr.tlb_pointer),%temp1)
3544	__(cmpq $0,INTERRUPT_LEVEL_BINDING_INDEX(%temp1))
3545	__(push INTERRUPT_LEVEL_BINDING_INDEX(%temp1))
3546	__(push $INTERRUPT_LEVEL_BINDING_INDEX)
3547	__(push rcontext(tcr.db_link))
3548	__(movq %rsp,rcontext(tcr.db_link))
3549	__(movq $0,INTERRUPT_LEVEL_BINDING_INDEX(%temp1))
3550	__(js 1f)
3551	0: __(jmp *%ra0)
3552	/* Interrupt level was negative; interrupt may be pending */
3553	1: __(check_pending_enabled_interrupt(2f))
3554	2: __(jmp *%ra0)
3555	_endsubp(bind_interrupt_level_0)
3556
3557
3558	/* Bind CCL::INTERRUPT-LEVEL to the fixnum -1. (This has the effect */
3559	/* of disabling interrupts.) */
3560
3561	_spentry(bind_interrupt_level_m1)
3562	__(movq rcontext(tcr.tlb_pointer),%temp1)
3563	__(push INTERRUPT_LEVEL_BINDING_INDEX(%temp1))
3564	__(push $INTERRUPT_LEVEL_BINDING_INDEX)
3565	__(push rcontext(tcr.db_link))
3566	__(movq %rsp,rcontext(tcr.db_link))
3567	__(movq $-1<<fixnumshift,INTERRUPT_LEVEL_BINDING_INDEX(%temp1))
3568	__(jmp *%ra0)
3569	_endsubp(bind_interrupt_level_m1)
3570
3571	/* Bind CCL::INTERRUPT-LEVEL to the value in arg_z. If that value's 0, */
3572	/* do what _SPbind_interrupt_level_0 does */
3573	_spentry(bind_interrupt_level)
3574	__(testq %arg_z,%arg_z)
3575	__(movq rcontext(tcr.tlb_pointer),%temp1)
3576	__(jz _SPbind_interrupt_level_0)
3577	__(push INTERRUPT_LEVEL_BINDING_INDEX(%temp1))
3578	__(push $INTERRUPT_LEVEL_BINDING_INDEX)
3579	__(push rcontext(tcr.db_link))
3580	__(movq %rsp,rcontext(tcr.db_link))
3581	__(movq %arg_z,INTERRUPT_LEVEL_BINDING_INDEX(%temp1))
3582	__(jmp *%ra0)
3583	_endsubp(bind_interrupt_level)
3584
3585	/* Unbind CCL::INTERRUPT-LEVEL. If the value changes from negative to */
3586	/* non-negative, check for pending interrupts. */
3587
3588	_spentry(unbind_interrupt_level)
3589	__(btq $TCR_FLAG_BIT_PENDING_SUSPEND,rcontext(tcr.flags))
3590	__(movq rcontext(tcr.db_link),%imm1)
3591	__(movq rcontext(tcr.tlb_pointer),%arg_x)
3592	__(movq INTERRUPT_LEVEL_BINDING_INDEX(%arg_x),%imm0)
3593	__(jc 5f)
3594	0: __(testq %imm0,%imm0)
3595	__(movq binding.val(%imm1),%temp0)
3596	__(movq binding.link(%imm1),%imm1)
3597	__(movq %temp0,INTERRUPT_LEVEL_BINDING_INDEX(%arg_x))
3598	__(movq %imm1,rcontext(tcr.db_link))
3599	__(js 3f)
3600	2: __(repret)
3601	3: __(testq %temp0,%temp0)
3602	__(js 2b)
3603	__(check_pending_enabled_interrupt(4f))
3604	4: __(repret)
3605	5: /* Missed a suspend request; force suspend now if we're restoring
3606	interrupt level to -1 or greater */
3607	__(cmpq $-2<<fixnumshift,%imm0)
3608	__(jne 0b)
3609	__(movq binding.val(%imm1),%temp0)
3610	__(cmpq %imm0,%temp0)
3611	__(je 0b)
3612	__(movq $-1<<fixnumshift,INTERRUPT_LEVEL_BINDING_INDEX(%arg_x))
3613	__(suspend_now())
3614	__(jmp 0b)
3615	_endsubp(unbind_interrupt_level)
3616
3617
3618	_spentry(progvrestore)
3619	__(movq rcontext(tcr.save_tsp),%imm0)
3620	__(movq tsp_frame.backlink(%imm0),%imm0) /* ignore .SPnthrowXXX values frame */
3621	__(movq tsp_frame.data_offset(%imm0),%imm0)
3622	__(shrq $fixnumshift,%imm0)
3623	__(jne _SPunbind_n)
3624	__(repret)
3625	_endsubp(progvrestore)
3626
3627
3628	/* %arg_z <- %arg_y + %arg_z. Do the fixnum case - including overflow - */
3629	/* inline. Call out otherwise. */
3630	_spentry(builtin_plus)
3631	__(movb %arg_z_b,%imm0_b)
3632	__(orb %arg_y_b,%imm0_b)
3633	__(testb $fixnummask,%imm0_b)
3634	__(jne 1f)
3635	__(addq %arg_y,%arg_z)
3636	__(jo C(fix_one_bit_overflow))
3637	__(repret)
3638	1: __(jump_builtin(_builtin_plus,2))
3639	_endsubp(builtin_plus)
3640
3641
3642	/* %arg_z <- %arg_z - %arg_y. Do the fixnum case - including overflow - */
3643	/* inline. Call out otherwise. */
3644	_spentry(builtin_minus)
3645	__(movb %arg_z_b,%imm0_b)
3646	__(orb %arg_y_b,%imm0_b)
3647	__(testb $fixnummask,%imm0_b)
3648	__(jne 1f)
3649	__(xchgq %arg_y,%arg_z)
3650	__(subq %arg_y,%arg_z)
3651	__(jo C(fix_one_bit_overflow))
3652	__(repret)
3653	1: __(jump_builtin(_builtin_minus,2))
3654	_endsubp(builtin_minus)
3655
3656	/* %arg_z <- %arg_z * %arg_y. Do the fixnum case - including overflow - */
3657	/* inline. Call out otherwise. */
3658	_spentry(builtin_times)
3659	__(movb %arg_z_b,%imm0_b)
3660	__(orb %arg_y_b,%imm0_b)
3661	__(testb $fixnummask,%imm0_b)
3662	__(jne 2f)
3663	__(unbox_fixnum(%arg_z,%imm0))
3664	/* 128-bit fixnum result in %imm1:%imm0. Overflow set if %imm1 */
3665	/* is significant */
3666	__(imul %arg_y)
3667	__(jo 1f)
3668	__(mov %imm0,%arg_z)
3669	__(ret)
3670	1: __(unbox_fixnum(%arg_z,%imm0))
3671	__(unbox_fixnum(%arg_y,%imm1))
3672	__(imul %imm1)
3673	__(jmp C(makes128))
3674	2: __(jump_builtin(_builtin_times,2))
3675	_endsubp(builtin_times)
3676
3677	_spentry(builtin_div)
3678	__(jump_builtin(_builtin_div,2))
3679
3680	/* %arg_z <- (= %arg_y %arg_z). */
3681	_spentry(builtin_eq)
3682	__(movb %arg_z_b,%imm0_b)
3683	__(orb %arg_y_b,%imm0_b)
3684	__(testb $fixnummask,%imm0_b)
3685	__(jne 1f)
3686	__(rcmpq(%arg_z,%arg_y))
3687	__(condition_to_boolean(e,%imm0,%arg_z))
3688	__(ret)
3689	1: __(jump_builtin(_builtin_eq,2))
3690	_endsubp(builtin_eq)
3691
3692	/* %arg_z <- (/= %arg_y %arg_z). */
3693	_spentry(builtin_ne)
3694	__(movb %arg_z_b,%imm0_b)
3695	__(orb %arg_y_b,%imm0_b)
3696	__(testb $fixnummask,%imm0_b)
3697	__(jne 1f)
3698	__(rcmpq(%arg_z,%arg_y))
3699	__(condition_to_boolean(ne,%imm0,%arg_z))
3700	__(ret)
3701	1: __(jump_builtin(_builtin_ne,2))
3702	_endsubp(builtin_ne)
3703
3704	/* %arg_z <- (> %arg_y %arg_z). */
3705	_spentry(builtin_gt)
3706	__(movb %arg_z_b,%imm0_b)
3707	__(orb %arg_y_b,%imm0_b)
3708	__(testb $fixnummask,%imm0_b)
3709	__(jne 1f)
3710	__(rcmpq(%arg_y,%arg_z))
3711	__(condition_to_boolean(g,%imm0,%arg_z))
3712	__(ret)
3713	1: __(jump_builtin(_builtin_gt,2))
3714	_endsubp(builtin_gt)
3715
3716	/* %arg_z <- (>= %arg_y %arg_z). */
3717	_spentry(builtin_ge)
3718	__(movb %arg_z_b,%imm0_b)
3719	__(orb %arg_y_b,%imm0_b)
3720	__(testb $fixnummask,%imm0_b)
3721	__(jne 1f)
3722	__(rcmpq(%arg_y,%arg_z))
3723	__(condition_to_boolean(ge,%imm0,%arg_z))
3724	__(ret)
3725	1: __(jump_builtin(_builtin_ge,2))
3726	_endsubp(builtin_ge)
3727
3728	/* %arg_z <- (< %arg_y %arg_z). */
3729	_spentry(builtin_lt)
3730	__(movb %arg_z_b,%imm0_b)
3731	__(orb %arg_y_b,%imm0_b)
3732	__(testb $fixnummask,%imm0_b)
3733	__(jne 1f)
3734	__(rcmpq(%arg_y,%arg_z))
3735	__(condition_to_boolean(l,%imm0,%arg_z))
3736	__(ret)
3737	1: __(jump_builtin(_builtin_lt,2))
3738	_endsubp(builtin_lt)
3739
3740	/* %arg_z <- (<= %arg_y %arg_z). */
3741	_spentry(builtin_le)
3742	__(movb %arg_z_b,%imm0_b)
3743	__(orb %arg_y_b,%imm0_b)
3744	__(testb $fixnummask,%imm0_b)
3745	__(jne 1f)
3746	__(rcmpq(%arg_y,%arg_z))
3747	__(condition_to_boolean(le,%imm0,%arg_z))
3748	__(ret)
3749	1: __(jump_builtin(_builtin_le,2))
3750	_endsubp(builtin_le)
3751
3752	_spentry(builtin_eql)
3753	0: __(cmpq %arg_y,%arg_z)
3754	__(je 8f)
3755	/* Not EQ. Could only possibly be EQL if both are tag-misc */
3756	/* and both have the same subtag */
3757	__(extract_lisptag(%arg_y,%imm0))
3758	__(extract_lisptag(%arg_z,%imm1))
3759	__(cmpb $tag_misc,%imm0_b)
3760	__(jne 9f)
3761	__(cmpb %imm0_b,%imm1_b)
3762	__(jne 9f)
3763	__(extract_subtag(%arg_y,%imm0))
3764	__(extract_subtag(%arg_z,%imm1))
3765	__(cmpb %imm0_b,%imm1_b)
3766	__(jne 9f)
3767	__(cmpb $subtag_macptr,%imm0_b)
3768	__(je 1f)
3769	__(cmpb $subtag_double_float,%imm0_b)
3770	__(jne 2f)
3771	1: __(movq misc_data_offset(%arg_y),%imm0)
3772	__(cmpq misc_data_offset(%arg_z),%imm0)
3773	__(movl $t_value,%arg_z_l)
3774	__(movl $nil_value,%imm0_l)
3775	__(cmovnel %imm0_l,%arg_z_l)
3776	__(ret)
3777	2: __(cmpb $subtag_ratio,%imm0_b)
3778	__(je 3f)
3779	__(cmpb $subtag_complex,%imm0_b)
3780	__(jne 5f)
3781	3: __(pushq %rbp)
3782	__(movq %rsp,%rsp)
3783	__(pushq ratio.denom(%arg_y))
3784	__(pushq ratio.denom(%arg_z))
3785	__(movq ratio.numer(%arg_y),%arg_y)
3786	__(movq ratio.numer(%arg_z),%arg_z)
3787	__(lea 4f(%rip),%ra0)
3788	__(pushq %ra0)
3789	__(jmp 0b)
3790	__(tra(4))
3791	__(compare_reg_to_nil(%arg_z))
3792	__(popq %arg_z)
3793	__(popq %arg_y)
3794	__(popq %rbp)
3795	__(je 9f)
3796	__(jmp 0b)
3797	5: __(cmpb $subtag_bignum,%imm0_b)
3798	__(jne 9f)
3799	__(getvheader(%arg_y,%imm2))
3800	__(cmpq misc_header_offset(%arg_z),%imm2)
3801	__(jne 9f)
3802	__(shrq $num_subtag_bits,%imm2)
3803	__(xorl %imm1_l,%imm1_l)
3804	6: __(movl misc_data_offset(%arg_y,%imm1,4),%imm0_l)
3805	__(cmpl misc_data_offset(%arg_z,%imm1,4),%imm0_l)
3806	__(jne 9f)
3807	__(addq $1,%imm1)
3808	__(cmpq %imm1,%imm2)
3809	__(jne 6b)
3810	8: __(movl $t_value,%arg_z_l)
3811	__(ret)
3812	9: __(movl $nil_value,%arg_z_l)
3813	__(ret)
3814	_endsubp(builtin_eql)
3815
3816	_spentry(builtin_length)
3817	__(extract_lisptag(%arg_z,%imm0))
3818	__(cmpb $tag_list,%imm0_b)
3819	__(jz 2f)
3820	__(cmpb $tag_misc,%imm0_b)
3821	__(jnz 8f)
3822	__(extract_subtag(%arg_z,%imm0))
3823	__(rcmpb(%imm0_b,$min_vector_subtag))
3824	__(jb 8f)
3825	__(je 1f)
3826	/* (simple-array * ()) /
3827	__(movq %arg_z,%arg_y)
3828	__(vector_length(%arg_y,%arg_z))
3829	__(ret)
3830	1: /* vector header */
3831	__(movq vectorH.logsize(%arg_z),%arg_z)
3832	__(ret)
3833	2: /* list. Maybe null, maybe dotted or circular. */
3834	__(movq $-fixnumone,%imm2)
3835	__(movq %arg_z,%temp0) /* fast pointer */
3836	__(movq %arg_z,%temp1) /* slow pointer */
3837	3: __(extract_lisptag(%temp0,%imm0))
3838	__(compare_reg_to_nil(%temp0))
3839	__(leaq fixnumone(%imm2),%imm2)
3840	__(je 9f)
3841	__(cmpb $tag_list,%imm0_b)
3842	__(jne 8f)
3843	__(extract_lisptag(%temp1,%imm1))
3844	__(testb $fixnumone,%imm2_b)
3845	__(_cdr(%temp0,%temp0))
3846	__(je 3b)
3847	__(cmpb $tag_list,%imm1_b)
3848	__(jne 8f)
3849	__(_cdr(%temp1,%temp1))
3850	__(cmpq %temp0,%temp1)
3851	__(jne 3b)
3852	8:
3853	__(jump_builtin(_builtin_length,1))
3854	9:
3855	__(movq %imm2,%arg_z)
3856	__(ret)
3857	_endsubp(builtin_length)
3858
3859
3860	_spentry(builtin_seqtype)
3861	__(extract_lisptag(%arg_z,%imm0))
3862	__(cmpb $tag_list,%imm0_b)
3863	__(jz 1f)
3864	__(cmpb $tag_misc,%imm0_b)
3865	__(jne 2f)
3866	__(movb misc_subtag_offset(%arg_z),%imm0_b)
3867	__(rcmpb(%imm0_b,$min_vector_subtag))
3868	__(jb 2f)
3869	__(movl $nil_value,%arg_z_l)
3870	__(ret)
3871	1: __(movl $t_value,%arg_z_l)
3872	__(ret)
3873	2:
3874	__(jump_builtin(_builtin_seqtype,1))
3875	_endsubp(builtin_seqtype)
3876
3877	_spentry(builtin_assq)
3878	__(cmpb $fulltag_nil,%arg_z_b)
3879	__(jz 5f)
3880	1: __(movb $tagmask,%imm0_b)
3881	__(andb %arg_z_b,%imm0_b)
3882	__(cmpb $tag_list,%imm0_b)
3883	__(jnz 2f)
3884	__(_car(%arg_z,%arg_x))
3885	__(_cdr(%arg_z,%arg_z))
3886	__(cmpb $fulltag_nil,%arg_x_b)
3887	__(jz 4f)
3888	__(movb $tagmask,%imm0_b)
3889	__(andb %arg_x_b,%imm0_b)
3890	__(cmpb $tag_list,%imm0_b)
3891	__(jnz 3f)
3892	__(_car(%arg_x,%temp0))
3893	__(cmpq %temp0,%arg_y)
3894	__(jnz 4f)
3895	__(movq %arg_x,%arg_z)
3896	__(ret)
3897	4: __(cmpb $fulltag_nil,%arg_z_b)
3898	5: __(jnz 1b)
3899	__(repret)
3900	2: __(uuo_error_reg_not_list(Rarg_z))
3901	3: __(uuo_error_reg_not_list(Rarg_x))
3902	_endsubp(builtin_assq)
3903
3904	_spentry(builtin_memq)
3905	__(cmpb $fulltag_nil,%arg_z_b)
3906	__(jmp 3f)
3907	1: __(movb $tagmask,%imm0_b)
3908	__(andb %arg_z_b,%imm0_b)
3909	__(cmpb $tag_list,%imm0_b)
3910	__(jnz 2f)
3911	__(_car(%arg_z,%arg_x))
3912	__(_cdr(%arg_z,%temp0))
3913	__(cmpq %arg_x,%arg_y)
3914	__(jz 4f)
3915	__(cmpb $fulltag_nil,%temp0_b)
3916	__(movq %temp0,%arg_z)
3917	3: __(jnz 1b)
3918	4: __(repret)
3919	2: __(uuo_error_reg_not_list(Rarg_z))
3920	_endsubp(builtin_memq)
3921
3922	__ifdef(`X8664')
3923	logbitp_max_bit = 61
3924	__else
3925	logbitp_max_bit = 30
3926	__endif
3927
3928	_spentry(builtin_logbitp)
3929	__(movb %arg_z_b,%imm0_b)
3930	__(orb %arg_y_b,%imm0_b)
3931	__(testb $fixnummask,%imm0_b)
3932	__(jnz 1f)
3933	__(unbox_fixnum(%arg_y,%imm0))
3934	__(movl $logbitp_max_bit-1+fixnumshift,%imm1_l)
3935	__(js 1f) /* bit number negative */
3936	__(addb $fixnumshift,%imm0_b)
3937	__(cmpq $logbitp_max_bit<<fixnumshift,%arg_y)
3938	__(cmovael %imm1_l,%imm0_l)
3939	__(bt %imm0,%arg_z)
3940	__(condition_to_boolean(b,%imm0,%arg_z))
3941	__(ret)
3942	1: __(jump_builtin(_builtin_logbitp,2))
3943	_endsubp(builtin_logbitp)
3944
3945	_spentry(builtin_logior)
3946	__(movb %arg_y_b,%imm0_b)
3947	__(orb %arg_z_b,%imm0_b)
3948	__(testb $fixnummask,%imm0_b)
3949	__(jne 1f)
3950	__(orq %arg_y,%arg_z)
3951	__(ret)
3952	1:
3953	__(jump_builtin(_builtin_logior,2))
3954
3955	_endsubp(builtin_logior)
3956
3957	_spentry(builtin_logand)
3958	__(movb %arg_y_b,%imm0_b)
3959	__(orb %arg_z_b,%imm0_b)
3960	__(testb $fixnummask,%imm0_b)
3961	__(jne 1f)
3962	__(andq %arg_y,%arg_z)
3963	__(ret)
3964	1:
3965	__(jump_builtin(_builtin_logand,2))
3966	_endsubp(builtin_logand)
3967
3968	_spentry(builtin_negate)
3969	__(testb $fixnummask,%arg_z_b)
3970	__(jne 1f)
3971	__(negq %arg_z)
3972	__(jo C(fix_one_bit_overflow))
3973	__(repret)
3974	1:
3975	__(jump_builtin(_builtin_negate,1))
3976	_endsubp(builtin_negate)
3977
3978	_spentry(builtin_logxor)
3979	__(movb %arg_y_b,%imm0_b)
3980	__(orb %arg_z_b,%imm0_b)
3981	__(testb $fixnummask,%imm0_b)
3982	__(jne 1f)
3983	__(xorq %arg_y,%arg_z)
3984	__(ret)
3985	1:
3986	__(jump_builtin(_builtin_logxor,2))
3987	_endsubp(builtin_logxor)
3988
3989
3990	_spentry(builtin_aset1)
3991	__(extract_typecode(%arg_x,%imm0))
3992	__(box_fixnum(%imm0,%temp0))
3993	__(cmpb $min_vector_subtag,%imm0_b)
3994	__(ja _SPsubtag_misc_set)
3995	__(jump_builtin(_builtin_aset1,3))
3996	_endsubp(builtin_aset1)
3997
3998
3999	_spentry(builtin_ash)
4000	__(movb %arg_y_b,%imm0_b)
4001	__(orb %arg_z_b,%imm0_b)
4002	__(testb $fixnummask,%imm0_b)
4003	__(jne 9f)
4004	__(unbox_fixnum(%arg_y,%imm1))
4005	__(unbox_fixnum(%arg_z,%imm0))
4006	/* Z flag set if zero ASH shift count */
4007	__(jnz 1f)
4008	__(movq %arg_y,%arg_z) /* shift by 0 */
4009	__(ret)
4010	1: __(jns 3f)
4011	__(rcmpq(%imm0,$-63))
4012	__(jg 2f)
4013	__(sar $63,%imm1)
4014	__(box_fixnum(%imm1,%arg_z))
4015	__(ret)
4016	2: /* Right-shift by small fixnum */
4017	__(negb %imm0_b)
4018	__(movzbl %imm0_b,%ecx)
4019	__(sar %cl,%imm1)
4020	__(box_fixnum(%imm1,%arg_z))
4021	__(ret)
4022	3: /* Left shift by fixnum. We cant shift by more than 63 bits, though */
4023	/* shifting by 64 is actually easy. */
4024	__(rcmpq(%imm0,$64))
4025	__(jg 9f)
4026	__(jne 4f)
4027	/* left-shift by 64-bits exactly */
4028	__(xorl %imm0_l,%imm0_l)
4029	__(jmp C(makes128))
4030	4: /* left-shift by 1..63 bits. Safe to move shift count to %rcx/%cl */
4031	__(movzbl %imm0_b,%ecx) /* zero-extending mov */
4032	__(movq %imm1,%imm0)
4033	__(sarq $63,%imm1)
4034	__(js 5f)
4035	__(shld %cl,%imm0,%imm1)
4036	__(shl %cl,%imm0)
4037	__(jmp C(makes128))
4038	5: __(shld %cl,%imm0,%imm1)
4039	__(shl %cl,%imm0)
4040	__(jmp C(makes128))
4041	9:
4042	__(jump_builtin(_builtin_ash,2))
4043	_endsubp(builtin_ash)
4044
4045	_spentry(builtin_aref1)
4046	__(extract_typecode(%arg_y,%imm0))
4047	__(cmpb $min_vector_subtag,%imm0_b)
4048	__(box_fixnum_no_flags(%imm0,%arg_x))
4049	__(ja _SPsubtag_misc_ref)
4050	__(jump_builtin(_builtin_aref1,2))
4051	_endsubp(builtin_aref1)
4052
4053	/* Arg_z is either a MACPTR containing the function address or a boxed fixnum. */
4054	/* %imm0.b (aka %al) contains the number (0-7) of args passed in FP regs. */
4055	/* On entry, the foreign stack contains a frame containing at least 8 words: */
4056
4057	/* * -> aligned on 16-byte boundary */
4058	/* backlink <- foreign %rsp /
4059	/* unused */
4060	/* scalar arg 0 passed in %rdi */
4061	/* scalar arg 1 passed in %rsi */
4062	/* scalar arg 2 passed in %rdx */
4063	/* scalar arg 3 passed in %rcx */
4064	/* scalar arg 4 passed in %r8 */
4065	/* scalar arg 5 passed in %r9 */
4066	/* address of first memory arg /
4067	/* ... */
4068	/* possible scratch space */
4069	/* previous %rsp value /
4070
4071	/* Any floating-point args will have been loaded into %xmm0-%xmm7 by the caller. */
4072	/* When this returns, the foreign %rsp will contain its previous value, and */
4073	/* the function result will be in %rax (and possibly %rdx) or %xmm0 (+ %xmm1). */
4074
4075	.globl C(ffcall_return)
4076	_spentry(ffcall)
4077	LocalLabelPrefix`'ffcall:
4078	/* Unbox %arg_z. It's either a fixnum or macptr (or bignum) ;
4079	if not a fixnum, get the first word */
4080	__(unbox_fixnum(%arg_z,%imm1))
4081	__(testb $fixnummask,%arg_z_b)
4082	__(je 0f)
4083	__(movq macptr.address(%arg_z),%imm1)
4084	0:
4085	/* Save lisp registers */
4086	__(push %rbp)
4087	__(movq %rsp,%rbp)
4088	__(push %temp0)
4089	__(push %temp1)
4090	__(push %temp2)
4091	__(push %arg_x)
4092	__(push %arg_y)
4093	__(push %arg_z)
4094	__(push %fn)
4095	__ifndef(`TCR_IN_GPR')
4096	__(push %save3)
4097	__endif
4098	__(push %save2)
4099	__(push %save1)
4100	__(push %save0) /* 10 or 11 registers pushed after %rbp */
4101	__(movq %rsp,rcontext(tcr.save_vsp))
4102	__(movq %rbp,rcontext(tcr.save_rbp))
4103	__(movq $TCR_STATE_FOREIGN,rcontext(tcr.valence))
4104	__(movq rcontext(tcr.foreign_sp),%rsp)
4105	__ifdef(`WINDOWS')
4106	__(stmxcsr rcontext(tcr.lisp_mxcsr))
4107	__else
4108	__(movq $0,rcontext(tcr.ffi_exception))
4109	__endif
4110	__(emms)
4111	__ifdef(`WINDOWS')
4112	__(ldmxcsr rcontext(tcr.foreign_mxcsr))
4113	__endif
4114	__(movq (%rsp),%rbp)
4115	__ifdef(`TCR_IN_GPR')
4116	/* Preserve TCR pointer */
4117	__(movq %rcontext_reg, %csave0)
4118	__endif
4119	LocalLabelPrefix`'ffcall_setup:
4120	__(addq $2*node_size,%rsp)
4121	__(movq %imm1,%r11)
4122	__ifdef(`WINDOWS')
4123	/* Leave 0x20 bytes of register spill area on stack */
4124	__(movq (%rsp),%carg0)
4125	__(movq 8(%rsp),%carg1)
4126	__(movq 16(%rsp),%carg2)
4127	__(movq 24(%rsp),%carg3)
4128	__else
4129	__(pop %carg0)
4130	__(pop %carg1)
4131	__(pop %carg2)
4132	__(pop %carg3)
4133	__(pop %carg4)
4134	__(pop %carg5)
4135	__endif
4136	LocalLabelPrefix`'ffcall_setup_end:
4137	LocalLabelPrefix`'ffcall_call:
4138	__(call *%r11)
4139	C(ffcall_return):
4140	LocalLabelPrefix`'ffcall_call_end:
4141	__ifdef(`WINDOWS')
4142	__(add $0x20,%rsp)
4143	__endif
4144	__(movq %rbp,%rsp)
4145	__ifdef(`TCR_IN_GPR')
4146	__(movq %csave0, %rcontext_reg)
4147	__endif
4148	__(movq %rsp,rcontext(tcr.foreign_sp))
4149	__ifndef(`TCR_IN_GPR')
4150	__(clr %save3)
4151	__endif
4152	__(clr %save2)
4153	__(clr %save1)
4154	__(clr %save0)
4155	__(clr %arg_z)
4156	__(clr %arg_y)
4157	__(clr %arg_x)
4158	__(clr %temp2)
4159	__(clr %temp1)
4160	__(clr %temp0)
4161	__(clr %fn)
4162	__(pxor %fpzero,%fpzero)
4163
4164	__ifndef(`WINDOWS')
4165	/* If we got a floating-point exception during the ff-call,
4166	our handler will have set a flag, preserved lisp's MXCSR,
4167	and resumed execution with fp exceptions masked. */
4168	__(btrq $TCR_FLAG_BIT_FOREIGN_FPE,rcontext(tcr.flags))
4169	__(jnc 1f)
4170	__endif
4171	__(cmpb $0,C(bogus_fp_exceptions)(%rip))
4172	__(je 0f)
4173	__(movl %arg_x_l,rcontext(tcr.ffi_exception))
4174	__(jmp 1f)
4175	0: __(stmxcsr rcontext(tcr.ffi_exception))
4176	__ifndef(`WINDOWS')
4177	__(ldmxcsr rcontext(tcr.lisp_mxcsr)) /* preserved by the handler */
4178	__endif
4179	1: __(movq rcontext(tcr.save_vsp),%rsp)
4180	__(movq rcontext(tcr.save_rbp),%rbp)
4181	__(movq $TCR_STATE_LISP,rcontext(tcr.valence))
4182	__(pop %save0)
4183	__(pop %save1)
4184	__(pop %save2)
4185	__ifndef(`TCR_IN_GPR')
4186	__(pop %save3)
4187	__endif
4188	__(pop %fn)
4189	__(pop %arg_z)
4190	__(pop %arg_y)
4191	__(pop %arg_x)
4192	__(pop %temp2)
4193	__(pop %temp1)
4194	__ifdef(`WINDOWS')
4195	__(ldmxcsr rcontext(tcr.lisp_mxcsr))
4196	__endif
4197	__(check_pending_interrupt(%temp0))
4198	__(pop %temp0)
4199	__(leave)
4200	__ifdef(`DARWIN')
4201	__(btrq $TCR_FLAG_BIT_FOREIGN_EXCEPTION,rcontext(tcr.flags))
4202	__(jc 0f)
4203	__endif
4204	__(ret)
4205	__ifdef(`DARWIN')
4206	0:
4207	/* Unboxed foreign exception (likely an NSException) in %imm0. */
4208	/* Box it, then signal a lisp error. */
4209	__(movq %imm0,%imm2)
4210	__(movq $macptr_header,%rax)
4211	__(Misc_Alloc_Fixed(%arg_z,macptr.size))
4212	__(movq %imm2,macptr.address(%arg_z))
4213	__(movq $XFOREIGNEXCEPTION,%arg_y)
4214	__(set_nargs(2))
4215	__(jmp _SPksignalerr)
4216	__endif
4217	__ifdef(`DARWIN')
4218	/* Handle exceptions, for ObjC 2.0 */
4219	LocalLabelPrefix`'ffcallLandingPad:
4220	__(movq %rax,%save1)
4221	__(cmpq $1,%rdx)
4222	__(je 1f)
4223	__(movq %rax,%rdi)
4224	LocalLabelPrefix`'ffcallUnwindResume:
4225	__(call *lisp_global(unwind_resume))
4226	LocalLabelPrefix`'ffcallUnwindResume_end:
4227	1: __(movq %save1,%rdi)
4228	LocalLabelPrefix`'ffcallBeginCatch:
4229	__(call *lisp_global(objc_2_begin_catch))
4230	LocalLabelPrefix`'ffcallBeginCatch_end:
4231	__(movq (%rax),%save1) /* indirection is necessary because we don't provide type info in lsda */
4232	LocalLabelPrefix`'ffcallEndCatch:
4233	__(call *lisp_global(objc_2_end_catch))
4234	LocalLabelPrefix`'ffcallEndCatch_end:
4235	__(ref_global(get_tcr,%rax))
4236	__(movq $1,%rdi)
4237	__(call *%rax)
4238	__(btsq $TCR_FLAG_BIT_FOREIGN_EXCEPTION,tcr.flags(%rax))
4239	__(movq %save1,%rax)
4240	__(jmp LocalLabelPrefix`'ffcall_call_end)
4241	LocalLabelPrefix`'ffcall_end:
4242	__endif
4243	_endsubp(ffcall)
4244
4245	__ifdef(`DARWIN')
4246	.section __DATA,__gcc_except_tab
4247	GCC_except_table0:
4248	.align 3
4249	LLSDA1:
4250	.byte 0xff /* @LPStart format (omit) */
4251	.byte 0x0 /* @TType format (absolute) */
4252	.byte 0x4d /* uleb128 0x4d; @TType base offset */
4253	.byte 0x3 /* call-site format (udata4) */
4254	.byte 0x41 /* uleb128 0x41; Call-site table length */
4255
4256	.long Lffcall_setup-Lffcall /* region 0 start */
4257	.long Lffcall_setup_end-Lffcall_setup /* length */
4258	.long 0x0 /* landing pad */
4259	.byte 0x0 /* uleb128 0x0; action */
4260
4261	.long Lffcall_call-Lffcall /* region 1 start */
4262	.long Lffcall_call_end-Lffcall_call /* length */
4263	.long LffcallLandingPad-Lffcall /* landing pad */
4264	.byte 0x1 /* uleb128 0x1; action */
4265
4266	.long LffcallUnwindResume-Lffcall /* region 2 start */
4267	.long LffcallUnwindResume_end-LffcallUnwindResume /* length */
4268	.long 0x0 /* landing pad */
4269	.byte 0x0 /* uleb128 0x0; action */
4270
4271	.long LffcallBeginCatch-Lffcall /* region 3 start */
4272	.long LffcallBeginCatch_end-LffcallBeginCatch /* length */
4273	.long 0 /* landing pad */
4274	.byte 0x0 /* uleb128 0x0; action */
4275
4276	.long LffcallEndCatch-Lffcall
4277	.long LffcallEndCatch_end-LffcallEndCatch /* length */
4278	.long 0x0 /* landing pad */
4279	.byte 0x0 /* uleb128 0x0; action */
4280	.byte 0x1 /* Action record table */
4281	.byte 0x0
4282	.align 3
4283	.quad 0 /* _OBJC_EHTYPE_$_NSException */
4284	.text
4285	__endif
4286
4287	_spentry(ffcall_return_registers)
4288	LocalLabelPrefix`'ffcall_return_registers:
4289	/* Unbox %arg_z. It's either a fixnum or macptr (or bignum) ;
4290	if not a fixnum, get the first word */
4291	__(unbox_fixnum(%arg_z,%imm1))
4292	__(testb $fixnummask,%arg_z_b)
4293	__(je 0f)
4294	__(movq macptr.address(%arg_z),%imm1)
4295	0:
4296	/* Save lisp registers */
4297	__(push %rbp)
4298	__(movq %rsp,%rbp)
4299	__(push %temp0)
4300	__(push %temp1)
4301	__(push %temp2)
4302	__(push %arg_x)
4303	__(push %arg_y)
4304	__(push %arg_z)
4305	__ifndef(`TCR_IN_GPR')
4306	__(push %save3)
4307	__endif
4308	__(push %save2)
4309	__(push %save1)
4310	__(push %save0)
4311	__(movq macptr.address(%arg_y),%csave0) /* %rbx non-volatile */
4312	__(push %fn)
4313	__(movq %rsp,rcontext(tcr.save_vsp))
4314	__(movq %rbp,rcontext(tcr.save_rbp))
4315	__(movq $TCR_STATE_FOREIGN,rcontext(tcr.valence))
4316	__(movq $0,rcontext(tcr.ffi_exception))
4317	__(movq rcontext(tcr.foreign_sp),%rsp)
4318	__(emms)
4319	__(movq (%rsp),%rbp)
4320	__ifdef(`TCR_IN_GPR')
4321	/* Preserve TCR pointer */
4322	__(movq %rcontext_reg, %csave1)
4323	__endif
4324	__(movq %imm1,%r11)
4325	LocalLabelPrefix`'ffcall_return_registers_setup:
4326	__(addq $2*node_size,%rsp)
4327	__(pop %carg0)
4328	__(pop %carg1)
4329	__(pop %carg2)
4330	__(pop %carg3)
4331	__ifdef(`WINDOWS')
4332	__(sub $0x20, %rsp) /* Make room for arg register spill */
4333	__else
4334	__(pop %carg4)
4335	__(pop %carg5)
4336	__endif
4337	LocalLabelPrefix`'ffcall_return_registers_setup_end:
4338	LocalLabelPrefix`'ffcall_return_registers_call:
4339	__(call *%r11)
4340	LocalLabelPrefix`'ffcall_return_registers_call_end:
4341	__ifdef(`WINDOWS')
4342	__(add $0x20, %rsp)
4343	__endif
4344	__(movq %rax,(%csave0))
4345	__(movq %rdx,8(%csave0))
4346	__(movsd %xmm0,16(%csave0))
4347	__(movsd %xmm1,24(%csave0))
4348	__(movq %rbp,%rsp)
4349	__ifdef(`TCR_IN_GPR')
4350	__(movq %csave1, %rcontext_reg)
4351	__endif
4352	__(movq %rsp,rcontext(tcr.foreign_sp))
4353	__ifndef(`TCR_IN_GPR')
4354	__(clr %save3)
4355	__endif
4356	__(clr %save2)
4357	__(clr %save1)
4358	__(clr %save0)
4359	__(clr %arg_z)
4360	__(clr %arg_y)
4361	__(clr %arg_x)
4362	__(clr %temp2)
4363	__(clr %temp1)
4364	__(clr %temp0)
4365	__(clr %fn)
4366	__(pxor %fpzero,%fpzero)
4367	/* Check for fp exceptions as in .SPffcall, above. */
4368	__(btrq $TCR_FLAG_BIT_FOREIGN_FPE,rcontext(tcr.flags))
4369	__(jnc 1f)
4370	__(cmpb $0,C(bogus_fp_exceptions)(%rip))
4371	__(je 0f)
4372	__(movl %arg_x_l,rcontext(tcr.ffi_exception))
4373	__(jmp 1f)
4374	0: __(stmxcsr rcontext(tcr.ffi_exception))
4375	__(ldmxcsr rcontext(tcr.lisp_mxcsr))
4376	1: __(movq rcontext(tcr.save_vsp),%rsp)
4377	__(movq rcontext(tcr.save_rbp),%rbp)
4378	__(movq $TCR_STATE_LISP,rcontext(tcr.valence))
4379	__(pop %fn)
4380	__(pop %save0)
4381	__(pop %save1)
4382	__(pop %save2)
4383	__ifndef(`TCR_IN_GPR')
4384	__(pop %save3)
4385	__endif
4386	__(pop %arg_z)
4387	__(pop %arg_y)
4388	__(pop %arg_x)
4389	__(pop %temp2)
4390	__(pop %temp1)
4391	__(check_pending_interrupt(%temp0))
4392	__(pop %temp0)
4393	__(leave)
4394	__ifdef(`DARWIN')
4395	__(btrq $TCR_FLAG_BIT_FOREIGN_EXCEPTION,rcontext(tcr.flags))
4396	__(jc 0f)
4397	__endif
4398	__(ret)
4399	__ifdef(`DARWIN')
4400	0:
4401	/* Unboxed foreign exception (likely an NSException) in %imm0. */
4402	/* Box it, then signal a lisp error. */
4403	__(movq %imm0,%imm2)
4404	__(movq $macptr_header,%rax)
4405	__(Misc_Alloc_Fixed(%arg_z,macptr.size))
4406	__(movq %imm2,macptr.address(%arg_z))
4407	__(movq $XFOREIGNEXCEPTION,%arg_y)
4408	__(set_nargs(2))
4409	__(jmp _SPksignalerr)
4410	__endif
4411	__ifdef(`DARWIN')
4412	/* Handle exceptions, for ObjC 2.0 */
4413	LocalLabelPrefix`'ffcall_return_registersLandingPad:
4414	__(movq %rax,%save1)
4415	__(cmpq $1,%rdx)
4416	__(je 1f)
4417	__(movq %rax,%rdi)
4418	LocalLabelPrefix`'ffcall_return_registersUnwindResume:
4419	__(call *lisp_global(unwind_resume))
4420	LocalLabelPrefix`'ffcall_return_registersUnwindResume_end:
4421	1: __(movq %save1,%rdi)
4422	LocalLabelPrefix`'ffcall_return_registersBeginCatch:
4423	__(call *lisp_global(objc_2_begin_catch))
4424	LocalLabelPrefix`'ffcall_return_registersBeginCatch_end:
4425	__(movq (%rax),%save1) /* indirection is necessary because we don't provide type info in lsda */
4426	LocalLabelPrefix`'ffcall_return_registersEndCatch:
4427	__(call *lisp_global(objc_2_end_catch))
4428	LocalLabelPrefix`'ffcall_return_registersEndCatch_end:
4429	__(ref_global(get_tcr,%rax))
4430	__(movq $1,%rdi)
4431	__(call *%rax)
4432	__(btsq $TCR_FLAG_BIT_FOREIGN_EXCEPTION,tcr.flags(%rax))
4433	__(movq %save1,%rax)
4434	__(jmp LocalLabelPrefix`'ffcall_return_registers_call_end)
4435	LocalLabelPrefix`'ffcall_return_registers_end:
4436	__endif
4437	_endsubp(ffcall_returning_registers)
4438
4439	__ifdef(`DARWIN')
4440	.section __DATA,__gcc_except_tab
4441	GCC_except_table1:
4442	.align 3
4443	LLSDA2:
4444	.byte 0xff /* @LPStart format (omit) */
4445	.byte 0x0 /* @TType format (absolute) */
4446	.byte 0x4d /* uleb128 0x4d; @TType base offset */
4447	.byte 0x3 /* call-site format (udata4) */
4448	.byte 0x41 /* uleb128 0x41; Call-site table length */
4449
4450	.long Lffcall_return_registers_setup-Lffcall_return_registers /* region 0 start */
4451	.long Lffcall_return_registers_setup_end-Lffcall_return_registers_setup /* length */
4452	.long 0x0 /* landing pad */
4453	.byte 0x0 /* uleb128 0x0; action */
4454
4455	.long Lffcall_return_registers_call-Lffcall_return_registers /* region 1 start */
4456	.long Lffcall_return_registers_call_end-Lffcall_return_registers_call /* length */
4457	.long Lffcall_return_registersLandingPad-Lffcall_return_registers /* landing pad */
4458	.byte 0x1 /* uleb128 0x1; action */
4459
4460	.long Lffcall_return_registersUnwindResume-Lffcall_return_registers /* region 2 start */
4461	.long Lffcall_return_registersUnwindResume_end-Lffcall_return_registersUnwindResume /* length */
4462	.long 0x0 /* landing pad */
4463	.byte 0x0 /* uleb128 0x0; action */
4464
4465	.long Lffcall_return_registersBeginCatch-Lffcall_return_registers /* region 3 start */
4466	.long Lffcall_return_registersBeginCatch_end-Lffcall_return_registersBeginCatch /* length */
4467	.long 0 /* landing pad */
4468	.byte 0x0 /* uleb128 0x0; action */
4469
4470	.long Lffcall_return_registersEndCatch-Lffcall_return_registers
4471	.long Lffcall_return_registersEndCatch_end-Lffcall_return_registersEndCatch /* length */
4472	.long 0x0 /* landing pad */
4473	.byte 0x0 /* uleb128 0x0; action */
4474	.byte 0x1 /* Action record table */
4475	.byte 0x0
4476	.align 3
4477	.quad 0 /* _OBJC_EHTYPE_$_NSException */
4478	.text
4479	__endif
4480
4481	_spentry(syscall)
4482	/* Save lisp registers */
4483	__(push %rbp)
4484	__(movq %rsp,%rbp)
4485	__(push %temp0)
4486	__(push %temp1)
4487	__(push %temp2)
4488	__(push %arg_x)
4489	__(push %arg_y)
4490	__(push %arg_z)
4491	__ifndef(`TCR_IN_GPR')
4492	__(push %save3)
4493	__endif
4494	__(push %save2)
4495	__(push %save1)
4496	__(push %save0)
4497	__(push %fn)
4498	__(movq %rsp,rcontext(tcr.save_vsp))
4499	__(movq %rbp,rcontext(tcr.save_rbp))
4500	__(movq $TCR_STATE_FOREIGN,rcontext(tcr.valence))
4501	__(movq rcontext(tcr.foreign_sp),%rsp)
4502	__(emms)
4503	__(movq (%rsp),%rbp)
4504	__(addq $2*node_size,%rsp)
4505	__ifdef(`TCR_IN_GPR')
4506	__(movq %rcontext_reg,%csave0)
4507	__endif
4508	__ifdef(`WINDOWS')
4509	__(pop %carg0)
4510	__(pop %carg1)
4511	__(pop %carg2)
4512	__(pop %carg3)
4513	__(subq $0x20,%rsp)
4514	__(orq $-1,%cret)
4515	__(addq $0x20,%rsp)
4516	__else
4517	__(unbox_fixnum(%arg_z,%rax))
4518	__(pop %rdi)
4519	__(pop %rsi)
4520	__(pop %rdx)
4521	__(pop %r10) /* syscalls take 4th param in %r10, not %rcx */
4522	__(pop %r8)
4523	__(pop %r9)
4524	__(syscall)
4525	__ifdef(`SYSCALL_SETS_CARRY_ON_ERROR')
4526	__(jnc 0f)
4527	__(negq %rax)
4528	0:
4529	__endif
4530	__endif
4531	__ifdef(`TCR_IN_GPR')
4532	__(movq %csave0,%rcontext_reg)
4533	__endif
4534	__(movq %rbp,%rsp)
4535	__(movq %rsp,rcontext(tcr.foreign_sp))
4536	__ifndef(`TCR_IN_GPR')
4537	__(clr %save3)
4538	__endif
4539	__(clr %save2)
4540	__(clr %save1)
4541	__(clr %save0)
4542	__(clr %arg_z)
4543	__(clr %arg_y)
4544	__(clr %arg_x)
4545	__(clr %temp2)
4546	__(clr %temp1)
4547	__(clr %temp0)
4548	__(clr %fn)
4549	__(pxor %fpzero,%fpzero)
4550	__(movq rcontext(tcr.save_vsp),%rsp)
4551	__(movq rcontext(tcr.save_rbp),%rbp)
4552	__(movq $TCR_STATE_LISP,rcontext(tcr.valence))
4553	__(pop %fn)
4554	__(pop %save0)
4555	__(pop %save1)
4556	__(pop %save2)
4557	__ifndef(`TCR_IN_GPR')
4558	__(pop %save3)
4559	__endif
4560	__(pop %arg_z)
4561	__(pop %arg_y)
4562	__(pop %arg_x)
4563	__(pop %temp2)
4564	__(pop %temp1)
4565	__(check_pending_interrupt(%temp0))
4566	__(pop %temp0)
4567	__(leave)
4568	__(ret)
4569	_endsubp(syscall)
4570
4571	/* We need to reserve a frame here if (a) nothing else was already pushed and (b) */
4572	/* we push something (e.g., more than 3 args in the lexpr) */
4573	_spentry(spread_lexprz)
4574	new_local_labels()
4575	__(movq (%arg_z),%imm0)
4576	__(testl %nargs,%nargs) /* anything pushed by caller ? */
4577	__(leaq node_size(%arg_z,%imm0),%imm1)
4578	__(jne 0f) /* yes, caller has already created frame. */
4579	__(cmpw $(nargregsnode_size),%imm0_w) / will we push anything ? */
4580	__(jbe 0f)
4581	__(push $reserved_frame_marker)
4582	__(push $reserved_frame_marker)
4583	0: __(addw %imm0_w,%nargs_w)
4584	__(cmpw $(nargregs*node_size),%imm0_w)
4585	__(jae 9f)
4586	__(cmpw $(2*node_size),%imm0_w)
4587	__(je 2f)
4588	__(testw %imm0_w,%imm0_w)
4589	__(jne 1f)
4590	/* lexpr count was 0; vpop the args that */
4591	/* were pushed by the caller */
4592	__(testl %nargs,%nargs)
4593	__(je local_label(all_args_popped))
4594	__(pop %arg_z)
4595	local_label(maybe_pop_yx):
4596	__(cmpl $(1*node_size),%nargs)
4597	__(je local_label(all_args_popped))
4598	__(pop %arg_y)
4599	__(cmpl $(2*node_size),%nargs)
4600	__(je local_label(all_args_popped))
4601	local_label(pop_arg_x):
4602	__(pop %arg_x)
4603	local_label(all_args_popped):
4604	/* If all args fit in registers but some were pushed */
4605	/* by the caller, discard the reserved frame that the caller */
4606	/* pushed. */
4607	__(cmpw %imm0_w,%nargs_w)
4608	__(je local_label(go))
4609	__(cmpl $(nargregs*node_size),%nargs)
4610	__(ja local_label(go))
4611	__(addq $(2*node_size),%rsp)
4612	local_label(go):
4613	__(jmp *%ra0)
4614	/* vpush args from the lexpr until we have only */
4615	/* three left, then assign them to arg_x, arg_y, */
4616	/* and arg_z. */
4617	8: __(cmpw $(4*node_size),%imm0_w)
4618	__(lea -1*node_size(%imm0),%imm0)
4619	__(push -node_size(%imm1))
4620	__(lea -1*node_size(%imm1),%imm1)
4621	9: __(jne 8b)
4622	__(movq -node_size*1(%imm1),%arg_x)
4623	__(movq -node_size*2(%imm1),%arg_y)
4624	__(movq -node_size*3(%imm1),%arg_z)
4625	__(jmp *%ra0)
4626
4627	/* lexpr count is two: set arg_y, arg_z from the */
4628	/* lexpr, maybe vpop arg_x */
4629	2: __(cmpl $(2*node_size),%nargs)
4630	__(movq -node_size*1(%imm1),%arg_y)
4631	__(movq -node_size*2(%imm1),%arg_z)
4632	__(jne local_label(pop_arg_x))
4633	__(jmp *%ra0)
4634	/* lexpr count is one: set arg_z from the lexpr, */
4635	/* maybe vpop arg_y, arg_x */
4636	1: __(movq -node_size*1(%imm1),%arg_z)
4637	__(jmp local_label(maybe_pop_yx))
4638	_endsubp(spread_lexprz)
4639
4640
4641
4642
4643	/* Callback index in %r11 */
4644	_spentry(callback)
4645	__(push %rbp)
4646	__(movq %rsp,%rbp)
4647	/* C scalar args */
4648	__(push %carg0) /* -8(%rbp) */
4649	__(push %carg1)
4650	__(push %carg2)
4651	__(push %carg3)
4652	__ifndef(`WINDOWS')
4653	__(push %carg4)
4654	__(push %carg5)
4655	__endif
4656	/* FP arg regs */
4657	__ifdef(`WINDOWS')
4658	__(subq $4*8,%rsp)
4659	__(movq %xmm0,38(%rsp)) / -40(%rbp) */
4660	__(movq %xmm1,2*8(%rsp))
4661	__(movq %xmm2,1*8(%rsp))
4662	__(movq %xmm3,0*8(%rsp))
4663	__else
4664	__(subq $8*8,%rsp)
4665	__(movq %xmm0,78(%rsp)) / -56(%rbp) */
4666	__(movq %xmm1,6*8(%rsp))
4667	__(movq %xmm2,5*8(%rsp))
4668	__(movq %xmm3,4*8(%rsp))
4669	__(movq %xmm4,3*8(%rsp))
4670	__(movq %xmm5,2*8(%rsp))
4671	__(movq %xmm6,1*8(%rsp))
4672	__(movq %xmm7,0*8(%rsp))
4673	__endif
4674	__ifndef(`WINDOWS')
4675	__endif
4676	/* Save caller's mxcsr */
4677	__(subq $16,%rsp)
4678	__(stmxcsr (%rsp))
4679	__(andb $~mxcsr_all_exceptions,(%rsp))
4680	/* C NVRs */
4681	__(push %csave0)
4682	__(push %csave1)
4683	__(push %csave2)
4684	__(push %csave3)
4685	__(push %csave4)
4686	__ifdef(`WINDOWS')
4687	__(push %csave5)
4688	__(push %csave6)
4689	__endif
4690	__(push %rbp)
4691	__(movq %r11,%csave0)
4692	__ifdef(`HAVE_TLS')
4693	/* TCR initialized for lisp ? */
4694	__ifndef(`TCR_IN_GPR') /* FIXME */
4695	__(movq %fs:current_tcr@TPOFF,%rax)
4696	__(testq %rax,%rax)
4697	__(jne 1f)
4698	__endif
4699	__endif
4700	__(ref_global(get_tcr,%rax))
4701	__(movq $1,%carg0)
4702	__ifdef(`WINDOWS')
4703	__(sub $0x20, %rsp)
4704	__endif
4705	__(call *%rax)
4706	__ifdef(`WINDOWS')
4707	__(add $0x20, %rsp)
4708	__endif
4709	__ifdef(`TCR_IN_GPR')
4710	__(movq %rax, %rcontext_reg)
4711	__endif
4712	1: /* Align foreign stack for lisp */
4713	__(pushq rcontext(tcr.save_rbp)) /* mark cstack frame's "owner" */
4714	__(pushq rcontext(tcr.foreign_sp))
4715	/* init lisp registers */
4716	__(movq %csave0,%rax)
4717	__(movq %rsp,rcontext(tcr.foreign_sp))
4718	__ifndef(`TCR_IN_GPR')
4719	__(clr %save3)
4720	__endif
4721	__(clr %save2)
4722	__(clr %save1)
4723	__(clr %save0)
4724	__(clr %arg_z)
4725	__(clr %arg_y)
4726	__(clr %arg_x)
4727	__(clr %temp2)
4728	__(clr %temp1)
4729	__(clr %temp0)
4730	__(clr %fn)
4731	__(pxor %fpzero,%fpzero)
4732	__(movq rcontext(tcr.save_vsp),%rsp)
4733	__(box_fixnum(%rax,%arg_y))
4734	__(movq %rbp,%arg_z)
4735	__(movq rcontext(tcr.save_rbp),%rbp)
4736	__(movq $TCR_STATE_LISP,rcontext(tcr.valence))
4737	__(movq (%rsp),%save0)
4738	__(movq 8(%rsp),%save1)
4739	__(movq 16(%rsp),%save2)
4740	__ifndef(`TCR_IN_GPR')
4741	__(movq 24(%rsp),%save3)
4742	__endif
4743	__(ldmxcsr rcontext(tcr.lisp_mxcsr))
4744	__(movq $nrs.callbacks,%fname)
4745	__(lea local_label(back_from_callback)(%rip),%ra0)
4746	__(set_nargs(2))
4747	__(push %ra0)
4748	__(jump_fname())
4749	__(tra(local_label(back_from_callback)))
4750	__(movq %rsp,rcontext(tcr.save_vsp))
4751	__(movq %rbp,rcontext(tcr.save_rbp))
4752	__(movq rcontext(tcr.foreign_sp),%rsp)
4753	__(stmxcsr rcontext(tcr.lisp_mxcsr))
4754	__(movq $TCR_STATE_FOREIGN,rcontext(tcr.valence))
4755	__(emms)
4756	__(pop rcontext(tcr.foreign_sp))
4757	__(addq $node_size,%rsp)
4758	__(ldmxcsr rcontext(tcr.foreign_mxcsr))
4759	__(pop %rbp)
4760	__ifdef(`WINDOWS')
4761	__(pop %csave6)
4762	__(pop %csave5)
4763	__endif
4764	__(pop %csave4)
4765	__(pop %csave3)
4766	__(pop %csave2)
4767	__(pop %csave1)
4768	__(pop %csave0)
4769	__(ldmxcsr (%rsp))
4770	__(addq $16,%rsp)
4771	__(movq -8(%rbp),%rax)
4772	__(movq -16(%rbp),%rdx)
4773	__(movq -24(%rbp),%xmm0)
4774	__(movq -32(%rbp),%xmm1)
4775	__(leave)
4776	__(ret)
4777	_endsubp(callback)
4778
4779	/* arg_x = array, arg_y = i, arg_z = j. Typecheck everything.
4780	We don't know whether the array is alleged to be simple or
4781	not, and don't know anythng about the element type. */
4782
4783	_spentry(aref2)
4784	__(testb $fixnummask,%arg_y_b)
4785	__(jne 0f)
4786
4787	__(testb $fixnummask,%arg_z_b)
4788	__(jne 1f)
4789	__(extract_typecode(%arg_x,%imm0))
4790	__(cmpb $subtag_arrayH,%imm0_b)
4791	__(jne 2f)
4792	__(cmpq $2<<fixnumshift,arrayH.rank(%arg_x))
4793	__(jne 2f)
4794	__(cmpq arrayH.dim0(%arg_x),%arg_y)
4795	__(jae 3f)
4796	__(movq arrayH.dim0+node_size(%arg_x),%imm0)
4797	__(cmpq %imm0,%arg_z)
4798	__(jae 4f)
4799	__(unbox_fixnum(%imm0,%imm0))
4800	__(mulq %arg_y) /* imm0 <- imm0 * arg_y */
4801	__(addq %imm0,%arg_z)
4802	__(movq %arg_x,%arg_y)
4803	6: __(addq arrayH.displacement(%arg_y),%arg_z)
4804	__(movq arrayH.data_vector(%arg_y),%arg_y)
4805	__(extract_subtag(%arg_y,%imm1))
4806	__(cmpb $subtag_vectorH,%imm1_b)
4807	__(ja C(misc_ref_common))
4808	__(jmp 6b)
4809	0: __(uuo_error_reg_not_fixnum(Rarg_y))
4810	1: __(uuo_error_reg_not_fixnum(Rarg_z))
4811	2: __(uuo_error_reg_not_type(Rarg_x,error_object_not_array_2d))
4812	3: __(uuo_error_array_bounds(Rarg_y,Rarg_x))
4813	4: __(uuo_error_array_bounds(Rarg_z,Rarg_x))
4814
4815	_endsubp(aref2)
4816
4817	/* %temp0 = array, %arg_x = i,%arg_y = j, %arg_z = k */
4818	_spentry(aref3)
4819	__(testb $fixnummask,%arg_x_b)
4820	__(jne 0f)
4821	__(testb $fixnummask,%arg_y_b)
4822	__(jne 1f)
4823	__(testb $fixnummask,%arg_z_b)
4824	__(jne 2f)
4825	__(extract_typecode(%temp0,%imm0))
4826	__(cmpb $subtag_arrayH,%imm0_b)
4827	__(jne 3f)
4828	__(cmpq $3<<fixnumshift,arrayH.rank(%temp0))
4829	__(jne 3f)
4830	__(cmpq arrayH.dim0(%temp0),%arg_x)
4831	__(jae 5f)
4832	__(movq arrayH.dim0+node_size(%temp0),%imm0)
4833	__(cmpq %imm0,%arg_y)
4834	__(jae 6f)
4835	__(unbox_fixnum(%imm0,%imm0))
4836	__(movq arrayH.dim0+(node_size*2)(%temp0),%imm1)
4837	__(cmpq %imm1,%arg_z)
4838	__(jae 7f)
4839	__(unbox_fixnum(%imm1,%imm1))
4840	__(imulq %imm1,%arg_y)
4841	__(mulq %imm1)
4842	__(imulq %imm0,%arg_x)
4843	__(addq %arg_x,%arg_z)
4844	__(addq %arg_y,%arg_z)
4845	__(movq %temp0,%arg_y)
4846	8: __(addq arrayH.displacement(%arg_y),%arg_z)
4847	__(movq arrayH.data_vector(%arg_y),%arg_y)
4848	__(extract_subtag(%arg_y,%imm1))
4849	__(cmpb $subtag_vectorH,%imm1_b)
4850	__(ja C(misc_ref_common))
4851	__(jmp 8b)
4852	0: __(uuo_error_reg_not_fixnum(Rarg_x))
4853	1: __(uuo_error_reg_not_fixnum(Rarg_y))
4854	2: __(uuo_error_reg_not_fixnum(Rarg_z))
4855	3: __(uuo_error_reg_not_type(Rtemp0,error_object_not_array_3d))
4856	5: __(uuo_error_array_bounds(Rarg_x,Rtemp0))
4857	6: __(uuo_error_array_bounds(Rarg_y,Rtemp0))
4858	7: __(uuo_error_array_bounds(Rarg_z,Rtemp0))
4859
4860	_endsubp(aref3)
4861
4862	/* As with aref2, but temp0 = array, arg_x = i, arg_y = j, arg_z = new_value */
4863	_spentry(aset2)
4864	__(testb $fixnummask,%arg_x_b)
4865	__(jne 0f)
4866	__(testb $fixnummask,%arg_y_b)
4867	__(jne 1f)
4868	__(extract_typecode(%temp0,%imm0))
4869	__(cmpb $subtag_arrayH,%imm0_b)
4870	__(jne 2f)
4871	__(cmpq $2<<fixnumshift,arrayH.rank(%temp0))
4872	__(jne 2f)
4873	__(cmpq arrayH.dim0(%temp0),%arg_x)
4874	__(jae 4f)
4875	__(movq arrayH.dim0+node_size(%temp0),%imm0)
4876	__(cmpq %imm0,%arg_y)
4877	__(jae 5f)
4878	__(unbox_fixnum(%imm0,%imm0))
4879	__(mulq %arg_x) /* imm0 <- imm0 * arg_x */
4880	__(addq %imm0,%arg_y)
4881	__(movq %temp0,%arg_x)
4882	6: __(addq arrayH.displacement(%arg_x),%arg_y)
4883	__(movq arrayH.data_vector(%arg_x),%arg_x)
4884	__(extract_subtag(%arg_x,%imm1))
4885	__(cmpb $subtag_vectorH,%imm1_b)
4886	__(ja C(misc_set_common))
4887	__(jmp 6b)
4888	0: __(uuo_error_reg_not_fixnum(Rarg_x))
4889	1: __(uuo_error_reg_not_fixnum(Rarg_y))
4890	2: __(uuo_error_reg_not_type(Rtemp0,error_object_not_array_2d))
4891	4: __(uuo_error_array_bounds(Rarg_x,Rtemp0))
4892	5: __(uuo_error_array_bounds(Rarg_y,Rtemp0))
4893	_endsubp(aset2)
4894
4895	/* %temp1 = array, %temp0 = i, %arg_x = j, %arg_y = k, %arg_y = newval. */
4896
4897	_spentry(aset3)
4898	__(testb $fixnummask,%temp0_b)
4899	__(jne 0f)
4900	__(testb $fixnummask,%arg_x_b)
4901	__(jne 1f)
4902	__(testb $fixnummask,%arg_y_b)
4903	__(jne 2f)
4904	__(extract_typecode(%temp1,%imm0))
4905	__(cmpb $subtag_arrayH,%imm0_b)
4906	__(jne 3f)
4907	__(cmpq $3<<fixnumshift,arrayH.rank(%temp1))
4908	__(jne 3f)
4909	__(cmpq arrayH.dim0(%temp1),%temp0)
4910	__(jae 5f)
4911	__(movq arrayH.dim0+node_size(%temp1),%imm0)
4912	__(cmpq %imm0,%arg_x)
4913	__(jae 6f)
4914	__(unbox_fixnum(%imm0,%imm0))
4915	__(movq arrayH.dim0+(node_size*2)(%temp1),%imm1)
4916	__(cmpq %imm1,%arg_y)
4917	__(jae 7f)
4918	__(unbox_fixnum(%imm1,%imm1))
4919	__(imulq %imm1,%arg_x)
4920	__(mulq %imm1)
4921	__(imulq %imm0,%temp0)
4922	__(addq %temp0,%arg_y)
4923	__(addq %arg_x,%arg_y)
4924	__(movq %temp1,%arg_x)
4925	8: __(addq arrayH.displacement(%arg_x),%arg_y)
4926	__(movq arrayH.data_vector(%arg_x),%arg_x)
4927	__(extract_subtag(%arg_x,%imm1))
4928	__(cmpb $subtag_vectorH,%imm1_b)
4929	__(ja C(misc_set_common))
4930	__(jmp 8b)
4931
4932	0: __(uuo_error_reg_not_fixnum(Rtemp0))
4933	1: __(uuo_error_reg_not_fixnum(Rarg_x))
4934	2: __(uuo_error_reg_not_fixnum(Rarg_y))
4935	3: __(uuo_error_reg_not_type(Rtemp1,error_object_not_array_3d))
4936	5: __(uuo_error_array_bounds(Rtemp0,Rtemp1))
4937	6: __(uuo_error_array_bounds(Rarg_x,Rtemp1))
4938	6: __(uuo_error_array_bounds(Rarg_x,Rtemp1))
4939	7: __(uuo_error_array_bounds(Rarg_y,Rtemp1))
4940
4941	_endsubp(aset3)
4942
4943
4944
4945
4946	/* Prepend all but the first five (4 words of code, inner fn) and last */
4947	/* (lfbits) elements of %fn to the "arglist". */
4948
4949	_spentry(call_closure)
4950	new_local_labels()
4951	__(subq $fulltag_function-fulltag_misc,%fn)
4952	__(vector_length(%fn,%imm0))
4953
4954	__(subq $6<<fixnumshift,%imm0) /* imm0 = inherited arg count */
4955	__(lea (%nargs_q,%imm0),%imm1)
4956	__(cmpl $nargregs<<fixnumshift,%imm1_l)
4957	__(jna local_label(regs_only))
4958	__(pop %ra0)
4959	__(cmpl $nargregs<<fixnumshift,%nargs)
4960	__(jna local_label(no_insert))
4961
4962	/* Some arguments have already been pushed. Push imm0's worth */
4963	/* of NILs, copy those arguments that have already been vpushed from */
4964	/* the old TOS to the new, then insert all of the inerited args */
4965	/* and go to the function. */
4966
4967	__(movq %imm0,%imm1)
4968	local_label(push_nil_loop):
4969	__(push $nil_value)
4970	__(sub $fixnumone,%imm1)
4971	__(jne local_label(push_nil_loop))
4972
4973	/* Need to use arg regs as temporaries here. */
4974	__(movq %rsp,%temp1)
4975	__(push %arg_z)
4976	__(push %arg_y)
4977	__(push %arg_x)
4978	__(lea 3*node_size(%rsp,%imm0),%arg_x)
4979	__(lea -nargregs<<fixnumshift(%nargs_q),%arg_y)
4980	local_label(copy_already_loop):
4981	__(movq (%arg_x),%arg_z)
4982	__(addq $fixnumone,%arg_x)
4983	__(movq %arg_z,(%temp1))
4984	__(addq $fixnumone,%temp1)
4985	__(subq $fixnumone,%arg_y)
4986	__(jne local_label(copy_already_loop))
4987
4988	__(movl $5<<fixnumshift,%imm1_l) /* skip code, new fn */
4989	local_label(insert_loop):
4990	__(movq misc_data_offset(%fn,%imm1),%arg_z)
4991	__(addq $node_size,%imm1)
4992	__(addl $fixnum_one,%nargs)
4993	__(subq $node_size,%arg_x)
4994	__(movq %arg_z,(%arg_x))
4995	__(subq $fixnum_one,%imm0)
4996	__(jne local_label(insert_loop))
4997
4998	/* Recover the argument registers, pushed earlier */
4999	__(pop %arg_x)
5000	__(pop %arg_y)
5001	__(pop %arg_z)
5002	__(jmp local_label(go))
5003
5004	/* Here if nothing was pushed by the caller. If we're */
5005	/* going to push anything, we have to reserve a stack */
5006	/* frame first. (We'll need to push something if the */
5007	/* sum of %nargs and %imm0 is greater than nargregs) */
5008
5009	local_label(no_insert):
5010	__(lea (%nargs_q,%imm0),%imm1)
5011	__(cmpq $nargregs<<fixnumshift,%imm1)
5012	__(jna local_label(no_insert_no_frame))
5013	/* Reserve space for a stack frame */
5014	__(push $reserved_frame_marker)
5015	__(push $reserved_frame_marker)
5016	local_label(no_insert_no_frame):
5017	/* nargregs or fewer args were already vpushed. */
5018	/* if exactly nargregs, vpush remaining inherited vars. */
5019	__(cmpl $nargregs<<fixnumshift,%nargs)
5020	__(movl $5<<fixnumshift,%imm1_l) /* skip code, new fn */
5021	__(leaq 5<<fixnumshift(%imm0),%temp1)
5022	__(jnz local_label(set_regs))
5023	local_label(vpush_remaining):
5024	__(push misc_data_offset(%fn,%imm1))
5025	__(addq $node_size,%imm1)
5026	__(addl $fixnumone,%nargs)
5027	__(subq $node_size,%imm0)
5028	__(jnz local_label(vpush_remaining))
5029	__(jmp local_label(go))
5030	local_label(set_regs):
5031	/* if nargs was > 1 (and we know that it was < 3), it must have */
5032	/* been 2. Set arg_x, then vpush the remaining args. */
5033	__(cmpl $fixnumone,%nargs)
5034	__(jle local_label(set_y_z))
5035	local_label(set_arg_x):
5036	__(subq $node_size,%temp1)
5037	__(movq misc_data_offset(%fn,%temp1),%arg_x)
5038	__(addl $fixnumone,%nargs)
5039	__(subq $fixnumone,%imm0)
5040	__(jne local_label(vpush_remaining))
5041	__(jmp local_label(go))
5042	/* Maybe set arg_y or arg_z, preceding args */
5043	local_label(set_y_z):
5044	__(jne local_label(set_arg_z))
5045	/* Set arg_y, maybe arg_x, preceding args */
5046	local_label(set_arg_y):
5047	__(subq $node_size,%temp1)
5048	__(movq misc_data_offset(%fn,%temp1),%arg_y)
5049	__(addl $fixnumone,%nargs)
5050	__(subq $fixnum_one,%imm0)
5051	__(jnz local_label(set_arg_x))
5052	__(jmp local_label(go))
5053	local_label(set_arg_z):
5054	__(subq $node_size,%temp1)
5055	__(movq misc_data_offset(%fn,%temp1),%arg_z)
5056	__(addl $fixnumone,%nargs)
5057	__(subq $fixnum_one,%imm0)
5058	__(jne local_label(set_arg_y))
5059	local_label(go):
5060	__(movq misc_data_offset+(4*node_size)(%fn),%fn)
5061	__(push %ra0)
5062	__(jmp *%fn)
5063	local_label(regs_only):
5064	__(leaq 5<<fixnumshift(%imm0),%temp1)
5065	__(testl %nargs,%nargs)
5066	__(jne local_label(some_args))
5067	__(cmpw $node_size,%imm0)
5068	__(movq misc_data_offset-node_size(%fn,%temp1),%arg_z)
5069	__(je local_label(rgo))
5070	__(cmpw $2*node_size,%imm0)
5071	__(movq misc_data_offset-(node_size*2)(%fn,%temp1),%arg_y)
5072	__(je local_label(rgo))
5073	__(movq misc_data_offset-(node_size*3)(%fn,%temp1),%arg_x)
5074	local_label(rgo):
5075	__(addw %imm0_w,%nargs_w)
5076	__(jmp misc_data_offset+(4node_size)(%fn))
5077	local_label(some_args):
5078	__(cmpl $2*node_size,%nargs)
5079	__(jz local_label(rtwo))
5080	/* One arg was passed, could be one or two inherited args */
5081	__(cmpw $node_size,%imm0)
5082	__(movq misc_data_offset-node_size(%fn,%temp1),%arg_y)
5083	__(je local_label(rgo))
5084	__(movq misc_data_offset-(node_size*2)(%fn,%temp1),%arg_x)
5085	__(jmp local_label(rgo))
5086	local_label(rtwo):
5087	__(movq misc_data_offset-node_size(%fn,%temp1),%arg_x)
5088	__(jmp local_label(rgo))
5089	_endsubp(call_closure)
5090
5091
5092	_spentry(poweropen_callbackX)
5093	_endsubp(poweropen_callbackX)
5094
5095
5096	_spentry(poweropen_ffcallX)
5097	_endsubp(poweropen_ffcallX)
5098
5099	_spentry(poweropen_syscall)
5100	_endsubp(poweropen_syscall)
5101
5102	_spentry(eabi_ff_call)
5103	_endsubp(eabi_ff_call)
5104
5105	_spentry(eabi_callback)
5106	_endsubp(eabi_callback)
5107
5108
5109	/* Unused, and often not used on PPC either */
5110	_spentry(callbuiltin)
5111	__(hlt)
5112	_endsubp(callbuiltin)
5113
5114	_spentry(callbuiltin0)
5115	__(hlt)
5116	_endsubp(callbuiltin0)
5117
5118	_spentry(callbuiltin1)
5119	__(hlt)
5120	_endsubp(callbuiltin1)
5121
5122	_spentry(callbuiltin2)
5123	__(hlt)
5124	_endsubp(callbuiltin2)
5125
5126	_spentry(callbuiltin3)
5127	__(hlt)
5128	_endsubp(callbuiltin3)
5129
5130	_spentry(restorefullcontext)
5131	__(hlt)
5132	_endsubp(restorefullcontext)
5133
5134	_spentry(savecontextvsp)
5135	__(hlt)
5136	_endsubp(savecontextvsp)
5137
5138	_spentry(savecontext0)
5139	__(hlt)
5140	_endsubp(savecontext0)
5141
5142	_spentry(restorecontext)
5143	__(hlt)
5144	_endsubp(restorecontext)
5145
5146	_spentry(stkconsyz)
5147	__(hlt)
5148	_endsubp(stkconsyz)
5149
5150	_spentry(stkvcell0)
5151	__(hlt)
5152	_endsubp(stkvcell0)
5153
5154	_spentry(stkvcellvsp)
5155	__(hlt)
5156	_endsubp(stkvcellvsp)
5157
5158	_spentry(breakpoint)
5159	__(hlt)
5160	_endsubp(breakpoint)
5161
5162
5163	__ifdef(`DARWIN')
5164	.if 1
5165	.globl C(lisp_objc_personality)
5166	C(lisp_objc_personality):
5167	jmp *lisp_global(objc_2_personality)
5168
5169	.section __TEXT,__eh_frame,coalesced,no_toc+strip_static_syms+live_support
5170	EH_frame1:
5171	.set L$set$12,LECIE1-LSCIE1
5172	.long L$set$12 /* Length of Common Information Entry */
5173	LSCIE1:
5174	.long 0x0 /* CIE Identifier Tag */
5175	.byte 0x1 /* CIE Version */
5176	.ascii "zPLR\0" /* CIE Augmentation */
5177	.byte 0x1 /* uleb128 0x1; CIE Code Alignment Factor */
5178	.byte 0x78 /* sleb128 -8; CIE Data Alignment Factor */
5179	.byte 0x10 /* CIE RA Column */
5180	.byte 0x7
5181	.byte 0x9b
5182	.long _lisp_objc_personality+4@GOTPCREL
5183	.byte 0x10 /* LSDA Encoding (pcrel) */
5184	.byte 0x10 /* FDE Encoding (pcrel) */
5185	.byte 0xc /* DW_CFA_def_cfa */
5186	.byte 0x7 /* uleb128 0x7 */
5187	.byte 0x8 /* uleb128 0x8 */
5188	.byte 0x90 /* DW_CFA_offset, column 0x10 */
5189	.byte 0x1 /* uleb128 0x1 */
5190	.align 3
5191	LECIE1:
5192	.globl _SPffcall.eh
5193	_SPffcall.eh:
5194	.long LEFDEffcall-LSFDEffcall
5195	LSFDEffcall:
5196	.long LSFDEffcall-EH_frame1 /* FDE CIE offset */
5197	.quad Lffcall-. /* FDE Initial Location */
5198	.quad Lffcall_end-Lffcall /* FDE address range */
5199	.byte 8 /* uleb128 0x8; Augmentation size */
5200	.quad LLSDA1-. /* Language Specific Data Area */
5201	.byte 0x4 /* DW_CFA_advance_loc4 */
5202	.long Lffcall_setup-Lffcall
5203	.byte 0xe /* DW_CFA_def_cfa_offset */
5204	.byte 0x10 /* uleb128 0x10 */
5205	.byte 0x86 /* DW_CFA_offset, column 0x6 */
5206	.byte 0x2 /* uleb128 0x2 */
5207	.byte 0x4 /* DW_CFA_advance_loc4 */
5208	.long Lffcall_setup_end-Lffcall_setup
5209	.byte 0xd /* DW_CFA_def_cfa_register */
5210	.byte 0x6 /* uleb128 0x6 */
5211	.byte 0x4 /* DW_CFA_advance_loc4 */
5212	.long Lffcall_call_end-Lffcall_call
5213	.byte 0x83 /* DW_CFA_offset, column 0x3 */
5214	.byte 0x3 /* uleb128 0x3 */
5215	.align 3
5216	LEFDEffcall:
5217	.globl _SPffcall_return_registers.eh
5218	_SPffcall_return_registers.eh:
5219	.long LEFDEffcall_return_registers-LSFDEffcall_return_registers
5220	LSFDEffcall_return_registers:
5221	.long LSFDEffcall_return_registers-EH_frame1 /* FDE CIE offset */
5222	.quad Lffcall_return_registers-. /* FDE Initial Location */
5223	.quad Lffcall_return_registers_end-Lffcall_return_registers /* FDE address range */
5224	.byte 8 /* uleb128 0x8; Augmentation size */
5225	.quad LLSDA2-. /* Language Specific Data Area */
5226	.byte 0x4 /* DW_CFA_advance_loc4 */
5227	.long Lffcall_return_registers_setup-Lffcall_return_registers
5228	.byte 0xe /* DW_CFA_def_cfa_offset */
5229	.byte 0x10 /* uleb128 0x10 */
5230	.byte 0x86 /* DW_CFA_offset, column 0x6 */
5231	.byte 0x2 /* uleb128 0x2 */
5232	.byte 0x4 /* DW_CFA_advance_loc4 */
5233	.long Lffcall_return_registers_setup_end-Lffcall_return_registers_setup
5234	.byte 0xd /* DW_CFA_def_cfa_register */
5235	.byte 0x6 /* uleb128 0x6 */
5236	.byte 0x4 /* DW_CFA_advance_loc4 */
5237	.long Lffcall_return_registers_call_end-Lffcall_return_registers_call
5238	.byte 0x83 /* DW_CFA_offset, column 0x3 */
5239	.byte 0x3 /* uleb128 0x3 */
5240	.align 3
5241	LEFDEffcall_return_registers:
5242	.text
5243	.endif
5244	__endif
5245
5246	_spentry(unused_5)
5247	__(hlt)
5248	Xspentry_end:
5249	_endsubp(unused_5)
5250
5251	.data
5252	.globl C(spentry_start)
5253	.globl C(spentry_end)
5254	C(spentry_start): .quad Xspentry_start
5255	C(spentry_end): .quad Xspentry_end

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: trunk/source/lisp-kernel/x86-spentry64.s @ 15572

Download in other formats: