Context Navigation

← Previous Revision
Next Revision →
Normal
Revision Log

close Warning: Can't use blame annotator:
No changeset 2899 in the repository

x8664-vinsns.lisp

Visit:

Last change on this file was 16772, checked in by R. Matthew Emerson, 8 years ago

Merge fix for ticket:1335 (r16675) to 1.11 release branch.

Closes ticket:1335.

Property svn:eol-style set to native
Property svn:keywords set to Author Date Id Revision

File size: 188.3 KB

Rev	Line
	1	;;-- Mode: Lisp; Package: CCL --
	2	;;;
	3	;;; Copyright 2005-2009 Clozure Associates
	4	;;;
	5	;;; Licensed under the Apache License, Version 2.0 (the "License");
	6	;;; you may not use this file except in compliance with the License.
	7	;;; You may obtain a copy of the License at
	8	;;;
	9	;;; http://www.apache.org/licenses/LICENSE-2.0
	10	;;;
	11	;;; Unless required by applicable law or agreed to in writing, software
	12	;;; distributed under the License is distributed on an "AS IS" BASIS,
	13	;;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	14	;;; See the License for the specific language governing permissions and
	15	;;; limitations under the License.
	16
	17	(in-package "CCL")
	18
	19	(eval-when (:compile-toplevel :load-toplevel :execute)
	20	(require "VINSN")
	21	(require "X8664-BACKEND"))
	22
	23	(eval-when (:compile-toplevel :execute)
	24	(require "X8664ENV"))
	25
	26	(defmacro define-x8664-vinsn (vinsn-name (results args &optional temps) &body body)
	27	(%define-vinsn x8664-backend vinsn-name results args temps body))
	28
	29
	30
	31	(define-x8664-vinsn scale-32bit-misc-index (((dest :u64))
	32	((idx :imm) ; A fixnum
	33	)
	34	())
	35	(movq (:%q idx) (:%q dest))
	36	(shrq (:$1 1) (:%q dest)))
	37
	38	(define-x8664-vinsn scale-16bit-misc-index (((dest :u64))
	39	((idx :imm) ; A fixnum
	40	)
	41	())
	42	(movq (:%q idx) (:%q dest))
	43	(shrq (:$ub 2) (:%q dest)))
	44
	45	(define-x8664-vinsn scale-8bit-misc-index (((dest :u64))
	46	((idx :imm) ; A fixnum
	47	)
	48	())
	49	(movq (:%q idx) (:%q dest))
	50	(shrq (:$ub 3) (:%q dest)))
	51
	52	;;; same as above, but looks better in bit vector contexts
	53	(define-x8664-vinsn scale-1bit-misc-index (((dest :u64))
	54	((idx :imm) ; A fixnum
	55	)
	56	())
	57	(movq (:%q idx) (:%q dest))
	58	(shrq (:$ub 3) (:%q dest)))
	59
	60	(define-x8664-vinsn misc-ref-u64 (((dest :u64))
	61	((v :lisp)
	62	(scaled-idx :imm)))
	63	(movq (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx)) (:%q dest)))
	64
	65	(define-x8664-vinsn misc-ref-double-float (((dest :double-float))
	66	((v :lisp)
	67	(scaled-idx :imm)))
	68	(movsd (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx)) (:%xmm dest)))
	69
	70	(define-x8664-vinsn misc-ref-complex-single-float (((dest :complex-single-float))
	71	((v :lisp)
	72	(scaled-idx :imm)))
	73	(movq (:@ x8664::complex-single-float.realpart (:%q v) (:%q scaled-idx)) (:%xmm dest)))
	74
	75	(define-x8664-vinsn misc-ref-complex-double-float (((dest :complex-double-float))
	76	((v :lisp)
	77	(scaled-idx :imm)))
	78	(movapd (:@ x8664::complex-double-float.realpart (:%q v) (:%q scaled-idx) 2) (:%xmm dest)))
	79
	80	(define-x8664-vinsn misc-ref-c-double-float (((dest :double-float))
	81	((v :lisp)
	82	(idx :s32const)))
	83	(movsd (:@ (:apply + x8664::misc-data-offset (:apply ash idx x8664::word-shift)) (:%q v)) (:%xmm dest)))
	84
	85
	86	(define-x8664-vinsn misc-ref-c-complex-single-float (((dest :complex-single-float))
	87	((v :lisp)
	88	(idx :s32const)))
	89	(movq (:@ (:apply + x8664::complex-single-float.realpart (:apply ash idx x8664::word-shift)) (:%q v)) (:%xmm dest)))
	90
	91	(define-x8664-vinsn misc-ref-node (((dest :lisp))
	92	((v :lisp)
	93	(scaled-idx :imm)))
	94	(movq (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx)) (:%q dest)))
	95
	96	(define-x8664-vinsn (push-misc-ref-node :push :node :vsp) (()
	97	((v :lisp)
	98	(scaled-idx :imm)))
	99	(pushq (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx))))
	100
	101	(define-x8664-vinsn misc-set-node (()
	102	((val :lisp)
	103	(v :lisp)
	104	(unscaled-idx :imm))
	105	())
	106	(movq (:%q val) (:@ x8664::misc-data-offset (:%q v) (:%q unscaled-idx))))
	107
	108	(define-x8664-vinsn misc-set-immediate-node (()
	109	((val :s32const)
	110	(v :lisp)
	111	(unscaled-idx :imm))
	112	())
	113	(movq (:$l val) (:@ x8664::misc-data-offset (:%q v) (:%q unscaled-idx))))
	114
	115
	116	(define-x8664-vinsn misc-set-double-float (()
	117	((val :double-float)
	118	(v :lisp)
	119	(unscaled-idx :imm))
	120	())
	121	(movsd (:%xmm val) (:@ x8664::misc-data-offset (:%q v) (:%q unscaled-idx))))
	122
	123	(define-x8664-vinsn misc-set-complex-single-float (()
	124	((val :complex-single-float)
	125	(v :lisp)
	126	(unscaled-idx :imm))
	127	())
	128	(movq (:%xmm val) (:@ x8664::complex-single-float.realpart (:%q v) (:%q unscaled-idx))))
	129
	130	(define-x8664-vinsn misc-set-complex-double-float (()
	131	((val :complex-double-float)
	132	(v :lisp)
	133	(unscaled-idx :imm))
	134	())
	135	(movapd (:%xmm val) (:@ x8664::complex-double-float.realpart (:%q v) (:%q unscaled-idx) 2)))
	136
	137	(define-x8664-vinsn misc-ref-u8 (((dest :u8))
	138	((v :lisp)
	139	(scaled-idx :s64)))
	140	(movzbl (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx)) (:%l dest)))
	141
	142	(define-x8664-vinsn misc-ref-s8 (((dest :s8))
	143	((v :lisp)
	144	(scaled-idx :s64)))
	145	(movsbq (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx)) (:%q dest)))
	146
	147	(define-x8664-vinsn misc-ref-u16 (((dest :u16))
	148	((v :lisp)
	149	(scaled-idx :s64)))
	150	(movzwl (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx)) (:%l dest)))
	151
	152	(define-x8664-vinsn misc-ref-u32 (((dest :u32))
	153	((v :lisp)
	154	(scaled-idx :s64)))
	155	(movl (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx)) (:%l dest)))
	156
	157
	158	(define-x8664-vinsn misc-ref-single-float (((dest :single-float))
	159	((v :lisp)
	160	(scaled-idx :s64)))
	161	(movss(:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx)) (:%xmm dest)))
	162
	163	(define-x8664-vinsn misc-ref-s32 (((dest :s32))
	164	((v :lisp)
	165	(scaled-idx :s64)))
	166	(movslq (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx)) (:%q dest)))
	167
	168	(define-x8664-vinsn misc-ref-s16 (((dest :s16))
	169	((v :lisp)
	170	(scaled-idx :s64)))
	171	(movswq (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx)) (:%q dest)))
	172
	173	(define-x8664-vinsn misc-ref-s64 (((dest :s64))
	174	((v :lisp)
	175	(scaled-idx :imm)))
	176	(movq (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx)) (:%q dest)))
	177
	178
	179	(define-x8664-vinsn misc-ref-c-node (((dest :lisp))
	180	((v :lisp)
	181	(idx :u32const)) ; sic
	182	())
	183	(movq (:@ (:apply + x8664::misc-data-offset (:apply ash idx x8664::word-shift)) (:%q v)) (:%q dest)))
	184
	185
	186	(define-x8664-vinsn (push-misc-ref-c-node :push :node :vsp)
	187	(()
	188	((v :lisp)
	189	(idx :u32const)) ; sic
	190	())
	191	(pushq (:@ (:apply + x8664::misc-data-offset (:apply ash idx x8664::word-shift)) (:%q v))))
	192
	193	(define-x8664-vinsn misc-ref-c-u64 (((dest :u64))
	194	((v :lisp)
	195	(idx :u32const)) ; sic
	196	())
	197	(movq (:@ (:apply + x8664::misc-data-offset (:apply ash idx x8664::word-shift)) (:%q v)) (:%q dest)))
	198
	199
	200	(define-x8664-vinsn misc-ref-c-s64 (((dest :s64))
	201	((v :lisp)
	202	(idx :s32const)) ; sic
	203	())
	204	(movq (:@ (:apply + x8664::misc-data-offset (:apply ash idx x8664::word-shift)) (:%q v)) (:%q dest)))
	205
	206
	207	(define-x8664-vinsn misc-ref-c-u32 (((dest :u32))
	208	((v :lisp)
	209	(idx :u32const)) ; sic
	210	())
	211	(movl (:@ (:apply + x8664::misc-data-offset (:apply ash idx 2)) (:%q v)) (:%l dest)))
	212
	213	(define-x8664-vinsn misc-ref-c-s32 (((dest :s32))
	214	((v :lisp)
	215	(idx :s32const)) ; sic
	216	())
	217	(movslq (:@ (:apply + x8664::misc-data-offset (:apply ash idx x8664::word-shift)) (:%q v)) (:%q dest)))
	218
	219	(define-x8664-vinsn misc-ref-c-single-float (((dest :single-float))
	220	((v :lisp)
	221	(idx :s32const)) ; sic
	222	())
	223	(movss (:@ (:apply + x8664::misc-data-offset (:apply ash idx 2)) (:%q v)) (:%xmm dest)))
	224
	225	(define-x8664-vinsn misc-ref-c-u8 (((dest :u64))
	226	((v :lisp)
	227	(idx :s32const)) ; sic
	228	())
	229	(movzbl (:@ (:apply + x8664::misc-data-offset idx) (:%q v)) (:%l dest)))
	230
	231	(define-x8664-vinsn misc-ref-c-s8 (((dest :s64))
	232	((v :lisp)
	233	(idx :s32const)) ; sic
	234	())
	235	(movsbq (:@ (:apply + x8664::misc-data-offset idx) (:%q v)) (:%q dest)))
	236
	237	(define-x8664-vinsn misc-set-u64 (()
	238	((val :u64)
	239	(v :lisp)
	240	(idx :u64)))
	241	(movq (:%q val) (:@ x8664::misc-data-offset (:%q v) (:%q idx))))
	242
	243	(define-x8664-vinsn misc-set-immediate-u64 (()
	244	((val :u32const)
	245	(v :lisp)
	246	(idx :u64)))
	247	(movq (:$l val) (:@ x8664::misc-data-offset (:%q v) (:%q idx))))
	248
	249	(define-x8664-vinsn misc-set-c-u64 (()
	250	((val :u64)
	251	(v :lisp)
	252	(idx :u32const)))
	253	(movq (:%q val) (:@ (:apply + x8664::misc-data-offset (:apply ash idx 3)) (:%q v))))
	254
	255	(define-x8664-vinsn misc-set-immediate-c-u64 (()
	256	((val :u32const)
	257	(v :lisp)
	258	(idx :u32const)))
	259	(movq (:$l val) (:@ (:apply + x8664::misc-data-offset (:apply ash idx 3)) (:%q v))))
	260
	261	(define-x8664-vinsn misc-set-s64 (()
	262	((val :s64)
	263	(v :lisp)
	264	(scaled-idx :imm)))
	265	(movq (:%q val) (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx))))
	266
	267	(define-x8664-vinsn misc-set-immediate-s64 (()
	268	((val :s32const)
	269	(v :lisp)
	270	(scaled-idx :imm)))
	271	(movq (:$l val) (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx))))
	272
	273
	274	(define-x8664-vinsn misc-set-c-s64 (()
	275	((val :s64)
	276	(v :lisp)
	277	(idx :s32const)))
	278	(movq (:%q val) (:@ (:apply + x8664::misc-data-offset (:apply ash idx 3)) (:%q v))))
	279
	280	(define-x8664-vinsn misc-set-immediate-c-s64 (()
	281	((val :s32const)
	282	(v :lisp)
	283	(idx :s32const)))
	284	(movq (:$l val) (:@ (:apply + x8664::misc-data-offset (:apply ash idx 3)) (:%q v))))
	285
	286
	287	(define-x8664-vinsn misc-set-c-node (()
	288	((val :lisp)
	289	(v :lisp)
	290	(idx :s32const)))
	291	(movq (:%q val) (:@ (:apply + x8664::misc-data-offset (:apply ash idx 3)) (:%q v))))
	292
	293
	294
	295
	296	(define-x8664-vinsn misc-set-immediate-c-node (()
	297	((val :s32const)
	298	(v :lisp)
	299	(idx :s32const)))
	300	(movq (:$l val) (:@ (:apply + x8664::misc-data-offset (:apply ash idx 3)) (:%q v))))
	301
	302	(define-x8664-vinsn set-closure-forward-reference (()
	303	((val :lisp)
	304	(closure :lisp)
	305	(idx :s32const)))
	306	(movq (:%q val) (:@ (:apply + x8664::misc-function-offset (:apply ash idx x8664::word-shift)) (:%q closure))))
	307
	308
	309	(define-x8664-vinsn misc-set-c-double-float (()
	310	((val :double-float)
	311	(v :lisp)
	312	(idx :s32const)))
	313	(movsd (:%xmm val) (:@ (:apply + x8664::misc-data-offset (:apply ash idx 3)) (:%q v))))
	314
	315
	316
	317	(define-x8664-vinsn (call-known-symbol :call) (((result (:lisp x8664::arg_z)))
	318	()
	319	((entry (:label 1))))
	320	(:talign 4)
	321	(call (:@ x8664::symbol.fcell (:% x8664::fname)))
	322	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	323
	324
	325	(define-x8664-vinsn (jump-known-symbol :jumplr) (()
	326	())
	327
	328	(jmp (:@ x8664::symbol.fcell (:% x8664::fname))))
	329
	330	(define-x8664-vinsn set-nargs (()
	331	((n :s16const)))
	332	((:pred = n 0)
	333	(xorl (:%l x8664::nargs ) (:%l x8664::nargs )))
	334	((:not (:pred = n 0))
	335	(movl (:$l (:apply ash n x8664::word-shift)) (:%l x8664::nargs ))))
	336
	337	(define-x8664-vinsn check-exact-nargs (()
	338	((n :u16const)))
	339	:resume
	340	((:pred = n 0)
	341	(testl (:%l x8664::nargs) (:%l x8664::nargs)))
	342	((:not (:pred = n 0))
	343	((:pred < n 16)
	344	(cmpl (:$b (:apply ash n x8664::word-shift)) (:%l x8664::nargs)))
	345	((:pred >= n 16)
	346	(cmpl (:$l (:apply ash n x8664::word-shift)) (:%l x8664::nargs))))
	347	(jne :bad)
	348	(:anchored-uuo-section :resume)
	349	:bad
	350	(:anchored-uuo (uuo-error-wrong-number-of-args)))
	351
	352	(define-x8664-vinsn check-min-nargs (()
	353	((min :u16const)))
	354	:resume
	355	((:pred = min 1)
	356	(testl (:%l x8664::nargs) (:%l x8664::nargs))
	357	(je :toofew))
	358	((:not (:pred = min 1))
	359	((:pred < min 16)
	360	(rcmpl (:%l x8664::nargs) (:$b (:apply ash min x8664::word-shift))))
	361	((:pred >= min 16)
	362	(rcmpl (:%l x8664::nargs) (:$l (:apply ash min x8664::word-shift))))
	363	(jb :toofew))
	364
	365	(:anchored-uuo-section :resume)
	366	:toofew
	367	(:anchored-uuo (uuo-error-too-few-args)))
	368
	369	(define-x8664-vinsn check-max-nargs (()
	370	((n :u16const)))
	371	:resume
	372	((:pred < n 16)
	373	(rcmpl (:%l x8664::nargs) (:$b (:apply ash n x8664::word-shift))))
	374	((:pred >= n 16)
	375	(rcmpl (:%l x8664::nargs) (:$l (:apply ash n x8664::word-shift))))
	376	(ja :bad)
	377
	378	(:anchored-uuo-section :resume)
	379	:bad
	380	(:anchored-uuo (uuo-error-too-many-args)))
	381
	382
	383	(define-x8664-vinsn check-min-max-nargs (()
	384	((min :u16const)
	385	(max :u16const)))
	386	:resume
	387	((:pred = min 1)
	388	(testl (:%l x8664::nargs) (:%l x8664::nargs))
	389	(je :toofew))
	390	((:not (:pred = min 1))
	391	((:pred < min 16)
	392	(rcmpl (:%l x8664::nargs) (:$b (:apply ash min x8664::word-shift))))
	393	((:pred >= min 16)
	394	(rcmpl (:%l x8664::nargs) (:$l (:apply ash min x8664::word-shift))))
	395	(jb :toofew))
	396	((:pred < max 16)
	397	(rcmpl (:%l x8664::nargs) (:$b (:apply ash max x8664::word-shift))))
	398	((:pred >= max 16)
	399	(rcmpl (:%l x8664::nargs) (:$l (:apply ash max x8664::word-shift))))
	400	(ja :toomany)
	401
	402	(:anchored-uuo-section :resume)
	403	:toofew
	404	(:anchored-uuo (uuo-error-too-few-args))
	405	(:anchored-uuo-section :resume)
	406	:toomany
	407	(:anchored-uuo (uuo-error-too-many-args)))
	408
	409
	410	(define-x8664-vinsn default-1-arg (()
	411	((min :u16const)))
	412	((:pred < min 16)
	413	(rcmpl (:%l x8664::nargs) (:$b (:apply ash min x8664::word-shift))))
	414	((:pred >= min 16)
	415	(rcmpl (:%l x8664::nargs) (:$l (:apply ash min x8664::word-shift))))
	416	(jne :done)
	417	((:pred >= min 3)
	418	(pushq (:%q x8664::arg_x)))
	419	((:pred >= min 2)
	420	(movq (:%q x8664::arg_y) (:%q x8664::arg_x)))
	421	((:pred >= min 1)
	422	(movq (:%q x8664::arg_z) (:%q x8664::arg_y)))
	423	(movq (:$l (:apply target-nil-value)) (:%q x8664::arg_z))
	424	:done)
	425
	426
	427	(define-x8664-vinsn default-2-args (()
	428	((min :u16const)))
	429	((:pred < (:apply 1+ min) 16)
	430	(rcmpl (:%l x8664::nargs ) (:$b (:apply ash (:apply 1+ min) x8664::word-shift))))
	431	((:pred >= (:apply 1+ min) 16)
	432	(rcmpl (:%l x8664::nargs ) (:$l (:apply ash (:apply 1+ min) x8664::word-shift))))
	433	(ja :done)
	434	(je :one)
	435	;; We got "min" args; arg_y & arg_z default to nil
	436	((:pred >= min 3)
	437	(pushq (:%q x8664::arg_x)))
	438	((:pred >= min 2)
	439	(pushq (:%q x8664::arg_y)))
	440	((:pred >= min 1)
	441	(movq (:%q x8664::arg_z) (:%q x8664::arg_x)))
	442	(movl (:$l (:apply target-nil-value)) (:%l x8664::arg_y))
	443	(jmp :last)
	444	:one
	445	;; We got min+1 args: arg_y was supplied, arg_z defaults to nil.
	446	((:pred >= min 2)
	447	(pushq (:%q x8664::arg_x)))
	448	((:pred >= min 1)
	449	(movq (:%q x8664::arg_y) (:%q x8664::arg_x)))
	450	(movq (:%q x8664::arg_z) (:%q x8664::arg_y))
	451	:last
	452	(movq (:$l (:apply target-nil-value)) (:%q x8664::arg_z))
	453	:done)
	454
	455	(define-x8664-vinsn default-3-args (()
	456	((min :u16const)))
	457	((:pred < (:apply + 2 min) 16)
	458	(rcmpl (:%l x8664::nargs ) (:$b (:apply ash (:apply + 2 min) x8664::word-shift))))
	459	((:pred >= (:apply + 2 min) 16)
	460	(rcmpl (:%l x8664::nargs ) (:$l (:apply ash (:apply + 2 min) x8664::word-shift))))
	461	(ja :done)
	462	(je :two)
	463	((:pred < min 16)
	464	(rcmpl (:%l x8664::nargs ) (:$b (:apply ash min x8664::word-shift))))
	465	((:pred >= min 16)
	466	(rcmpl (:%l x8664::nargs ) (:$l (:apply ash min x8664::word-shift))))
	467	(je :none)
	468	;; The first (of three) &optional args was supplied.
	469	((:pred >= min 2)
	470	(pushq (:%q x8664::arg_x)))
	471	((:pred >= min 1)
	472	(pushq (:%q x8664::arg_y)))
	473	(movq (:%q x8664::arg_z) (:%q x8664::arg_x))
	474	(jmp :last-2)
	475	:two
	476	;; The first two (of three) &optional args were supplied.
	477	((:pred >= min 1)
	478	(pushq (:%q x8664::arg_x)))
	479	(movq (:%q x8664::arg_y) (:%q x8664::arg_x))
	480	(movq (:%q x8664::arg_z) (:%q x8664::arg_y))
	481	(jmp :last-1)
	482	;; None of the three &optional args was provided.
	483	:none
	484	((:pred >= min 3)
	485	(pushq (:%q x8664::arg_x)))
	486	((:pred >= min 2)
	487	(pushq (:%q x8664::arg_y)))
	488	((:pred >= min 1)
	489	(pushq (:%q x8664::arg_z)))
	490	(movl (:$l (:apply target-nil-value)) (:%l x8664::arg_x))
	491	:last-2
	492	(movl (:$l (:apply target-nil-value)) (:%l x8664::arg_y))
	493	:last-1
	494	(movl (:$l (:apply target-nil-value)) (:%l x8664::arg_z))
	495	:done)
	496
	497
	498	(define-x8664-vinsn default-optionals (()
	499	((n :u16const))
	500	((temp :u64)))
	501	((:pred < n 16)
	502	(rcmpl (:%l x8664::nargs) (:$b (:apply ash n x8664::word-shift))))
	503	((:pred >= n 16)
	504	(rcmpl (:%l x8664::nargs) (:$l (:apply ash n x8664::word-shift))))
	505	(movl (:%l x8664::nargs) (:%l temp))
	506	(jae :done)
	507	:loop
	508	(addl (:$b x8664::fixnumone) (:%l temp))
	509	((:pred < n 16)
	510	(cmpl (:$b (:apply ash n x8664::word-shift)) (:%l temp)))
	511	((:pred >= n 16)
	512	(cmpl (:$l (:apply ash n x8664::word-shift)) (:%l temp)))
	513	(pushq (:$l (:apply target-nil-value)))
	514	(jne :loop)
	515	:done)
	516
	517
	518	(define-x8664-vinsn save-lisp-context-no-stack-args (()
	519	())
	520	(pushq (:%q x8664::rbp))
	521	(movq (:%q x8664::rsp) (:%q x8664::rbp)))
	522
	523
	524	(define-x8664-vinsn save-lisp-context-offset (()
	525	((nbytes-pushed :s32const)))
	526	(movq (:%q x8664::rbp) (:@ (:apply + nbytes-pushed x8664::node-size) (:%q x8664::rsp)))
	527	(leaq (:@ (:apply + nbytes-pushed x8664::node-size) (:%q x8664::rsp)) (:%q x8664::rbp))
	528	(popq (:@ x8664::node-size (:%q x8664::rbp))))
	529
	530	(define-x8664-vinsn save-lisp-context-variable-arg-count (()
	531	()
	532	((temp :u64)))
	533	(movl (:%l x8664::nargs) (:%l temp))
	534	(subq (:$b (* $numx8664argregs x8664::node-size)) (:%q temp))
	535	(jle :push)
	536	(movq (:%q x8664::rbp) (:@ x8664::node-size (:%q x8664::rsp) (:%q temp)))
	537	(leaq (:@ x8664::node-size (:%q x8664::rsp) (:%q temp)) (:%q x8664::rbp))
	538	(popq (:@ 8 (:%q x8664::rbp)))
	539	(jmp :done)
	540	:push
	541	(pushq (:%q x8664::rbp))
	542	(movq (:%q x8664::rsp) (:%q x8664::rbp))
	543	:done)
	544
	545	;;; We know that some args were pushed, but don't know how many were
	546	;;; passed.
	547	(define-x8664-vinsn save-lisp-context-in-frame (()
	548	()
	549	((temp :u64)))
	550	(movl (:%l x8664::nargs) (:%l temp))
	551	(subq (:$b (* $numx8664argregs x8664::node-size)) (:%q temp))
	552	(movq (:%q x8664::rbp) (:@ x8664::node-size (:%q x8664::rsp) (:%q temp)))
	553	(leaq (:@ x8664::node-size (:%q x8664::rsp) (:%q temp)) (:%q x8664::rbp))
	554	(popq (:@ x8664::node-size (:%q x8664::rbp))))
	555
	556
	557	(define-x8664-vinsn (vpush-register :push :node :vsp)
	558	(()
	559	((reg :lisp)))
	560	(pushq (:% reg)))
	561
	562	(define-x8664-vinsn (vpush-fixnum :push :node :vsp)
	563	(()
	564	((const :s32const)))
	565	((:and (:pred < const 128) (:pred >= const -128))
	566	(pushq (:$b const)))
	567	((:not (:and (:pred < const 128) (:pred >= const -128)))
	568	(pushq (:$l const))))
	569
	570
	571
	572	(define-x8664-vinsn vframe-load (((dest :lisp))
	573	((frame-offset :u16const)
	574	(cur-vsp :u16const)))
	575	(movq (:@ (:apply - (:apply + frame-offset x8664::word-size-in-bytes)) (:%q x8664::rbp)) (:%q dest)))
	576
	577	(define-x8664-vinsn compare-vframe-offset-to-nil (()
	578	((frame-offset :u16const)
	579	(cur-vsp :u16const)))
	580	(cmpb (:$b x8664::fulltag-nil) (:@ (:apply - (:apply + frame-offset x8664::word-size-in-bytes)) (:%q x8664::rbp))))
	581
	582	(define-x8664-vinsn compare-vframe-offset-to-fixnum (()
	583	((frame-offset :u16const)
	584	(fixval :s32const)))
	585	((:and (:pred < fixval 128) (:pred >= fixval -128))
	586	(cmpq (:$b fixval) (:@ (:apply - (:apply + frame-offset x8664::word-size-in-bytes)) (:%q x8664::rbp))))
	587	((:not (:and (:pred < fixval 128) (:pred >= fixval -128)))
	588	(cmpq (:$l fixval) (:@ (:apply - (:apply + frame-offset x8664::word-size-in-bytes)) (:%q x8664::rbp)))))
	589
	590
	591	(define-x8664-vinsn add-constant-to-vframe-offset (()
	592	((frame-offset :u16const)
	593	(constant :s32const)))
	594	((:and (:pred < constant 128) (:pred >= constant -128))
	595	(addq (:$b constant) (:@ (:apply - (:apply + frame-offset x8664::word-size-in-bytes)) (:%q x8664::rbp))))
	596	((:not (:and (:pred < constant 128) (:pred >= constant -128)))
	597	(addq (:$l constant) (:@ (:apply - (:apply + frame-offset x8664::word-size-in-bytes)) (:%q x8664::rbp)))))
	598
	599
	600	(define-x8664-vinsn compare-value-cell-to-nil (()
	601	((vcell :lisp)))
	602	(cmpb (:$b x8664::fulltag-nil) (:@ x8664::value-cell.value (:%q vcell))))
	603
	604	(define-x8664-vinsn lcell-load (((dest :lisp))
	605	((cell :lcell)
	606	(top :lcell)))
	607	(movq (:@ (:apply - (:apply + (:apply calc-lcell-offset cell) x8664::word-size-in-bytes)) (:%q x8664::rbp)) (:%q dest)))
	608
	609	(define-x8664-vinsn (vframe-push :push :node :vsp)
	610	(()
	611	((frame-offset :u16const)
	612	(cur-vsp :u16const)))
	613	(pushq (:@ (:apply - (:apply + frame-offset x8664::word-size-in-bytes)) (:%q x8664::rbp))))
	614
	615	(define-x8664-vinsn vframe-store (()
	616	((src :lisp)
	617	(frame-offset :u16const)
	618	(cur-vsp :u16const)))
	619	(movq (:%q src) (:@ (:apply - (:apply + frame-offset x8664::word-size-in-bytes)) (:%q x8664::rbp))))
	620
	621	(define-x8664-vinsn lcell-store (()
	622	((src :lisp)
	623	(cell :lcell)
	624	(top :lcell)))
	625	(movq (:%q src) (:@ (:apply - (:apply + (:apply calc-lcell-offset cell) x8664::word-size-in-bytes)) (:%q x8664::rbp))))
	626
	627	(define-x8664-vinsn (popj :lispcontext :pop :vsp :lrRestore :jumpLR)
	628	(()
	629	())
	630	(leave)
	631	(ret))
	632
	633	(define-x8664-vinsn (popj-via-jump :lispcontext :pop :vsp :lrRestore :jumpLR)
	634	(()
	635	((lab :label)))
	636	(jmp lab))
	637
	638	(define-x8664-vinsn (restore-full-lisp-context :lispcontext :pop :vsp )
	639	(()
	640	())
	641	(leave))
	642
	643	(define-x8664-vinsn compare-to-nil (()
	644	((arg0 t)))
	645	(cmpb (:$b x8664::fulltag-nil) (:%b arg0)))
	646
	647	(define-x8664-vinsn compare-to-t (()
	648	((arg0 t)))
	649	(cmpq (:$l (:apply target-t-value)) (:%q arg0)))
	650
	651	(define-x8664-vinsn ref-constant (((dest :lisp))
	652	((lab :label)))
	653	(movq (:@ (:^ lab) (:%q x8664::fn)) (:%q dest)))
	654
	655	(define-x8664-vinsn compare-constant-to-register (()
	656	((lab :label)
	657	(reg :lisp)))
	658	(cmpq (:@ (:^ lab) (:%q x8664::fn)) (:%q reg)))
	659
	660
	661	(define-x8664-vinsn (vpush-constant :push :node :vsp) (()
	662	((lab :label)))
	663	(pushq (:@ (:^ lab) (:%q x8664::fn))))
	664
	665
	666	(define-x8664-vinsn (jump :jump)
	667	(()
	668	((label :label)))
	669	(jmp label))
	670
	671	(define-x8664-vinsn (cbranch-true :branch) (()
	672	((label :label)
	673	(crbit :u8const)))
	674	(jcc (:$ub crbit) label))
	675
	676	(define-x8664-vinsn (cbranch-false :branch) (()
	677	((label :label)
	678	(crbit :u8const)))
	679	(jcc (:$ub (:apply logxor 1 crbit)) label))
	680
	681
	682	(define-x8664-vinsn (lri :constant-ref) (((dest :imm))
	683	((intval :s64const))
	684	())
	685	((:pred = intval 0)
	686	(xorl (:%l dest) (:%l dest)))
	687	((:and (:pred /= intval 0)
	688	(:pred >= intval -2147483648)
	689	(:pred <= intval #xffffffff))
	690	((:pred > intval 0)
	691	(movl (:$l intval) (:%l dest)))
	692	((:pred < intval 0)
	693	(movq (:$l intval) (:%q dest))))
	694	((:or (:pred < intval -2147483648)
	695	(:pred > intval #xffffffff))
	696	(movq (:$q (:apply logand #xffffffffffffffff intval)) (:%q dest))))
	697
	698	(define-x8664-vinsn (lriu :constant-ref) (((dest :imm))
	699	((intval :u64const))
	700	())
	701	((:pred = intval 0)
	702	(xorl (:%l dest) (:%l dest)))
	703	((:and (:pred > intval 0)
	704	(:pred <= intval #xffffffff))
	705	(movl (:$l intval) (:%l dest)))
	706	((:pred > intval #xffffffff)
	707	(movq (:$q (:apply logand #xffffffffffffffff intval)) (:%q dest))))
	708
	709	(define-x8664-vinsn trap-unless-bit (()
	710	((value :lisp)))
	711	:resume
	712	(testq (:$l (lognot x8664::fixnumone)) (:%q value))
	713	(jne :bad)
	714
	715	(:anchored-uuo-section :resume)
	716	:bad
	717	(:anchored-uuo (uuo-error-reg-not-type (:%q value) (:$ub arch::error-object-not-bit)))
	718	)
	719
	720	(define-x8664-vinsn trap-unless-list (()
	721	((object :lisp))
	722	((tag :u8)))
	723	:resume
	724	(movl (:%l object) (:%l tag))
	725	(andl (:$b x8664::tagmask) (:%l tag))
	726	(cmpl (:$b x8664::tag-list) (:%l tag))
	727	(jne :bad)
	728
	729	(:anchored-uuo-section :resume)
	730	:bad
	731	(:anchored-uuo (uuo-error-reg-not-list (:%q object))))
	732
	733
	734
	735	(define-x8664-vinsn trap-unless-cons (()
	736	((object :lisp))
	737	((tag :u8)))
	738	:resume
	739	(movl (:%l object) (:%l tag))
	740	(andl (:$b x8664::fulltagmask) (:%l tag))
	741	(cmpl (:$b x8664::fulltag-cons) (:%l tag))
	742	(jne :bad)
	743
	744	(:anchored-uuo-section :resume)
	745	:bad
	746	(:anchored-uuo (uuo-error-reg-not-tag (:%q object) (:$ub x8664::fulltag-cons))))
	747
	748	(define-x8664-vinsn set-z-flag-if-consp (()
	749	((object :lisp))
	750	((tag :u8)))
	751	(movl (:%l object) (:%l tag))
	752	(andl (:$b x8664::fulltagmask) (:%l tag))
	753	(cmpl (:$b x8664::fulltag-cons) (:%l tag)))
	754
	755	(define-x8664-vinsn trap-unless-uvector (()
	756	((object :lisp))
	757	((tag :u8)))
	758	:resume
	759	(movl (:%l object) (:%l tag))
	760	(andl (:$b x8664::tagmask) (:%l tag))
	761	(cmpl (:$b x8664::tag-misc) (:%l tag))
	762	(jne :bad)
	763
	764	(:anchored-uuo-section :resume)
	765	:bad
	766	(:anchored-uuo (uuo-error-reg-not-tag (:%q object) (:$ub x8664::tag-misc))))
	767
	768	(define-x8664-vinsn trap-unless-single-float (()
	769	((object :lisp)))
	770	:resume
	771	(cmpb (:$b x8664::tag-single-float) (:%b object))
	772	(jne :bad)
	773
	774	(:anchored-uuo-section :resume)
	775	:bad
	776	(:anchored-uuo (uuo-error-reg-not-tag (:%q object) (:$ub x8664::tag-single-float))))
	777
	778	(define-x8664-vinsn trap-unless-character (()
	779	((object :lisp)))
	780	:resume
	781	(cmpb (:$b x8664::subtag-character) (:%b object))
	782	(jne :bad)
	783
	784	(:anchored-uuo-section :resume)
	785	:bad
	786	(:anchored-uuo (uuo-error-reg-not-tag (:%q object) (:$ub x8664::subtag-character))))
	787
	788	(define-x8664-vinsn trap-unless-fixnum (()
	789	((object :lisp))
	790	())
	791	:resume
	792	(testb (:$b x8664::tagmask) (:%b object))
	793	(jne :bad)
	794
	795	(:anchored-uuo-section :resume)
	796	:bad
	797	(:anchored-uuo (uuo-error-reg-not-fixnum (:%q object))))
	798
	799	(define-x8664-vinsn set-flags-from-lisptag (()
	800	((reg :lisp)))
	801	(testb (:$b x8664::tagmask) (:%b reg)))
	802
	803
	804	(define-x8664-vinsn trap-unless-typecode= (()
	805	((object :lisp)
	806	(tagval :u16const))
	807	((tag :u8)))
	808	:resume
	809	(movl (:%l object) (:%l tag))
	810	(andl (:$b x8664::tagmask) (:%l tag))
	811	(cmpl (:$b x8664::tag-misc) (:%l tag))
	812	(jne :have-tag)
	813	;; This needs to be a sign-extending mov, since the cmpl below
	814	;; will sign-extend the 8-bit constant operand.
	815	(movsbl (:@ x8664::misc-subtag-offset (:%q object)) (:%l tag))
	816	:have-tag
	817	(cmpl (:$b tagval) (:%l tag))
	818	(jne :bad)
	819	(:anchored-uuo-section :resume)
	820	:bad
	821	(:anchored-uuo (uuo-error-reg-not-tag (:%q object) (:$ub tagval))))
	822
	823	(define-x8664-vinsn trap-unless-double-float (()
	824	((object :lisp))
	825	((tag :u8)))
	826	:resume
	827	(movl (:%l object) (:%l tag))
	828	(andl (:$b x8664::tagmask) (:%l tag))
	829	(cmpl (:$b x8664::tag-misc) (:%l tag))
	830	(jne :have-tag)
	831	(movsbl (:@ x8664::misc-subtag-offset (:%q object)) (:%l tag))
	832	:have-tag
	833	(cmpl (:$b x8664::subtag-double-float) (:%l tag))
	834	(jne :bad)
	835	(:anchored-uuo-section :resume)
	836	:bad
	837	(:anchored-uuo (uuo-error-reg-not-tag (:%q object) (:$ub x8664::subtag-double-float))))
	838
	839	(define-x8664-vinsn trap-unless-complex-double-float (()
	840	((object :lisp))
	841	((tag :u8)))
	842	:resume
	843	(movl (:%l object) (:%l tag))
	844	(andl (:$b x8664::tagmask) (:%l tag))
	845	(cmpl (:$b x8664::tag-misc) (:%l tag))
	846	(jne :have-tag)
	847	(movsbl (:@ x8664::misc-subtag-offset (:%q object)) (:%l tag))
	848	:have-tag
	849	(cmpl (:$b x8664::subtag-complex-double-float) (:%l tag))
	850	(jne :bad)
	851	(:anchored-uuo-section :resume)
	852	:bad
	853	(:anchored-uuo (uuo-error-reg-not-tag (:%q object) (:$ub x8664::subtag-complex-double-float))))
	854
	855	(define-x8664-vinsn trap-unless-complex-single-float (()
	856	((object :lisp))
	857	((tag :u8)))
	858	:resume
	859	(movl (:%l object) (:%l tag))
	860	(andl (:$b x8664::tagmask) (:%l tag))
	861	(cmpl (:$b x8664::tag-misc) (:%l tag))
	862	(jne :have-tag)
	863	(movsbl (:@ x8664::misc-subtag-offset (:%q object)) (:%l tag))
	864	:have-tag
	865	(cmpl (:$b x8664::subtag-complex-single-float) (:%l tag))
	866	(jne :bad)
	867	(:anchored-uuo-section :resume)
	868	:bad
	869	(:anchored-uuo (uuo-error-reg-not-tag (:%q object) (:$ub x8664::subtag-complex-single-float))))
	870
	871
	872
	873
	874	(define-x8664-vinsn trap-unless-macptr (()
	875	((object :lisp))
	876	((tag :u8)))
	877	:resume
	878	(movl (:%l object) (:%l tag))
	879	(andl (:$b x8664::tagmask) (:%l tag))
	880	(cmpl (:$b x8664::tag-misc) (:%l tag))
	881	(jne :have-tag)
	882	(movzbl (:@ x8664::misc-subtag-offset (:%q object)) (:%l tag))
	883	:have-tag
	884	(cmpb (:$b x8664::subtag-macptr) (:%b tag))
	885	(jne :bad)
	886
	887	(:anchored-uuo-section :resume)
	888	:bad
	889	(:anchored-uuo (uuo-error-reg-not-tag (:%q object) (:$ub x8664::subtag-macptr))))
	890
	891
	892	(define-x8664-vinsn check-misc-bound (()
	893	((idx :imm)
	894	(v :lisp))
	895	((temp :u64)))
	896	:resume
	897	(movq (:@ x8664::misc-header-offset (:%q v)) (:%q temp))
	898	(shrq (:$ub x8664::num-subtag-bits) (:%q temp))
	899	(shlq (:$ub x8664::fixnumshift) (:%q temp))
	900	(rcmpq (:%q idx) (:%q temp))
	901	(jae :bad)
	902
	903	(:anchored-uuo-section :resume)
	904	:bad
	905	(:anchored-uuo (uuo-error-vector-bounds (:%q idx) (:%q v))))
	906
	907	(define-x8664-vinsn check-vector-header-bound (()
	908	((v :lisp)
	909	(idx :imm)))
	910	:resume
	911	(cmpq (:@ x8664::vectorH.physsize (:%q v)) (:%q idx))
	912	(jae :bad)
	913	(:anchored-uuo-section :resume)
	914	:bad
	915	(:anchored-uuo (uuo-error-vector-bounds (:%q idx) (:%q v))))
	916
	917	(define-x8664-vinsn deref-vector-header (((vector :lisp)
	918	(index :lisp))
	919	((vector :lisp)
	920	(index :lisp)))
	921	:again
	922	(addq (:@ x8664::vectorH.displacement (:%q vector)) (:%q index))
	923	(btw (:$ub (+ x8664::fixnumshift $arh_disp_bit)) (:@ x8664::vectorH.flags (:%q vector)))
	924	(movq (:@ x8664::vectorH.data-vector (:%q vector)) (:%q vector))
	925	(jb :again))
	926
	927
	928	(define-x8664-vinsn %cdr (((dest :lisp))
	929	((src :lisp)))
	930	(movq (:@ x8664::cons.cdr (:%q src)) (:%q dest)))
	931
	932	(define-x8664-vinsn (%vpush-cdr :push :node :vsp)
	933	(()
	934	((src :lisp)))
	935	(pushq (:@ x8664::cons.cdr (:%q src))))
	936
	937	(define-x8664-vinsn %car (((dest :lisp))
	938	((src :lisp)))
	939	(movq (:@ x8664::cons.car (:%q src)) (:%q dest)))
	940
	941	(define-x8664-vinsn (%vpush-car :push :node :vsp)
	942	(()
	943	((src :lisp)))
	944	(pushq (:@ x8664::cons.car (:%q src))))
	945
	946
	947	(define-x8664-vinsn u32->char (((dest :lisp)
	948	(src :u8))
	949	((src :u8))
	950	())
	951	(shll (:$ub x8664::charcode-shift) (:%l src))
	952	(leaq (:@ x8664::subtag-character (:%q src)) (:%q dest)))
	953
	954
	955	(define-x8664-vinsn (load-nil :constant-ref) (((dest t))
	956	())
	957	(movl (:$l (:apply target-nil-value)) (:%l dest)))
	958
	959
	960	(define-x8664-vinsn (load-t :constant-ref) (((dest t))
	961	())
	962	(movl(:$l (:apply target-t-value)) (:%l dest)))
	963
	964
	965	(define-x8664-vinsn extract-tag (((tag :u8))
	966	((object :lisp)))
	967	(movl (:%l object) (:%l tag))
	968	(andl (:$b x8664::tagmask) (:%l tag)))
	969
	970	(define-x8664-vinsn extract-tag-fixnum (((tag :imm))
	971	((object :lisp)))
	972	((:pred =
	973	(:apply %hard-regspec-value tag)
	974	(:apply %hard-regspec-value object))
	975	(shll (:$ub x8664::fixnumshift) (:%l object)))
	976	((:not (:pred =
	977	(:apply %hard-regspec-value tag)
	978	(:apply %hard-regspec-value object)))
	979	(imull (:$b x8664::fixnumone) (:%l object) (:%l tag)))
	980	(andl (:$b (ash x8664::tagmask x8664::fixnumshift)) (:%l tag)))
	981
	982	(define-x8664-vinsn extract-fulltag (((tag :u8))
	983	((object :lisp)))
	984	(movl (:%l object) (:%l tag))
	985	(andl (:$b x8664::fulltagmask) (:%l tag)))
	986
	987	(define-x8664-vinsn extract-fulltag-fixnum (((tag :imm))
	988	((object :lisp)))
	989	((:pred =
	990	(:apply %hard-regspec-value tag)
	991	(:apply %hard-regspec-value object))
	992	(shll (:$ub x8664::fixnumshift) (:%l object)))
	993	((:not (:pred =
	994	(:apply %hard-regspec-value tag)
	995	(:apply %hard-regspec-value object)))
	996	(imull (:$b x8664::fixnumone) (:%l object) (:%l tag)))
	997	(andl (:$b (ash x8664::fulltagmask x8664::fixnumshift)) (:%l tag)))
	998
	999	(define-x8664-vinsn extract-typecode (((tag :u32))
	1000	((object :lisp)))
	1001	(movl (:%l object) (:%l tag))
	1002	(andl (:$b x8664::tagmask) (:%l tag))
	1003	(cmpl (:$b x8664::tag-misc) (:%l tag))
	1004	(jne :have-tag)
	1005	(movzbl (:@ x8664::misc-subtag-offset (:%q object)) (:%l tag))
	1006	:have-tag)
	1007
	1008	(define-x8664-vinsn extract-typecode-fixnum (((tag :imm))
	1009	((object :lisp))
	1010	((temp :u32)))
	1011	(movl (:%l object) (:%l temp))
	1012	(andl (:$b x8664::tagmask) (:%l temp))
	1013	(cmpl (:$b x8664::tag-misc) (:%l temp))
	1014	(jne :have-tag)
	1015	(movzbl (:@ x8664::misc-subtag-offset (:%q object)) (:%l temp))
	1016	:have-tag
	1017	(imulq (:$b x8664::fixnumone) (:%q temp) (:%q tag)))
	1018
	1019
	1020	(define-x8664-vinsn compare-reg-to-zero (()
	1021	((reg :imm)))
	1022	(testq (:%q reg) (:%q reg)))
	1023
	1024	(define-x8664-vinsn compare-u8-reg-to-zero (()
	1025	((reg :u8)))
	1026	(testb (:%b reg) (:%b reg)))
	1027
	1028	(define-x8664-vinsn cr-bit->boolean (((dest :lisp))
	1029	((crbit :u8const)))
	1030	(movl (:$l (:apply target-nil-value)) (:%l dest))
	1031	(cmovccl (:$ub crbit) (:@ (+ x8664::t-offset x8664::symbol.vcell) (:%l dest)) (:%l dest)))
	1032
	1033
	1034
	1035
	1036
	1037	(define-x8664-vinsn compare-s32-constant (()
	1038	((val :imm)
	1039	(const :s32const)))
	1040	((:or (:pred < const -128) (:pred > const 127))
	1041	(rcmpq (:%q val) (:$l const)))
	1042	((:not (:or (:pred < const -128) (:pred > const 127)))
	1043	(rcmpq (:%q val) (:$b const))))
	1044
	1045	(define-x8664-vinsn compare-u31-constant (()
	1046	((val :u64)
	1047	(const :u32const)))
	1048	((:pred > const 127)
	1049	(rcmpq (:%q val) (:$l const)))
	1050	((:not (:pred > const 127))
	1051	(rcmpq (:%q val) (:$b const))))
	1052
	1053	(define-x8664-vinsn compare-u8-constant (()
	1054	((val :u8)
	1055	(const :u8const)))
	1056	#\|
	1057	((:pred logbitp 7 const)
	1058	(movzbl (:%b val) (:%l val))
	1059	(rcmpw (:%w val) (:$w const)))
	1060	((:not (:pred logbitp 7 const))
	1061	(rcmpb (:%b val) (:$b const)))
	1062	\|\|#
	1063	(rcmpb (:%b val) (:$b const))
	1064	)
	1065
	1066
	1067	(define-x8664-vinsn cons (((dest :lisp))
	1068	((car :lisp)
	1069	(cdr :lisp))
	1070	((allocptr (:lisp #.x8664::allocptr))))
	1071	(subq (:$b (- x8664::cons.size x8664::fulltag-cons)) (:rcontext x8664::tcr.save-allocptr))
	1072	(movq (:rcontext x8664::tcr.save-allocptr) (:%q allocptr))
	1073	(rcmpq (:%q allocptr) (:rcontext x8664::tcr.save-allocbase))
	1074	(:byte #x77) (:byte #x02) ;(ja :no-trap)
	1075	(uuo-alloc)
	1076	:no-trap
	1077	(andb (:$b (lognot x8664::fulltagmask)) (:rcontext x8664::tcr.save-allocptr))
	1078	(movq (:%q car) (:@ x8664::cons.car (:%q allocptr)))
	1079	(movq (:%q cdr) (:@ x8664::cons.cdr (:%q allocptr)))
	1080	((:pred /= (:apply %hard-regspec-value dest) (:apply %hard-regspec-value x8664::allocptr))
	1081	(movq (:%q allocptr) (:%q dest))))
	1082
	1083	(define-x8664-vinsn unbox-u8 (((dest :u8))
	1084	((src :lisp)))
	1085	:resume
	1086	(movq (:$l (lognot (ash #xff x8664::fixnumshift))) (:%q dest))
	1087	(andq (:% src) (:% dest))
	1088	(jne :bad)
	1089	(movq (:%q src) (:%q dest))
	1090	(shrq (:$ub x8664::fixnumshift) (:%q dest))
	1091
	1092	(:anchored-uuo-section :resume)
	1093	:bad
	1094	(:anchored-uuo (uuo-error-reg-not-type (:%q src) (:$ub arch::error-object-not-unsigned-byte-8))))
	1095
	1096	(define-x8664-vinsn %unbox-u8 (((dest :u8))
	1097	((src :lisp)))
	1098	(movl (:%l src) (:%l dest))
	1099	(shrl (:$ub x8664::fixnumshift) (:%l dest))
	1100	(movzbl (:%b dest) (:%l dest)))
	1101
	1102	(define-x8664-vinsn unbox-s8 (((dest :s8))
	1103	((src :lisp)))
	1104	:resume
	1105	(movq (:%q src) (:%q dest))
	1106	(shlq (:$ub (- x8664::nbits-in-word (+ 8 x8664::fixnumshift))) (:%q dest))
	1107	(sarq (:$ub (- x8664::nbits-in-word (+ 8 x8664::fixnumshift))) (:%q dest))
	1108	(cmpq (:%q src) (:%q dest))
	1109	(jne :bad)
	1110	(testb (:$b x8664::fixnummask) (:%b dest))
	1111	(jne :bad)
	1112	(sarq (:$ub x8664::fixnumshift) (:%q dest))
	1113
	1114	(:anchored-uuo-section :resume)
	1115	:bad
	1116	(:anchored-uuo (uuo-error-reg-not-type (:%q src) (:$ub arch::error-object-not-signed-byte-8))))
	1117
	1118	(define-x8664-vinsn %unbox-s8 (((dest :s8))
	1119	((src :lisp)))
	1120	(movq (:%q src) (:%q dest))
	1121	(sarq (:$ub x8664::fixnumshift) (:%q dest)))
	1122
	1123	(define-x8664-vinsn unbox-u16 (((dest :u16))
	1124	((src :lisp)))
	1125	:resume
	1126	(testq (:$l (lognot (ash #xffff x8664::fixnumshift))) (:% src))
	1127	(movq (:%q src) (:%q dest))
	1128	(jne :bad)
	1129	(shrq (:$ub x8664::fixnumshift) (:%q dest))
	1130
	1131	(:anchored-uuo-section :resume)
	1132	:bad
	1133	(:anchored-uuo (uuo-error-reg-not-type (:%q src) (:$ub arch::error-object-not-unsigned-byte-16))))
	1134
	1135	(define-x8664-vinsn %unbox-u16 (((dest :u16))
	1136	((src :lisp)))
	1137	(movq (:%q src) (:%q dest))
	1138	(shrq (:$ub x8664::fixnumshift) (:%q dest)))
	1139
	1140	(define-x8664-vinsn unbox-s16 (((dest :s16))
	1141	((src :lisp)))
	1142	:resume
	1143	(movq (:%q src) (:%q dest))
	1144	(shlq (:$ub (- x8664::nbits-in-word (+ 16 x8664::fixnumshift))) (:%q dest))
	1145	(sarq (:$ub (- x8664::nbits-in-word (+ 16 x8664::fixnumshift))) (:%q dest))
	1146	(cmpq (:%q src) (:%q dest))
	1147	(jne :bad)
	1148	(testb (:$b x8664::fixnummask) (:%b dest))
	1149	(jne :bad)
	1150	(sarq (:$ub x8664::fixnumshift) (:%q dest))
	1151
	1152	(:anchored-uuo-section :resume)
	1153	:bad
	1154	(:anchored-uuo (uuo-error-reg-not-type (:%q src) (:$ub arch::error-object-not-signed-byte-16))))
	1155
	1156	(define-x8664-vinsn %unbox-s16 (((dest :s16))
	1157	((src :lisp)))
	1158	(movq (:%q src) (:%q dest))
	1159	(sarq (:$ub x8664::fixnumshift) (:%q dest)))
	1160
	1161	(define-x8664-vinsn unbox-u32 (((dest :u32))
	1162	((src :lisp)))
	1163	:resume
	1164	(movq (:$q (lognot (ash #xffffffff x8664::fixnumshift))) (:%q dest))
	1165	(testq (:% src) (:% dest))
	1166	(jne :bad)
	1167	(movq (:%q src) (:%q dest))
	1168	(shrq (:$ub x8664::fixnumshift) (:%q dest))
	1169
	1170	(:anchored-uuo-section :resume)
	1171	:bad
	1172	(:anchored-uuo (uuo-error-reg-not-type (:%q src) (:$ub arch::error-object-not-unsigned-byte-32))))
	1173
	1174	(define-x8664-vinsn %unbox-u32 (((dest :u32))
	1175	((src :lisp)))
	1176
	1177	(movq (:%q src) (:%q dest))
	1178	(shrq (:$ub x8664::fixnumshift) (:%q dest)))
	1179
	1180	(define-x8664-vinsn unbox-s32 (((dest :s32))
	1181	((src :lisp)))
	1182	:resume
	1183	(movq (:%q src) (:%q dest))
	1184	(shlq (:$ub (- x8664::nbits-in-word (+ 32 x8664::fixnumshift))) (:%q dest))
	1185	(sarq (:$ub (- x8664::nbits-in-word (+ 32 x8664::fixnumshift))) (:%q dest))
	1186	(cmpq (:%q src) (:%q dest))
	1187	(jne :bad)
	1188	(testb (:$b x8664::fixnummask) (:%b dest))
	1189	(jne :bad)
	1190	(sarq (:$ub x8664::fixnumshift) (:%q dest))
	1191
	1192	(:anchored-uuo-section :resume)
	1193	:bad
	1194	(:anchored-uuo (uuo-error-reg-not-type (:%q src) (:$ub arch::error-object-not-signed-byte-32))))
	1195
	1196	(define-x8664-vinsn %unbox-s32 (((dest :s32))
	1197	((src :lisp)))
	1198	(movq (:%q src) (:%q dest))
	1199	(sarq (:$ub x8664::fixnumshift) (:%q dest)))
	1200
	1201
	1202	(define-x8664-vinsn unbox-u64 (((dest :u64))
	1203	((src :lisp)))
	1204	:resume
	1205	(movq (:$q (lognot (ash x8664::target-most-positive-fixnum x8664::fixnumshift))) (:%q dest))
	1206	(testq (:%q dest) (:%q src))
	1207	(movq (:%q src) (:%q dest))
	1208	(jnz :maybe-bignum)
	1209	(shrq (:$ub x8664::fixnumshift) (:%q dest))
	1210	(jmp :done)
	1211	:maybe-bignum
	1212	(andl (:$b x8664::tagmask) (:%l dest))
	1213	(cmpl (:$b x8664::tag-misc) (:%l dest))
	1214	(jne :have-tag)
	1215	(movzbl (:@ x8664::misc-subtag-offset (:%q src)) (:%l dest))
	1216	:have-tag
	1217	(cmpl (:$b x8664::subtag-bignum) (:%l dest))
	1218	(jne :bad)
	1219	(movq (:@ x8664::misc-header-offset (:%q src)) (:%q dest))
	1220	(cmpq (:$l x8664::three-digit-bignum-header) (:%q dest))
	1221	(je :three)
	1222	(cmpq (:$l x8664::two-digit-bignum-header) (:%q dest))
	1223	(jne :bad)
	1224	(movq (:@ x8664::misc-data-offset (:%q src)) (:%q dest))
	1225	(testq (:%q dest) (:%q dest))
	1226	(js :bad)
	1227	(jmp :done)
	1228
	1229	:three
	1230	(movl (:@ (+ 8 x8664::misc-data-offset) (:%q src)) (:%l dest))
	1231	(testl (:%l dest) (:%l dest))
	1232	(movq (:@ x8664::misc-data-offset (:%q src)) (:%q dest))
	1233	(jne :bad)
	1234	:done
	1235
	1236	(:anchored-uuo-section :resume)
	1237	:bad
	1238	(:anchored-uuo (uuo-error-reg-not-type (:%q src) (:$ub arch::error-object-not-unsigned-byte-64))))
	1239
	1240	(define-x8664-vinsn unbox-s64 (((dest :s64))
	1241	((src :lisp)))
	1242	:resume
	1243	(movq (:%q src) (:%q dest))
	1244	(sarq (:$ub x8664::fixnumshift) (:%q dest))
	1245	;; Was it a fixnum ?
	1246	(testb (:$b x8664::fixnummask) (:%b src))
	1247	(je :done)
	1248	;; May be a 2-digit bignum
	1249	(movl (:%l src) (:%l dest))
	1250	(andl (:$b x8664::tagmask) (:%l dest))
	1251	(cmpl (:$b x8664::tag-misc) (:%l dest))
	1252	(jne :bad)
	1253	(cmpq (:$l x8664::two-digit-bignum-header) (:@ x8664::misc-header-offset (:%q src)))
	1254	(movq (:@ x8664::misc-data-offset (:%q src)) (:%q dest))
	1255	(jne :bad)
	1256	:done
	1257
	1258	(:anchored-uuo-section :resume)
	1259	:bad
	1260	(:anchored-uuo (uuo-error-reg-not-type (:%q src) (:$ub arch::error-object-not-signed-byte-64))))
	1261
	1262	(define-x8664-vinsn sign-extend-s8 (((dest :s64))
	1263	((src :s8)))
	1264	(movsbq (:%b src) (:%q dest)))
	1265
	1266	(define-x8664-vinsn sign-extend-s16 (((dest :s64))
	1267	((src :s16)))
	1268	(movswq (:%w src) (:%q dest)))
	1269
	1270	(define-x8664-vinsn sign-extend-s32 (((dest :s64))
	1271	((src :s32)))
	1272	(movslq (:%l src) (:%q dest)))
	1273
	1274
	1275	(define-x8664-vinsn zero-extend-u8 (((dest :s64))
	1276	((src :u8)))
	1277	(movzbl (:%b src) (:%l dest)))
	1278
	1279	(define-x8664-vinsn zero-extend-u16 (((dest :s64))
	1280	((src :u16)))
	1281	(movzwl (:%w src) (:%l dest)))
	1282
	1283	(define-x8664-vinsn zero-extend-u32 (((dest :s64))
	1284	((src :u32)))
	1285	(movl (:%l src) (:%l dest)))
	1286
	1287	(define-x8664-vinsn (jump-subprim :jumpLR) (()
	1288	((spno :s32const)))
	1289	(jmp (:@ spno)))
	1290
	1291	;;; Call a subprimitive using a tail-aligned CALL instruction.
	1292	(define-x8664-vinsn (call-subprim :call :subprim) (()
	1293	((spno :s32const))
	1294	((entry (:label 1))))
	1295	(:talign 4)
	1296	(call (:@ spno))
	1297	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	1298
	1299
	1300	(define-x8664-vinsn %logand-c (((dest t)
	1301	(val t))
	1302	((val t)
	1303	(const :s32const)))
	1304	((:and (:pred >= const -128) (:pred <= const 127))
	1305	(andq (:$b const) (:%q val)))
	1306	((:not (:and (:pred >= const -128) (:pred <= const 127)))
	1307	(andq (:$l const) (:%q val))))
	1308
	1309	(define-x8664-vinsn %logior-c (((dest t)
	1310	(val t))
	1311	((val t)
	1312	(const :s32const)))
	1313	((:and (:pred >= const -128) (:pred <= const 127))
	1314	(orq (:$b const) (:%q val)))
	1315	((:not (:and (:pred >= const -128) (:pred <= const 127)))
	1316	(orq (:$l const) (:%q val))))
	1317
	1318	(define-x8664-vinsn %logxor-c (((dest t)
	1319	(val t))
	1320	((val t)
	1321	(const :s32const)))
	1322	((:and (:pred >= const -128) (:pred <= const 127))
	1323	(xorq (:$b const) (:%q val)))
	1324	((:not (:and (:pred >= const -128) (:pred <= const 127)))
	1325	(xorq (:$l const) (:%q val))))
	1326
	1327	(define-x8664-vinsn character->fixnum (((dest :lisp))
	1328	((src :lisp))
	1329	())
	1330	((:not (:pred =
	1331	(:apply %hard-regspec-value dest)
	1332	(:apply %hard-regspec-value src)))
	1333	(movq (:%q src) (:%q dest)))
	1334	(shrq (:$ub (- x8664::charcode-shift x8664::fixnumshift)) (:%q dest)))
	1335
	1336	(define-x8664-vinsn compare (()
	1337	((x t)
	1338	(y t)))
	1339	(rcmpq (:%q x) (:%q y)))
	1340
	1341	(define-x8664-vinsn negate-fixnum (((val :lisp))
	1342	((val :imm)))
	1343	(negq (:% val)))
	1344
	1345	(define-x8664-vinsn handle-fixnum-overflow-inline
	1346	(()
	1347	((val :lisp))
	1348	((header (:u64 #.x8664::imm0))
	1349	(scaled-size (:u64 #.x8664::imm1))
	1350	(bigits (:u64 #.x8664::imm2))
	1351	(freeptr (:lisp #.x8664::allocptr))))
	1352	(movq (:%q val) (:%q bigits))
	1353	(btcq (:$ub 63) (:%q bigits))
	1354	(sarq (:$ub x8664::fixnumshift) (:%q bigits))
	1355	(btcq (:$ub 60) (:%q bigits))
	1356	(movl (:$l x8664::two-digit-bignum-header) (:%l header))
	1357	(xorl (:%l scaled-size) (:%l scaled-size))
	1358	(movb (:$b (- 16 x8664::fulltag-misc)) (:%b scaled-size))
	1359	(subq (:%q scaled-size) (:rcontext x8664::tcr.save-allocptr))
	1360	(movq (:rcontext x8664::tcr.save-allocptr) (:%q freeptr))
	1361	(rcmpq (:%q freeptr) (:rcontext x8664::tcr.save-allocbase))
	1362	(:byte #x77) (:byte #x02) ;(ja :no-trap)
	1363	(uuo-alloc)
	1364	:no-trap
	1365	(movq (:%q header) (:@ x8664::misc-header-offset (:%q freeptr)))
	1366	(andb (:$b (lognot x8664::fulltagmask)) (:rcontext x8664::tcr.save-allocptr))
	1367	((:not (:pred = freeptr
	1368	(:apply %hard-regspec-value val)))
	1369	(movq (:%q freeptr) (:%q val)))
	1370	(movq (:%q bigits) (:@ x8664::misc-data-offset (:%q val))))
	1371
	1372	(define-x8664-vinsn %set-z-flag-if-s64-fits-in-fixnum (((dest :imm))
	1373	((src :s64))
	1374	((temp :s64)))
	1375	(movq (:%q src) (:%q temp))
	1376	(shlq (:$ub x8664::fixnumshift) (:%q temp))
	1377	(movq (:%q temp) (:%q dest)) ; tagged as a fixnum
	1378	(sarq (:$ub x8664::fixnumshift) (:%q temp))
	1379	(cmpq (:%q src) (:%q temp)))
	1380
	1381	(define-x8664-vinsn %set-z-flag-if-u64-fits-in-fixnum (((dest :imm))
	1382	((src :u64)))
	1383	(movq (:$q (ash #xf 60)) (:%q dest))
	1384	(testq (:%q dest) (:%q src))
	1385	(leaq (:@ (:%q src) x8664::fixnumone) (:% dest)))
	1386
	1387
	1388	(define-x8664-vinsn setup-bignum-alloc-for-s64-overflow (()
	1389	((src :s64)))
	1390	(movq (:%q src) (:%q x8664::imm2))
	1391	(movl (:$l x8664::two-digit-bignum-header) (:%l x8664::imm0.l))
	1392	(movl (:$l (- 16 x8664::fulltag-misc)) (:%l x8664::imm1.l)))
	1393
	1394
	1395	;;; If the sign bit is set in SRC, need to make a 3-digit bignum
	1396	;;; that requires 32 bytes of aligned memory
	1397	(define-x8664-vinsn setup-bignum-alloc-for-u64-overflow (()
	1398	((src :s64)))
	1399	(testq (:%q src) (:%q src))
	1400	(movq (:%q src) (:%q x8664::imm2))
	1401	(movl (:$l x8664::two-digit-bignum-header) (:%l x8664::imm0.l))
	1402	(movl (:$l (- 16 x8664::fulltag-misc)) (:%l x8664::imm1.l))
	1403	(jns :done)
	1404	(movl (:$l x8664::three-digit-bignum-header) (:%l x8664::imm0.l))
	1405	(movl (:$l (- 32 x8664::fulltag-misc)) (:%l x8664::imm1.l))
	1406	:done)
	1407
	1408
	1409
	1410	(define-x8664-vinsn %allocate-uvector (((dest :lisp))
	1411	()
	1412	((header (:u64 #.x8664::imm0))
	1413	(freeptr (:lisp #.x8664::allocptr))))
	1414	(subq (:%q x8664::imm1) (:rcontext x8664::tcr.save-allocptr))
	1415	(movq (:rcontext x8664::tcr.save-allocptr) (:%q freeptr))
	1416	(rcmpq (:%q freeptr) (:rcontext x8664::tcr.save-allocbase))
	1417	(:byte #x77) (:byte #x02) ;(ja :no-trap)
	1418	(uuo-alloc)
	1419	:no-trap
	1420	(movq (:%q header) (:@ x8664::misc-header-offset (:%q freeptr)))
	1421	(andb (:$b (lognot x8664::fulltagmask)) (:rcontext x8664::tcr.save-allocptr))
	1422	((:not (:pred = freeptr
	1423	(:apply %hard-regspec-value dest)))
	1424	(movq (:%q freeptr) (:%q dest))))
	1425
	1426	(define-x8664-vinsn set-bigits-after-fixnum-overflow (()
	1427	((bignum :lisp)))
	1428	(movq (:%q x8664::imm2) (:@ x8664::misc-data-offset (:%q bignum))))
	1429
	1430
	1431	(define-x8664-vinsn box-fixnum (((dest :imm))
	1432	((src :s8)))
	1433	(imulq (:$b x8664::fixnumone) (:%q src)(:%q dest)))
	1434
	1435
	1436	(define-x8664-vinsn (return-or-fix-overflow :jumpLR)(()
	1437	())
	1438	(jo :fix)
	1439	(:byte #xf3) (ret)
	1440	:fix
	1441	(jmp (:@ .SPfix-overflow)))
	1442
	1443	(define-x8664-vinsn add-constant (((dest :imm))
	1444	((dest :imm)
	1445	(const :s32const)))
	1446	((:and (:pred >= const -128) (:pred <= const 127))
	1447	(addq (:$b const) (:%q dest)))
	1448	((:not (:and (:pred >= const -128) (:pred <= const 127)))
	1449	(addq (:$l const) (:%q dest))))
	1450
	1451	(define-x8664-vinsn add-constant3 (((dest :imm))
	1452	((src :imm)
	1453	(const :s32const)))
	1454	((:pred = (:apply %hard-regspec-value dest)
	1455	(:apply %hard-regspec-value src))
	1456	((:and (:pred >= const -128) (:pred <= const 127))
	1457	(addq (:$b const) (:%q dest)))
	1458	((:not (:and (:pred >= const -128) (:pred <= const 127)))
	1459	(addq (:$l const) (:%q dest))))
	1460	((:not (:pred = (:apply %hard-regspec-value dest)
	1461	(:apply %hard-regspec-value src)))
	1462	(leaq (:@ const (:%q src)) (:%q dest))))
	1463
	1464
	1465
	1466	(define-x8664-vinsn fixnum-add2 (((dest :imm))
	1467	((dest :imm)
	1468	(other :imm)))
	1469	(addq (:%q other) (:%q dest)))
	1470
	1471	(define-x8664-vinsn fixnum-sub2 (((dest :imm))
	1472	((x :imm)
	1473	(y :imm))
	1474	((temp :imm)))
	1475	((:pred = (:apply %hard-regspec-value x) (:apply %hard-regspec-value dest))
	1476	(subq (:%q y) (:%q dest)))
	1477	((:not (:pred = (:apply %hard-regspec-value x) (:apply %hard-regspec-value dest)))
	1478	((:pred = (:apply %hard-regspec-value y) (:apply %hard-regspec-value dest))
	1479	(movq (:%q x) (:%q temp))
	1480	(subq (:%q y) (:%q temp))
	1481	(movq (:%q temp) (:%q dest)))
	1482	((:not (:pred = (:apply %hard-regspec-value y) (:apply %hard-regspec-value dest)))
	1483	(movq (:%q x) (:%q dest))
	1484	(subq (:%q y) (:%q dest)))))
	1485
	1486
	1487
	1488	(define-x8664-vinsn fixnum-add3 (((dest :imm))
	1489	((x :imm)
	1490	(y :imm)))
	1491
	1492	((:pred =
	1493	(:apply %hard-regspec-value x)
	1494	(:apply %hard-regspec-value dest))
	1495	(addq (:%q y) (:%q dest)))
	1496	((:not (:pred =
	1497	(:apply %hard-regspec-value x)
	1498	(:apply %hard-regspec-value dest)))
	1499	((:pred =
	1500	(:apply %hard-regspec-value y)
	1501	(:apply %hard-regspec-value dest))
	1502	(addq (:%q x) (:%q dest)))
	1503	((:not (:pred =
	1504	(:apply %hard-regspec-value y)
	1505	(:apply %hard-regspec-value dest)))
	1506	(leaq (:@ (:%q x) (:%q y)) (:%q dest)))))
	1507
	1508	(define-x8664-vinsn copy-gpr (((dest t))
	1509	((src t)))
	1510	((:not (:pred =
	1511	(:apply %hard-regspec-value dest)
	1512	(:apply %hard-regspec-value src)))
	1513	(movq (:%q src) (:%q dest))))
	1514
	1515	(define-x8664-vinsn (vpop-register :pop :node :vsp)
	1516	(((dest :lisp))
	1517	())
	1518	(popq (:%q dest)))
	1519
	1520	(define-x8664-vinsn (vpop-gvector-element :pop :node :vsp)
	1521	(()
	1522	((vector :lisp)
	1523	(idx :u32const))
	1524	())
	1525	(popq (:@ (:apply + x8664::misc-data-offset (:apply ash idx x8664::word-shift)) (:%q vector))))
	1526
	1527	(define-x8664-vinsn (push-argregs :push :node :vsp) (()
	1528	())
	1529	(testl (:%l x8664::nargs) (:%l x8664::nargs))
	1530	(jz :done)
	1531	(rcmpl (:%l x8664::nargs) (:$b (* 2 x8664::node-size)))
	1532	(jb :one)
	1533	(je :two)
	1534	(pushq (:%q x8664::arg_x))
	1535	:two
	1536	(pushq (:%q x8664::arg_y))
	1537	:one
	1538	(pushq (:%q x8664::arg_z))
	1539	:done)
	1540
	1541	(define-x8664-vinsn (push-max-argregs :push :node :vsp) (()
	1542	((max :u32const)))
	1543	((:pred >= max 3)
	1544	(testl (:%l x8664::nargs) (:%l x8664::nargs))
	1545	(jz :done)
	1546	(rcmpl (:%l x8664::nargs) (:$b (* 2 x8664::node-size)))
	1547	(jb :one)
	1548	(je :two)
	1549	(pushq (:%q x8664::arg_x))
	1550	:two
	1551	(pushq (:%q x8664::arg_y))
	1552	:one
	1553	(pushq (:%q x8664::arg_z))
	1554	:done)
	1555	((:pred = max 2)
	1556	(rcmpl (:%l x8664::nargs) (:$b (* 1 x8664::node-size)))
	1557	(jb :done)
	1558	(je :one)
	1559	(pushq (:%q x8664::arg_y))
	1560	:one
	1561	(pushq (:%q x8664::arg_z))
	1562	:done)
	1563	((:pred = max 1)
	1564	(testl (:%l x8664::nargs) (:%l x8664::nargs))
	1565	(je :done)
	1566	(pushq (:%q x8664::arg_z))
	1567	:done))
	1568
	1569	(define-x8664-vinsn (call-label :call) (()
	1570	((label :label))
	1571	((entry (:label 1))))
	1572	(:talign 4)
	1573	(call label)
	1574	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	1575
	1576	(define-x8664-vinsn double-float-compare (()
	1577	((arg0 :double-float)
	1578	(arg1 :double-float)))
	1579	(comisd (:%xmm arg1) (:%xmm arg0)))
	1580
	1581	(define-x8664-vinsn single-float-compare (()
	1582	((arg0 :single-float)
	1583	(arg1 :single-float)))
	1584	(comiss (:%xmm arg1) (:%xmm arg0)))
	1585
	1586
	1587	(define-x8664-vinsn double-float+-2 (((result :double-float))
	1588	((x :double-float)
	1589	(y :double-float)))
	1590	((:pred =
	1591	(:apply %hard-regspec-value result)
	1592	(:apply %hard-regspec-value x))
	1593	(addsd (:%xmm y) (:%xmm result)))
	1594	((:and (:not (:pred =
	1595	(:apply %hard-regspec-value result)
	1596	(:apply %hard-regspec-value x)))
	1597	(:pred =
	1598	(:apply %hard-regspec-value result)
	1599	(:apply %hard-regspec-value y)))
	1600	(addsd (:%xmm x) (:%xmm result)))
	1601	((:and (:not (:pred =
	1602	(:apply %hard-regspec-value result)
	1603	(:apply %hard-regspec-value x)))
	1604	(:not (:pred =
	1605	(:apply %hard-regspec-value result)
	1606	(:apply %hard-regspec-value y))))
	1607	(movsd (:%xmm x) (:%xmm result))
	1608	(addsd (:%xmm y) (:%xmm result))))
	1609
	1610	;;; Caller guarantees (not (eq y result))
	1611	(define-x8664-vinsn double-float--2 (((result :double-float))
	1612	((x :double-float)
	1613	(y :double-float)))
	1614	((:not (:pred = (:apply %hard-regspec-value result)
	1615	(:apply %hard-regspec-value x)))
	1616	(movsd (:%xmm x) (:%xmm result)))
	1617	(subsd (:%xmm y) (:%xmm result)))
	1618
	1619	(define-x8664-vinsn double-float*-2 (((result :double-float))
	1620	((x :double-float)
	1621	(y :double-float)))
	1622	((:pred =
	1623	(:apply %hard-regspec-value result)
	1624	(:apply %hard-regspec-value x))
	1625	(mulsd (:%xmm y) (:%xmm result)))
	1626	((:and (:not (:pred =
	1627	(:apply %hard-regspec-value result)
	1628	(:apply %hard-regspec-value x)))
	1629	(:pred =
	1630	(:apply %hard-regspec-value result)
	1631	(:apply %hard-regspec-value y)))
	1632	(mulsd (:%xmm x) (:%xmm result)))
	1633	((:and (:not (:pred =
	1634	(:apply %hard-regspec-value result)
	1635	(:apply %hard-regspec-value x)))
	1636	(:not (:pred =
	1637	(:apply %hard-regspec-value result)
	1638	(:apply %hard-regspec-value y))))
	1639	(movsd (:%xmm x) (:%xmm result))
	1640	(mulsd (:%xmm y) (:%xmm result))))
	1641
	1642	;;; Caller guarantees (not (eq y result))
	1643	(define-x8664-vinsn double-float/-2 (((result :double-float))
	1644	((x :double-float)
	1645	(y :double-float)))
	1646	((:not (:pred = (:apply %hard-regspec-value result)
	1647	(:apply %hard-regspec-value x)))
	1648	(movsd (:%xmm x) (:%xmm result)))
	1649	(divsd (:%xmm y) (:%xmm result)))
	1650
	1651
	1652	(define-x8664-vinsn complex-double-float+-2 (((result :complex-double-float))
	1653	((x :complex-double-float)
	1654	(y :complex-double-float)))
	1655	((:pred =
	1656	(:apply %hard-regspec-value result)
	1657	(:apply %hard-regspec-value x))
	1658	(addpd (:%xmm y) (:%xmm result)))
	1659	((:and (:not (:pred =
	1660	(:apply %hard-regspec-value result)
	1661	(:apply %hard-regspec-value x)))
	1662	(:pred =
	1663	(:apply %hard-regspec-value result)
	1664	(:apply %hard-regspec-value y)))
	1665	(addpd (:%xmm x) (:%xmm result)))
	1666	((:and (:not (:pred =
	1667	(:apply %hard-regspec-value result)
	1668	(:apply %hard-regspec-value x)))
	1669	(:not (:pred =
	1670	(:apply %hard-regspec-value result)
	1671	(:apply %hard-regspec-value y))))
	1672	(movapd (:%xmm x) (:%xmm result))
	1673	(addpd (:%xmm y) (:%xmm result))))
	1674
	1675	;;; Caller guarantees (not (eq y result))
	1676	(define-x8664-vinsn complex-double-float--2 (((result :complex-double-float))
	1677	((x :complex-double-float)
	1678	(y :complex-double-float)))
	1679	((:not (:pred = (:apply %hard-regspec-value result)
	1680	(:apply %hard-regspec-value x)))
	1681	(movapd (:%xmm x) (:%xmm result)))
	1682	(subpd (:%xmm y) (:%xmm result)))
	1683
	1684	(define-x8664-vinsn complex-double-float*-2 (((result :complex-double-float))
	1685	((x :complex-double-float)
	1686	(y :complex-double-float))
	1687	((b :double-float)
	1688	(ix :double-float)
	1689	(iy :double-float)))
	1690	(movapd (:%xmm x) (:%xmm ix))
	1691	(shufpd (:$ub 1) (:%xmm x8664::fpzero) (:%xmm ix))
	1692	(movapd (:%xmm y) (:%xmm iy))
	1693	(shufpd (:$ub 1) (:%xmm x8664::fpzero) (:%xmm iy))
	1694	(movsd (:%xmm y) (:%xmm result))
	1695	(mulsd (:%xmm x) (:%xmm result))
	1696	(movsd (:%xmm iy) (:%xmm b))
	1697	(mulsd (:%xmm ix) (:%xmm b))
	1698	(subsd (:%xmm b) (:%xmm result))
	1699	(mulsd (:%xmm x) (:%xmm iy))
	1700	(mulsd (:%xmm y) (:%xmm ix))
	1701	(addsd (:%xmm ix) (:%xmm iy))
	1702	(shufpd (:$ub 0) (:%xmm iy) (:%xmm result)))
	1703
	1704
	1705
	1706
	1707
	1708
	1709	(define-x8664-vinsn single-float+-2 (((result :single-float))
	1710	((x :single-float)
	1711	(y :single-float)))
	1712	((:pred =
	1713	(:apply %hard-regspec-value result)
	1714	(:apply %hard-regspec-value x))
	1715	(addss (:%xmm y) (:%xmm result)))
	1716	((:and (:not (:pred =
	1717	(:apply %hard-regspec-value result)
	1718	(:apply %hard-regspec-value x)))
	1719	(:pred =
	1720	(:apply %hard-regspec-value result)
	1721	(:apply %hard-regspec-value y)))
	1722	(addss (:%xmm x) (:%xmm result)))
	1723	((:and (:not (:pred =
	1724	(:apply %hard-regspec-value result)
	1725	(:apply %hard-regspec-value x)))
	1726	(:not (:pred =
	1727	(:apply %hard-regspec-value result)
	1728	(:apply %hard-regspec-value y))))
	1729	(movss (:%xmm x) (:%xmm result))
	1730	(addss (:%xmm y) (:%xmm result))))
	1731
	1732	;;; Caller guarantees (not (eq y result))
	1733	(define-x8664-vinsn single-float--2 (((result :single-float))
	1734	((x :single-float)
	1735	(y :single-float)))
	1736	((:not (:pred = (:apply %hard-regspec-value result)
	1737	(:apply %hard-regspec-value x)))
	1738	(movss (:%xmm x) (:%xmm result)))
	1739	(subss (:%xmm y) (:%xmm result)))
	1740
	1741	(define-x8664-vinsn single-float*-2 (((result :single-float))
	1742	((x :single-float)
	1743	(y :single-float)))
	1744	((:pred =
	1745	(:apply %hard-regspec-value result)
	1746	(:apply %hard-regspec-value x))
	1747	(mulss (:%xmm y) (:%xmm result)))
	1748	((:and (:not (:pred =
	1749	(:apply %hard-regspec-value result)
	1750	(:apply %hard-regspec-value x)))
	1751	(:pred =
	1752	(:apply %hard-regspec-value result)
	1753	(:apply %hard-regspec-value y)))
	1754	(mulss (:%xmm x) (:%xmm result)))
	1755	((:and (:not (:pred =
	1756	(:apply %hard-regspec-value result)
	1757	(:apply %hard-regspec-value x)))
	1758	(:not (:pred =
	1759	(:apply %hard-regspec-value result)
	1760	(:apply %hard-regspec-value y))))
	1761	(movss (:%xmm x) (:%xmm result))
	1762	(mulss (:%xmm y) (:%xmm result))))
	1763
	1764	;;; Caller guarantees (not (eq y result))
	1765	(define-x8664-vinsn single-float/-2 (((result :single-float))
	1766	((x :single-float)
	1767	(y :single-float)))
	1768	((:not (:pred = (:apply %hard-regspec-value result)
	1769	(:apply %hard-regspec-value x)))
	1770	(movss (:%xmm x) (:%xmm result)))
	1771	(divss (:%xmm y) (:%xmm result)))
	1772
	1773	(define-x8664-vinsn complex-single-float+-2 (((result :complex-single-float))
	1774	((x :complex-single-float)
	1775	(y :complex-single-float)))
	1776	((:pred =
	1777	(:apply %hard-regspec-value result)
	1778	(:apply %hard-regspec-value x))
	1779	(addps (:%xmm y) (:%xmm result)))
	1780	((:and (:not (:pred =
	1781	(:apply %hard-regspec-value result)
	1782	(:apply %hard-regspec-value x)))
	1783	(:pred =
	1784	(:apply %hard-regspec-value result)
	1785	(:apply %hard-regspec-value y)))
	1786	(addps (:%xmm x) (:%xmm result)))
	1787	((:and (:not (:pred =
	1788	(:apply %hard-regspec-value result)
	1789	(:apply %hard-regspec-value x)))
	1790	(:not (:pred =
	1791	(:apply %hard-regspec-value result)
	1792	(:apply %hard-regspec-value y))))
	1793	(movq (:%xmm x) (:%xmm result))
	1794	(addps (:%xmm y) (:%xmm result))))
	1795
	1796	;;; Caller guarantees (not (eq y result))
	1797	(define-x8664-vinsn complex-single-float--2 (((result :complex-single-float))
	1798	((x :complex-single-float)
	1799	(y :complex-single-float)))
	1800	((:not (:pred = (:apply %hard-regspec-value result)
	1801	(:apply %hard-regspec-value x)))
	1802	(movq (:%xmm x) (:%xmm result)))
	1803	(subps (:%xmm y) (:%xmm result)))
	1804
	1805	(define-x8664-vinsn complex-single-float*-2 (((result :complex-single-float))
	1806	((x :complex-single-float)
	1807	(y :complex-single-float))
	1808	((b :single-float)
	1809	(ix :single-float)
	1810	(iy :single-float)))
	1811	(movq (:%xmm x) (:%xmm ix))
	1812	(psrlq (:$ub 32) (:%xmm ix))
	1813	(movq (:%xmm y) (:%xmm iy))
	1814	(psrlq (:$ub 32) (:%xmm iy))
	1815	(movss(:%xmm y) (:%xmm result))
	1816	(mulss (:%xmm x) (:%xmm result))
	1817	(movss (:%xmm iy) (:%xmm b))
	1818	(mulss (:%xmm ix) (:%xmm b))
	1819	(subss (:%xmm b) (:%xmm result))
	1820	(mulss (:%xmm x) (:%xmm iy))
	1821	(mulss (:%xmm y) (:%xmm ix))
	1822	(addss (:%xmm ix) (:%xmm iy))
	1823	(unpcklps (:%xmm iy) (:%xmm result)))
	1824
	1825	(define-x8664-vinsn get-single (((result :single-float))
	1826	((source :lisp)))
	1827	(movd (:%q source) (:%xmm result))
	1828	(psrlq (:$ub 32) (:%xmm result)))
	1829
	1830	(define-x8664-vinsn get-double (((result :double-float))
	1831	((source :lisp)))
	1832	(movsd (:@ x8664::double-float.value (:%q source)) (:%xmm result)))
	1833
	1834	(define-x8664-vinsn get-complex-double-float (((result :complex-double-float))
	1835	((source :lisp)))
	1836	(movdqa (:@ x8664::complex-double-float.realpart (:%q source)) (:%xmm result)))
	1837
	1838	(define-x8664-vinsn get-complex-single-float (((result :complex-single-float))
	1839	((source :lisp)))
	1840	(movq (:@ x8664::complex-single-float.realpart (:%q source)) (:%xmm result)))
	1841
	1842
	1843	;;; Extract a double-float value, typechecking in the process.
	1844	;;; IWBNI we could simply call the "trap-unless-typecode=" vinsn here,
	1845	;;; instead of replicating it ..
	1846
	1847	(define-x8664-vinsn get-double? (((target :double-float))
	1848	((source :lisp))
	1849	((tag :u8)))
	1850	:resume
	1851	(movl (:%l source) (:%l tag))
	1852	(andl (:$b x8664::tagmask) (:%l tag))
	1853	(cmpl (:$b x8664::tag-misc) (:%l tag))
	1854	(jne :have-tag)
	1855	(movsbl (:@ x8664::misc-subtag-offset (:%q source)) (:%l tag))
	1856	:have-tag
	1857	(cmpl (:$b x8664::subtag-double-float) (:%l tag))
	1858	(jne :bad)
	1859	(movsd (:@ x8664::double-float.value (:%q source)) (:%xmm target))
	1860
	1861	(:anchored-uuo-section :resume)
	1862	:bad
	1863	(:anchored-uuo (uuo-error-reg-not-tag (:%q source) (:$ub x8664::subtag-double-float))))
	1864
	1865	(define-x8664-vinsn single->node (((result :lisp)
	1866	(source :single-float))
	1867	((source :single-float)))
	1868	(psllq (:$ub 32) (:%xmm source))
	1869	(movd (:%xmm source) (:%q result))
	1870	(movb (:$b x8664::tag-single-float) (:%b result)))
	1871
	1872	(define-x8664-vinsn copy-double-float (((dest :double-float))
	1873	((src :double-float)))
	1874	(movsd (:%xmm src) (:%xmm dest)))
	1875
	1876	(define-x8664-vinsn copy-single-float (((dest :single-float))
	1877	((src :single-float)))
	1878	(movss (:%xmm src) (:%xmm dest)))
	1879
	1880	(define-x8664-vinsn copy-complex-single-float (((dest :complex-single-float))
	1881	((src :complex-single-float)))
	1882	(movq (:%xmm src) (:%xmm dest)))
	1883
	1884	(define-x8664-vinsn copy-complex-double-float (((dest :complex-double-float))
	1885	((src :complex-double-float)))
	1886	(movapd (:%xmm src) (:%xmm dest)))
	1887
	1888
	1889	(define-x8664-vinsn copy-single-to-double (((dest :double-float))
	1890	((src :single-float)))
	1891	(cvtss2sd (:%xmm src) (:%xmm dest)))
	1892
	1893	(define-x8664-vinsn copy-double-to-single (((dest :single-float))
	1894	((src :double-float)))
	1895	(cvtsd2ss (:%xmm src) (:%xmm dest)))
	1896
	1897	(define-x8664-vinsn u8->fixnum (((result :imm))
	1898	((val :u8))
	1899	())
	1900	(leaq (:@ (:%q val) 8) (:%q result)))
	1901
	1902	(define-x8664-vinsn fitvals (()
	1903	((n :u16const))
	1904	((imm :u16)))
	1905	((:pred = n 0)
	1906	(xorl (:%l imm) (:%l imm)))
	1907	((:not (:pred = n 0))
	1908	(movl (:$l (:apply ash n x8664::fixnumshift)) (:%l imm)))
	1909	(subl (:%l x8664::nargs) (:%l imm))
	1910	(jae :push-more)
	1911	(movslq (:%l imm) (:%q imm))
	1912	(subq (:%q imm) (:%q x8664::rsp))
	1913	(jmp :done)
	1914	:push-loop
	1915	(pushq (:$l (:apply target-nil-value)))
	1916	(addl (:$b x8664::node-size) (:%l x8664::nargs))
	1917	(subl (:$b x8664::node-size) (:%l imm))
	1918	:push-more
	1919	(jne :push-loop)
	1920	:done)
	1921
	1922	(define-x8664-vinsn (nvalret :jumpLR) (()
	1923	())
	1924
	1925	(jmp (:@ .SPnvalret)))
	1926
	1927
	1928	(define-x8664-vinsn lisp-word-ref (((dest t))
	1929	((base t)
	1930	(offset t)))
	1931	(movq (:@ (:%q base) (:%q offset)) (:%q dest)))
	1932
	1933
	1934	(define-x8664-vinsn lisp-word-ref-c (((dest t))
	1935	((base t)
	1936	(offset :s32const)))
	1937	((:pred = offset 0)
	1938	(movq (:@ (:%q base)) (:%q dest)))
	1939	((:not (:pred = offset 0))
	1940	(movq (:@ offset (:%q base)) (:%q dest))))
	1941
	1942
	1943	(define-x8664-vinsn (vpush-label :push :node :vsp) (()
	1944	((label :label))
	1945	((temp :lisp)))
	1946	(leaq (:@ (:^ label) (:%q x8664::fn)) (:%q temp))
	1947	(pushq (:%q temp)))
	1948
	1949	;; ????
	1950	(define-x8664-vinsn (emit-aligned-label :align) (()
	1951	((label :label)))
	1952	(:align 3)
	1953	(:long (:^ label)))
	1954
	1955	;;; %ra0 is pointing into %fn, so no need to copy %fn here.
	1956	(define-x8664-vinsn (pass-multiple-values-symbol :jumplr) (()
	1957	())
	1958	(pushq (:@ (:apply + (:apply target-nil-value) (x8664::%kernel-global 'x86::ret1valaddr))))
	1959	(jmp (:@ x8664::symbol.fcell (:% x8664::fname))))
	1960
	1961	;;; It'd be good to have a variant that deals with a known function
	1962	;;; as well as this.
	1963	(define-x8664-vinsn (pass-multiple-values :jumplr) (()
	1964	()
	1965	((tag :u8)))
	1966	:resume
	1967	(movl (:%l x8664::temp0) (:%l tag))
	1968	(andl (:$b x8664::fulltagmask) (:%l tag))
	1969	(cmpl (:$b x8664::fulltag-symbol) (:%l tag))
	1970	(cmovgq (:%q x8664::temp0) (:%q x8664::fn))
	1971	(jl :bad)
	1972	(cmoveq (:@ x8664::symbol.fcell (:%q x8664::fname)) (:%q x8664::fn))
	1973	(pushq (:@ (:apply + (:apply target-nil-value) (x8664::%kernel-global 'x86::ret1valaddr))))
	1974	(jmp (:%q x8664::fn))
	1975
	1976	(:anchored-uuo-section :resume)
	1977	:bad
	1978	(:anchored-uuo (uuo-error-not-callable)))
	1979
	1980	(define-x8664-vinsn (pass-multiple-values-known-function :jumplr) (((fnreg :lisp))
	1981	())
	1982	(pushq (:@ (:apply + (:apply target-nil-value) (x8664::%kernel-global 'x86::ret1valaddr))))
	1983	(jmp (:%q fnreg)))
	1984
	1985
	1986	(define-x8664-vinsn reserve-outgoing-frame (()
	1987	())
	1988	(pushq (:$b x8664::reserved-frame-marker))
	1989	(pushq (:$b x8664::reserved-frame-marker)))
	1990
	1991
	1992	(define-x8664-vinsn (call-known-function :call) (()
	1993	()
	1994	((entry (:label 1))))
	1995	(:talign 4)
	1996	(call (:%q x8664::temp0))
	1997	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	1998
	1999	(define-x8664-vinsn (jump-known-function :jumplr) (()
	2000	()
	2001	((xfn (:lisp #.x8664::xfn))))
	2002	(movq (:%q x8664::fn) (:%q x8664::xfn))
	2003	(movq (:%q x8664::temp0) (:%q x8664::fn))
	2004	(jmp (:%q x8664::fn)))
	2005
	2006	(define-x8664-vinsn (list :call) (()
	2007	()
	2008	((entry (:label 1))))
	2009	(leaq (:@ (:^ :back) (:%q x8664::fn)) (:%q x8664::ra0))
	2010	(:talign 4)
	2011	(jmp (:@ .SPconslist))
	2012	:back
	2013	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	2014
	2015
	2016	(define-x8664-vinsn make-tsp-cons (((dest :lisp))
	2017	((car :lisp) (cdr :lisp))
	2018	((temp :imm)
	2019	(stack-temp :imm)))
	2020	(subq (:$b (+ x8664::cons.size x8664::dnode-size)) (:rcontext x8664::tcr.next-tsp))
	2021	(movq (:rcontext x8664::tcr.next-tsp) (:%q temp))
	2022	(movapd (:%xmm x8664::fpzero) (:@ (:%q temp)))
	2023	(movapd (:%xmm x8664::fpzero) (:@ 16 (:%q temp)))
	2024	(movq (:rcontext x8664::tcr.save-tsp) (:%q stack-temp))
	2025	(movq (:%q stack-temp) (:@ (:%q temp)))
	2026	(movq (:% x8664::rbp) (:@ x8664::tsp-frame.rbp (:%q temp)))
	2027	(movq (:%q temp) (:rcontext x8664::tcr.save-tsp))
	2028	(leaq (:@ (+ x8664::dnode-size x8664::fulltag-cons) (:%q temp)) (:%q temp))
	2029	(movq (:%q car) (:@ x8664::cons.car (:%q temp)))
	2030	(movq (:%q cdr) (:@ x8664::cons.cdr (:%q temp)))
	2031	(movq (:%q temp) (:%q dest)))
	2032
	2033	(define-x8664-vinsn make-fixed-stack-gvector (((dest :lisp))
	2034	((aligned-size :u32const)
	2035	(header :s32const))
	2036	((tempa :imm)
	2037	(tempb :imm)
	2038	(stack-temp :imm)))
	2039	((:and (:pred >= (:apply + aligned-size x8664::dnode-size) -128)
	2040	(:pred <= (:apply + aligned-size x8664::dnode-size) 127))
	2041	(subq (:$b (:apply + aligned-size x8664::dnode-size))
	2042	(:rcontext x8664::tcr.next-tsp)))
	2043	((:not (:and (:pred >= (:apply + aligned-size x8664::dnode-size) -128)
	2044	(:pred <= (:apply + aligned-size x8664::dnode-size) 127)))
	2045	(subq (:$l (:apply + aligned-size x8664::dnode-size))
	2046	(:rcontext x8664::tcr.next-tsp)))
	2047	(movq (:rcontext x8664::tcr.save-tsp) (:%q tempb))
	2048	(movq (:rcontext x8664::tcr.next-tsp) (:%q tempa))
	2049	(movq (:%q tempb) (:%q stack-temp))
	2050	:loop
	2051	(movapd (:%xmm x8664::fpzero) (:@ -16 (:%q tempb)))
	2052	(subq (:$b x8664::dnode-size) (:%q tempb))
	2053	(cmpq (:%q tempa) (:%q tempb))
	2054	(jnz :loop)
	2055	(movq (:%q stack-temp) (:@ (:%q tempa)))
	2056	(movq (:% x8664::rbp) (:@ x8664::tsp-frame.rbp (:%q tempa)))
	2057	(movq (:%q tempa) (:rcontext x8664::tcr.save-tsp))
	2058	(movl (:$l header) (:@ x8664::dnode-size (:%q tempa)))
	2059	(leaq (:@ (+ x8664::dnode-size x8664::fulltag-misc) (:%q tempa)) (:%q dest)))
	2060
	2061
	2062	(define-x8664-vinsn (discard-temp-frame :tsp :pop :discard) (()
	2063	()
	2064	((temp :imm)))
	2065	(movq (:rcontext x8664::tcr.save-tsp) (:%q temp))
	2066	(movq (:@ (:%q temp)) (:%q temp))
	2067	(movq (:%q temp) (:rcontext x8664::tcr.save-tsp))
	2068	(movq (:%q temp) (:rcontext x8664::tcr.next-tsp))
	2069	)
	2070
	2071	(define-x8664-vinsn (discard-c-frame :csp :pop :discard) (()
	2072	()
	2073	((temp :imm)))
	2074	(movq (:rcontext x8664::tcr.foreign-sp) (:%q temp))
	2075	(movq (:@ (:%q temp)) (:%q temp))
	2076	(movq (:%q temp) (:rcontext x8664::tcr.foreign-sp)))
	2077
	2078
	2079	(define-x8664-vinsn (vstack-discard :vsp :pop :discard) (()
	2080	((nwords :u32const)))
	2081	((:not (:pred = nwords 0))
	2082	((:pred < nwords 16)
	2083	(addq (:$b (:apply ash nwords x8664::word-shift)) (:%q x8664::rsp)))
	2084	((:not (:pred < nwords 16))
	2085	(addq (:$l (:apply ash nwords x8664::word-shift)) (:%q x8664::rsp)))))
	2086
	2087
	2088	(defmacro define-x8664-subprim-lea-jmp-vinsn ((name &rest other-attrs) spno)
	2089	`(define-x8664-vinsn (,name :call :subprim ,@other-attrs) (() () ((entry (:label 1))))
	2090	(leaq (:@ (:^ :back) (:%q x8664::fn)) (:%q x8664::ra0))
	2091	(:talign 4)
	2092	(jmp (:@ ,spno))
	2093	:back
	2094	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn))))
	2095
	2096	(defmacro define-x8664-subprim-call-vinsn ((name &rest other-attrs) spno)
	2097	`(define-x8664-vinsn (,name :call :subprim ,@other-attrs) (() () ((entry (:label 1))))
	2098	(:talign 4)
	2099	(call (:@ ,spno))
	2100	:back
	2101	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn))))
	2102
	2103	(defmacro define-x8664-subprim-jump-vinsn ((name &rest other-attrs) spno)
	2104	`(define-x8664-vinsn (,name :jumpLR ,@other-attrs) (() ())
	2105	(jmp (:@ ,spno))))
	2106
	2107	(define-x8664-vinsn (nthrowvalues :call :subprim) (()
	2108	((lab :label)))
	2109	(leaq (:@ (:^ lab) (:%q x8664::fn)) (:%q x8664::ra0))
	2110	(jmp (:@ .SPnthrowvalues)))
	2111
	2112	(define-x8664-vinsn (nthrow1value :call :subprim) (()
	2113	((lab :label)))
	2114	(leaq (:@ (:^ lab) (:%q x8664::fn)) (:%q x8664::ra0))
	2115	(jmp (:@ .SPnthrow1value)))
	2116
	2117
	2118
	2119	(define-x8664-subprim-lea-jmp-vinsn (bind-interrupt-level-0) .SPbind-interrupt-level-0)
	2120
	2121
	2122	(define-x8664-vinsn bind-interrupt-level-0-inline (()
	2123	()
	2124	((temp :imm)))
	2125	(movq (:rcontext x8664::tcr.tlb-pointer) (:%q temp))
	2126	(cmpq (:$b 0) (:@ x8664::interrupt-level-binding-index (:%q temp)))
	2127	(pushq (:@ x8664::interrupt-level-binding-index (:%q temp)))
	2128	(pushq (:$b x8664::interrupt-level-binding-index))
	2129	(pushq (:rcontext x8664::tcr.db-link))
	2130	(movq (:$l 0) (:@ x8664::interrupt-level-binding-index (:%q temp)))
	2131	(movq (:%q x8664::rsp) (:rcontext x8664::tcr.db-link))
	2132	(jns :done)
	2133	(btrq (:$ub 63) (:rcontext x8664::tcr.interrupt-pending))
	2134	(jae :done)
	2135	(ud2a)
	2136	(:byte 2)
	2137	:done)
	2138
	2139
	2140
	2141	(define-x8664-subprim-lea-jmp-vinsn (bind-interrupt-level-m1) .SPbind-interrupt-level-m1)
	2142
	2143	(define-x8664-vinsn bind-interrupt-level-m1-inline (()
	2144	()
	2145	((temp :imm)))
	2146	(movq (:rcontext x8664::tcr.tlb-pointer) (:%q temp))
	2147	(pushq (:@ x8664::interrupt-level-binding-index (:%q temp)))
	2148	(pushq (:$b x8664::interrupt-level-binding-index))
	2149	(pushq (:rcontext x8664::tcr.db-link))
	2150	(movq (:$l (ash -1 x8664::fixnumshift)) (:@ x8664::interrupt-level-binding-index (:%q temp)))
	2151	(movq (:%q x8664::rsp) (:rcontext x8664::tcr.db-link)))
	2152
	2153	(define-x8664-subprim-lea-jmp-vinsn (bind-interrupt-level) .SPbind-interrupt-level)
	2154
	2155	(define-x8664-subprim-call-vinsn (unbind-interrupt-level) .SPunbind-interrupt-level)
	2156
	2157	(define-x8664-vinsn unbind-interrupt-level-inline (()
	2158	()
	2159	((link :imm)
	2160	(curval :imm)
	2161	(oldval :imm)
	2162	(tlb :imm)))
	2163	(movq (:rcontext x8664::tcr.tlb-pointer) (:%q tlb))
	2164	(movq (:rcontext x8664::tcr.db-link) (:%q link))
	2165	(movq (:@ x8664::interrupt-level-binding-index (:%q tlb)) (:%q curval))
	2166	(testq (:%q curval) (:%q curval))
	2167	(movq (:@ 16 #\|binding.val\|# (:%q link)) (:%q oldval))
	2168	(movq (:@ #\|binding.link\|# (:%q link)) (:%q link))
	2169	(movq (:%q oldval) (:@ x8664::interrupt-level-binding-index (:%q tlb)))
	2170	(movq (:%q link) (:rcontext x8664::tcr.db-link))
	2171	(jns :done)
	2172	(testq (:%q oldval) (:%q oldval))
	2173	(js :done)
	2174	(btrq (:$ub 63) (:rcontext x8664::tcr.interrupt-pending))
	2175	(jae :done)
	2176	(ud2a)
	2177	(:byte 2)
	2178	:done)
	2179
	2180	(define-x8664-vinsn (jump-return-pc :jumpLR)
	2181	(()
	2182	())
	2183	(ret))
	2184
	2185	(define-x8664-vinsn (nmkcatchmv :call :subprim) (()
	2186	((lab :label))
	2187	((entry (:label 1))))
	2188	(leaq (:@ (:^ lab) (:%q x8664::fn)) (:%q x8664::xfn))
	2189	(:talign 4)
	2190	(call (:@ .SPmkcatchmv))
	2191	:back
	2192	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	2193
	2194	(define-x8664-vinsn (nmkcatch1v :call :subprim) (()
	2195	((lab :label))
	2196	((entry (:label 1))))
	2197	(leaq (:@ (:^ lab) (:%q x8664::fn)) (:%q x8664::xfn))
	2198	(:talign 4)
	2199	(call (:@ .SPmkcatch1v))
	2200	:back
	2201	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	2202
	2203
	2204	(define-x8664-vinsn (make-simple-unwind :call :subprim) (()
	2205	((protform-lab :label)
	2206	(cleanup-lab :label)))
	2207	(leaq (:@ (:^ protform-lab) (:%q x8664::fn)) (:%q x8664::ra0))
	2208	(leaq (:@ (:^ cleanup-lab) (:%q x8664::fn)) (:%q x8664::xfn))
	2209	(jmp (:@ .SPmkunwind)))
	2210
	2211	(define-x8664-vinsn (nmkunwind :call :subprim) (()
	2212	((protform-lab :label)
	2213	(cleanup-lab :label)))
	2214	(leaq (:@ (:^ protform-lab) (:%q x8664::fn)) (:%q x8664::ra0))
	2215	(leaq (:@ (:^ cleanup-lab) (:%q x8664::fn)) (:%q x8664::xfn))
	2216	(jmp (:@ .SPnmkunwind)))
	2217
	2218	;;; "old" mkunwind. Used by PROGV, since the binding of interrupt-level
	2219	;;; on entry to the new mkunwind confuses the issue.
	2220
	2221	(define-x8664-vinsn (mkunwind :call :subprim) (()
	2222	((protform-lab :label)
	2223	(cleanup-lab :label)))
	2224	(leaq (:@ (:^ protform-lab) (:%q x8664::fn)) (:%q x8664::ra0))
	2225	(leaq (:@ (:^ cleanup-lab) (:%q x8664::fn)) (:%q x8664::xfn))
	2226	(jmp (:@ .SPmkunwind)))
	2227
	2228	(define-x8664-subprim-lea-jmp-vinsn (gvector) .SPgvector)
	2229
	2230	(define-x8664-subprim-call-vinsn (getu64) .SPgetu64)
	2231
	2232	;;; Call something callable and obtain the single value that it
	2233	;;; returns.
	2234	(define-x8664-vinsn (funcall :call) (()
	2235	()
	2236	((tag :u8)
	2237	(entry (:label 1))))
	2238	:resume
	2239	(movl (:%l x8664::temp0) (:%l tag))
	2240	(andl (:$b x8664::fulltagmask) (:%l tag))
	2241	(cmpl (:$b x8664::fulltag-symbol) (:%l tag))
	2242	(cmovgq (:%q x8664::temp0) (:%q x8664::xfn))
	2243	(jl :bad)
	2244	(cmoveq (:@ x8664::symbol.fcell (:%q x8664::fname)) (:%q x8664::xfn))
	2245	(:talign 4)
	2246	(call (:%q x8664::xfn))
	2247	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn))
	2248	(:anchored-uuo-section :resume)
	2249	:bad
	2250	(:anchored-uuo (uuo-error-not-callable)))
	2251
	2252	(define-x8664-vinsn (tail-funcall :jumplr) (()
	2253	()
	2254	((tag (:u8 #.x8664::imm0))))
	2255	:resume
	2256	(movl (:%l x8664::temp0) (:%l tag))
	2257	(andl (:$b x8664::fulltagmask) (:%l tag))
	2258	(cmpl (:$b x8664::fulltag-symbol) (:%l tag))
	2259	(cmovgq (:%q x8664::temp0) (:%q x8664::xfn))
	2260	(jl :bad)
	2261	(cmoveq (:@ x8664::symbol.fcell (:%q x8664::fname)) (:%q x8664::xfn))
	2262	(jmp (:%q x8664::xfn))
	2263
	2264	(:anchored-uuo-section :resume)
	2265	:bad
	2266	(:anchored-uuo (uuo-error-not-callable)))
	2267
	2268
	2269
	2270	;;; Magic numbers in here include the address of .SPcall-closure.
	2271	(define-x8664-vinsn init-nclosure (()
	2272	((closure :lisp))
	2273	((imm0 :u64)))
	2274	(movq (:$q #x24fffffffff92d8d) (:%q imm0))
	2275	(movb (:$b 4) (:@ x8664::misc-data-offset (:%q closure))) ; code word count
	2276	(movb (:$b #x4c) (:@ (+ x8664::misc-data-offset 7) (:%q closure))) ; 1st byte of lea
	2277	(movq (:%q imm0) (:@ (+ x8664::misc-data-offset 8) (:%q closure))) ; rest of lea, start of jmp
	2278	(movl (:$l #x01516825) (:@ (+ x8664::misc-data-offset 16) (:%q closure)))
	2279	(movb (:$b x8664::function-boundary-marker) (:@ (+ x8664::misc-data-offset 24) (:%q closure))))
	2280
	2281
	2282	(define-x8664-vinsn finalize-closure (((closure :lisp))
	2283	((closure :lisp)))
	2284	(addq (:$b (- x8664::fulltag-function x8664::fulltag-misc)) (:%q closure)))
	2285
	2286
	2287	(define-x8664-vinsn (ref-symbol-value :call :subprim)
	2288	(((val :lisp))
	2289	((sym (:lisp (:ne val))))
	2290	((entry (:label 1))))
	2291	(:talign 4)
	2292	(call (:@ .SPspecrefcheck))
	2293	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)) )
	2294
	2295	(define-x8664-vinsn ref-symbol-value-inline (((dest :lisp))
	2296	((src (:lisp (:ne dest))))
	2297	((table :imm)
	2298	(idx :imm)))
	2299	:resume
	2300	(movq (:@ x8664::symbol.binding-index (:%q src)) (:%q idx))
	2301	;; The entry for binding-index 0 in the table always
	2302	;; contains a no-thread-local-binding marker; treat
	2303	;; out-of-bounds indices as 0 to avoid branches
	2304	(xorl (:%l table) (:%l table))
	2305	(rcmpq (:%q idx) (:rcontext x8664::tcr.tlb-limit))
	2306	(cmovael (:%l table) (:%l idx))
	2307	(movq (:rcontext x8664::tcr.tlb-pointer) (:%q table))
	2308	(movq (:@ (:%q table) (:%q idx)) (:%q dest))
	2309	(cmpl (:$b x8664::subtag-no-thread-local-binding) (:%l dest))
	2310	(cmoveq (:@ x8664::symbol.vcell (:%q src)) (:%q dest))
	2311	:test
	2312	(cmpl (:$b x8664::unbound-marker) (:%l dest))
	2313	(je :bad)
	2314
	2315	(:anchored-uuo-section :resume)
	2316	:bad
	2317	(:anchored-uuo (uuo-error-unbound (:%q src))))
	2318
	2319
	2320	(define-x8664-vinsn (%ref-symbol-value :call :subprim)
	2321	(((val :lisp))
	2322	((sym (:lisp (:ne val))))
	2323	((entry (:label 1))))
	2324	(:talign 4)
	2325	(call (:@ .SPspecref))
	2326	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	2327
	2328	(define-x8664-vinsn %ref-symbol-value-inline (((dest :lisp))
	2329	((src (:lisp (:ne dest))))
	2330	((table :imm)
	2331	(idx :imm)))
	2332	;; Treat out-of-bounds indices as index 0
	2333	(movq (:@ x8664::symbol.binding-index (:%q src)) (:%q idx))
	2334	(xorl (:%l table) (:%l table))
	2335	(rcmpq (:%q idx) (:rcontext x8664::tcr.tlb-limit))
	2336	(cmovael (:%l table) (:%l idx))
	2337	(movq (:rcontext x8664::tcr.tlb-pointer) (:%q table))
	2338	(movq (:@ (:%q table) (:%q idx)) (:%q dest))
	2339	(cmpb (:$b x8664::subtag-no-thread-local-binding) (:%b dest))
	2340	(cmoveq (:@ x8664::symbol.vcell (:%q src)) (:%q dest)))
	2341
	2342	(define-x8664-vinsn ref-interrupt-level (((dest :imm))
	2343	()
	2344	((temp :u64)))
	2345	(movq (:rcontext x8664::tcr.tlb-pointer) (:%q temp))
	2346	(movq (:@ x8664::INTERRUPT-LEVEL-BINDING-INDEX (:%q temp)) (:%q dest)))
	2347
	2348
	2349
	2350
	2351	(define-x8664-vinsn setup-double-float-allocation (()
	2352	())
	2353	(movl (:$l (arch::make-vheader x8664::double-float.element-count x8664::subtag-double-float)) (:%l x8664::imm0.l))
	2354	(movl (:$l (- x8664::double-float.size x8664::fulltag-misc)) (:%l x8664::imm1.l)))
	2355
	2356	(define-x8664-vinsn setup-complex-double-float-allocation (()
	2357	())
	2358	(movl (:$l (arch::make-vheader 6 x8664::subtag-complex-double-float)) (:%l x8664::imm0.l))
	2359	(movl (:$l (- x8664::complex-double-float.size x8664::fulltag-misc)) (:%l x8664::imm1.l)))
	2360
	2361	(define-x8664-vinsn setup-complex-single-float-allocation (()
	2362	())
	2363	(movl (:$l (arch::make-vheader 2 x8664::subtag-complex-single-float)) (:%l x8664::imm0.l))
	2364	(movl (:$l (- x8664::complex-single-float.size x8664::fulltag-misc)) (:%l x8664::imm1.l)))
	2365
	2366	(define-x8664-vinsn set-double-float-value (()
	2367	((node :lisp)
	2368	(val :double-float)))
	2369	(movsd (:%xmm val) (:@ x8664::double-float.value (:%q node))))
	2370
	2371	(define-x8664-vinsn set-complex-double-float-value (()
	2372	((node :lisp)
	2373	(val :complex-double-float)))
	2374	(movdqa (:%xmm val) (:@ x8664::complex-double-float.realpart (:%q node))))
	2375
	2376	(define-x8664-vinsn set-complex-single-float-value (()
	2377	((node :lisp)
	2378	(val :complex-single-float)))
	2379	(movq (:%xmm val) (:@ x8664::complex-single-float.realpart (:%q node))))
	2380
	2381
	2382	(define-x8664-vinsn word-index-and-bitnum-from-index (((word-index :u64)
	2383	(bitnum :u8))
	2384	((index :imm)))
	2385	(movq (:%q index) (:%q word-index))
	2386	(shrq (:$ub x8664::fixnumshift) (:%q word-index))
	2387	(movl (:$l 63) (:%l bitnum))
	2388	(andl (:%l word-index) (:%l bitnum))
	2389	(shrq (:$ub 6) (:%q word-index)))
	2390
	2391	(define-x8664-vinsn ref-bit-vector-fixnum (((dest :imm)
	2392	(bitnum :u8))
	2393	((bitnum :u8)
	2394	(bitvector :lisp)
	2395	(word-index :u64)))
	2396	(btq (:%q bitnum) (:@ x8664::misc-data-offset (:%q bitvector) (:%q word-index) 8))
	2397	(setb (:%b bitnum))
	2398	(negb (:%b bitnum))
	2399	(andl (:$l x8664::fixnumone) (:%l bitnum))
	2400	(movl (:%l bitnum) (:%l dest)))
	2401
	2402	(define-x8664-vinsn nref-bit-vector-fixnum (((dest :imm)
	2403	(bitnum :s64))
	2404	((bitnum :s64)
	2405	(bitvector :lisp))
	2406	())
	2407	(btq (:%q bitnum) (:@ x8664::misc-data-offset (:%q bitvector)))
	2408	(setc (:%b bitnum))
	2409	(movzbl (:%b bitnum) (:%l bitnum))
	2410	(imull (:$b x8664::fixnumone) (:%l bitnum) (:%l dest)))
	2411
	2412
	2413	(define-x8664-vinsn nref-bit-vector-flags (()
	2414	((bitnum :s64)
	2415	(bitvector :lisp))
	2416	())
	2417	(btq (:%q bitnum) (:@ x8664::misc-data-offset (:%q bitvector))))
	2418
	2419	(define-x8664-vinsn misc-ref-c-bit-fixnum (((dest :imm))
	2420	((src :lisp)
	2421	(idx :u64const))
	2422	((temp :u8)))
	2423	(btq (:$ub (:apply logand 63 idx))
	2424	(:@ (:apply + x8664::misc-data-offset (:apply ash (:apply ash idx -6) x8664::word-shift)) (:%q src)))
	2425	(setc (:%b temp))
	2426	(movzbl (:%b temp) (:%l temp))
	2427	(imull (:$b x8664::fixnumone) (:%l temp) (:%l dest)))
	2428
	2429
	2430	(define-x8664-vinsn misc-ref-c-bit-flags (()
	2431	((src :lisp)
	2432	(idx :u64const))
	2433	)
	2434	(btq (:$ub (:apply logand 63 idx))
	2435	(:@ (:apply + x8664::misc-data-offset (:apply ash (:apply ash idx -6) x8664::word-shift)) (:%q src))))
	2436
	2437	(define-x8664-vinsn deref-macptr (((addr :address))
	2438	((src :lisp))
	2439	())
	2440	(movq (:@ x8664::macptr.address (:%q src)) (:%q addr)))
	2441
	2442	(define-x8664-vinsn save-nfp (()
	2443	()
	2444	((temp :imm)))
	2445	((:pred > (:apply x862-max-nfp-depth) 0)
	2446	(movq (:rcontext x8664::tcr.foreign-sp) (:%mmx x8664::stack-temp))
	2447	(:if (:pred < (:apply + 16 (:apply x862-max-nfp-depth)) 128)
	2448	(subq (:$b (:apply + 16 (:apply x862-max-nfp-depth))) (:rcontext x8664::tcr.foreign-sp))
	2449	(subq (:$l (:apply + 16 (:apply x862-max-nfp-depth))) (:rcontext x8664::tcr.foreign-sp)))
	2450	(movq (:rcontext x8664::tcr.foreign-sp) (:%q temp))
	2451	(movq (:%mmx x8664::stack-temp) (:@ (:%q temp)))
	2452	(movq (:rcontext x8664::tcr.nfp) (:%mmx x8664::stack-temp))
	2453	(movq (:%mmx x8664::stack-temp) (:@ 8 (:%q temp)))
	2454	(movq (:% temp) (:rcontext x8664::tcr.nfp))))
	2455
	2456
	2457	(define-x8664-vinsn load-nfp (((reg :imm))
	2458	())
	2459	(movq (:rcontext x8664::tcr.nfp) (:%q reg)))
	2460
	2461	(define-x8664-vinsn restore-nfp (()
	2462	()
	2463	((temp :imm)))
	2464	((:pred > (:apply x862-max-nfp-depth) 0)
	2465	(movq (:rcontext x8664::tcr.nfp) (:%q temp))
	2466	(movq (:@ 8 (:%q temp)) (:%mmx x8664::stack-temp))
	2467	(movq (:@ (:%q temp)) (:%q temp))
	2468	(movq (:%q temp) (:rcontext x8664::tcr.foreign-sp))
	2469	(movq (:%mmx x8664::stack-temp)(:rcontext x8664::tcr.nfp))))
	2470
	2471	(define-x8664-vinsn (nfp-store-unboxed-word :nfp :set) (()
	2472	((val :u64)
	2473	(offset :u16const)))
	2474	(movd (:%q val) (:%mmx x8664::stack-temp))
	2475	(movq (:rcontext x8664::tcr.nfp) (:%q val))
	2476	(movq (:%mmx x8664::stack-temp)(:@ (:apply + 16 offset) (:% val)))
	2477	(movd (:%mmx x8664::stack-temp) (:%q val)))
	2478
	2479
	2480	(define-x8664-vinsn (nfp-load-unboxed-word :nfp :ref) (((val :u64))
	2481	((offset :u16const))
	2482	)
	2483	(movq (:rcontext x8664::tcr.nfp) (:%q val))
	2484	(movq (:@ (:apply + 16 offset) (:% val)) (:%q val)))
	2485
	2486	(define-x8664-vinsn (nfp-store-single-float :nfp :set) (()
	2487	((val :single-float)
	2488	(offset :u16const))
	2489	)
	2490	(movd (:%q x8664::imm0) (:%mmx x8664::stack-temp))
	2491	(movq (:rcontext x8664::tcr.nfp) (:%q x8664::imm0))
	2492	(movss (:%xmm val) (:@ (:apply + 16 offset) (:% x8664::imm0)))
	2493	(movd (:%mmx x8664::stack-temp) (:%q x8664::imm0)))
	2494
	2495	(define-x8664-vinsn (nfp-store-double-float :nfp :set) (()
	2496	((val :double-float)
	2497	(offset :u16const)))
	2498	(movd (:%q x8664::imm0) (:%mmx x8664::stack-temp))
	2499	(movq (:rcontext x8664::tcr.nfp) (:%q x8664::imm0))
	2500	(movsd (:%xmm val) (:@ (:apply + 16 offset) (:% x8664::imm0)))
	2501	(movd (:%mmx x8664::stack-temp) (:%q x8664::imm0)))
	2502
	2503
	2504	(define-x8664-vinsn (nfp-load-double-float :nfp :ref) (((val :double-float))
	2505	((offset :u16const)))
	2506	(movd (:%q x8664::imm0) (:%mmx x8664::stack-temp))
	2507	(movq (:rcontext x8664::tcr.nfp) (:%q x8664::imm0))
	2508	(movsd (:@ (:apply + 16 offset) (:% x8664::imm0)) (:%xmm val))
	2509	(movd (:%mmx x8664::stack-temp) (:%q x8664::imm0)))
	2510
	2511
	2512	(define-x8664-vinsn (nfp-load-single-float :nfp :ref) (((val :single-float))
	2513	((offset :u16const)))
	2514	(movd (:%q x8664::imm0) (:%mmx x8664::stack-temp))
	2515	(movq (:rcontext x8664::tcr.nfp) (:%q x8664::imm0))
	2516	(movss (:@ (:apply + 16 offset) (:% x8664::imm0)) (:%xmm val))
	2517	(movd (:%mmx x8664::stack-temp) (:%q x8664::imm0)))
	2518
	2519	(define-x8664-vinsn (nfp-store-complex-single-float :nfp :set) (()
	2520	((val :complex-single-float)
	2521	(offset :u16const)))
	2522
	2523	(movd (:%q x8664::imm0) (:%mmx x8664::stack-temp))
	2524	(movq (:rcontext x8664::tcr.nfp) (:%q x8664::imm0))
	2525	(movq (:%xmm val) (:@ (:apply + 16 offset) (:%q x8664::imm0)))
	2526	(movd (:%mmx x8664::stack-temp) (:%q x8664::imm0)))
	2527
	2528
	2529	(define-x8664-vinsn (nfp-load-complex-single-float :nfp :ref) (((val :complex-single-float))
	2530	((offset :u16const)))
	2531	(movd (:%q x8664::imm0) (:%mmx x8664::stack-temp))
	2532	(movq (:rcontext x8664::tcr.nfp) (:%q x8664::imm0))
	2533	(movq (:@ (:apply + 16 offset) (:% x8664::imm0)) (:%xmm val))
	2534	(movd (:%mmx x8664::stack-temp) (:%q x8664::imm0)))
	2535
	2536
	2537	(define-x8664-vinsn (nfp-store-complex-double-float :nfp :set) (()
	2538	((val :complex-double-float)
	2539	(offset :u16const)))
	2540	(movd (:%q x8664::imm0) (:%mmx x8664::stack-temp))
	2541	(movq (:rcontext x8664::tcr.nfp) (:%q x8664::imm0))
	2542	(movdqu (:%xmm val) (:@ (:apply + 16 offset) (:% x8664::imm0)))
	2543	(movd (:%mmx x8664::stack-temp) (:%q x8664::imm0)))
	2544
	2545	(define-x8664-vinsn (nfp-load-complex-double-float :nfp :ref) (((val :complex-double-float))
	2546	((offset :u16const)))
	2547	(movd (:%q x8664::imm0) (:%mmx x8664::stack-temp))
	2548	(movq (:rcontext x8664::tcr.nfp) (:%q x8664::imm0))
	2549	(movdqu (:@ (:apply + 16 offset) (:% x8664::imm0)) (:%xmm val))
	2550	(movd (:%mmx x8664::stack-temp) (:%q x8664::imm0)))
	2551
	2552	(define-x8664-vinsn (temp-push-unboxed-word :push :word :csp)
	2553	(()
	2554	((w :u64))
	2555	((temp :imm)
	2556	(stack-temp :imm)))
	2557	(movq (:rcontext x8664::tcr.foreign-sp) (:%q stack-temp))
	2558	(subq (:$b (* 2 x8664::dnode-size)) (:rcontext x8664::tcr.foreign-sp))
	2559	(movq (:rcontext x8664::tcr.foreign-sp) (:%q temp))
	2560	(movq (:%q stack-temp) (:@ (:%q temp)))
	2561	(movq (:% x8664::rbp) (:@ x8664::csp-frame.rbp (:%q temp)))
	2562	(movq (:%q w) (:@ x8664::dnode-size (:%q temp))))
	2563
	2564
	2565	(define-x8664-vinsn (temp-push-node :push :word :tsp)
	2566	(()
	2567	((w :lisp))
	2568	((temp :imm)
	2569	(stack-temp :imm)))
	2570	(subq (:$b (* 2 x8664::dnode-size)) (:rcontext x8664::tcr.next-tsp))
	2571	(movq (:rcontext x8664::tcr.save-tsp) (:%q stack-temp))
	2572	(movq (:rcontext x8664::tcr.next-tsp) (:%q temp))
	2573	(movapd (:%xmm x8664::fpzero) (:@ (:%q temp)))
	2574	(movapd (:%xmm x8664::fpzero) (:@ 16 (:%q temp)))
	2575	(movq (:%q stack-temp) (:@ (:%q temp)))
	2576	(movq (:% x8664::rbp) (:@ x8664::tsp-frame.rbp (:%q temp)))
	2577	(movq (:%q temp) (:rcontext x8664::tcr.save-tsp))
	2578	(movq (:%q w) (:@ x8664::dnode-size (:%q temp))))
	2579
	2580	(define-x8664-vinsn (temp-push-double-float :push :word :csp)
	2581	(()
	2582	((f :double-float))
	2583	((temp :imm)
	2584	(stack-temp :imm)))
	2585	(movq (:rcontext x8664::tcr.foreign-sp) (:%q stack-temp))
	2586	(subq (:$b (* 2 x8664::dnode-size)) (:rcontext x8664::tcr.foreign-sp))
	2587	(movq (:rcontext x8664::tcr.foreign-sp) (:%q temp))
	2588	(movq (:%q stack-temp) (:@ (:%q temp)))
	2589	(movq (:% x8664::rbp) (:@ x8664::csp-frame.rbp (:%q temp)))
	2590	(movapd (:%xmm f) (:@ x8664::dnode-size (:%q temp))))
	2591
	2592
	2593	(define-x8664-vinsn (vpush-single-float :push :word :vsp)
	2594	(()
	2595	((f :single-float)))
	2596	(pushq (:$b x8664::tag-single-float))
	2597	(movss (:%xmm f) (:@ 4 (:%q x8664::rsp))))
	2598
	2599	(define-x8664-vinsn (vpop-single-float :pop :word :vsp)
	2600	(()
	2601	((f :single-float)))
	2602	(movss (:@ 4 (:%q x8664::rsp)) (:%xmm f))
	2603	(addq (:$b x8664::node-size) (:%q x8664::rsp)))
	2604
	2605	(define-x8664-vinsn (temp-pop-unboxed-word :pop :word :csp)
	2606	(((w :u64))
	2607	()
	2608	((temp (:u64 #.x8664::ra0))))
	2609	(movq (:rcontext x8664::tcr.foreign-sp) (:%q x8664::ra0))
	2610	(movq (:@ x8664::dnode-size (:%q x8664::ra0)) (:%q w))
	2611	(addq (:$b (* 2 x8664::dnode-size)) (:rcontext x8664::tcr.foreign-sp)))
	2612
	2613
	2614	(define-x8664-vinsn (temp-pop-node :pop :word :tsp)
	2615	(((w :lisp))
	2616	()
	2617	((temp :imm)))
	2618	(movq (:rcontext x8664::tcr.save-tsp) (:%q temp))
	2619	(movq (:@ x8664::dnode-size (:%q temp)) (:%q w))
	2620	(movq (:@ (:%q temp)) (:%q temp))
	2621	(movq (:%q temp) (:rcontext x8664::tcr.save-tsp))
	2622	(movq (:%q temp) (:rcontext x8664::tcr.next-tsp)))
	2623
	2624	(define-x8664-vinsn (temp-pop-double-float :pop :word :csp)
	2625	(((f :double-float))
	2626	()
	2627	((temp (:u64 #.x8664::ra0))))
	2628	(movq (:rcontext x8664::tcr.foreign-sp) (:%q x8664::ra0))
	2629	(movapd (:@ x8664::dnode-size (:%q x8664::ra0)) (:%xmm f))
	2630	(addq (:$b (* 2 x8664::dnode-size)) (:rcontext x8664::tcr.foreign-sp)))
	2631
	2632
	2633
	2634	(define-x8664-vinsn macptr->stack (((dest :lisp))
	2635	((ptr :address))
	2636	((temp :imm)
	2637	(stack-temp :imm)))
	2638	(movq (:rcontext x8664::tcr.foreign-sp) (:%q stack-temp))
	2639	(subq (:$b (+ x8664::dnode-size x8664::macptr.size)) (:rcontext x8664::tcr.foreign-sp))
	2640	(movq (:rcontext x8664::tcr.foreign-sp) (:%q temp))
	2641	(movq (:%q stack-temp) (:@ (:%q temp)))
	2642	(movq (:% x8664::rbp) (:@ x8664::csp-frame.rbp (:%q temp)))
	2643	(leaq (:@ (+ x8664::dnode-size x8664::fulltag-misc) (:%q temp)) (:%q dest))
	2644	(movq (:$l x8664::macptr-header) (:@ x8664::macptr.header (:%q dest)))
	2645	(movq (:%q ptr) (:@ x8664::macptr.address (:%q dest)))
	2646	(movapd (:%xmm x8664::fpzero) (:@ x8664::macptr.domain (:%q dest))))
	2647
	2648	(define-x8664-vinsn fixnum->signed-natural (((dest :s64))
	2649	((src :imm)))
	2650	(movq (:%q src) (:%q dest))
	2651	(sarq (:$ub x8664::fixnumshift) (:%q dest)))
	2652
	2653	(define-x8664-vinsn mem-set-double-float (()
	2654	((val :double-float)
	2655	(src :address)
	2656	(index :s64)))
	2657	(movsd (:%xmm val) (:@ (:%q src) (:%q index))))
	2658
	2659	(define-x8664-vinsn mem-set-single-float (()
	2660	((val :single-float)
	2661	(src :address)
	2662	(index :s64)))
	2663	(movss (:%xmm val) (:@ (:%q src) (:%q index))))
	2664
	2665
	2666
	2667	(define-x8664-vinsn mem-set-c-doubleword (()
	2668	((val :u64)
	2669	(dest :address)
	2670	(offset :s32const)))
	2671	((:pred = offset 0)
	2672	(movq (:%q val) (:@ (:%q dest))))
	2673	((:not (:pred = offset 0))
	2674	(movq (:%q val) (:@ offset (:%q dest)))))
	2675
	2676	(define-x8664-vinsn mem-set-c-fullword (()
	2677	((val :u32)
	2678	(dest :address)
	2679	(offset :s32const)))
	2680	((:pred = offset 0)
	2681	(movl (:%l val) (:@ (:%q dest))))
	2682	((:not (:pred = offset 0))
	2683	(movl (:%l val) (:@ offset (:%q dest)))))
	2684
	2685	(define-x8664-vinsn mem-set-c-halfword (()
	2686	((val :u16)
	2687	(dest :address)
	2688	(offset :s32const)))
	2689	((:pred = offset 0)
	2690	(movw (:%w val) (:@ (:%q dest))))
	2691	((:not (:pred = offset 0))
	2692	(movw (:%w val) (:@ offset (:%q dest)))))
	2693
	2694	(define-x8664-vinsn mem-set-c-byte (()
	2695	((val :u8)
	2696	(dest :address)
	2697	(offset :s32const)))
	2698	((:pred = offset 0)
	2699	(movb (:%b val) (:@ (:%q dest))))
	2700	((:not (:pred = offset 0))
	2701	(movb (:%b val) (:@ offset (:%q dest)))))
	2702
	2703	(define-x8664-vinsn mem-set-c-constant-doubleword (()
	2704	((val :s32const)
	2705	(dest :address)
	2706	(offset :s32const)))
	2707	((:pred = offset 0)
	2708	(movq (:$l val) (:@ (:%q dest))))
	2709	((:not (:pred = offset 0))
	2710	(movq (:$l val) (:@ offset (:%q dest)))))
	2711
	2712	(define-x8664-vinsn mem-set-c-constant-fullword (()
	2713	((val :s32const)
	2714	(dest :address)
	2715	(offset :s32const)))
	2716	((:pred = offset 0)
	2717	(movl (:$l val) (:@ (:%q dest))))
	2718	((:not (:pred = offset 0))
	2719	(movl (:$l val) (:@ offset (:%q dest)))))
	2720
	2721	(define-x8664-vinsn mem-set-c-constant-halfword (()
	2722	((val :s16const)
	2723	(dest :address)
	2724	(offset :s32const)))
	2725	((:pred = offset 0)
	2726	(movw (:$w val) (:@ (:%q dest))))
	2727	((:not (:pred = offset 0))
	2728	(movw (:$w val) (:@ offset (:%q dest)))))
	2729
	2730	(define-x8664-vinsn mem-set-c-constant-byte (()
	2731	((val :s8const)
	2732	(dest :address)
	2733	(offset :s32const)))
	2734	((:pred = offset 0)
	2735	(movb (:$b val) (:@ (:%q dest))))
	2736	((:not (:pred = offset 0))
	2737	(movb (:$b val) (:@ offset (:%q dest)))))
	2738
	2739
	2740
	2741
	2742
	2743
	2744	(define-x8664-vinsn mem-ref-natural (((dest :u64))
	2745	((src :address)
	2746	(index :s64)))
	2747	(movq (:@ (:%q src) (:%q index)) (:%q dest)))
	2748
	2749	(define-x8664-vinsn setup-macptr-allocation (()
	2750	((src :address)))
	2751	(movq (:%q src) (:%q x8664::imm2))
	2752	(movl (:$l x8664::macptr-header) (:%l x8664::imm0.l))
	2753	(movl (:$l (- x8664::macptr.size x8664::fulltag-misc)) (:%l x8664::imm1.l)))
	2754
	2755	(define-x8664-vinsn %set-new-macptr-value (()
	2756	((ptr :lisp)))
	2757	(movq (:%q x8664::imm2) (:@ x8664::macptr.address (:%q ptr))))
	2758
	2759	(define-x8664-vinsn mem-ref-c-fullword (((dest :u32))
	2760	((src :address)
	2761	(index :s32const)))
	2762	((:pred = index 0)
	2763	(movl (:@ (:%q src)) (:%l dest)))
	2764	((:not (:pred = index 0))
	2765	(movl (:@ index (:%q src)) (:%l dest))))
	2766
	2767	(define-x8664-vinsn mem-ref-c-signed-fullword (((dest :s32))
	2768	((src :address)
	2769	(index :s32const)))
	2770	((:pred = index 0)
	2771	(movslq (:@ (:%q src)) (:%q dest)))
	2772	((:not (:pred = index 0))
	2773	(movslq (:@ index (:%q src)) (:%q dest))))
	2774
	2775
	2776	(define-x8664-vinsn mem-ref-c-single-float (((dest :single-float))
	2777	((src :address)
	2778	(index :s32const)))
	2779	((:pred = index 0)
	2780	(movss (:@ (:%q src)) (:%xmm dest)))
	2781	((:not (:pred = index 0))
	2782	(movss (:@ index (:%q src)) (:%xmm dest))))
	2783
	2784	(define-x8664-vinsn mem-set-c-single-float (()
	2785	((val :single-float)
	2786	(src :address)
	2787	(index :s16const)))
	2788	((:pred = index 0)
	2789	(movss (:%xmm val) (:@ (:%q src))))
	2790	((:not (:pred = index 0))
	2791	(movss (:%xmm val) (:@ index (:%q src)))))
	2792
	2793	(define-x8664-vinsn mem-ref-c-doubleword (((dest :u64))
	2794	((src :address)
	2795	(index :s32const)))
	2796	((:pred = index 0)
	2797	(movq (:@ (:%q src)) (:%q dest)))
	2798	((:not (:pred = index 0))
	2799	(movq (:@ index (:%q src)) (:%q dest))))
	2800
	2801	(define-x8664-vinsn mem-ref-c-signed-doubleword (((dest :s64))
	2802	((src :address)
	2803	(index :s32const)))
	2804	((:pred = index 0)
	2805	(movq (:@ (:%q src)) (:%q dest)))
	2806	((:not (:pred = index 0))
	2807	(movq (:@ index (:%q src)) (:%q dest))))
	2808
	2809	(define-x8664-vinsn mem-ref-c-natural (((dest :u64))
	2810	((src :address)
	2811	(index :s32const)))
	2812	((:pred = index 0)
	2813	(movq (:@ (:%q src)) (:%q dest)))
	2814	((:not (:pred = index 0))
	2815	(movq (:@ index (:%q src)) (:%q dest))))
	2816
	2817	(define-x8664-vinsn mem-ref-c-double-float (((dest :double-float))
	2818	((src :address)
	2819	(index :s32const)))
	2820	((:pred = index 0)
	2821	(movsd (:@ (:%q src)) (:%xmm dest)))
	2822	((:not (:pred = index 0))
	2823	(movsd (:@ index (:%q src)) (:%xmm dest))))
	2824
	2825	(define-x8664-vinsn mem-set-c-double-float (()
	2826	((val :double-float)
	2827	(src :address)
	2828	(index :s16const)))
	2829	((:pred = index 0)
	2830	(movsd (:%xmm val) (:@ (:%q src))))
	2831	((:not (:pred = index 0))
	2832	(movsd (:%xmm val) (:@ index (:%q src)))))
	2833
	2834	(define-x8664-vinsn mem-ref-fullword (((dest :u32))
	2835	((src :address)
	2836	(index :s64)))
	2837	(movl (:@ (:%q src) (:%q index)) (:%l dest)))
	2838
	2839	(define-x8664-vinsn mem-ref-signed-fullword (((dest :s32))
	2840	((src :address)
	2841	(index :s64)))
	2842	(movslq (:@ (:%q src) (:%q index)) (:%q dest)))
	2843
	2844	(define-x8664-vinsn mem-ref-doubleword (((dest :u64))
	2845	((src :address)
	2846	(index :s64)))
	2847	(movq (:@ (:%q src) (:%q index)) (:%q dest)))
	2848
	2849	(define-x8664-vinsn mem-ref-natural (((dest :u64))
	2850	((src :address)
	2851	(index :s64)))
	2852	(movq (:@ (:%q src) (:%q index)) (:%q dest)))
	2853
	2854	(define-x8664-vinsn mem-ref-signed-doubleword (((dest :s64))
	2855	((src :address)
	2856	(index :s64)))
	2857	(movq (:@ (:%q src) (:%q index)) (:%q dest)))
	2858
	2859	(define-x8664-vinsn mem-ref-c-u16 (((dest :u16))
	2860	((src :address)
	2861	(index :s32const)))
	2862	((:pred = index 0)
	2863	(movzwq (:@ (:%q src)) (:%q dest)))
	2864	((:not (:pred = index 0))
	2865	(movzwq (:@ index (:%q src)) (:%q dest))))
	2866
	2867	(define-x8664-vinsn mem-ref-u16 (((dest :u16))
	2868	((src :address)
	2869	(index :s64)))
	2870	(movzwq (:@ (:%q src) (:%q index)) (:%q dest)))
	2871
	2872
	2873	(define-x8664-vinsn mem-ref-c-s16 (((dest :s16))
	2874	((src :address)
	2875	(index :s32const)))
	2876	((:pred = index 0)
	2877	(movswq (:@ (:%q src)) (:%q dest)))
	2878	((:not (:pred = index 0))
	2879	(movswq (:@ index (:%q src)) (:%q dest))))
	2880
	2881	(define-x8664-vinsn mem-ref-s16 (((dest :s16))
	2882	((src :address)
	2883	(index :s32)))
	2884	(movswq (:@ (:%q src) (:%q index)) (:%q dest)))
	2885
	2886	(define-x8664-vinsn mem-ref-c-u8 (((dest :u8))
	2887	((src :address)
	2888	(index :s16const)))
	2889	((:pred = index 0)
	2890	(movzbq (:@ (:%q src)) (:%q dest)))
	2891	((:not (:pred = index 0))
	2892	(movzbq (:@ index (:%q src)) (:%q dest))))
	2893
	2894	(define-x8664-vinsn mem-ref-u8 (((dest :u8))
	2895	((src :address)
	2896	(index :s32)))
	2897	(movzbq (:@ (:%q src) (:%q index)) (:%q dest)))
	2898
	2899	(define-x8664-vinsn mem-ref-c-s8 (((dest :s8))
	2900	((src :address)
	2901	(index :s16const)))
	2902	((:pred = index 0)
	2903	(movsbq (:@ (:%q src)) (:%q dest)))
	2904	((:not (:pred = index 0))
	2905	(movsbq (:@ index (:%q src)) (:%q dest))))
	2906
	2907	(define-x8664-vinsn misc-set-c-s8 (((val :s8))
	2908	((v :lisp)
	2909	(idx :u32const))
	2910	())
	2911	(movb (:%b val) (:@ (:apply + x8664::misc-data-offset idx) (:%q v))))
	2912
	2913	(define-x8664-vinsn misc-set-s8 (((val :s8))
	2914	((v :lisp)
	2915	(scaled-idx :s64))
	2916	())
	2917	(movb (:%b val) (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx))))
	2918
	2919	(define-x8664-vinsn mem-ref-s8 (((dest :s8))
	2920	((src :address)
	2921	(index :s32)))
	2922	(movsbq (:@ (:%q src) (:%q index)) (:%q dest)))
	2923
	2924	(define-x8664-vinsn mem-set-constant-doubleword (()
	2925	((val :s32const)
	2926	(ptr :address)
	2927	(offset :s64)))
	2928	(movq (:$l val) (:@ (:%q ptr) (:%q offset))))
	2929
	2930	(define-x8664-vinsn mem-set-constant-fullword (()
	2931	((val :s32const)
	2932	(ptr :address)
	2933	(offset :s64)))
	2934	(movl (:$l val) (:@ (:%q ptr) (:%q offset))))
	2935
	2936
	2937	(define-x8664-vinsn mem-set-constant-halfword (()
	2938	((val :s16const)
	2939	(ptr :address)
	2940	(offset :s64)))
	2941	(movw (:$w val) (:@ (:%q ptr) (:%q offset))))
	2942
	2943	(define-x8664-vinsn mem-set-constant-byte (()
	2944	((val :s8const)
	2945	(ptr :address)
	2946	(offset :s64)))
	2947	(movb (:$b val) (:@ (:%q ptr) (:%q offset))))
	2948
	2949	(define-x8664-vinsn misc-set-c-u8 (((val :u8))
	2950	((v :lisp)
	2951	(idx :u32const))
	2952	())
	2953	(movb (:%b val) (:@ (:apply + x8664::misc-data-offset idx) (:%q v))))
	2954
	2955	(define-x8664-vinsn misc-set-u8 (((val :u8))
	2956	((v :lisp)
	2957	(scaled-idx :s64))
	2958	())
	2959	(movb (:%b val) (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx))))
	2960
	2961	(define-x8664-vinsn misc-set-c-u8 (((val :u8))
	2962	((v :lisp)
	2963	(idx :s32const))
	2964	())
	2965	(movb (:%b val) (:@ (:apply + x8664::misc-data-offset idx) (:%q v))))
	2966
	2967	(define-x8664-vinsn misc-set-u8 (()
	2968	((val :u8)
	2969	(v :lisp)
	2970	(scaled-idx :s64))
	2971	())
	2972	(movb (:%b val) (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx))))
	2973
	2974	(define-x8664-vinsn misc-set-c-u16 (()
	2975	((val :u16)
	2976	(v :lisp)
	2977	(idx :s32const))
	2978	())
	2979	(movw (:%w val) (:@ (:apply + x8664::misc-data-offset (:apply * 2 idx)) (:%q v))))
	2980
	2981
	2982	(define-x8664-vinsn misc-set-u16 (()
	2983	((val :u16)
	2984	(v :lisp)
	2985	(scaled-idx :s64))
	2986	())
	2987	(movw (:%w val) (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx))))
	2988
	2989	(define-x8664-vinsn misc-set-c-s16 (()
	2990	((val :s16)
	2991	(v :lisp)
	2992	(idx :s32const))
	2993	())
	2994	(movw (:%w val) (:@ (:apply + x8664::misc-data-offset (:apply * 2 idx)) (:%q v))))
	2995
	2996
	2997	(define-x8664-vinsn misc-set-s16 (()
	2998	((val :s16)
	2999	(v :lisp)
	3000	(scaled-idx :s64))
	3001	())
	3002	(movw (:%w val) (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx))))
	3003
	3004	(define-x8664-vinsn misc-set-c-u32 (()
	3005	((val :u32)
	3006	(v :lisp)
	3007	(idx :u32const)) ; sic
	3008	())
	3009	(movl (:%l val) (:@ (:apply + x8664::misc-data-offset (:apply ash idx 2)) (:%q v))))
	3010
	3011	(define-x8664-vinsn misc-set-u32 (()
	3012	((val :u32)
	3013	(v :lisp)
	3014	(scaled-idx :s64))
	3015	())
	3016	(movl (:%l val) (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx))))
	3017
	3018	(define-x8664-vinsn misc-set-c-s32 (()
	3019	((val :s32)
	3020	(v :lisp)
	3021	(idx :u32const)) ; sic
	3022	())
	3023	(movl (:%l val) (:@ (:apply + x8664::misc-data-offset (:apply ash idx 2)) (:%q v))))
	3024
	3025	(define-x8664-vinsn misc-set-s32 (()
	3026	((val :s32)
	3027	(v :lisp)
	3028	(scaled-idx :s64))
	3029	())
	3030	(movl (:%l val) (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx))))
	3031
	3032	(define-x8664-vinsn %iasr (((dest :imm))
	3033	((count :imm)
	3034	(src :imm))
	3035	((temp :s64)
	3036	(shiftcount (:s64 #.x8664::rcx))))
	3037	(movq (:%q count) (:%q temp))
	3038	(sarq (:$ub x8664::fixnumshift) (:%q temp))
	3039	(movl (:$l 63) (:%l shiftcount))
	3040	(rcmpq (:%q temp) (:%q shiftcount))
	3041	(cmovbel (:%l temp) (:%l shiftcount))
	3042	(movq (:%q src) (:%q temp))
	3043	(sarq (:%shift x8664::cl) (:%q temp))
	3044	(andq (:$b (lognot x8664::fixnummask)) (:%q temp))
	3045	(movq (:%q temp) (:%q dest)))
	3046
	3047	(define-x8664-vinsn %ilsr (((dest :imm))
	3048	((count :imm)
	3049	(src :imm))
	3050	((temp :s64)
	3051	(shiftcount (:s64 #.x8664::rcx))))
	3052	(movq (:%q count) (:%q temp))
	3053	(sarq (:$ub x8664::fixnumshift) (:%q temp))
	3054	(movl (:$l 63) (:%l shiftcount))
	3055	(rcmpq (:%q temp) (:%q shiftcount))
	3056	(cmovbel (:%l temp) (:%l shiftcount))
	3057	(movq (:%q src) (:%q temp))
	3058	(shrq (:%shift x8664::cl) (:%q temp))
	3059	(andq (:$b (lognot x8664::fixnummask)) (:%q temp))
	3060	(movq (:%q temp) (:%q dest)))
	3061
	3062
	3063	(define-x8664-vinsn %iasr-c (((dest :imm))
	3064	((count :u8const)
	3065	(src :imm))
	3066	((temp :s64)))
	3067	(movq (:%q src) (:%q temp))
	3068	(sarq (:$ub count) (:%q temp))
	3069	(andq (:$b (lognot x8664::fixnummask)) (:%q temp))
	3070	(movq (:%q temp) (:%q dest)))
	3071
	3072	(define-x8664-vinsn %ilsr-c (((dest :imm))
	3073	((count :u8const)
	3074	(src :imm))
	3075	((temp :s64)))
	3076	(movq (:%q src) (:%q temp))
	3077	(shrq (:$ub count) (:%q temp))
	3078	(andb (:$b (lognot x8664::fixnummask)) (:%b temp))
	3079	(movq (:%q temp) (:%q dest)))
	3080
	3081	(define-x8664-vinsn %ilsl (((dest :imm))
	3082	((count :imm)
	3083	(src :imm))
	3084	((shiftcount (:s64 #.x8664::rcx))))
	3085	(movl (:$l (ash 63 x8664::fixnumshift)) (:%l shiftcount))
	3086	(rcmpq (:%q count) (:%q shiftcount))
	3087	(cmovbl (:%l count) (:%l shiftcount))
	3088	(sarl (:$ub x8664::fixnumshift) (:%l shiftcount))
	3089	((:not (:pred =
	3090	(:apply %hard-regspec-value src)
	3091	(:apply %hard-regspec-value dest)))
	3092	(movq (:%q src) (:%q dest)))
	3093	(shlq (:%shift x8664::cl) (:%q dest)))
	3094
	3095	(define-x8664-vinsn %ilsl-c (((dest :imm))
	3096	((count :u8const)
	3097	(src :imm)))
	3098	((:not (:pred =
	3099	(:apply %hard-regspec-value src)
	3100	(:apply %hard-regspec-value dest)))
	3101	(movq (:%q src) (:%q dest)))
	3102	(shlq (:$ub count) (:%q dest)))
	3103
	3104	(define-x8664-vinsn fixnum-ash-left (((dest :lisp))
	3105	((num :lisp)
	3106	(amt :lisp))
	3107	((shiftcount (:s64 #.x8664::rcx))))
	3108	(movq (:%q amt) (:%q shiftcount))
	3109	(sarq (:$ub x8664::fixnumshift) (:%q shiftcount))
	3110	((:not (:pred =
	3111	(:apply %hard-regspec-value num)
	3112	(:apply %hard-regspec-value dest)))
	3113	(movq (:%q num) (:%q dest)))
	3114	(shlq (:%shift x8664::cl) (:%q dest)))
	3115
	3116	(define-x8664-vinsn fixnum-ash (((dest :lisp))
	3117	((num :lisp)
	3118	(amt :lisp))
	3119	((shiftcount (:s64 #.x8664::rcx))
	3120	(temp (:s64))))
	3121	(movq (:%q amt) (:%q shiftcount))
	3122	(sarq (:$ub x8664::fixnumshift) (:%q shiftcount))
	3123	(jns :left)
	3124	(negq (:%q shiftcount))
	3125	(movq (:%q num) (:%q temp))
	3126	(sarq (:$ub x8664::fixnumshift) (:%q temp))
	3127	(sarq (:%shift x8664::cl) (:%q temp))
	3128	(imulq (:$b x8664::fixnumone) (:%q temp)(:%q dest))
	3129	(jmp :done)
	3130	:left
	3131	((:not (:pred =
	3132	(:apply %hard-regspec-value num)
	3133	(:apply %hard-regspec-value dest)))
	3134	(movq (:%q num) (:%q dest)))
	3135	(shlq (:%shift x8664::cl) (:%q dest))
	3136	:done)
	3137
	3138
	3139	;;; In safe code, something else has ensured that the value is of type
	3140	;;; BIT.
	3141	(define-x8664-vinsn set-variable-bit-to-variable-value (()
	3142	((vec :lisp)
	3143	(word-index :s64)
	3144	(bitnum :u8)
	3145	(value :lisp)))
	3146	(testb (:%b value) (:%b value))
	3147	(je :clr)
	3148	(btsq (:%q bitnum) (:@ x8664::misc-data-offset (:%q vec) (:%q word-index) 8))
	3149	(jmp :done)
	3150	:clr
	3151	(btrq (:%q bitnum) (:@ x8664::misc-data-offset (:%q vec) (:%q word-index) 8))
	3152	:done)
	3153
	3154	(define-x8664-vinsn set-variable-bit-to-zero (()
	3155	((vec :lisp)
	3156	(word-index :s64)
	3157	(bitnum :u8)))
	3158	(btrq (:%q bitnum) (:@ x8664::misc-data-offset (:%q vec) (:%q word-index) 8)))
	3159
	3160	(define-x8664-vinsn set-variable-bit-to-one (()
	3161	((vec :lisp)
	3162	(word-index :s64)
	3163	(bitnum :u8)))
	3164	(btsq (:%q bitnum) (:@ x8664::misc-data-offset (:%q vec) (:%q word-index) 8)))
	3165
	3166	;;; In safe code, something else has ensured that the value is of type
	3167	;;; BIT.
	3168	(define-x8664-vinsn nset-variable-bit-to-variable-value (()
	3169	((vec :lisp)
	3170	(index :s64)
	3171	(value :lisp)))
	3172	(testb (:%b value) (:%b value))
	3173	(je :clr)
	3174	(btsq (:%q index) (:@ x8664::misc-data-offset (:%q vec)))
	3175	(jmp :done)
	3176	:clr
	3177	(btrq (:%q index) (:@ x8664::misc-data-offset (:%q vec)))
	3178	:done)
	3179
	3180	(define-x8664-vinsn nset-variable-bit-to-zero (()
	3181	((vec :lisp)
	3182	(index :s64)))
	3183	(btrq (:%q index) (:@ x8664::misc-data-offset (:%q vec))))
	3184
	3185	(define-x8664-vinsn nset-variable-bit-to-one (()
	3186	((vec :lisp)
	3187	(index :s64)))
	3188	(btsq (:%q index) (:@ x8664::misc-data-offset (:%q vec))))
	3189
	3190	(define-x8664-vinsn set-constant-bit-to-zero (()
	3191	((src :lisp)
	3192	(idx :u64const)))
	3193	(btrq (:$ub (:apply logand 63 idx))
	3194	(:@ (:apply + x8664::misc-data-offset (:apply ash (:apply ash idx -6) x8664::word-shift)) (:%q src))))
	3195
	3196	(define-x8664-vinsn set-constant-bit-to-one (()
	3197	((src :lisp)
	3198	(idx :u64const)))
	3199	(btsq (:$ub (:apply logand 63 idx))
	3200	(:@ (:apply + x8664::misc-data-offset (:apply ash (:apply ash idx -6) x8664::word-shift)) (:%q src))))
	3201
	3202	(define-x8664-vinsn set-constant-bit-to-variable-value (()
	3203	((src :lisp)
	3204	(idx :u64const)
	3205	(value :lisp)))
	3206	(testb (:%b value) (:%b value))
	3207	(je :clr)
	3208	(btsq (:$ub (:apply logand 63 idx))
	3209	(:@ (:apply + x8664::misc-data-offset (:apply ash (:apply ash idx -6) x8664::word-shift)) (:%q src)))
	3210	(jmp :done)
	3211	:clr
	3212	(btrq (:$ub (:apply logand 63 idx))
	3213	(:@ (:apply + x8664::misc-data-offset (:apply ash (:apply ash idx -6) x8664::word-shift)) (:%q src)))
	3214	:done)
	3215
	3216
	3217	(define-x8664-vinsn require-fixnum (()
	3218	((object :lisp)))
	3219	:again
	3220	(testl (:$l x8664::fixnummask) (:%l object))
	3221	(jne :bad)
	3222	(:anchored-uuo-section :again)
	3223	:bad
	3224	(:anchored-uuo (uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-fixnum))))
	3225
	3226	(define-x8664-vinsn require-integer (()
	3227	((object :lisp))
	3228	((tag :u8)))
	3229	:again
	3230	(movl (:%l object) (:%l tag))
	3231	(andl (:$b x8664::fixnummask) (:%l tag))
	3232	(je :got-it)
	3233	(cmpl (:$b x8664::tag-misc) (:%l tag))
	3234	(jne :bad)
	3235	(cmpb (:$b x8664::subtag-bignum) (:@ x8664::misc-subtag-offset (:%q object)))
	3236	(jne :bad)
	3237	:got-it
	3238
	3239	(:anchored-uuo-section :again)
	3240	:bad
	3241	(:anchored-uuo (uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-integer))))
	3242
	3243	(define-x8664-vinsn require-simple-vector (()
	3244	((object :lisp))
	3245	((tag :u8)))
	3246	:again
	3247	(movl (:%l object) (:%l tag))
	3248	(andl (:$b x8664::fixnummask) (:%l tag))
	3249	(cmpl (:$b x8664::tag-misc) (:%l tag))
	3250	(jne :bad)
	3251	(cmpb (:$b x8664::subtag-simple-vector) (:@ x8664::misc-subtag-offset (:%q object)))
	3252	(jne :bad)
	3253
	3254	(:anchored-uuo-section :again)
	3255	:bad
	3256	(:anchored-uuo (uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-simple-vector))))
	3257
	3258	(define-x8664-vinsn require-simple-string (()
	3259	((object :lisp))
	3260	((tag :u8)))
	3261	:again
	3262	(movl (:%l object) (:%l tag))
	3263	(andl (:$b x8664::fixnummask) (:%l tag))
	3264	(cmpl (:$b x8664::tag-misc) (:%l tag))
	3265	(jne :bad)
	3266	(cmpb (:$b x8664::subtag-simple-base-string) (:@ x8664::misc-subtag-offset (:%q object)))
	3267	(jne :bad)
	3268
	3269	(:anchored-uuo-section :again)
	3270	:bad
	3271	(:anchored-uuo (uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-simple-string))))
	3272
	3273	(define-x8664-vinsn require-real (()
	3274	((object :lisp))
	3275	((tag :u8)
	3276	(mask :u64)))
	3277	(movq (:$q (logior (ash 1 x8664::tag-fixnum)
	3278	(ash 1 x8664::tag-single-float)
	3279	(ash 1 x8664::subtag-double-float)
	3280	(ash 1 x8664::subtag-bignum)
	3281	(ash 1 x8664::subtag-ratio)))
	3282	(:%q mask))
	3283	:again
	3284	(movl (:%l object) (:%l tag))
	3285	(andl (:$b x8664::tagmask) (:%l tag))
	3286	(cmpl (:$b x8664::tag-misc) (:%l tag))
	3287	(jne :have-tag)
	3288	(movzbl (:@ x8664::misc-subtag-offset (:%q object)) (:%l tag))
	3289	:have-tag
	3290	(rcmpl (:%l tag) (:$b 64))
	3291	(jae :bad)
	3292	(btq (:%q tag) (:%q mask))
	3293	(jae :bad)
	3294
	3295	(:anchored-uuo-section :again)
	3296	:bad
	3297	(:anchored-uuo (uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-real))))
	3298
	3299	(define-x8664-vinsn require-number (()
	3300	((object :lisp))
	3301	((tag :u8)
	3302	(mask :u64)))
	3303	(movq (:$q (logior (ash 1 x8664::tag-fixnum)
	3304	(ash 1 x8664::tag-single-float)
	3305	(ash 1 x8664::subtag-double-float)
	3306	(ash 1 x8664::subtag-bignum)
	3307	(ash 1 x8664::subtag-ratio)
	3308	(ash 1 x8664::subtag-complex)))
	3309	(:%q mask))
	3310	:again
	3311	(movl (:%l object) (:%l tag))
	3312	(andl (:$b x8664::tagmask) (:%l tag))
	3313	(cmpl (:$b x8664::tag-misc) (:%l tag))
	3314	(jne :have-tag)
	3315	(movzbl (:@ x8664::misc-subtag-offset (:%q object)) (:%l tag))
	3316	:have-tag
	3317	(cmpl (:$b x8664::subtag-complex-single-float) (:%l tag))
	3318	(jz :good)
	3319	(cmpl (:$b x8664::subtag-complex-double-float) (:%l tag))
	3320	(jz :good)
	3321	(rcmpl (:%l tag) (:$b 64))
	3322	(jae :bad)
	3323	(btq (:%q tag) (:%q mask))
	3324	(jae :bad)
	3325	:good
	3326	(:anchored-uuo-section :again)
	3327	:bad
	3328	(:anchored-uuo (uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-number))))
	3329
	3330	(define-x8664-vinsn require-list (()
	3331	((object :lisp))
	3332	((tag :u8)))
	3333	:again
	3334	(movl (:%l object) (:%l tag))
	3335	(andl (:$b x8664::tagmask) (:%l tag))
	3336	(cmpl (:$b x8664::tag-list) (:%l tag))
	3337	(jne :bad)
	3338
	3339	(:anchored-uuo-section :again)
	3340	:bad
	3341	(:anchored-uuo (uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-list))))
	3342
	3343	(define-x8664-vinsn require-symbol (()
	3344	((object :lisp))
	3345	((tag :u8)))
	3346	:again
	3347	(movzbl (:%b object) (:%l tag))
	3348	(cmpl (:$b x8664::fulltag-nil) (:%l tag))
	3349	(je :good)
	3350	(andl (:$b x8664::tagmask) (:%l tag))
	3351	(cmpl (:$b x8664::tag-symbol) (:%l tag))
	3352	(jne :bad)
	3353	:good
	3354
	3355	(:anchored-uuo-section :again)
	3356	:bad
	3357	(:anchored-uuo (uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-symbol))))
	3358
	3359	(define-x8664-vinsn require-character (()
	3360	((object :lisp)))
	3361	:again
	3362	(cmpb (:$b x8664::subtag-character) (:%b object))
	3363	(jne :bad)
	3364	(:anchored-uuo-section :again)
	3365	:bad
	3366	(:anchored-uuo (uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-character))))
	3367
	3368	(define-x8664-vinsn require-s8 (()
	3369	((object :lisp))
	3370	((tag :u32)))
	3371	:again
	3372	(movq (:%q object) (:%q tag))
	3373	(shlq (:$ub (- x8664::nbits-in-word (+ 8 x8664::fixnumshift))) (:%q tag))
	3374	(sarq (:$ub (- x8664::nbits-in-word 8)) (:%q tag))
	3375	(shlq (:$ub x8664::fixnumshift) (:%q tag))
	3376	(cmpq (:%q object) (:%q tag))
	3377	(jne :bad)
	3378
	3379	(:anchored-uuo-section :again)
	3380	:bad
	3381	(:anchored-uuo (uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-signed-byte-8))))
	3382
	3383	(define-x8664-vinsn require-u8 (()
	3384	((object :lisp))
	3385	((tag :u32)))
	3386	:again
	3387	(movq (:$l (lognot (ash #xff x8664::fixnumshift))) (:%q tag))
	3388	(andq (:% object) (:% tag))
	3389	(jne :bad)
	3390	(:anchored-uuo-section :again)
	3391	:bad
	3392	(:anchored-uuo (uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-unsigned-byte-8))))
	3393
	3394	(define-x8664-vinsn require-s16 (()
	3395	((object :lisp))
	3396	((tag :s64)))
	3397	:again
	3398	(movq (:%q object) (:%q tag))
	3399	(shlq (:$ub (- x8664::nbits-in-word (+ 16 x8664::fixnumshift))) (:%q tag))
	3400	(sarq (:$ub (- x8664::nbits-in-word 16)) (:%q tag))
	3401	(shlq (:$ub x8664::fixnumshift) (:%q tag))
	3402	(cmpq (:%q object) (:%q tag))
	3403	(jne :bad)
	3404	(:anchored-uuo-section :again)
	3405	:bad
	3406	(:anchored-uuo (uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-signed-byte-16))))
	3407
	3408	(define-x8664-vinsn require-u16 (()
	3409	((object :lisp))
	3410	((tag :u32)))
	3411	:again
	3412	(movq (:$l (lognot (ash #xffff x8664::fixnumshift))) (:%q tag))
	3413	(andq (:% object) (:% tag))
	3414	(jne :bad)
	3415	(:anchored-uuo-section :again)
	3416	:bad
	3417	(:anchored-uuo (uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-unsigned-byte-16))))
	3418
	3419	(define-x8664-vinsn require-s32 (()
	3420	((object :lisp))
	3421	((tag :s64)))
	3422	:again
	3423	(movq (:%q object) (:%q tag))
	3424	(shlq (:$ub (- x8664::nbits-in-word (+ 32 x8664::fixnumshift))) (:%q tag))
	3425	(sarq (:$ub (- x8664::nbits-in-word 32)) (:%q tag))
	3426	(shlq (:$ub x8664::fixnumshift) (:%q tag))
	3427	(cmpq (:%q object) (:%q tag))
	3428	(jne :bad)
	3429	(testl (:$l x8664::fixnummask) (:%l object))
	3430	(jne :bad)
	3431	(:anchored-uuo-section :again)
	3432	:bad
	3433	(:anchored-uuo (uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-signed-byte-32))))
	3434
	3435	(define-x8664-vinsn require-u32 (()
	3436	((object :lisp))
	3437	((tag :u32)))
	3438	:again
	3439	(movq (:$q (lognot (ash #xffffffff x8664::fixnumshift))) (:%q tag))
	3440	(andq (:% object) (:% tag))
	3441	(jne :bad)
	3442	(:anchored-uuo-section :again)
	3443	:bad
	3444	(:anchored-uuo (uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-unsigned-byte-32))))
	3445
	3446	(define-x8664-vinsn require-s64 (()
	3447	((object :lisp))
	3448	((tag :s64)))
	3449	:again
	3450	(testl (:$l x8664::fixnummask) (:%l object))
	3451	(movl (:%l object) (:%l tag))
	3452	(je :ok)
	3453	(andl (:$b x8664::fulltagmask) (:%l tag))
	3454	(cmpl (:$b x8664::fulltag-misc) (:%l tag))
	3455	(jne :bad)
	3456	(cmpq (:$l x8664::two-digit-bignum-header) (:@ x8664::misc-header-offset (:%q object)))
	3457	(jne :bad)
	3458	:ok
	3459	(:anchored-uuo-section :again)
	3460	:bad
	3461	(:anchored-uuo (uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-signed-byte-64))))
	3462
	3463	(define-x8664-vinsn require-u64 (()
	3464	((object :lisp))
	3465	((tag :s64)))
	3466	:again
	3467	(testl (:$l x8664::fixnummask) (:%l object))
	3468	(movq (:%q object) (:%q tag))
	3469	(je :ok-if-non-negative)
	3470	(andl (:$b x8664::fulltagmask) (:%l tag))
	3471	(cmpl (:$b x8664::fulltag-misc) (:%l tag))
	3472	(jne :bad)
	3473	(cmpq (:$l x8664::two-digit-bignum-header) (:@ x8664::misc-header-offset (:%q object)))
	3474	(je :two)
	3475	(cmpq (:$l x8664::three-digit-bignum-header) (:@ x8664::misc-header-offset (:%q object)))
	3476	(jne :bad)
	3477	(cmpl (:$b 0) (:@ (+ x8664::misc-data-offset 8) (:%q object)))
	3478	(je :ok)
	3479	(jmp :bad)
	3480	:two
	3481	(movq (:@ x8664::misc-data-offset (:%q object)) (:%q tag))
	3482	:ok-if-non-negative
	3483	(testq (:%q tag) (:%q tag))
	3484	(js :bad)
	3485	:ok
	3486	(:anchored-uuo-section :again)
	3487	:bad
	3488	(:anchored-uuo (uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-unsigned-byte-64))))
	3489
	3490	(define-x8664-vinsn require-char-code (()
	3491	((object :lisp))
	3492	((tag :u32)))
	3493	:again
	3494	(testl (:$l x8664::fixnummask) (:%l object))
	3495	(jne :bad)
	3496	(cmpq (:$l (ash #x110000 x8664::fixnumshift)) (:%q object))
	3497	(jae :bad)
	3498	(:anchored-uuo-section :again)
	3499	:bad
	3500	(:anchored-uuo (uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-mod-char-code-limit))))
	3501
	3502
	3503	;;; set DEST to
	3504	(define-x8664-vinsn mask-base-char (((dest :u8))
	3505	((src :lisp)))
	3506	(movzbl (:%b src) (:%l dest)))
	3507
	3508	(define-x8664-vinsn single-float-bits (((dest :u32))
	3509	((src :lisp)))
	3510	(movq (:%q src) (:%q dest))
	3511	(shrq (:$ub 32) (:%q dest)))
	3512
	3513	(define-x8664-vinsn zero-double-float-register (((dest :double-float))
	3514	())
	3515	(movsd (:%xmm x8664::fpzero) (:%xmm dest)))
	3516
	3517	(define-x8664-vinsn zero-single-float-register (((dest :single-float))
	3518	())
	3519	(movss (:%xmm x8664::fpzero) (:%xmm dest)))
	3520
	3521	(define-x8664-subprim-lea-jmp-vinsn (heap-rest-arg) .SPheap-rest-arg)
	3522	(define-x8664-subprim-lea-jmp-vinsn (stack-rest-arg) .SPstack-rest-arg)
	3523	(define-x8664-subprim-lea-jmp-vinsn (req-stack-rest-arg) .SPreq-stack-rest-arg)
	3524
	3525	(define-x8664-subprim-call-vinsn (stack-misc-alloc) .SPstack-misc-alloc)
	3526
	3527	(define-x8664-vinsn misc-element-count-fixnum (((dest :imm))
	3528	((src :lisp))
	3529	((temp :u64)))
	3530	(movq (:@ x8664::misc-header-offset (:%q src)) (:%q temp))
	3531	(shrq (:$ub x8664::num-subtag-bits) (:%q temp))
	3532	(imulq (:$b x8664::fixnumone) (:%q temp)(:%q dest)))
	3533
	3534	(define-x8664-vinsn %logior2 (((dest :imm))
	3535	((x :imm)
	3536	(y :imm)))
	3537	((:pred =
	3538	(:apply %hard-regspec-value x)
	3539	(:apply %hard-regspec-value dest))
	3540	(orq (:%q y) (:%q dest)))
	3541	((:not (:pred =
	3542	(:apply %hard-regspec-value x)
	3543	(:apply %hard-regspec-value dest)))
	3544	((:pred =
	3545	(:apply %hard-regspec-value y)
	3546	(:apply %hard-regspec-value dest))
	3547	(orq (:%q x) (:%q dest)))
	3548	((:not (:pred =
	3549	(:apply %hard-regspec-value y)
	3550	(:apply %hard-regspec-value dest)))
	3551	(movq (:%q x) (:%q dest))
	3552	(orq (:%q y) (:%q dest)))))
	3553
	3554	(define-x8664-vinsn %logand2 (((dest :imm))
	3555	((x :imm)
	3556	(y :imm)))
	3557	((:pred =
	3558	(:apply %hard-regspec-value x)
	3559	(:apply %hard-regspec-value dest))
	3560	(andq (:%q y) (:%q dest)))
	3561	((:not (:pred =
	3562	(:apply %hard-regspec-value x)
	3563	(:apply %hard-regspec-value dest)))
	3564	((:pred =
	3565	(:apply %hard-regspec-value y)
	3566	(:apply %hard-regspec-value dest))
	3567	(andq (:%q x) (:%q dest)))
	3568	((:not (:pred =
	3569	(:apply %hard-regspec-value y)
	3570	(:apply %hard-regspec-value dest)))
	3571	(movq (:%q x) (:%q dest))
	3572	(andq (:%q y) (:%q dest)))))
	3573
	3574	(define-x8664-vinsn %logxor2 (((dest :imm))
	3575	((x :imm)
	3576	(y :imm)))
	3577	((:pred =
	3578	(:apply %hard-regspec-value x)
	3579	(:apply %hard-regspec-value dest))
	3580	(xorq (:%q y) (:%q dest)))
	3581	((:not (:pred =
	3582	(:apply %hard-regspec-value x)
	3583	(:apply %hard-regspec-value dest)))
	3584	((:pred =
	3585	(:apply %hard-regspec-value y)
	3586	(:apply %hard-regspec-value dest))
	3587	(xorq (:%q x) (:%q dest)))
	3588	((:not (:pred =
	3589	(:apply %hard-regspec-value y)
	3590	(:apply %hard-regspec-value dest)))
	3591	(movq (:%q x) (:%q dest))
	3592	(xorq (:%q y) (:%q dest)))))
	3593
	3594
	3595	(define-x8664-vinsn vcell-ref (((dest :lisp))
	3596	((vcell :lisp)))
	3597	(movq (:@ x8664::misc-data-offset (:%q vcell)) (:%q dest)))
	3598
	3599	(define-x8664-vinsn (call-subprim-3 :call :subprim) (((dest t))
	3600	((spno :s32const)
	3601	(x t)
	3602	(y t)
	3603	(z t))
	3604	((entry (:label 1))))
	3605	(:talign 4)
	3606	(call (:@ spno))
	3607	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	3608
	3609	(define-x8664-vinsn setup-vcell-allocation (()
	3610	())
	3611	(movl (:$l x8664::value-cell-header) (:%l x8664::imm0))
	3612	(movl (:$l (- x8664::value-cell.size x8664::fulltag-misc)) (:%l x8664::imm1)))
	3613
	3614	(define-x8664-vinsn %init-vcell (()
	3615	((vcell :lisp)
	3616	(closed :lisp)))
	3617	(movq (:%q closed) (:@ x8664::value-cell.value (:%q vcell))))
	3618
	3619	(define-x8664-subprim-call-vinsn (progvsave) .SPprogvsave)
	3620
	3621	(define-x8664-subprim-jump-vinsn (progvrestore) .SPprogvrestore)
	3622
	3623	(define-x8664-subprim-lea-jmp-vinsn (simple-keywords) .SPsimple-keywords)
	3624
	3625	(define-x8664-subprim-lea-jmp-vinsn (keyword-args) .SPkeyword-args)
	3626
	3627	(define-x8664-subprim-lea-jmp-vinsn (keyword-bind) .SPkeyword-bind)
	3628
	3629	(define-x8664-vinsn scale-nargs (()
	3630	((nfixed :s16const)))
	3631	((:pred > nfixed 0)
	3632	((:pred < nfixed 16)
	3633	(subl (:$b (:apply ash nfixed x8664::word-shift)) (:%l x8664::nargs)))
	3634	((:pred >= nfixed 16)
	3635	(subl (:$l (:apply ash nfixed x8664::word-shift)) (:%l x8664::nargs)))))
	3636
	3637	(define-x8664-vinsn opt-supplied-p (()
	3638	((num-opt :u16const))
	3639	((nargs (:u64 #.x8664::nargs))
	3640	(imm :imm)))
	3641	(xorl (:%l imm) (:%l imm))
	3642	(movl (:$l (:apply target-nil-value)) (:%l x8664::arg_y))
	3643	:loop
	3644	(rcmpl (:%l imm) (:%l nargs))
	3645	(movl (:%l x8664::arg_y) (:%l x8664::arg_z))
	3646	(cmovll (:@ (+ x8664::t-offset x8664::symbol.vcell) (:%l x8664::arg_y)) (:%l x8664::arg_z))
	3647	(addl (:$b x8664::node-size) (:%l imm))
	3648	(rcmpl (:%l imm) (:$l (:apply ash num-opt x8664::fixnumshift)))
	3649	(pushq (:%q x8664::arg_z))
	3650	(jne :loop))
	3651
	3652	(define-x8664-vinsn one-opt-supplied-p (()
	3653	()
	3654	((temp :u64)))
	3655	(testl (:%l x8664::nargs) (:%l x8664::nargs))
	3656	(movl (:$l (:apply target-nil-value)) (:%l temp))
	3657	(cmovnel (:@ (+ x8664::t-offset x8664::symbol.vcell) (:%l temp)) (:%l temp))
	3658	(pushq (:%q temp)))
	3659
	3660	(define-x8664-vinsn two-opt-supplied-p (()
	3661	()
	3662	((temp0 :u64)
	3663	(temp1 :u64)))
	3664	(rcmpl (:%l x8664::nargs) (:$b x8664::node-size))
	3665	(movl (:$l (:apply target-nil-value)) (:%l temp0))
	3666	(movl (:%l temp0) (:%l temp1))
	3667	(cmovael (:@ (+ x8664::t-offset x8664::symbol.vcell) (:%l temp0)) (:%l temp0))
	3668	(cmoval (:@ (+ x8664::t-offset x8664::symbol.vcell) (:%l temp1)) (:%l temp1))
	3669	(pushq (:%q temp0))
	3670	(pushq (:%q temp1)))
	3671
	3672
	3673	(define-x8664-vinsn set-c-flag-if-constant-logbitp (()
	3674	((bit :u8const)
	3675	(int :imm)))
	3676	(btq (:$ub bit) (:%q int)))
	3677
	3678	(define-x8664-vinsn set-c-flag-if-variable-logbitp (()
	3679	((bit :imm)
	3680	(int :imm))
	3681	((temp0 :u8)
	3682	(temp1 :u8)))
	3683	(movl (:$l 63) (:%l temp1))
	3684	(movq (:%q bit) (:%q temp0))
	3685	(sarq (:$ub x8664::fixnumshift) (:%q temp0))
	3686	(addq (:$b x8664::fixnumshift) (:%q temp0))
	3687	(rcmpq (:%q temp0) (:%q temp1))
	3688	(cmoval (:%l temp1) (:%l temp0))
	3689	(btq (:%q temp0) (:%q int)))
	3690
	3691	(define-x8664-vinsn multiply-immediate (((dest :imm))
	3692	((src :imm)
	3693	(const :s32const)))
	3694	((:and (:pred >= const -128) (:pred <= const 127))
	3695	(imulq (:$b const) (:%q src) (:%q dest)))
	3696	((:not (:and (:pred >= const -128) (:pred <= const 127)))
	3697	(imulq (:$l const) (:%q src) (:%q dest))))
	3698
	3699	(define-x8664-vinsn multiply-fixnums (((dest :imm))
	3700	((x :imm)
	3701	(y :imm))
	3702	((unboxed :s64)))
	3703	((:pred =
	3704	(:apply %hard-regspec-value x)
	3705	(:apply %hard-regspec-value dest))
	3706	(movq (:%q y) (:%q unboxed))
	3707	(sarq (:$ub x8664::fixnumshift) (:%q unboxed))
	3708	(imulq (:%q unboxed) (:%q dest)))
	3709	((:and (:not (:pred =
	3710	(:apply %hard-regspec-value x)
	3711	(:apply %hard-regspec-value dest)))
	3712	(:pred =
	3713	(:apply %hard-regspec-value y)
	3714	(:apply %hard-regspec-value dest)))
	3715	(movq (:%q x) (:%q unboxed))
	3716	(sarq (:$ub x8664::fixnumshift) (:%q unboxed))
	3717	(imulq (:%q unboxed) (:%q dest)))
	3718	((:and (:not (:pred =
	3719	(:apply %hard-regspec-value x)
	3720	(:apply %hard-regspec-value dest)))
	3721	(:not (:pred =
	3722	(:apply %hard-regspec-value y)
	3723	(:apply %hard-regspec-value dest))))
	3724	(movq (:%q y) (:%q dest))
	3725	(movq (:%q x) (:%q unboxed))
	3726	(sarq (:$ub x8664::fixnumshift) (:%q unboxed))
	3727	(imulq (:%q unboxed) (:%q dest))))
	3728
	3729
	3730	(define-x8664-vinsn save-lexpr-argregs (()
	3731	((min-fixed :u16const)))
	3732	((:pred >= min-fixed $numx8664argregs)
	3733	(pushq (:%q x8664::arg_x))
	3734	(pushq (:%q x8664::arg_y))
	3735	(pushq (:%q x8664::arg_z)))
	3736	((:pred = min-fixed 2) ; at least 2 args
	3737	(cmpl (:$b (ash 2 x8664::word-shift)) (:%l x8664::nargs))
	3738	(je :yz2) ; skip arg_x if exactly 2
	3739	(pushq (:%q x8664::arg_x))
	3740	:yz2
	3741	(pushq (:%q x8664::arg_y))
	3742	(pushq (:%q x8664::arg_z)))
	3743	((:pred = min-fixed 1) ; at least one arg
	3744	(rcmpl (:%l x8664::nargs) (:$b (ash 2 x8664::word-shift)))
	3745	(jl :z1) ; branch if exactly one
	3746	(je :yz1) ; branch if exactly two
	3747	(pushq (:%q x8664::arg_x))
	3748	:yz1
	3749	(pushq (:%q x8664::arg_y))
	3750	:z1
	3751	(pushq (:%q x8664::arg_z)))
	3752	((:pred = min-fixed 0)
	3753	(testl (:%l x8664::nargs) (:%l x8664::nargs))
	3754	(je :none) ; exactly zero
	3755	(rcmpl (:%l x8664::nargs) (:$b (ash 2 x8664::word-shift)))
	3756	(je :yz0) ; exactly two
	3757	(jl :z0) ; one
	3758	; Three or more ...
	3759	(pushq (:%q x8664::arg_x))
	3760	:yz0
	3761	(pushq (:%q x8664::arg_y))
	3762	:z0
	3763	(pushq (:%q x8664::arg_z))
	3764	:none
	3765	)
	3766	((:not (:pred = min-fixed 0))
	3767	(leaq (:@ (:apply - (:apply ash min-fixed x8664::word-shift)) (:%q x8664::nargs))
	3768	(:%q x8664::nargs)))
	3769	(pushq (:%q x8664::nargs))
	3770	(movq (:%q x8664::rsp) (:%q x8664::arg_z)))
	3771
	3772
	3773
	3774
	3775	;;; The frame that was built (by SAVE-LISP-CONTEXT-VARIABLE-ARG-COUNT
	3776	;;; and SAVE-LEXPR-ARGREGS) contains an unknown number of arguments
	3777	;;; followed by the count of non-required arguments; the count is on
	3778	;;; top of the stack and its address is in %arg_z. We need to build a
	3779	;;; frame so that the function can address its arguments (copies of
	3780	;;; the required arguments and the lexpr) and locals; when the
	3781	;;; function returns, it should one or more values (depending on how
	3782	;;; it was called) and discard the hidden lexpr frame. At this point,
	3783	;;; %ra0 still contains the "real" return address. If it's not the
	3784	;;; magic multiple-value address, we can make the function return to
	3785	;;; something that does a single-value return (.SPpopj); otherwise, we
	3786	;;; need to make it return multiple values to the real caller. (Unlike
	3787	;;; the PPC, this case only involves creating one frame here, but that
	3788	;;; frame has two return addresses.)
	3789	(define-x8664-vinsn build-lexpr-frame (()
	3790	()
	3791	((temp :imm)))
	3792	(movq (:@ (:apply + (:apply target-nil-value) (x8664::%kernel-global 'x86::ret1valaddr)))
	3793	(:%q temp))
	3794	(cmpq (:%q temp)
	3795	(:%q x8664::ra0))
	3796	(je :multiple)
	3797	(pushq (:@ (:apply + (:apply target-nil-value) (x8664::%kernel-global 'x86::lexpr-return1v))))
	3798	(jmp :finish)
	3799	:multiple
	3800	(pushq (:@ (:apply + (:apply target-nil-value) (x8664::%kernel-global 'x86::lexpr-return))))
	3801	(pushq (:%q temp))
	3802	:finish
	3803	(pushq (:%q x8664::rbp))
	3804	(movq (:%q x8664::rsp) (:%q x8664::rbp)))
	3805
	3806
	3807	(define-x8664-vinsn copy-lexpr-argument (()
	3808	((n :u16const))
	3809	((temp :imm)))
	3810	(movq (:@ (:%q x8664::arg_z)) (:%q temp))
	3811	(pushq (:@ (:apply ash n x8664::word-shift) (:%q x8664::arg_z) (:%q temp))))
	3812
	3813
	3814	(define-x8664-vinsn %current-tcr (((dest :lisp))
	3815	())
	3816	(movq (:rcontext x8664::tcr.linear) (:%q dest)))
	3817
	3818	(define-x8664-vinsn (setq-special :call :subprim)
	3819	(()
	3820	((sym :lisp)
	3821	(val :lisp))
	3822	((entry (:label 1))))
	3823	(:talign 4)
	3824	(call (:@ .SPspecset))
	3825	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	3826
	3827	(define-x8664-vinsn set-z-flag-if-istruct-typep (()
	3828	((val :lisp)
	3829	(type :lisp))
	3830	((tag :u8)
	3831	(valtype :lisp)))
	3832	(xorl (:%l valtype) (:%l valtype))
	3833	(movl (:%l val) (:%l tag))
	3834	(andl (:$b x8664::tagmask) (:%l tag))
	3835	(cmpl (:$b x8664::tag-misc) (:%l tag))
	3836	(jne :have-tag)
	3837	(movzbl (:@ x8664::misc-subtag-offset (:%q val)) (:%l tag))
	3838	:have-tag
	3839	(cmpl (:$b x8664::subtag-istruct) (:%l tag))
	3840	(jne :do-compare)
	3841	(movq (:@ x8664::misc-data-offset (:%q val)) (:%q valtype))
	3842	:do-compare
	3843	(cmpq (:%q valtype) (:%q type)))
	3844
	3845	(define-x8664-subprim-call-vinsn (misc-ref) .SPmisc-ref)
	3846
	3847	(define-x8664-subprim-call-vinsn (ksignalerr) .SPksignalerr)
	3848
	3849	(define-x8664-subprim-call-vinsn (misc-alloc-init) .SPmisc-alloc-init)
	3850
	3851	(define-x8664-subprim-call-vinsn (misc-alloc) .SPmisc-alloc)
	3852
	3853	(define-x8664-subprim-lea-jmp-vinsn (make-stack-gvector) .SPstkgvector)
	3854
	3855	(define-x8664-vinsn load-character-constant (((dest :lisp))
	3856	((code :u32const))
	3857	())
	3858	(movl (:$l (:apply logior (:apply ash code 8) x8664::subtag-character))
	3859	(:%l dest)))
	3860
	3861	(define-x8664-vinsn %scharcode8 (((code :imm))
	3862	((str :lisp)
	3863	(idx :imm))
	3864	((imm :u64)))
	3865	(movq (:%q idx) (:%q imm))
	3866	(sarq (:$ub x8664::fixnumshift) (:%q imm))
	3867	(movzbl (:@ x8664::misc-data-offset (:%q str) (:%q imm)) (:%l imm))
	3868	(imulq (:$b x8664::fixnumone) (:%q imm)(:%q code)))
	3869
	3870	(define-x8664-vinsn %scharcode32 (((code :imm))
	3871	((str :lisp)
	3872	(idx :imm))
	3873	((imm :u64)))
	3874	(movq (:%q idx) (:%q imm))
	3875	(sarq (:$ub 1) (:%q imm))
	3876	(movl (:@ x8664::misc-data-offset (:%q str) (:%q imm)) (:%l imm))
	3877	(imulq (:$b x8664::fixnumone) (:%q imm)(:%q code)))
	3878
	3879	(define-x8664-subprim-jump-vinsn (tail-call-sym-slide) .SPtcallsymslide)
	3880
	3881	(define-x8664-subprim-jump-vinsn (tail-call-sym-vsp) .SPtcallsymvsp)
	3882
	3883
	3884	(define-x8664-vinsn character->code (((dest :u32))
	3885	((src :lisp)))
	3886	(movq (:%q src) (:%q dest))
	3887	(sarq (:$ub x8664::charcode-shift) (:%q dest)))
	3888
	3889	(define-x8664-vinsn (adjust-vsp :vsp :pop :discard)
	3890	(()
	3891	((amount :s32const)))
	3892	((:and (:pred >= amount -128) (:pred <= amount 127))
	3893	(addq (:$b amount) (:%q x8664::rsp)))
	3894	((:not (:and (:pred >= amount -128) (:pred <= amount 127)))
	3895	(addq (:$l amount) (:%q x8664::rsp))))
	3896
	3897	(define-x8664-vinsn (call-subprim-2 :call :subprim) (((dest t))
	3898	((spno :s32const)
	3899	(y t)
	3900	(z t))
	3901	((entry (:label 1))))
	3902	(:talign 4)
	3903	(call (:@ spno))
	3904	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	3905
	3906	(define-x8664-vinsn (call-subprim-1 :call :subprim) (((dest t))
	3907	((spno :s32const)
	3908	(x t))
	3909	((entry (:label 1))))
	3910	(:talign 4)
	3911	(call (:@ spno))
	3912	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	3913
	3914
	3915
	3916	(define-x8664-vinsn set-macptr-address (()
	3917	((addr :address)
	3918	(src :lisp))
	3919	())
	3920	(movq (:%q addr) (:@ x8664::macptr.address (:%q src))))
	3921
	3922	(define-x8664-vinsn %symbol->symptr (((dest :lisp))
	3923	((src :lisp))
	3924	((tag :u8)))
	3925	:begin
	3926	(movl (:$l (:apply + (:apply target-nil-value) x8664::nilsym-offset)) (:%l tag))
	3927	(cmpb (:$b x8664::fulltag-nil) (:%b src))
	3928	(cmoveq (:%q tag) (:%q dest))
	3929	(movl (:%l src) (:%l tag))
	3930	(je :ok)
	3931	(andl (:$b x8664::tagmask) (:%l tag))
	3932	(cmpl (:$b x8664::tag-symbol) (:%l tag))
	3933	(jne :bad)
	3934
	3935	((:not (:pred =
	3936	(:apply %hard-regspec-value dest)
	3937	(:apply %hard-regspec-value src)))
	3938	(movq (:% src) (:% dest)))
	3939	:ok
	3940	(:anchored-uuo-section :begin)
	3941	:bad
	3942	(:anchored-uuo (uuo-error-reg-not-tag (:%q src) (:$ub x8664::fulltag-symbol))))
	3943
	3944	(define-x8664-vinsn symbol-function (((val :lisp))
	3945	((sym (:lisp (:ne val))))
	3946	((tag :u8)))
	3947	:anchor
	3948	(movq (:@ x8664::symbol.fcell (:%q sym)) (:%q val))
	3949	(movl (:%l val) (:%l tag))
	3950	(andl (:$b x8664::tagmask) (:%l tag))
	3951	(cmpl (:$b x8664::tag-function) (:%l tag))
	3952	(jne :bad)
	3953
	3954	(:anchored-uuo-section :anchor)
	3955	:bad
	3956	(:anchored-uuo (uuo-error-udf (:%q sym))))
	3957
	3958	(define-x8664-subprim-jump-vinsn (tail-call-fn-slide) .SPtcallnfnslide)
	3959
	3960	(define-x8664-vinsn load-double-float-constant (((dest :double-float))
	3961	((lab :label)
	3962	))
	3963	(movsd (:@ (:^ lab) (:%q x8664::fn)) (:%xmm dest)))
	3964
	3965	(define-x8664-vinsn load-single-float-constant (((dest :single-float))
	3966	((lab :label)
	3967	))
	3968	(movss (:@ (:^ lab) (:%q x8664::fn)) (:%xmm dest)))
	3969
	3970	(define-x8664-subprim-call-vinsn (misc-set) .SPmisc-set)
	3971
	3972	(define-x8664-subprim-lea-jmp-vinsn (slide-values) .SPmvslide)
	3973
	3974	(define-x8664-subprim-lea-jmp-vinsn (spread-list) .SPspreadargz)
	3975
	3976	;;; Even though it's implemented by calling a subprim, THROW is really
	3977	;;; a JUMP (to a possibly unknown destination). If the destination's
	3978	;;; really known, it should probably be inlined (stack-cleanup, value
	3979	;;; transfer & jump ...)
	3980	(define-x8664-vinsn (throw :jump-unknown) (()
	3981	()
	3982	((entry (:label 1))))
	3983	(leaq (:@ (:^ :back) (:%q x8664::fn)) (:%q x8664::ra0))
	3984	(:talign 4)
	3985	(jmp (:@ .SPthrow))
	3986	:back
	3987	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn))
	3988	(uuo-error-reg-not-tag (:%q x8664::temp0) (:$ub x8664::subtag-catch-frame)))
	3989
	3990
	3991
	3992	(define-x8664-vinsn unbox-base-char (((dest :u64))
	3993	((src :lisp)))
	3994	:anchor
	3995	(movq (:%q src) (:%q dest))
	3996	(shrq (:$ub x8664::charcode-shift) (:%q dest))
	3997	(cmpb (:$b x8664::subtag-character) (:%b src))
	3998	(jne :bad)
	3999	(:anchored-uuo-section :anchor)
	4000	:bad
	4001	(:anchored-uuo (uuo-error-reg-not-tag (:%q src) (:$ub x8664::subtag-character))))
	4002
	4003	(define-x8664-subprim-lea-jmp-vinsn (save-values) .SPsave-values)
	4004
	4005	(define-x8664-subprim-lea-jmp-vinsn (recover-values) .SPrecover-values)
	4006
	4007	(define-x8664-subprim-lea-jmp-vinsn (recover-values-for-mvcall) .SPrecover-values-for-mvcall)
	4008
	4009	(define-x8664-subprim-lea-jmp-vinsn (add-values) .SPadd-values)
	4010
	4011	(define-x8664-subprim-call-vinsn (make-stack-block) .SPmakestackblock)
	4012
	4013	(define-x8664-subprim-call-vinsn (make-stack-block0) .Spmakestackblock0)
	4014
	4015	;;; "dest" is preallocated, presumably on a stack somewhere.
	4016	(define-x8664-vinsn store-double (()
	4017	((dest :lisp)
	4018	(source :double-float))
	4019	())
	4020	(movsd (:%xmm source) (:@ x8664::double-float.value (:%q dest))))
	4021
	4022	(define-x8664-vinsn fixnum->char (((dest :lisp))
	4023	((src :imm))
	4024	((temp :u32)))
	4025	(movl (:%l src) (:%l temp))
	4026	(sarl (:$ub (+ x8664::fixnumshift 11)) (:%l temp))
	4027	(cmpl (:$b (ash #xd800 -11))(:%l temp))
	4028	(movl (:$l (:apply target-nil-value)) (:%l temp))
	4029	(cmovel (:%l temp) (:%l dest))
	4030	(je :done)
	4031	((:not (:pred =
	4032	(:apply %hard-regspec-value dest)
	4033	(:apply %hard-regspec-value src)))
	4034	(movl (:%l src) (:%l dest)))
	4035	(shll (:$ub (- x8664::charcode-shift x8664::fixnumshift)) (:%l dest))
	4036	(addl (:$b x8664::subtag-character) (:%l dest))
	4037	:done)
	4038
	4039	;;; src is known to be a code for which CODE-CHAR returns non-nil.
	4040	(define-x8664-vinsn code-char->char (((dest :lisp))
	4041	((src :imm))
	4042	())
	4043	((:not (:pred =
	4044	(:apply %hard-regspec-value dest)
	4045	(:apply %hard-regspec-value src)))
	4046	(movl (:%l src) (:%l dest)))
	4047	(shll (:$ub (- x8664::charcode-shift x8664::fixnumshift)) (:%l dest))
	4048	(addl (:$b x8664::subtag-character) (:%l dest))
	4049	:done)
	4050
	4051
	4052	(define-x8664-vinsn sign-extend-halfword (((dest :imm))
	4053	((src :imm)))
	4054	(movq (:%q src ) (:%q dest))
	4055	(shlq (:$ub (- 48 x8664::fixnumshift)) (:%q dest))
	4056	(sarq (:$ub (- 48 x8664::fixnumshift)) (:%q dest)))
	4057
	4058	(define-x8664-subprim-jump-vinsn (tail-funcall-gen) .SPtfuncallgen)
	4059
	4060	(define-x8664-subprim-call-vinsn (gets64) .SPgets64)
	4061
	4062	(define-x8664-subprim-call-vinsn (getu64) .SPgetu64)
	4063
	4064	(define-x8664-vinsn %init-gvector (()
	4065	((v :lisp)
	4066	(nbytes :u32const))
	4067	((count :imm)))
	4068	(movl (:$l nbytes) (:%l count))
	4069	(jmp :test)
	4070	:loop
	4071	(popq (:@ x8664::misc-data-offset (:%q v) (:%q count)))
	4072	:test
	4073	(subq (:$b x8664::node-size) (:%q count))
	4074	(jge :loop))
	4075
	4076	(define-x8664-subprim-jump-vinsn (tail-funcall-slide) .SPtfuncallslide)
	4077
	4078	(define-x8664-vinsn nth-value (((result :lisp))
	4079	()
	4080	((imm0 :u64)))
	4081	(leaq (:@ (:%q x8664::rsp) (:%q x8664::nargs)) (:%q imm0))
	4082	(subq (:@ (:%q imm0)) (:%q x8664::nargs))
	4083	(movl (:$l (:apply target-nil-value)) (:%l result))
	4084	(jle :done)
	4085	;; I -think- that a CMOV would be safe here, assuming that N wasn't
	4086	;; extremely large. Don't know if we can assume that.
	4087	(movq (:@ (- x8664::node-size) (:%q x8664::rsp) (:%q x8664::nargs)) (:%q result))
	4088	:done
	4089	(leaq (:@ x8664::node-size (:%q imm0)) (:%q x8664::rsp)))
	4090
	4091
	4092	(define-x8664-subprim-lea-jmp-vinsn (req-heap-rest-arg) .SPreq-heap-rest-arg)
	4093
	4094	(define-x8664-subprim-call-vinsn (stack-misc-alloc-init) .SPstack-misc-alloc-init)
	4095
	4096	(define-x8664-vinsn fixnum->unsigned-natural (((dest :u64))
	4097	((src :imm)))
	4098	(movq (:%q src) (:%q dest))
	4099	(shrq (:$ub x8664::fixnumshift) (:%q dest)))
	4100
	4101	(define-x8664-vinsn %debug-trap (()
	4102	())
	4103	(uuo-error-debug-trap))
	4104
	4105	(define-x8664-vinsn double-to-single (((result :single-float))
	4106	((arg :double-float)))
	4107	(cvtsd2ss (:%xmm arg) (:%xmm result)))
	4108
	4109	(define-x8664-vinsn single-to-double (((result :double-float))
	4110	((arg :single-float)))
	4111	(cvtss2sd (:%xmm arg) (:%xmm result)))
	4112
	4113
	4114	(define-x8664-vinsn alloc-c-frame (()
	4115	((nwords :u32const))
	4116	((temp :imm)
	4117	(stack-temp :imm)))
	4118	(movq (:rcontext x8664::tcr.foreign-sp) (:%q stack-temp))
	4119	((:pred < (:apply ash (:apply logandc2 (:apply + nwords 9) 1) x8664::word-shift) 128)
	4120	(subq (:$b (:apply ash (:apply logandc2 (:apply + nwords 9) 1) x8664::word-shift)) (:rcontext x8664::tcr.foreign-sp)))
	4121	((:not (:pred < (:apply ash (:apply logandc2 (:apply + nwords 9) 1) x8664::word-shift) 128))
	4122	(subq (:$l (:apply ash (:apply logandc2 (:apply + nwords 9) 1) x8664::word-shift)) (:rcontext x8664::tcr.foreign-sp)))
	4123	(movq (:rcontext x8664::tcr.foreign-sp) (:%q temp))
	4124	(movq (:%q stack-temp) (:@ (:%q temp)))
	4125	(movq (:% x8664::rbp) (:@ x8664::csp-frame.rbp (:%q temp))))
	4126
	4127	(define-x8664-vinsn alloc-variable-c-frame (()
	4128	((nwords :imm))
	4129	((size :s64)
	4130	(temp :imm)
	4131	(stack-temp :imm)))
	4132	(leaq (:@ (* 9 x8664::node-size) (:%q nwords)) (:%q size))
	4133	(andb (:$b (lognot x8664::fulltagmask)) (:%b size))
	4134
	4135	(movq (:rcontext x8664::tcr.foreign-sp) (:%q stack-temp))
	4136	(subq (:%q size) (:rcontext x8664::tcr.foreign-sp))
	4137	(movq (:rcontext x8664::tcr.foreign-sp) (:%q temp))
	4138	(movq (:%q stack-temp) (:@ (:%q temp)))
	4139	(movq (:% x8664::rbp) (:@ x8664::csp-frame.rbp (:%q temp))))
	4140
	4141	(define-x8664-vinsn set-c-arg (()
	4142	((arg :u64)
	4143	(offset :u32const))
	4144	((temp (:u64 #.x8664::ra0))))
	4145	(movq (:rcontext x8664::tcr.foreign-sp) (:%q x8664::ra0))
	4146	(movq (:%q arg) (:@ (:apply + 16 (:apply ash offset 3)) (:%q x8664::ra0))))
	4147
	4148	(define-x8664-vinsn set-single-c-arg (()
	4149	((arg :single-float)
	4150	(offset :u32const))
	4151	((temp (:u64 #.x8664::ra0))))
	4152	(movq (:rcontext x8664::tcr.foreign-sp) (:%q x8664::ra0))
	4153	(movss (:%xmm arg) (:@ (:apply + 16 (:apply ash offset 3)) (:%q x8664::ra0))))
	4154
	4155	(define-x8664-vinsn reload-single-c-arg (((arg :single-float))
	4156	((offset :u32const))
	4157	((u64 (:u64 #.x8664::ra0))))
	4158	(movq (:rcontext x8664::tcr.foreign-sp) (:%q x8664::ra0))
	4159	(movss (:@ (:apply + 16 (:apply ash offset 3)) (:%q x8664::ra0)) (:%xmm arg)))
	4160
	4161	(define-x8664-vinsn set-double-c-arg (()
	4162	((arg :double-float)
	4163	(offset :u32const))
	4164	((temp (:u64 #.x8664::ra0))))
	4165	(movq (:rcontext x8664::tcr.foreign-sp) (:%q x8664::ra0))
	4166	(movsd (:%xmm arg) (:@ (:apply + 16 (:apply ash offset 3)) (:%q x8664::ra0))))
	4167
	4168	(define-x8664-vinsn reload-double-c-arg (((arg :double-float))
	4169	((offset :u32const))
	4170	((temp (:u64 #.x8664::ra0))))
	4171	(movq (:rcontext x8664::tcr.foreign-sp) (:%q x8664::ra0))
	4172	(movsd (:@ (:apply + 16 (:apply ash offset 3)) (:%q x8664::ra0)) (:%xmm arg)))
	4173
	4174	(define-x8664-subprim-call-vinsn (ff-call) .SPffcall)
	4175
	4176	(define-x8664-subprim-call-vinsn (ff-call-return-registers) .SPffcall-return-registers)
	4177
	4178	(define-x8664-subprim-call-vinsn (syscall) .SPsyscall)
	4179
	4180	(define-x8664-subprim-call-vinsn (setqsym) .SPsetqsym)
	4181
	4182	(define-x8664-vinsn recover-fn-from-rip (()
	4183	())
	4184	(leaq (:@ (:apply - (:^ :disp)) (:%q x8664::rip)) (:%q x8664::fn))
	4185	:disp)
	4186
	4187
	4188
	4189	(define-x8664-subprim-call-vinsn (makeu64) .SPmakeu64)
	4190
	4191	(define-x8664-subprim-call-vinsn (makes64) .SPmakes64)
	4192
	4193	(define-x8664-subprim-lea-jmp-vinsn (stack-cons-list*) .SPstkconslist-star)
	4194
	4195	(define-x8664-subprim-lea-jmp-vinsn (list*) .SPconslist-star)
	4196
	4197	(define-x8664-vinsn make-tsp-vcell (((dest :lisp))
	4198	((closed :lisp))
	4199	((temp :imm)
	4200	(stack-temp :imm)))
	4201	(subq (:$b (+ x8664::value-cell.size x8664::dnode-size)) (:rcontext x8664::tcr.next-tsp))
	4202	(movq (:rcontext x8664::tcr.save-tsp) (:%q stack-temp))
	4203	(movq (:rcontext x8664::tcr.next-tsp) (:%q temp))
	4204	(movapd (:%xmm x8664::fpzero) (:@ (:%q temp)))
	4205	(movapd (:%xmm x8664::fpzero) (:@ x8664::dnode-size (:%q temp)))
	4206	(movq (:%q stack-temp) (:@ (:%q temp)))
	4207	(movq (:% x8664::rbp) (:@ x8664::tsp-frame.rbp (:%q temp)))
	4208	(movq (:%q temp) (:rcontext x8664::tcr.save-tsp))
	4209	(movq (:$l x8664::value-cell-header) (:@ x8664::dnode-size (:%q temp)))
	4210	(movq (:%q closed) (:@ (+ x8664::dnode-size x8664::node-size) (:%q temp)))
	4211	(leaq (:@ (+ x8664::dnode-size x8664::fulltag-misc) (:%q temp)) (:%q dest)))
	4212
	4213	(define-x8664-subprim-lea-jmp-vinsn (bind-nil) .SPbind-nil)
	4214
	4215	(define-x8664-subprim-lea-jmp-vinsn (bind-self) .SPbind-self)
	4216
	4217	(define-x8664-subprim-lea-jmp-vinsn (bind-self-boundp-check) .SPbind-self-boundp-check)
	4218
	4219	(define-x8664-subprim-lea-jmp-vinsn (bind) .SPbind)
	4220
	4221	(define-x8664-vinsn (dpayback :call) (()
	4222	((n :s16const))
	4223	((temp (:u32 #.x8664::imm0))
	4224	(entry (:label 1))))
	4225	((:pred > n 0)
	4226	((:pred > n 1)
	4227	(movl (:$l n) (:%l temp))
	4228	(:talign 4)
	4229	(call (:@ .SPunbind-n)))
	4230	((:pred = n 1)
	4231	(:talign 4)
	4232	(call (:@ .SPunbind)))
	4233	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn))))
	4234
	4235	(define-x8664-subprim-jump-vinsn (tail-call-sym-gen) .SPtcallsymgen)
	4236
	4237	(define-x8664-subprim-call-vinsn (make-stack-list) .Spmakestacklist)
	4238
	4239	(define-x8664-vinsn node-slot-ref (((dest :lisp))
	4240	((node :lisp)
	4241	(cellno :u32const)))
	4242	(movq (:@ (:apply + x8664::misc-data-offset (:apply ash cellno 3))
	4243	(:%q node)) (:%q dest)))
	4244
	4245	(define-x8664-subprim-lea-jmp-vinsn (stack-cons-list) .SPstkconslist)
	4246
	4247
	4248	(define-x8664-vinsn %slot-ref (((dest :lisp))
	4249	((instance (:lisp (:ne dest)))
	4250	(index :lisp)))
	4251	(movq (:@ x8664::misc-data-offset (:%q instance) (:%q index)) (:%q dest))
	4252	(cmpl (:$b x8664::slot-unbound-marker) (:%l dest))
	4253	(je :bad)
	4254	:ok
	4255	(:anchored-uuo-section :ok)
	4256	:bad
	4257	(:anchored-uuo (uuo-error-slot-unbound (:%q dest) (:%q instance) (:%q index))))
	4258
	4259	(define-x8664-vinsn eep.address (((dest t))
	4260	((src (:lisp (:ne dest )))))
	4261	(movq (:@ (+ (ash 1 x8664::word-shift) x8664::misc-data-offset) (:%q src))
	4262	(:%q dest))
	4263	(cmpb (:$b x8664::fulltag-nil) (:%b dest))
	4264	(je :bad)
	4265	:ok
	4266	(:anchored-uuo-section :ok)
	4267	:bad
	4268	(:anchored-uuo (uuo-error-eep-unresolved (:%q src) (:%q dest))))
	4269
	4270
	4271	(define-x8664-subprim-lea-jmp-vinsn (heap-cons-rest-arg) .SPheap-cons-rest-arg)
	4272
	4273	(define-x8664-subprim-lea-jmp-vinsn (stack-cons-rest-arg) .SPstack-cons-rest-arg)
	4274
	4275	(define-x8664-subprim-lea-jmp-vinsn (make-stack-vector) .SPmkstackv)
	4276
	4277	(define-x8664-vinsn %current-frame-ptr (((dest :imm))
	4278	())
	4279	(movq (:%q x8664::rbp) (:%q dest)))
	4280
	4281	(define-x8664-vinsn %foreign-stack-pointer (((dest :imm))
	4282	())
	4283	(movq (:rcontext x8664::tcr.foreign-sp) (:%q dest)))
	4284
	4285
	4286	(define-x8664-vinsn %set-scharcode8 (()
	4287	((str :lisp)
	4288	(idx :imm)
	4289	(code :imm))
	4290	((imm :u64)
	4291	(imm1 :u64)))
	4292	(movq (:%q code) (:%q imm1))
	4293	(movq (:%q idx) (:%q imm))
	4294	(shrq (:$ub x8664::fixnumshift) (:%q imm1))
	4295	(shrq (:$ub x8664::word-shift) (:%q imm))
	4296	(movb (:%b imm1) (:@ x8664::misc-data-offset (:%q str) (:%q imm))))
	4297
	4298
	4299	(define-x8664-vinsn %set-scharcode32 (()
	4300	((str :lisp)
	4301	(idx :imm)
	4302	(code :imm))
	4303	((imm :u64)
	4304	(imm1 :u64)))
	4305	(movq (:%q code) (:%q imm1))
	4306	(movq (:%q idx) (:%q imm))
	4307	(shrq (:$ub x8664::fixnumshift) (:%q imm1))
	4308	(shrq (:$ub 1) (:%q imm))
	4309	(movl (:%l imm1) (:@ x8664::misc-data-offset (:%q str) (:%q imm))))
	4310
	4311
	4312
	4313
	4314	(define-x8664-vinsn pop-argument-registers (()
	4315	())
	4316	(testl (:%l x8664::nargs) (:%l x8664::nargs))
	4317	(je :done)
	4318	(rcmpl (:%l x8664::nargs) (:$b (ash 2 x8664::word-shift)))
	4319	(popq (:%q x8664::arg_z))
	4320	(jb :done)
	4321	(popq (:%q x8664::arg_y))
	4322	(je :done)
	4323	(popq (:%q x8664::arg_x))
	4324	:done)
	4325
	4326	(define-x8664-vinsn %symptr->symvector (((target :lisp))
	4327	((target :lisp)))
	4328	(subb (:$b (- x8664::fulltag-symbol x8664::fulltag-misc)) (:%b target)))
	4329
	4330	(define-x8664-vinsn %symvector->symptr (((target :lisp))
	4331	((target :lisp)))
	4332	(addb (:$b (- x8664::fulltag-symbol x8664::fulltag-misc)) (:%b target)))
	4333
	4334
	4335	(define-x8664-subprim-lea-jmp-vinsn (spread-lexpr) .SPspread-lexpr-z)
	4336
	4337	(define-x8664-vinsn mem-ref-double-float (((dest :double-float))
	4338	((src :address)
	4339	(index :s64)))
	4340	(movsd (:@ (:%q src) (:%q index)) (:%xmm dest)))
	4341
	4342	(define-x8664-vinsn mem-ref-single-float (((dest :single-float))
	4343	((src :address)
	4344	(index :s64)))
	4345	(movss (:@ (:%q src) (:%q index)) (:%xmm dest)))
	4346
	4347	(define-x8664-vinsn zero-extend-nargs (()
	4348	())
	4349	(movzwl (:%w x8664::nargs) (:%l x8664::nargs)))
	4350
	4351	(define-x8664-vinsn load-adl (()
	4352	((n :u32const)))
	4353	(movl (:$l n) (:%l x8664::nargs)))
	4354
	4355	(define-x8664-subprim-lea-jmp-vinsn (macro-bind) .SPmacro-bind)
	4356
	4357	(define-x8664-subprim-lea-jmp-vinsn (destructuring-bind-inner) .SPdestructuring-bind-inner)
	4358
	4359	(define-x8664-subprim-lea-jmp-vinsn (destructuring-bind) .SPdestructuring-bind)
	4360
	4361	(define-x8664-vinsn symbol-ref (((dest :lisp))
	4362	((src :lisp)
	4363	(cellno :u32const)))
	4364	(movq (:@ (:apply + (- x8664::node-size x8664::fulltag-symbol)
	4365	(:apply ash cellno 3))
	4366	(:%q src)) (:%q dest)))
	4367
	4368	(define-x8664-vinsn mem-ref-c-bit-fixnum (((dest :lisp))
	4369	((src :address)
	4370	(offset :s32const))
	4371	((temp :u32)))
	4372	((:pred = 0 (:apply ash offset -6))
	4373	(btq (:$ub (:apply logand 63 offset))
	4374	(:@ (:%q src))))
	4375	((:not (:pred = 0 (:apply ash offset -6)))
	4376	(btq (:$ub (:apply logand 63 offset))
	4377	(:@ (:apply ash (:apply ash offset -6) 3) (:%q src))))
	4378	(movl (:$l x8664::fixnumone) (:%l temp))
	4379	(leaq (:@ (- x8664::fixnumone) (:%q temp)) (:%q dest))
	4380	(cmovbl (:%l temp) (:%l dest)))
	4381
	4382	(define-x8664-vinsn mem-ref-c-bit (((dest :lisp))
	4383	((src :address)
	4384	(offset :s32const))
	4385	((temp :u32)))
	4386	((:pred = 0 (:apply ash offset -6))
	4387	(btq (:$ub (:apply logand 63 offset))
	4388	(:@ (:%q src))))
	4389	((:not (:pred = 0 (:apply ash offset -6)))
	4390	(btq (:$ub (:apply logand 63 offset))
	4391	(:@ (:apply ash (:apply ash offset -6) 3) (:%q src))))
	4392	(setb (:%b temp))
	4393	(movzbl (:%b temp) (:%l dest)))
	4394
	4395	(define-x8664-vinsn mem-ref-bit-fixnum (((dest :lisp)
	4396	(src :address))
	4397	((src :address)
	4398	(offset :lisp))
	4399	((temp :u32)))
	4400	(movq (:%q offset) (:%q temp))
	4401	(shrq (:$ub (+ 6 x8664::fixnumshift)) (:%q temp))
	4402	(leaq (:@ (:%q src) (:%q temp) 8) (:%q src))
	4403	(movq (:%q offset) (:%q temp))
	4404	(shrq (:$ub x8664::fixnumshift) (:%q temp))
	4405	(andl (:$l 63) (:%l temp))
	4406	(btq (:%q temp) (:@ (:%q src)))
	4407	(movl (:$l x8664::fixnumone) (:%l temp))
	4408	(leaq (:@ (- x8664::fixnumone) (:%q temp)) (:%q dest))
	4409	(cmovbl (:%l temp) (:%l dest)))
	4410
	4411	(define-x8664-vinsn mem-ref-bit (((dest :lisp)
	4412	(src :address))
	4413	((src :address)
	4414	(offset :lisp))
	4415	((temp :u32)))
	4416	(movq (:%q offset) (:%q temp))
	4417	(shrq (:$ub (+ 6 x8664::fixnumshift)) (:%q temp))
	4418	(leaq (:@ (:%q src) (:%q temp) 8) (:%q src))
	4419	(movq (:%q offset) (:%q temp))
	4420	(shrq (:$ub x8664::fixnumshift) (:%q temp))
	4421	(andl (:$l 63) (:%l temp))
	4422	(btq (:%q temp) (:@ (:%q src)))
	4423	(setb (:%b temp))
	4424	(movzbl (:%b temp) (:%l dest)))
	4425
	4426
	4427	(define-x8664-vinsn mem-set-c-bit-0 (()
	4428	((src :address)
	4429	(offset :s32const)))
	4430
	4431	((:pred = 0 (:apply ash offset -6))
	4432	(btrq (:$ub (:apply logand 63 offset))
	4433	(:@ (:%q src))))
	4434	((:not (:pred = 0 (:apply ash offset -6)))
	4435	(btrq (:$ub (:apply logand 63 offset))
	4436	(:@ (:apply ash (:apply ash offset -6) 3) (:%q src)))))
	4437
	4438	(define-x8664-vinsn mem-set-c-bit-1 (()
	4439	((src :address)
	4440	(offset :s32const)))
	4441
	4442	((:pred = 0 (:apply ash offset -6))
	4443	(btsq (:$ub (:apply logand 63 offset))
	4444	(:@ (:%q src))))
	4445	((:not (:pred = 0 (:apply ash offset -6)))
	4446	(btsq (:$ub (:apply logand 63 offset))
	4447	(:@ (:apply ash (:apply ash offset -6) 3) (:%q src)))))
	4448
	4449	(define-x8664-vinsn mem-set-c-bit-variable-value (()
	4450	((src :address)
	4451	(offset :s32const)
	4452	(value :lisp)))
	4453	(testq (:%q value) (:%q value))
	4454	(jne :set)
	4455	((:pred = 0 (:apply ash offset -6))
	4456	(btrq (:$ub (:apply logand 63 offset))
	4457	(:@ (:%q src))))
	4458	((:not (:pred = 0 (:apply ash offset -6)))
	4459	(btrq (:$ub (:apply logand 63 offset))
	4460	(:@ (:apply ash (:apply ash offset -6) 3) (:%q src))))
	4461	(jmp :done)
	4462	:set
	4463	((:pred = 0 (:apply ash offset -6))
	4464	(btsq (:$ub (:apply logand 63 offset))
	4465	(:@ (:%q src))))
	4466	((:not (:pred = 0 (:apply ash offset -6)))
	4467	(btsq (:$ub (:apply logand 63 offset))
	4468	(:@ (:apply ash (:apply ash offset -6) 3) (:%q src))))
	4469	:done)
	4470
	4471
	4472	(define-x8664-vinsn mem-set-bit-0 (((src :address))
	4473	((src :address)
	4474	(offset :lisp))
	4475	((temp :u32)))
	4476	(movq (:%q offset) (:%q temp))
	4477	(shrq (:$ub (+ 6 x8664::fixnumshift)) (:%q temp))
	4478	(leaq (:@ (:%q src) (:%q temp) 8) (:%q src))
	4479	(movq (:%q offset) (:%q temp))
	4480	(shrq (:$ub x8664::fixnumshift) (:%q temp))
	4481	(andl (:$l 63) (:%l temp))
	4482	(btrq (:%q temp) (:@ (:%q src))))
	4483
	4484	(define-x8664-vinsn mem-set-bit-1 (((src :address))
	4485	((src :address)
	4486	(offset :lisp))
	4487	((temp :u32)))
	4488	(movq (:%q offset) (:%q temp))
	4489	(shrq (:$ub (+ 6 x8664::fixnumshift)) (:%q temp))
	4490	(leaq (:@ (:%q src) (:%q temp) 8) (:%q src))
	4491	(movq (:%q offset) (:%q temp))
	4492	(shrq (:$ub x8664::fixnumshift) (:%q temp))
	4493	(andl (:$l 63) (:%l temp))
	4494	(btsq (:%q temp) (:@ (:%q src))))
	4495
	4496
	4497	(define-x8664-vinsn mem-set-bit-variable-value (((src :address))
	4498	((src :address)
	4499	(offset :lisp)
	4500	(value :lisp))
	4501	((temp :u32)))
	4502	(movq (:%q offset) (:%q temp))
	4503	(shrq (:$ub (+ 6 x8664::fixnumshift)) (:%q temp))
	4504	(leaq (:@ (:%q src) (:%q temp) 8) (:%q src))
	4505	(movq (:%q offset) (:%q temp))
	4506	(shrq (:$ub x8664::fixnumshift) (:%q temp))
	4507	(andl (:$l 63) (:%l temp))
	4508	(testq (:%q value) (:%q value))
	4509	(jne :set)
	4510	(btrq (:%q temp) (:@ (:%q src)))
	4511	(jmp :done)
	4512	:set
	4513	(btsq (:%q temp) (:@ (:%q src)))
	4514	:done)
	4515
	4516	(define-x8664-vinsn %natural+ (((result :u64))
	4517	((result :u64)
	4518	(other :u64)))
	4519	(addq (:%q other) (:%q result)))
	4520
	4521	(define-x8664-vinsn %natural+-c (((result :u64))
	4522	((result :u64)
	4523	(constant :s32const)))
	4524	(addq (:$l constant) (:%q result)))
	4525
	4526	(define-x8664-vinsn %natural- (((result :u64))
	4527	((result :u64)
	4528	(other :u64)))
	4529	(subq (:%q other) (:%q result)))
	4530
	4531	(define-x8664-vinsn %natural--c (((result :u64))
	4532	((result :u64)
	4533	(constant :s32const)))
	4534	(subq (:$l constant) (:%q result)))
	4535
	4536	(define-x8664-vinsn %natural-logior (((result :u64))
	4537	((result :u64)
	4538	(other :u64)))
	4539	(orq (:%q other) (:%q result)))
	4540
	4541	(define-x8664-vinsn %natural-logior-c (((result :u64))
	4542	((result :u64)
	4543	(constant :s32const)))
	4544	(orq (:$l constant) (:%q result)))
	4545
	4546	(define-x8664-vinsn %natural-logand (((result :u64))
	4547	((result :u64)
	4548	(other :u64)))
	4549	(andq (:%q other) (:%q result)))
	4550
	4551	(define-x8664-vinsn %natural-logand-c (((result :u64))
	4552	((result :u64)
	4553	(constant :s32const)))
	4554	(andq (:$l constant) (:%q result)))
	4555
	4556	(define-x8664-vinsn %natural-logxor (((result :u64))
	4557	((result :u64)
	4558	(other :u64)))
	4559	(xorq (:%q other) (:%q result)))
	4560
	4561	(define-x8664-vinsn %natural-logxor-c (((result :u64))
	4562	((result :u64)
	4563	(constant :s32const)))
	4564	(xorq (:$l constant) (:%q result)))
	4565
	4566	(define-x8664-vinsn natural-shift-left (((dest :u64))
	4567	((dest :u64)
	4568	(amt :u8const)))
	4569	(shlq (:$ub amt) (:%q dest)))
	4570
	4571	(define-x8664-vinsn natural-shift-right (((dest :u64))
	4572	((dest :u64)
	4573	(amt :u8const)))
	4574	(shrq (:$ub amt) (:%q dest)))
	4575
	4576	(define-x8664-vinsn set-z-if-uvector-type (((crf :crf))
	4577	((src :lisp)
	4578	(type :u8const))
	4579	((tag :u8)))
	4580	(movl (:%l src) (:%l tag))
	4581	(andl (:$b x8664::tagmask) (:%l tag))
	4582	(cmpl (:$b x8664::tag-misc) (:%l tag))
	4583	(jne :done)
	4584	(cmpb (:$b type) (:@ x8664::misc-subtag-offset (:%q src)))
	4585	:done)
	4586
	4587	(define-x8664-vinsn set-z-if-header-type (((crf :crf))
	4588	((src :lisp)
	4589	(type :u8const))
	4590	((flags :u32)))
	4591	(movl (:@ x8664::vectorH.flags (:%q src)) (:%l flags))
	4592	(shrl (:$ub (+ x8664::fixnumshift 8)) (:%l flags))
	4593	(cmpb (:$b type) (:%b flags))
	4594	)
	4595
	4596	(define-x8664-vinsn trap-unless-simple-array-2 (()
	4597	((object :lisp)
	4598	(expected-flags :u32const)
	4599	(type-error :u8const))
	4600	((tag :u8)))
	4601	:again
	4602	(movl (:%l object) (:%l tag))
	4603	(andl (:$b x8664::tagmask) (:%l tag))
	4604	(cmpl (:$b x8664::tag-misc) (:%l tag))
	4605	(jne :bad)
	4606	(cmpb (:$b x8664::subtag-arrayH) (:@ x8664::misc-subtag-offset (:%q object)))
	4607	(jne :bad)
	4608	(cmpq (:$b (ash 2 x8664::fixnumshift)) (:@ x8664::arrayH.rank (:%q object)))
	4609	(jne :bad)
	4610	(cmpq (:$l (:apply ash expected-flags x8664::fixnumshift)) (:@ x8664::arrayH.flags (:%q object)))
	4611	(jne :bad)
	4612
	4613	(:anchored-uuo-section :again)
	4614	:bad
	4615	(:anchored-uuo (uuo-error-reg-not-type (:%q object) (:$ub type-error))))
	4616
	4617	(define-x8664-vinsn set-z-if-typed-array (((crf :crf))
	4618	((object :lisp)
	4619	(expected-type :u8const)
	4620	(rank :u8const))
	4621	((tag :u8)))
	4622	:again
	4623	(movl (:%l object) (:%l tag))
	4624	(andl (:$b x8664::tagmask) (:%l tag))
	4625	(cmpl (:$b x8664::tag-misc) (:%l tag))
	4626	(jne :notz)
	4627	(cmpb (:$b x8664::subtag-arrayH) (:@ x8664::misc-subtag-offset (:%q object)))
	4628	(jne :notz)
	4629	(cmpq (:$b (:apply ash rank x8664::fixnumshift)) (:@ x8664::arrayH.rank (:%q object)))
	4630	(jne :notz)
	4631	(movl (:@ x8664::arrayH.flags (:%q object)) (:%l tag))
	4632	(shrl (:$ub (+ x8664::fixnumshift 8)) (:%l tag))
	4633	(cmpb (:$b expected-type) (:%b tag))
	4634	:notz
	4635	)
	4636
	4637
	4638
	4639	(define-x8664-vinsn trap-unless-simple-array-3 (()
	4640	((object :lisp)
	4641	(expected-flags :u32const)
	4642	(type-error :u8const))
	4643	((tag :u8)))
	4644	:again
	4645	(movl (:%l object) (:%l tag))
	4646	(andl (:$b x8664::tagmask) (:%l tag))
	4647	(cmpl (:$b x8664::tag-misc) (:%l tag))
	4648	(jne :bad)
	4649	(cmpb (:$b x8664::subtag-arrayH) (:@ x8664::misc-subtag-offset (:%q object)))
	4650	(jne :bad)
	4651	(cmpq (:$b (ash 3 x8664::fixnumshift)) (:@ x8664::arrayH.rank (:%q object)))
	4652	(jne :bad)
	4653	(cmpq (:$l (:apply ash expected-flags x8664::fixnumshift)) (:@ x8664::arrayH.flags (:%q object)))
	4654	(jne :bad)
	4655	(:anchored-uuo-section :again)
	4656	:bad
	4657	(:anchored-uuo (uuo-error-reg-not-type (:%q object) (:$ub type-error))))
	4658
	4659	(define-x8664-vinsn trap-unless-array-header (()
	4660	((object :lisp))
	4661	((tag :u8)))
	4662	:again
	4663	(movl (:%l object) (:%l tag))
	4664	(andl (:$b x8664::tagmask) (:%l tag))
	4665	(cmpl (:$b x8664::tag-misc) (:%l tag))
	4666	(jne :trap)
	4667	(cmpb (:$b x8664::subtag-arrayH) (:@ x8664::misc-subtag-offset (:%q object)))
	4668	(jne :trap)
	4669
	4670	(:anchored-uuo-section :again)
	4671	:trap
	4672	(:anchored-uuo (uuo-error-reg-not-tag (:%q object) (:$ub x8664::subtag-arrayH))))
	4673
	4674	(define-x8664-vinsn check-arrayH-rank (()
	4675	((header :lisp)
	4676	(expected :u32const))
	4677	((rank :imm)))
	4678	:anchor
	4679	(movl (:$l (:apply ash expected x8664::fixnumshift)) (:%l rank))
	4680	(cmpq (:@ x8664::arrayH.rank (:%q header)) (:%q rank))
	4681	(jne :bad)
	4682	(:anchored-uuo-section :anchor)
	4683	:bad
	4684	(:anchored-uuo (uuo-error-array-rank (:%q header) (:%q rank))))
	4685
	4686	(define-x8664-vinsn check-arrayH-flags (()
	4687	((header :lisp)
	4688	(expected :u32const)
	4689	(type-error :u8const)))
	4690	:anchor
	4691	(cmpq (:$l (:apply ash expected x8664::fixnumshift))
	4692	(:@ x8664::arrayH.flags (:%q header)))
	4693	(jne :bad)
	4694	(:anchored-uuo-section :anchor)
	4695	:bad
	4696	(:anchored-uuo (uuo-error-reg-not-type (:%q header) (:$ub type-error))))
	4697
	4698	(define-x8664-vinsn misc-ref-c-u16 (((dest :u16))
	4699	((v :lisp)
	4700	(idx :u32const))
	4701	())
	4702	(movzwl (:@ (:apply + x8664::misc-data-offset (:apply ash idx 1)) (:%q v)) (:%l dest)))
	4703
	4704	(define-x8664-vinsn misc-ref-c-s16 (((dest :s16))
	4705	((v :lisp)
	4706	(idx :u32const))
	4707	())
	4708	(movswq (:@ (:apply + x8664::misc-data-offset (:apply ash idx 1)) (:%q v)) (:%q dest)))
	4709
	4710	(define-x8664-vinsn misc-set-single-float (()
	4711	((val :single-float)
	4712	(v :lisp)
	4713	(scaled-idx :u32)))
	4714	(movss (:%xmm val) (:@ x8664::misc-data-offset (:% v) (:% scaled-idx))))
	4715
	4716	(define-x8664-vinsn u16->u32 (((dest :u32))
	4717	((src :u16)))
	4718	(movzwl (:%w src) (:%l dest)))
	4719
	4720	(define-x8664-vinsn u8->u32 (((dest :u32))
	4721	((src :u8)))
	4722	(movzbl (:%b src) (:%l dest)))
	4723
	4724
	4725	(define-x8664-vinsn s16->s32 (((dest :s32))
	4726	((src :s16)))
	4727	(movswl (:%w src) (:%l dest)))
	4728
	4729	(define-x8664-vinsn s8->s32 (((dest :s32))
	4730	((src :s8)))
	4731	(movsbl (:%b src) (:%l dest)))
	4732
	4733	(define-x8664-subprim-jump-vinsn (tail-call-fn-gen) .SPtcallnfngen)
	4734
	4735	(define-x8664-subprim-jump-vinsn (tail-call-fn-vsp) .SPtcallnfnvsp)
	4736
	4737	(define-x8664-vinsn set-eq-bit (()
	4738	())
	4739	(testb (:%b x8664::arg_z) (:%b x8664::arg_z)))
	4740
	4741	(define-x8664-vinsn %schar8 (((char :imm))
	4742	((str :lisp)
	4743	(idx :imm))
	4744	((imm :u32)))
	4745	(movq (:%q idx) (:%q imm))
	4746	(shrq (:$ub x8664::fixnumshift) (:%q imm))
	4747	(movzbl (:@ x8664::misc-data-offset (:%q str) (:%q imm)) (:%l imm))
	4748	(shll (:$ub x8664::charcode-shift) (:%l imm))
	4749	(leaq (:@ x8664::subtag-character (:%q imm)) (:%q char)))
	4750
	4751	(define-x8664-vinsn %schar32 (((char :imm))
	4752	((str :lisp)
	4753	(idx :imm))
	4754	((imm :u32)))
	4755	(movq (:%q idx) (:%q imm))
	4756	(shrq (:$ub 1) (:%q imm))
	4757	(movl (:@ x8664::misc-data-offset (:%q str) (:%q imm)) (:%l imm))
	4758	(shll (:$ub x8664::charcode-shift) (:%l imm))
	4759	(leaq (:@ x8664::subtag-character (:%q imm)) (:%q char)))
	4760
	4761
	4762	(define-x8664-vinsn %set-schar8 (()
	4763	((str :lisp)
	4764	(idx :imm)
	4765	(char :imm))
	4766	((imm0 :u64)
	4767	(imm1 :u64)))
	4768	(movq (:%q idx) (:%q imm0))
	4769	(movl (:%l char) (:%l imm1))
	4770	(shrq (:$ub x8664::fixnumshift) (:%q imm0))
	4771	(shrl (:$ub x8664::charcode-shift) (:%l imm1))
	4772	(movb (:%b imm1) (:@ x8664::misc-data-offset (:%q str) (:%q imm0))))
	4773
	4774	(define-x8664-vinsn %set-schar32 (()
	4775	((str :lisp)
	4776	(idx :imm)
	4777	(char :imm))
	4778	((imm0 :u64)
	4779	(imm1 :u64)))
	4780	(movq (:%q idx) (:%q imm0))
	4781	(movl (:%l char) (:%l imm1))
	4782	(shrq (:$ub 1) (:%q imm0))
	4783	(shrl (:$ub x8664::charcode-shift) (:%l imm1))
	4784	(movl (:%l imm1) (:@ x8664::misc-data-offset (:%q str) (:%q imm0))))
	4785
	4786	(define-x8664-vinsn misc-set-c-single-float (((val :single-float))
	4787	((v :lisp)
	4788	(idx :u32const)))
	4789	(movss (:%xmm val) (:@ (:apply + x8664::misc-data-offset (:apply ash idx 2)) (:%q v))))
	4790
	4791	(define-x8664-vinsn array-data-vector-ref (((dest :lisp))
	4792	((header :lisp)))
	4793	(movq (:@ x8664::arrayH.data-vector (:%q header)) (:%q dest)))
	4794
	4795	(define-x8664-subprim-call-vinsn (subtag-misc-ref) .SPsubtag-misc-ref)
	4796
	4797	(define-x8664-subprim-call-vinsn (subtag-misc-set) .SPsubtag-misc-set)
	4798
	4799	(define-x8664-vinsn mem-ref-c-absolute-u8 (((dest :u8))
	4800	((addr :s32const)))
	4801	(movzbl (:@ addr) (:%l dest)))
	4802
	4803	(define-x8664-vinsn mem-ref-c-absolute-s8 (((dest :s8))
	4804	((addr :s32const)))
	4805	(movsbq (:@ addr) (:%q dest)))
	4806
	4807	(define-x8664-vinsn mem-ref-c-absolute-u16 (((dest :u16))
	4808	((addr :s32const)))
	4809	(movzwl (:@ addr) (:%l dest)))
	4810
	4811	(define-x8664-vinsn mem-ref-c-absolute-s16 (((dest :s16))
	4812	((addr :s32const)))
	4813	(movswq (:@ addr) (:%q dest)))
	4814
	4815	(define-x8664-vinsn mem-ref-c-absolute-fullword (((dest :u32))
	4816	((addr :s32const)))
	4817	(movl (:@ addr) (:%l dest)))
	4818
	4819	(define-x8664-vinsn mem-ref-c-absolute-signed-fullword (((dest :s32))
	4820	((addr :s32const)))
	4821	(movslq (:@ addr) (:%q dest)))
	4822
	4823	(define-x8664-vinsn mem-ref-c-absolute-doubleword (((dest :s64))
	4824	((addr :s32const)))
	4825	(movq (:@ addr) (:%q dest)))
	4826
	4827	(define-x8664-vinsn mem-ref-c-absolute-signed-doubleword (((dest :s64))
	4828	((addr :s32const)))
	4829	(movq (:@ addr) (:%q dest)))
	4830
	4831	(define-x8664-vinsn mem-ref-c-absolute-natural (((dest :u64))
	4832	((addr :s32const)))
	4833	(movq (:@ addr) (:%q dest)))
	4834
	4835	(define-x8664-vinsn event-poll (()
	4836	())
	4837	(btrq (:$ub 63) (:rcontext x8664::tcr.interrupt-pending))
	4838	(jae :no-interrupt)
	4839	(ud2a)
	4840	(:byte 2)
	4841	:no-interrupt)
	4842
	4843	;;; Return dim1 (unboxed)
	4844	(define-x8664-vinsn check-2d-bound (((dim :u64))
	4845	((i :imm)
	4846	(j :imm)
	4847	(header :lisp)))
	4848	:anchor
	4849	(cmpq (:@ (+ x8664::misc-data-offset (* 8 x8664::arrayH.dim0-cell)) (:%q header)) (:%q i))
	4850	(jae :bad-i)
	4851	(movq (:@ (+ x8664::misc-data-offset (* 8 (1+ x8664::arrayH.dim0-cell))) (:%q header))
	4852	(:%q dim))
	4853	(cmpq (:%q dim) (:%q j))
	4854	(jae :bad-j)
	4855	(sarq (:$ub x8664::fixnumshift) (:%q dim))
	4856	(:anchored-uuo-section :anchor)
	4857	:bad-i
	4858	(:anchored-uuo (uuo-error-array-bounds (:%q i) (:%q header)))
	4859	(:anchored-uuo-section :anchor)
	4860	:bad-j
	4861	(:anchored-uuo (uuo-error-array-bounds (:%q j) (:%q header))))
	4862
	4863	;;; Return dim1, dim2 (unboxed)
	4864	(define-x8664-vinsn check-3d-bound (((dim1 :u64)
	4865	(dim2 :u64))
	4866	((i :imm)
	4867	(j :imm)
	4868	(k :imm)
	4869	(header :lisp)))
	4870	:anchor
	4871	(cmpq (:@ (+ x8664::misc-data-offset (* 8 x8664::arrayH.dim0-cell)) (:%q header)) (:%q i))
	4872	(jae :bad-i)
	4873	(movq (:@ (+ x8664::misc-data-offset (* 8 (1+ x8664::arrayH.dim0-cell))) (:%q header)) (:%q dim1))
	4874	(cmpq (:%q dim1) (:%q j))
	4875	(jae :bad-j)
	4876	(sarq (:$ub x8664::fixnumshift) (:%q dim1))
	4877	(movq (:@ (+ x8664::misc-data-offset (* 8 (+ 2 x8664::arrayH.dim0-cell))) (:%q header)) (:%q dim2))
	4878	(cmpq (:%q dim2) (:%q k))
	4879	(jae :bad-k)
	4880	(sarq (:$ub x8664::fixnumshift) (:%q dim2))
	4881	(:anchored-uuo-section :anchor)
	4882	:bad-i
	4883	(:anchored-uuo (uuo-error-array-bounds (:%q i) (:%q header)))
	4884	(:anchored-uuo-section :anchor)
	4885	:bad-j
	4886	(:anchored-uuo (uuo-error-array-bounds (:%q j) (:%q header)))
	4887	(:anchored-uuo-section :anchor)
	4888	:bad-k
	4889	(:anchored-uuo (uuo-error-array-bounds (:%q k) (:%q header)))
	4890	)
	4891
	4892
	4893	(define-x8664-vinsn 2d-dim1 (((dest :u64))
	4894	((header :lisp)))
	4895	(movq (:@ (+ x8664::misc-data-offset (* 8 (1+ x8664::arrayH.dim0-cell))) (:%q header))
	4896	(:%q dest))
	4897	(sarq (:$ub x8664::fixnumshift) (:%q dest)))
	4898
	4899
	4900	(define-x8664-vinsn 3d-dims (((dim1 :u64)
	4901	(dim2 :u64))
	4902	((header :lisp)))
	4903	(movq (:@ (+ x8664::misc-data-offset (* 8 (1+ x8664::arrayH.dim0-cell))) (:%q header)) (:%q dim1))
	4904	(movq (:@ (+ x8664::misc-data-offset (* 8 (+ 2 x8664::arrayH.dim0-cell))) (:%q header)) (:%q dim2))
	4905	(sarq (:$ub x8664::fixnumshift) (:%q dim1))
	4906	(sarq (:$ub x8664::fixnumshift) (:%q dim2)))
	4907
	4908	(define-x8664-vinsn 2d-unscaled-index (((dest :imm)
	4909	(dim1 :u64))
	4910	((dim1 :u64)
	4911	(i :imm)
	4912	(j :imm)))
	4913
	4914	(imulq (:%q i) (:%q dim1))
	4915	(leaq (:@ (:%q j) (:%q dim1)) (:%q dest)))
	4916
	4917
	4918	;; dest <- (+ (* i dim1 dim2) (* j dim2) k)
	4919	(define-x8664-vinsn 3d-unscaled-index (((dest :imm)
	4920	(dim1 :u64)
	4921	(dim2 :u64))
	4922	((dim1 :u64)
	4923	(dim2 :u64)
	4924	(i :imm)
	4925	(j :imm)
	4926	(k :imm)))
	4927	(imulq (:%q dim2) (:%q dim1))
	4928	(imulq (:%q j) (:%q dim2))
	4929	(imulq (:%q i) (:%q dim1))
	4930	(addq (:%q dim1) (:%q dim2))
	4931	(leaq (:@ (:%q k) (:%q dim2)) (:%q dest)))
	4932
	4933	(define-x8664-vinsn set-z-flag-if-arg-fixnum (()
	4934	((arg :imm)))
	4935	(testb (:$b x8664::fixnummask) (:%b arg)))
	4936
	4937	(define-x8664-vinsn set-z-flag-if-both-args-fixnums (()
	4938	((a :imm)
	4939	(b :imm))
	4940	((tag :u32)))
	4941	(movl (:%l a) (:%l tag))
	4942	(orl (:%l b) (:%l tag))
	4943	(testb (:$b x8664::fixnummask) (:%b tag)))
	4944
	4945
	4946
	4947
	4948
	4949	(define-x8664-vinsn fixnum->single-float (((f :single-float))
	4950	((arg :lisp))
	4951	((unboxed :s64)))
	4952	(movq (:%q arg) (:%q unboxed))
	4953	(sarq (:$ub x8664::fixnumshift) (:%q unboxed))
	4954	(cvtsi2ssq (:%q unboxed) (:%xmm f)))
	4955
	4956	(define-x8664-vinsn fixnum->double-float (((f :double-float))
	4957	((arg :lisp))
	4958	((unboxed :s64)))
	4959	(movq (:%q arg) (:%q unboxed))
	4960	(sarq (:$ub x8664::fixnumshift) (:%q unboxed))
	4961	(cvtsi2sdq (:%q unboxed) (:%xmm f)))
	4962
	4963
	4964	(define-x8664-vinsn xchg-registers (()
	4965	((a t)
	4966	(b t)))
	4967	(xchgq (:%q a) (:%q b)))
	4968
	4969	(define-x8664-vinsn establish-fn (()
	4970	()
	4971	((entry (:label 1))))
	4972	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	4973
	4974	(define-x8664-vinsn %ilognot (((dest :imm)
	4975	(src :imm))
	4976	((src :imm)))
	4977	(xorq (:$b (- x8664::fixnumone)) (:%q dest)))
	4978
	4979
	4980	(define-x8664-vinsn align-loop-head (()
	4981	()
	4982	()))
	4983
	4984	(define-x8664-vinsn double-float-negate (((reg :double-float))
	4985	((reg :double-float))
	4986	((tmp :double-float)))
	4987	(movsd (:@ (:^ :const) (:% x8664::fn)) (:%xmm tmp))
	4988	(pxor (:%xmm tmp) (:%xmm reg))
	4989	(:uuo-section)
	4990	:const
	4991	(:long 0)
	4992	(:long #x-80000000))
	4993
	4994	(define-x8664-vinsn single-float-negate (((reg :single-float))
	4995	((reg :single-float))
	4996	((tmp :single-float)))
	4997	(movss (:@ (:^ :const) (:% x8664::fn)) (:%xmm tmp))
	4998	(pxor (:%xmm tmp) (:%xmm reg))
	4999	(:uuo-section)
	5000	:const
	5001	(:long #x80000000))
	5002
	5003	(define-x8664-vinsn fixnum-ref-c-double-float (((dest :double-float))
	5004	((base :imm)
	5005	(idx :u32const)))
	5006	(movsd (:@ (:apply ash idx 3) (:%q base)) (:%xmm dest)))
	5007
	5008	(define-x8664-vinsn fixnum-ref-double-float (((dest :double-float))
	5009	((base :imm)
	5010	(idx :imm)))
	5011	(movsd (:@ (:%q base) (:%q idx)) (:%xmm dest)))
	5012
	5013	(define-x8664-vinsn fixnum-set-c-double-float (()
	5014	((base :imm)
	5015	(idx :u32const)
	5016	(val :double-float)))
	5017	(movsd (:%xmm val) (:@ (:apply ash idx 3) (:%q base))))
	5018
	5019	(define-x8664-vinsn fixnum-set-double-float (()
	5020	((base :imm)
	5021	(idx :imm)
	5022	(val :double-float)))
	5023	(movsd (:%xmm val) (:@ (:%q base) (:%q idx))))
	5024
	5025
	5026	(define-x8664-vinsn pop-outgoing-arg (((n :u16const))
	5027	())
	5028	(popq (:@ (:apply * n (- x8664::node-size)) (:%q x8664::rbp))))
	5029
	5030	(define-x8664-vinsn slide-nth-arg (()
	5031	((n :u16const)
	5032	(nstackargs :u16const)
	5033	(temp :lisp)))
	5034	(movq (:@ (:apply * (:apply - nstackargs (:apply + 1 n)) x8664::node-size) (:%q x8664::rsp)) (:%q temp))
	5035	(movq (:%q temp) (:@ (:apply * (:apply + n 1) (- x8664::node-size)) (:%q x8664::rbp))))
	5036
	5037
	5038	(define-x8664-vinsn set-tail-vsp (((nargs :u16const))
	5039	())
	5040	((:pred = 0 nargs)
	5041	(movq (:%q x8664::rbp) (:%q x8664::rsp)))
	5042	((:not (:pred = 0 nargs))
	5043	(leaq (:@ (:apply * nargs (- x8664::node-size)) (:%q x8664::rbp)) (:%q x8664::rsp))))
	5044
	5045
	5046	;;; If we've used one of the fixed-stack-args !slideN vinsns above
	5047	;;; and are calling some function (rather than jumping to an internal
	5048	;;; entry point), we need to push the caller's return address and unlink
	5049	;;; its frame pointer.
	5050	(define-x8664-vinsn prepare-tail-call (()
	5051	())
	5052	(pushq (:@ x8664::node-size (:%q x8664::rbp)))
	5053	(movq (:@ (:%q x8664::rbp)) (:%q x8664::rbp)))
	5054
	5055	(define-x8664-vinsn set-carry-if-fixnum-in-range
	5056	(((idx :u32))
	5057	((reg :imm)
	5058	(minval :s32const)
	5059	(maxval :u32const)))
	5060	(movq (:%q reg) (:%q idx))
	5061	(sarq (:$ub x8664::fixnumshift) (:%q idx))
	5062	((:not (:pred zerop minval))
	5063	((:and (:pred < minval 128) (:pred >= minval -128))
	5064	(subq (:$b minval) (:%q idx)))
	5065	((:not (:and (:pred < minval 128) (:pred >= minval -128)))
	5066	(subq (:$l minval) (:%q idx))))
	5067	((:pred < maxval 128)
	5068	(cmpq (:$b maxval) (:%q idx)))
	5069	((:pred >= maxval 128)
	5070	(cmpq (:$l maxval) (:%q idx))))
	5071
	5072	(define-x8664-vinsn (ijmp :branch) (((idx :u32))
	5073	((idx :u32)
	5074	(count :u32const))
	5075	((rjmp :lisp)))
	5076	(movl (:@ (:^ :jtab) (:%q x8664::fn) (:%q idx) 4) (:%l idx))
	5077	(leaq (:@ (:%q x8664::fn) (:%q idx)) (:%q rjmp))
	5078	(jmp (:%q rjmp))
	5079	(:uuo-section)
	5080	(:align 2)
	5081	(:long count)
	5082	:jtab)
	5083
	5084
	5085	(define-x8664-vinsn jtabentry (()
	5086	((label :label)))
	5087	(:uuo-section)
	5088	(:long (:^ label)))
	5089
	5090
	5091	(define-x8664-vinsn ivector-typecode-p (((dest :lisp))
	5092	((src :lisp))
	5093	((temp :u32)
	5094	(mask :u32)))
	5095	(movl (:%l src) (:%l temp))
	5096	(shrl (:$ub x8664::fixnumshift) (:%l temp))
	5097	(andl (:$b x8664::fulltagmask) (:%l temp))
	5098	(movl (:$l (logior (ash 1 x8664::fulltag-immheader-0)
	5099	(ash 1 x8664::fulltag-immheader-1)
	5100	(ash 1 x8664::fulltag-immheader-2))) (:%l mask))
	5101	(btl (:%l temp) (:%l mask))
	5102	(movl (:$l 0) (:%l mask))
	5103	((:not (:pred =
	5104	(:apply %hard-regspec-value dest)
	5105	(:apply %hard-regspec-value src)))
	5106	(movl (:%l src) (:%l dest)))
	5107	(cmovael (:%l mask) (:%l dest)))
	5108
	5109	(define-x8664-vinsn gvector-typecode-p (((dest :lisp))
	5110	((src :lisp))
	5111	((temp :u32)
	5112	(mask :u32)))
	5113	(movl (:%l src) (:%l temp))
	5114	(shrl (:$ub x8664::fixnumshift) (:%l temp))
	5115	(andl (:$b x8664::fulltagmask) (:%l temp))
	5116	(movl (:$l (logior (ash 1 x8664::fulltag-nodeheader-0)
	5117	(ash 1 x8664::fulltag-nodeheader-1))) (:%l mask))
	5118	(btl (:%l temp) (:%l mask))
	5119	(movl (:$l 0) (:%l mask))
	5120	((:not (:pred =
	5121	(:apply %hard-regspec-value dest)
	5122	(:apply %hard-regspec-value src)))
	5123	(movl (:%l src) (:%l dest)))
	5124	(cmovael (:%l mask) (:%l dest)))
	5125
	5126	(define-x8664-vinsn %complex-single-float-realpart
	5127	(((dest :single-float))
	5128	((src :complex-single-float)))
	5129	((:not (:pred =
	5130	(:apply %hard-regspec-value dest)
	5131	(:apply %hard-regspec-value src)))
	5132	(movss (:%xmm src) (:%xmm dest))))
	5133
	5134	(define-x8664-vinsn %complex-single-float-imagpart
	5135	(((dest :single-float))
	5136	((src :complex-single-float)))
	5137	((:not (:pred = (:apply %hard-regspec-value src ) (:apply %hard-regspec-value dest)))
	5138	(movapd (:%xmm src) (:%xmm dest)))
	5139	(psrlq (:$ub 32) (:%xmm dest)))
	5140
	5141	(define-x8664-vinsn %complex-double-float-realpart
	5142	(((dest :double-float))
	5143	((src :complex-double-float)))
	5144	((:not (:pred =
	5145	(:apply %hard-regspec-value dest)
	5146	(:apply %hard-regspec-value src)))
	5147	(movsd (:%xmm src) (:%xmm dest))))
	5148
	5149	(define-x8664-vinsn %complex-double-float-imagpart
	5150	(((dest :double-float))
	5151	((src :complex-double-float)))
	5152	((:not (:pred = (:apply %hard-regspec-value src ) (:apply %hard-regspec-value dest)))
	5153	(movapd (:%xmm src) (:%xmm dest)))
	5154	(shufpd (:$ub 1) (:%xmm target::fpzero) (:%xmm dest)))
	5155
	5156
	5157	(define-x8664-vinsn %make-complex-single-float
	5158	(((dest :complex-single-float))
	5159	((r :single-float)
	5160	(i :single-float)))
	5161	((:not (:pred = (:apply %hard-regspec-value r) (:apply %hard-regspec-value dest)))
	5162	(movss (:%xmm r) (:%xmm dest)))
	5163	(unpcklps (:%xmm i) (:%xmm dest)))
	5164
	5165	(define-x8664-vinsn %make-complex-double-float
	5166	(((dest :complex-double-float))
	5167	((r :double-float)
	5168	(i :double-float)))
	5169	((:not (:pred = (:apply %hard-regspec-value r) (:apply %hard-regspec-value dest)))
	5170	(:movupd (:%xmm r)ï¿œ(:%xmm dest)))
	5171	(shufpd (:$ub 0) (:%xmm i) (:%xmm dest)))
	5172
	5173
	5174
	5175
	5176	(queue-fixup
	5177	(fixup-x86-vinsn-templates
	5178	x8664-vinsn-templates
	5179	x86::x86-opcode-template-lists x8664-backend))
	5180
	5181	(provide "X8664-VINSNS")
	5182

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: release/1.11/source/compiler/X86/X8664/x8664-vinsns.lisp

Download in other formats: