Context Navigation

← Previous Revision
Next Revision →
Normal
Revision Log

close Warning: Can't use blame annotator:
No changeset 2899 in the repository

x8664-vinsns.lisp

Visit:

Last change on this file was 7466, checked in by Gary Byers, 17 years ago
don't trust assembler not to relax branch in CONS, %ALLOCATE-UVECTOR
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 151.7 KB

Rev	Line
	1	;;;-- Mode: Lisp; Package: (X86 :use CL) --
	2	;;;
	3	;;; Copyright (C) 2005 Clozure Associates and contributors.
	4	;;; This file is part of OpenMCL.
	5	;;;
	6	;;; OpenMCL is licensed under the terms of the Lisp Lesser GNU Public
	7	;;; License known as the LLGPL and distributed with OpenMCL as the
	8	;;; file "LICENSE". The LLGPL consists of a preamble and the LGPL
	9	;;; which is distributed with OpenMCL as the file "LGPL". Where these
	10	;;; conflict the preamble takes precedence.
	11	;;;
	12	;;; OpenMCL is referenced in the preamble as the "LIBRARY."
	13	;;;
	14	;;; The LLGPL is also available online at
	15	;;; http://opensource.franz.com/preamble.html
	16
	17	(in-package "CCL")
	18
	19	(eval-when (:compile-toplevel :load-toplevel :execute)
	20	(require "VINSN")
	21	(require "X8664-BACKEND"))
	22
	23	(eval-when (:compile-toplevel :execute)
	24	(require "X8664ENV"))
	25
	26	(defmacro define-x8664-vinsn (vinsn-name (results args &optional temps) &body body)
	27	(%define-vinsn x8664-backend vinsn-name results args temps body))
	28
	29
	30
	31	(define-x8664-vinsn scale-32bit-misc-index (((dest :u64))
	32	((idx :imm) ; A fixnum
	33	)
	34	())
	35	(movq (:%q idx) (:%q dest))
	36	(shrq (:$1 1) (:%q dest)))
	37
	38	(define-x8664-vinsn scale-16bit-misc-index (((dest :u64))
	39	((idx :imm) ; A fixnum
	40	)
	41	())
	42	(movq (:%q idx) (:%q dest))
	43	(shrq (:$ub 2) (:%q dest)))
	44
	45	(define-x8664-vinsn scale-8bit-misc-index (((dest :u64))
	46	((idx :imm) ; A fixnum
	47	)
	48	())
	49	(movq (:%q idx) (:%q dest))
	50	(shrq (:$ub 3) (:%q dest)))
	51
	52
	53	(define-x8664-vinsn misc-ref-u64 (((dest :u64))
	54	((v :lisp)
	55	(scaled-idx :imm)))
	56	(movq (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx)) (:%q dest)))
	57
	58	(define-x8664-vinsn misc-ref-double-float (((dest :double-float))
	59	((v :lisp)
	60	(scaled-idx :imm)))
	61	(movsd (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx)) (:%xmm dest)))
	62
	63	(define-x8664-vinsn misc-ref-c-double-float (((dest :double-float))
	64	((v :lisp)
	65	(idx :s32const)))
	66	(movsd (:@ (:apply + x8664::misc-data-offset (:apply ash idx x8664::word-shift)) (:%q v)) (:%xmm dest)))
	67
	68
	69	(define-x8664-vinsn misc-ref-node (((dest :lisp))
	70	((v :lisp)
	71	(scaled-idx :imm)))
	72	(movq (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx)) (:%q dest)))
	73
	74	(define-x8664-vinsn (push-misc-ref-node :push :node :vsp) (()
	75	((v :lisp)
	76	(scaled-idx :imm)))
	77	(pushq (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx))))
	78
	79	(define-x8664-vinsn misc-set-node (()
	80	((val :lisp)
	81	(v :lisp)
	82	(unscaled-idx :imm))
	83	())
	84	(movq (:%q val) (:@ x8664::misc-data-offset (:%q v) (:%q unscaled-idx))))
	85
	86	(define-x8664-vinsn misc-set-immediate-node (()
	87	((val :s32const)
	88	(v :lisp)
	89	(unscaled-idx :imm))
	90	())
	91	(movq (:$l val) (:@ x8664::misc-data-offset (:%q v) (:%q unscaled-idx))))
	92
	93
	94	(define-x8664-vinsn misc-set-double-float (()
	95	((val :double-float)
	96	(v :lisp)
	97	(unscaled-idx :imm))
	98	())
	99	(movsd (:%xmm val) (:@ x8664::misc-data-offset (:%q v) (:%q unscaled-idx))))
	100
	101	(define-x8664-vinsn misc-ref-u8 (((dest :u8))
	102	((v :lisp)
	103	(scaled-idx :s64)))
	104	(movzbl (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx)) (:%l dest)))
	105
	106	(define-x8664-vinsn misc-ref-s8 (((dest :s8))
	107	((v :lisp)
	108	(scaled-idx :s64)))
	109	(movsbq (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx)) (:%q dest)))
	110
	111	(define-x8664-vinsn misc-ref-u16 (((dest :u16))
	112	((v :lisp)
	113	(scaled-idx :s64)))
	114	(movzwl (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx)) (:%l dest)))
	115
	116	(define-x8664-vinsn misc-ref-u32 (((dest :u32))
	117	((v :lisp)
	118	(scaled-idx :s64)))
	119	(movl (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx)) (:%l dest)))
	120
	121
	122	(define-x8664-vinsn misc-ref-single-float (((dest :single-float))
	123	((v :lisp)
	124	(scaled-idx :s64)))
	125	(movss(:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx)) (:%xmm dest)))
	126
	127	(define-x8664-vinsn misc-ref-s32 (((dest :s32))
	128	((v :lisp)
	129	(scaled-idx :s64)))
	130	(movslq (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx)) (:%q dest)))
	131
	132	(define-x8664-vinsn misc-ref-s16 (((dest :s16))
	133	((v :lisp)
	134	(scaled-idx :s64)))
	135	(movswq (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx)) (:%q dest)))
	136
	137	(define-x8664-vinsn misc-ref-s64 (((dest :s64))
	138	((v :lisp)
	139	(scaled-idx :imm)))
	140	(movq (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx)) (:%q dest)))
	141
	142
	143	(define-x8664-vinsn misc-ref-c-node (((dest :lisp))
	144	((v :lisp)
	145	(idx :u32const)) ; sic
	146	())
	147	(movq (:@ (:apply + x8664::misc-data-offset (:apply ash idx x8664::word-shift)) (:%q v)) (:%q dest)))
	148
	149
	150	(define-x8664-vinsn (push-misc-ref-c-node :push :node :vsp)
	151	(()
	152	((v :lisp)
	153	(idx :u32const)) ; sic
	154	())
	155	(pushq (:@ (:apply + x8664::misc-data-offset (:apply ash idx x8664::word-shift)) (:%q v))))
	156
	157	(define-x8664-vinsn misc-ref-c-u64 (((dest :u64))
	158	((v :lisp)
	159	(idx :u32const)) ; sic
	160	())
	161	(movq (:@ (:apply + x8664::misc-data-offset (:apply ash idx x8664::word-shift)) (:%q v)) (:%q dest)))
	162
	163
	164	(define-x8664-vinsn misc-ref-c-s64 (((dest :s64))
	165	((v :lisp)
	166	(idx :s32const)) ; sic
	167	())
	168	(movq (:@ (:apply + x8664::misc-data-offset (:apply ash idx x8664::word-shift)) (:%q v)) (:%q dest)))
	169
	170
	171	(define-x8664-vinsn misc-ref-c-u32 (((dest :u32))
	172	((v :lisp)
	173	(idx :u32const)) ; sic
	174	())
	175	(movl (:@ (:apply + x8664::misc-data-offset (:apply ash idx 2)) (:%q v)) (:%l dest)))
	176
	177	(define-x8664-vinsn misc-ref-c-s32 (((dest :s32))
	178	((v :lisp)
	179	(idx :s32const)) ; sic
	180	())
	181	(movslq (:@ (:apply + x8664::misc-data-offset (:apply ash idx x8664::word-shift)) (:%q v)) (:%q dest)))
	182
	183	(define-x8664-vinsn misc-ref-c-single-float (((dest :single-float))
	184	((v :lisp)
	185	(idx :s32const)) ; sic
	186	())
	187	(movss (:@ (:apply + x8664::misc-data-offset (:apply ash idx x8664::word-shift)) (:%q v)) (:%xmm dest)))
	188
	189	(define-x8664-vinsn misc-ref-c-u8 (((dest :u64))
	190	((v :lisp)
	191	(idx :s32const)) ; sic
	192	())
	193	(movzbl (:@ (:apply + x8664::misc-data-offset idx) (:%q v)) (:%l dest)))
	194
	195	(define-x8664-vinsn misc-ref-c-s8 (((dest :s64))
	196	((v :lisp)
	197	(idx :s32const)) ; sic
	198	())
	199	(movsbq (:@ (:apply + x8664::misc-data-offset idx) (:%q v)) (:%q dest)))
	200
	201	(define-x8664-vinsn misc-set-u64 (()
	202	((val :u64)
	203	(v :lisp)
	204	(idx :u64)))
	205	(movq (:%q val) (:@ x8664::misc-data-offset (:%q v) (:%q idx))))
	206
	207	(define-x8664-vinsn misc-set-immediate-u64 (()
	208	((val :u32const)
	209	(v :lisp)
	210	(idx :u64)))
	211	(movq (:$l val) (:@ x8664::misc-data-offset (:%q v) (:%q idx))))
	212
	213	(define-x8664-vinsn misc-set-c-u64 (()
	214	((val :u64)
	215	(v :lisp)
	216	(idx :u32const)))
	217	(movq (:%q val) (:@ (:apply + x8664::misc-data-offset (:apply ash idx 3)) (:%q v))))
	218
	219	(define-x8664-vinsn misc-set-immediate-c-u64 (()
	220	((val :u32const)
	221	(v :lisp)
	222	(idx :u32const)))
	223	(movq (:$l val) (:@ (:apply + x8664::misc-data-offset (:apply ash idx 3)) (:%q v))))
	224
	225	(define-x8664-vinsn misc-set-s64 (()
	226	((val :s64)
	227	(v :lisp)
	228	(scaled-idx :imm)))
	229	(movq (:%q val) (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx))))
	230
	231	(define-x8664-vinsn misc-set-immediate-s64 (()
	232	((val :s32const)
	233	(v :lisp)
	234	(scaled-idx :imm)))
	235	(movq (:$l val) (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx))))
	236
	237
	238	(define-x8664-vinsn misc-set-c-s64 (()
	239	((val :s64)
	240	(v :lisp)
	241	(idx :s32const)))
	242	(movq (:%q val) (:@ (:apply + x8664::misc-data-offset (:apply ash idx 3)) (:%q v))))
	243
	244	(define-x8664-vinsn misc-set-immediate-c-s64 (()
	245	((val :s32const)
	246	(v :lisp)
	247	(idx :s32const)))
	248	(movq (:$l val) (:@ (:apply + x8664::misc-data-offset (:apply ash idx 3)) (:%q v))))
	249
	250
	251	(define-x8664-vinsn misc-set-c-node (()
	252	((val :lisp)
	253	(v :lisp)
	254	(idx :s32const)))
	255	(movq (:%q val) (:@ (:apply + x8664::misc-data-offset (:apply ash idx 3)) (:%q v))))
	256
	257	(define-x8664-vinsn misc-set-immediate-c-node (()
	258	((val :s32const)
	259	(v :lisp)
	260	(idx :s32const)))
	261	(movq (:$l val) (:@ (:apply + x8664::misc-data-offset (:apply ash idx 3)) (:%q v))))
	262
	263	(define-x8664-vinsn set-closure-forward-reference (()
	264	((val :lisp)
	265	(closure :lisp)
	266	(idx :s32const)))
	267	(movq (:%q val) (:@ (:apply + x8664::misc-function-offset (:apply ash idx x8664::word-shift)) (:%q closure))))
	268
	269
	270	(define-x8664-vinsn misc-set-c-double-float (()
	271	((val :double-float)
	272	(v :lisp)
	273	(idx :s32const)))
	274	(movsd (:%xmm val) (:@ (:apply + x8664::misc-data-offset (:apply ash idx 3)) (:%q v))))
	275
	276
	277
	278	(define-x8664-vinsn (call-known-symbol :call) (((result (:lisp x8664::arg_z)))
	279	()
	280	((entry (:label 1))))
	281	(:talign 4)
	282	(call (:@ x8664::symbol.fcell (:% x8664::fname)))
	283	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	284
	285
	286	(define-x8664-vinsn (jump-known-symbol :jumplr) (()
	287	())
	288
	289	(jmp (:@ x8664::symbol.fcell (:% x8664::fname))))
	290
	291	(define-x8664-vinsn set-nargs (()
	292	((n :s16const)))
	293	((:pred = n 0)
	294	(xorw (:%w x8664::nargs ) (:%w x8664::nargs )))
	295	((:not (:pred = n 0))
	296	(movw (:$w (:apply ash n x8664::word-shift)) (:%w x8664::nargs ))))
	297
	298	(define-x8664-vinsn check-exact-nargs (()
	299	((n :u16const)))
	300	((:pred = n 0)
	301	(testw (:%w x8664::nargs) (:%w x8664::nargs)))
	302	((:not (:pred = n 0))
	303	(cmpw (:$w (:apply ash n x8664::word-shift)) (:%w x8664::nargs)))
	304	(jz.pt :ok)
	305	(uuo-error-wrong-number-of-args)
	306	:ok)
	307
	308	(define-x8664-vinsn check-min-nargs (()
	309	((n :u16const)))
	310	(rcmpw (:%w x8664::nargs) (:$w (:apply ash n x8664::word-shift)))
	311	(jae.pt :ok)
	312	(uuo-error-too-few-args)
	313	:ok)
	314
	315	(define-x8664-vinsn check-max-nargs (()
	316	((n :u16const)))
	317	(rcmpw (:%w x8664::nargs) (:$w (:apply ash n x8664::word-shift)))
	318	(jbe.pt :ok)
	319	(uuo-error-too-many-args)
	320	:ok)
	321
	322
	323	(define-x8664-vinsn default-1-arg (()
	324	((min :u16const)))
	325	(rcmpw (:%w x8664::nargs) (:$w (:apply ash min x8664::word-shift)))
	326	(jne :done)
	327	((:pred >= min 3)
	328	(pushq (:%q x8664::arg_x)))
	329	((:pred >= min 2)
	330	(movq (:%q x8664::arg_y) (:%q x8664::arg_x)))
	331	((:pred >= min 1)
	332	(movq (:%q x8664::arg_z) (:%q x8664::arg_y)))
	333	(movq (:$l x8664::nil-value) (:%q x8664::arg_z))
	334	:done)
	335
	336
	337	(define-x8664-vinsn default-2-args (()
	338	((min :u16const)))
	339	(rcmpw (:%w x8664::nargs ) (:$w (:apply ash (:apply 1+ min) x8664::word-shift)))
	340	(ja :done)
	341	(je :one)
	342	;; We got "min" args; arg_y & arg_z default to nil
	343	((:pred >= min 3)
	344	(pushq (:%q x8664::arg_x)))
	345	((:pred >= min 2)
	346	(pushq (:%q x8664::arg_y)))
	347	((:pred >= min 1)
	348	(movq (:%q x8664::arg_z) (:%q x8664::arg_x)))
	349	(movl (:$l x8664::nil-value) (:%l x8664::arg_y))
	350	(jmp :last)
	351	:one
	352	;; We got min+1 args: arg_y was supplied, arg_z defaults to nil.
	353	((:pred >= min 2)
	354	(pushq (:%q x8664::arg_x)))
	355	((:pred >= min 1)
	356	(movq (:%q x8664::arg_y) (:%q x8664::arg_x)))
	357	(movq (:%q x8664::arg_z) (:%q x8664::arg_y))
	358	:last
	359	(movq (:$l x8664::nil-value) (:%q x8664::arg_z))
	360	:done)
	361
	362	(define-x8664-vinsn default-3-args (()
	363	((min :u16const)))
	364	(rcmpw (:%w x8664::nargs ) (:$w (:apply ash (:apply + 2 min) x8664::word-shift)))
	365	(ja :done)
	366	(je :two)
	367	(rcmpw (:%w x8664::nargs ) (:$w (:apply ash min x8664::word-shift)))
	368	(je :none)
	369	;; The first (of three) &optional args was supplied.
	370	((:pred >= min 2)
	371	(pushq (:%q x8664::arg_x)))
	372	((:pred >= min 1)
	373	(pushq (:%q x8664::arg_y)))
	374	(movq (:%q x8664::arg_z) (:%q x8664::arg_x))
	375	(jmp :last-2)
	376	:two
	377	;; The first two (of three) &optional args were supplied.
	378	((:pred >= min 1)
	379	(pushq (:%q x8664::arg_x)))
	380	(movq (:%q x8664::arg_y) (:%q x8664::arg_x))
	381	(movq (:%q x8664::arg_z) (:%q x8664::arg_y))
	382	(jmp :last-1)
	383	;; None of the three &optional args was provided.
	384	:none
	385	((:pred >= min 3)
	386	(pushq (:%q x8664::arg_x)))
	387	((:pred >= min 2)
	388	(pushq (:%q x8664::arg_y)))
	389	((:pred >= min 1)
	390	(pushq (:%q x8664::arg_z)))
	391	(movl (:$l x8664::nil-value) (:%l x8664::arg_x))
	392	:last-2
	393	(movl (:$l x8664::nil-value) (:%l x8664::arg_y))
	394	:last-1
	395	(movl (:$l x8664::nil-value) (:%l x8664::arg_z))
	396	:done)
	397
	398
	399	(define-x8664-vinsn default-optionals (()
	400	((n :u16const))
	401	((temp :u64)))
	402	(rcmpw (:%w x8664::nargs) (:$w (:apply ash n x8664::word-shift)))
	403	(movw (:%w x8664::nargs) (:%w temp))
	404	(jae :done)
	405	:loop
	406	(addw (:$w x8664::fixnumone) (:%w temp))
	407	(cmpw (:$w (:apply ash n x8664::word-shift)) (:%w temp))
	408	(pushq (:$l x8664::nil-value))
	409	(jne :loop)
	410	:done)
	411
	412
	413	(define-x8664-vinsn save-lisp-context-no-stack-args (()
	414	())
	415	(pushq (:%q x8664::rbp))
	416	(movq (:%q x8664::rsp) (:%q x8664::rbp)))
	417
	418
	419	(define-x8664-vinsn save-lisp-context-offset (()
	420	((nbytes-pushed :s32const)))
	421	(movq (:%q x8664::rbp) (:@ (:apply + nbytes-pushed x8664::node-size) (:%q x8664::rsp)))
	422	(leaq (:@ (:apply + nbytes-pushed x8664::node-size) (:%q x8664::rsp)) (:%q x8664::rbp))
	423	(popq (:@ x8664::node-size (:%q x8664::rbp))))
	424
	425	(define-x8664-vinsn save-lisp-context-variable-arg-count (()
	426	()
	427	((temp :u64)))
	428	(movzwl (:%w x8664::nargs) (:%l temp))
	429	(subq (:$b (* $numx8664argregs x8664::node-size)) (:%q temp))
	430	(jle :push)
	431	(movq (:%q x8664::rbp) (:@ x8664::node-size (:%q x8664::rsp) (:%q temp)))
	432	(leaq (:@ x8664::node-size (:%q x8664::rsp) (:%q temp)) (:%q x8664::rbp))
	433	(popq (:@ 8 (:%q x8664::rbp)))
	434	(jmp :done)
	435	:push
	436	(pushq (:%q x8664::rbp))
	437	(movq (:%q x8664::rsp) (:%q x8664::rbp))
	438	:done)
	439
	440	;;; We know that some args were pushed, but don't know how many were
	441	;;; passed.
	442	(define-x8664-vinsn save-lisp-context-in-frame (()
	443	()
	444	((temp :u64)))
	445	(movzwl (:%w x8664::nargs) (:%l temp))
	446	(subq (:$b (* $numx8664argregs x8664::node-size)) (:%q temp))
	447	(movq (:%q x8664::rbp) (:@ x8664::node-size (:%q x8664::rsp) (:%q temp)))
	448	(leaq (:@ x8664::node-size (:%q x8664::rsp) (:%q temp)) (:%q x8664::rbp))
	449	(popq (:@ x8664::node-size (:%q x8664::rbp))))
	450
	451
	452	(define-x8664-vinsn (vpush-register :push :node :vsp)
	453	(()
	454	((reg :lisp)))
	455	(pushq (:% reg)))
	456
	457	(define-x8664-vinsn (vpush-fixnum :push :node :vsp)
	458	(()
	459	((const :s32const)))
	460	((:and (:pred < const 128) (:pred >= const -128))
	461	(pushq (:$b const)))
	462	((:not (:and (:pred < const 128) (:pred >= const -128)))
	463	(pushq (:$l const))))
	464
	465
	466
	467	(define-x8664-vinsn vframe-load (((dest :lisp))
	468	((frame-offset :u16const)
	469	(cur-vsp :u16const)))
	470	(movq (:@ (:apply - (:apply + frame-offset x8664::word-size-in-bytes)) (:%q x8664::rbp)) (:%q dest)))
	471
	472	(define-x8664-vinsn compare-vframe-offset-to-nil (()
	473	((frame-offset :u16const)
	474	(cur-vsp :u16const)))
	475	(cmpb (:$b x8664::fulltag-nil) (:@ (:apply - (:apply + frame-offset x8664::word-size-in-bytes)) (:%q x8664::rbp))))
	476
	477
	478	(define-x8664-vinsn compare-value-cell-to-nil (()
	479	((vcell :lisp)))
	480	(cmpb (:$b x8664::fulltag-nil) (:@ x8664::value-cell.value (:%q vcell))))
	481
	482	(define-x8664-vinsn lcell-load (((dest :lisp))
	483	((cell :lcell)
	484	(top :lcell)))
	485	(movq (:@ (:apply - (:apply + (:apply calc-lcell-offset cell) x8664::word-size-in-bytes)) (:%q x8664::rbp)) (:%q dest)))
	486
	487	(define-x8664-vinsn (vframe-push :push :node :vsp)
	488	(()
	489	((frame-offset :u16const)
	490	(cur-vsp :u16const)))
	491	(pushq (:@ (:apply - (:apply + frame-offset x8664::word-size-in-bytes)) (:%q x8664::rbp))))
	492
	493	(define-x8664-vinsn vframe-store (()
	494	((src :lisp)
	495	(frame-offset :u16const)
	496	(cur-vsp :u16const)))
	497	(movq (:%q src) (:@ (:apply - (:apply + frame-offset x8664::word-size-in-bytes)) (:%q x8664::rbp))))
	498
	499	(define-x8664-vinsn lcell-store (()
	500	((src :lisp)
	501	(cell :lcell)
	502	(top :lcell)))
	503	(movq (:%q src) (:@ (:apply - (:apply + (:apply calc-lcell-offset cell) x8664::word-size-in-bytes)) (:%q x8664::rbp))))
	504
	505	(define-x8664-vinsn (popj :lispcontext :pop :csp :lrRestore :jumpLR)
	506	(()
	507	())
	508	(leave)
	509	(ret))
	510
	511	(define-x8664-vinsn (restore-full-lisp-context :lispcontext :pop :vsp )
	512	(()
	513	())
	514	(leave))
	515
	516	(define-x8664-vinsn compare-to-nil (()
	517	((arg0 t)))
	518	(cmpb (:$b x8664::fulltag-nil) (:%b arg0)))
	519
	520
	521	(define-x8664-vinsn ref-constant (((dest :lisp))
	522	((lab :label)))
	523	(movq (:@ (:^ lab) (:%q x8664::fn)) (:%q dest)))
	524
	525	(define-x8664-vinsn (vpush-constant :push :node :vsp) (()
	526	((lab :label)))
	527	(pushq (:@ (:^ lab) (:%q x8664::fn))))
	528
	529
	530	(define-x8664-vinsn (jump :jump)
	531	(()
	532	((label :label)))
	533	(jmp label))
	534
	535	(define-x8664-vinsn (cbranch-true :branch) (()
	536	((label :label)
	537	(crbit :u8const)))
	538	(jcc (:$ub crbit) label))
	539
	540	(define-x8664-vinsn (cbranch-false :branch) (()
	541	((label :label)
	542	(crbit :u8const)))
	543	(jcc (:$ub (:apply logxor 1 crbit)) label))
	544
	545
	546	(define-x8664-vinsn (lri :constant-ref) (((dest :imm))
	547	((intval :s64const))
	548	())
	549	((:pred = intval 0)
	550	(xorl (:%l dest) (:%l dest)))
	551	((:and (:pred /= intval 0)
	552	(:pred >= intval -2147483648)
	553	(:pred <= intval 2147483647))
	554	(movq (:$l intval) (:%q dest)))
	555	((:or (:pred < intval -2147483648)
	556	(:pred > intval 2147483647))
	557	(movq (:$q (:apply logand #xffffffffffffffff intval)) (:%q dest))))
	558
	559
	560	(define-x8664-vinsn trap-unless-bit (()
	561	((value :lisp)))
	562
	563	(testq (:$l (lognot x8664::fixnumone)) (:%q value))
	564	(je.pt :ok)
	565	(uuo-error-reg-not-type (:%q value) (:$ub arch::error-object-not-bit))
	566	:ok
	567	)
	568
	569	(define-x8664-vinsn trap-unless-list (()
	570	((object :lisp))
	571	((tag :u8)))
	572	(movb (:%b object) (:%b tag))
	573	(andb (:$b x8664::tagmask) (:%b tag))
	574	(cmpb (:$b x8664::tag-list) (:%b tag))
	575	(je.pt :ok)
	576	(uuo-error-reg-not-list (:%q object))
	577	:ok)
	578
	579	(define-x8664-vinsn trap-unless-cons (()
	580	((object :lisp))
	581	((tag :u8)))
	582	(movb (:%b object) (:%b tag))
	583	(andb (:$b x8664::fulltagmask) (:%b tag))
	584	(cmpb (:$b x8664::fulltag-cons) (:%b tag))
	585	(je.pt :ok)
	586	(uuo-error-reg-not-tag (:%q object) (:$ub x8664::fulltag-cons))
	587	:ok)
	588
	589	(define-x8664-vinsn trap-unless-uvector (()
	590	((object :lisp))
	591	((tag :u8)))
	592	(movb (:%b object) (:%b tag))
	593	(andb (:$b x8664::tagmask) (:%b tag))
	594	(cmpb (:$b x8664::tag-misc) (:%b tag))
	595	(jz.pt :ok)
	596	(uuo-error-reg-not-tag (:%q object) (:$ub x8664::tag-misc))
	597	:ok)
	598
	599	(define-x8664-vinsn trap-unless-single-float (()
	600	((object :lisp)))
	601	(cmpb (:$b x8664::tag-single-float) (:%b object))
	602	(je.pt :ok)
	603	(uuo-error-reg-not-tag (:%q object) (:$ub x8664::tag-single-float))
	604	:ok)
	605
	606	(define-x8664-vinsn trap-unless-character (()
	607	((object :lisp)))
	608	(cmpb (:$b x8664::subtag-character) (:%b object))
	609	(je.pt :ok)
	610	(uuo-error-reg-not-tag (:%q object) (:$ub x8664::subtag-character))
	611	:ok)
	612
	613	(define-x8664-vinsn trap-unless-fixnum (()
	614	((object :lisp))
	615	())
	616	(testb (:$b x8664::tagmask) (:%b object))
	617	(je.pt :ok)
	618	(uuo-error-reg-not-fixnum (:%q object))
	619	:ok)
	620
	621	(define-x8664-vinsn set-flags-from-lisptag (()
	622	((reg :lisp)))
	623	(testb (:$b x8664::tagmask) (:%b reg)))
	624
	625
	626	(define-x8664-vinsn trap-unless-typecode= (()
	627	((object :lisp)
	628	(tagval :u16const))
	629	((tag :u8)))
	630	(movb (:%b object) (:%b tag))
	631	(andb (:$b x8664::tagmask) (:%b tag))
	632	(cmpb (:$b x8664::tag-misc) (:%b tag))
	633	(jne :have-tag)
	634	(movb (:@ x8664::misc-subtag-offset (:%q object)) (:%b tag))
	635	:have-tag
	636	(cmpb (:$b tagval) (:%b tag))
	637	(je.pt :ok)
	638	(uuo-error-reg-not-tag (:%q object) (:$ub tagval))
	639	:ok)
	640
	641	(define-x8664-vinsn trap-unless-double-float (()
	642	((object :lisp))
	643	((tag :u8)))
	644	(movb (:%b object) (:%b tag))
	645	(andb (:$b x8664::tagmask) (:%b tag))
	646	(cmpb (:$b x8664::tag-misc) (:%b tag))
	647	(jne :have-tag)
	648	(movb (:@ x8664::misc-subtag-offset (:%q object)) (:%b tag))
	649	:have-tag
	650	(cmpb (:$b x8664::subtag-double-float) (:%b tag))
	651	(je.pt :ok)
	652	(uuo-error-reg-not-tag (:%q object) (:$ub x8664::subtag-double-float))
	653	:ok)
	654
	655	(define-x8664-vinsn trap-unless-macptr (()
	656	((object :lisp))
	657	((tag :u8)))
	658	(movb (:%b object) (:%b tag))
	659	(andb (:$b x8664::tagmask) (:%b tag))
	660	(cmpb (:$b x8664::tag-misc) (:%b tag))
	661	(jne :have-tag)
	662	(movb (:@ x8664::misc-subtag-offset (:%q object)) (:%b tag))
	663	:have-tag
	664	(cmpb (:$b x8664::subtag-macptr) (:%b tag))
	665	(je.pt :ok)
	666	(uuo-error-reg-not-tag (:%q object) (:$ub x8664::subtag-macptr))
	667	:ok)
	668
	669
	670	(define-x8664-vinsn check-misc-bound (()
	671	((idx :imm)
	672	(v :lisp))
	673	((temp :u64)))
	674	(movq (:@ x8664::misc-header-offset (:%q v)) (:%q temp))
	675	(xorb (:%b temp) (:%b temp))
	676	(shrq (:$ub (- x8664::num-subtag-bits x8664::fixnumshift)) (:%q temp))
	677	(rcmpq (:%q idx) (:%q temp))
	678	(jb.pt :ok)
	679	(uuo-error-vector-bounds (:%q idx) (:%q v))
	680	:ok)
	681
	682
	683
	684	(define-x8664-vinsn %cdr (((dest :lisp))
	685	((src :lisp)))
	686	(movq (:@ x8664::cons.cdr (:%q src)) (:%q dest)))
	687
	688	(define-x8664-vinsn (%vpush-cdr :push :node :vsp)
	689	(()
	690	((src :lisp)))
	691	(pushq (:@ x8664::cons.cdr (:%q src))))
	692
	693	(define-x8664-vinsn %car (((dest :lisp))
	694	((src :lisp)))
	695	(movq (:@ x8664::cons.car (:%q src)) (:%q dest)))
	696
	697	(define-x8664-vinsn (%vpush-car :push :node :vsp)
	698	(()
	699	((src :lisp)))
	700	(pushq (:@ x8664::cons.car (:%q src))))
	701
	702
	703	(define-x8664-vinsn u32->char (((dest :lisp)
	704	(src :u8))
	705	((src :u8))
	706	())
	707	(shll (:$ub x8664::charcode-shift) (:%l src))
	708	(leaq (:@ x8664::subtag-character (:%q src)) (:%q dest)))
	709
	710
	711	(define-x8664-vinsn (load-nil :constant-ref) (((dest t))
	712	())
	713	(movl (:$l x8664::nil-value) (:%l dest)))
	714
	715
	716	(define-x8664-vinsn (load-t :constant-ref) (((dest t))
	717	())
	718	(movl(:$l x8664::t-value) (:%l dest)))
	719
	720
	721	(define-x8664-vinsn extract-tag (((tag :u8))
	722	((object :lisp)))
	723	(movzbl (:%b object) (:%l tag))
	724	(andb (:$b x8664::tagmask) (:%b tag)))
	725
	726	(define-x8664-vinsn extract-tag-fixnum (((tag :imm))
	727	((object :lisp)))
	728	(leal (:@ (:%q object) 8) (:%l tag))
	729	(andl (:$b (ash x8664::tagmask x8664::fixnumshift)) (:%l tag)))
	730
	731	(define-x8664-vinsn extract-fulltag (((tag :u8))
	732	((object :lisp)))
	733	(movzbl (:%b object) (:%l tag))
	734	(andb (:$b x8664::fulltagmask) (:%b tag)))
	735
	736	(define-x8664-vinsn extract-fulltag-fixnum (((tag :imm))
	737	((object :lisp)))
	738	(leal (:@ (:%q object) 8) (:%l tag))
	739	(andl (:$b (ash x8664::fulltagmask x8664::fixnumshift)) (:%l tag)))
	740
	741	(define-x8664-vinsn extract-typecode (((tag :u32))
	742	((object :lisp)))
	743	(movzbl (:%b object) (:%l tag))
	744	(andb (:$b x8664::tagmask) (:%b tag))
	745	(cmpb (:$b x8664::tag-misc) (:%b tag))
	746	(jne :have-tag)
	747	(movb (:@ x8664::misc-subtag-offset (:%q object)) (:%b tag))
	748	:have-tag)
	749
	750	(define-x8664-vinsn extract-typecode-fixnum (((tag :imm))
	751	((object :lisp))
	752	((temp :u32)))
	753	(movzbl (:%b object) (:%l temp))
	754	(andb (:$b x8664::tagmask) (:%b temp))
	755	(cmpb (:$b x8664::tag-misc) (:%b temp))
	756	(jne :have-tag)
	757	(movb (:@ x8664::misc-subtag-offset (:%q object)) (:%b temp))
	758	:have-tag
	759	(leal (:@ (:%q temp) 8) (:%l tag)))
	760
	761
	762	(define-x8664-vinsn compare-reg-to-zero (()
	763	((reg :imm)))
	764	(testq (:%q reg) (:%q reg)))
	765
	766	(define-x8664-vinsn compare-u8-reg-to-zero (()
	767	((reg :u8)))
	768	(testb (:%b reg) (:%b reg)))
	769
	770	(define-x8664-vinsn cr-bit->boolean (((dest :lisp))
	771	((crbit :u8const))
	772	((temp :u32)))
	773	(movl (:$l x8664::t-value) (:%l temp))
	774	(leaq (:@ (- x8664::t-offset) (:%q temp)) (:%q dest))
	775	(cmovccl (:$ub crbit) (:%l temp) (:%l dest)))
	776
	777
	778
	779	(define-x8664-vinsn compare-s32-constant (()
	780	((val :imm)
	781	(const :s32const)))
	782	((:or (:pred < const -128) (:pred > const 127))
	783	(rcmpq (:%q val) (:$l const)))
	784	((:not (:or (:pred < const -128) (:pred > const 127)))
	785	(rcmpq (:%q val) (:$b const))))
	786
	787	(define-x8664-vinsn compare-u31-constant (()
	788	((val :u64)
	789	(const :u32const)))
	790	((:pred > const 127)
	791	(rcmpq (:%q val) (:$l const)))
	792	((:not (:pred > const 127))
	793	(rcmpq (:%q val) (:$b const))))
	794
	795	(define-x8664-vinsn compare-u8-constant (()
	796	((val :u8)
	797	(const :u8const)))
	798	#\|
	799	((:pred logbitp 7 const)
	800	(movzbl (:%b val) (:%l val))
	801	(rcmpw (:%w val) (:$w const)))
	802	((:not (:pred logbitp 7 const))
	803	(rcmpb (:%b val) (:$b const)))
	804	\|\|#
	805	(rcmpb (:%b val) (:$b const))
	806	)
	807
	808
	809	(define-x8664-vinsn cons (((dest :lisp))
	810	((car :lisp)
	811	(cdr :lisp)))
	812	(subq (:$b (- x8664::cons.size x8664::fulltag-cons)) (:@ (:%seg :rcontext) x8664::tcr.save-allocptr))
	813	(movq (:@ (:%seg :rcontext) x8664::tcr.save-allocptr) (:%q x8664::allocptr))
	814	(rcmpq (:%q x8664::allocptr) (:@ (:%seg :rcontext) x8664::tcr.save-allocbase))
	815	(:byte #x7f) (:byte #x02) ;(jg :no-trap)
	816	(uuo-alloc)
	817	:no-trap
	818	(andb (:$b (lognot x8664::fulltagmask)) (:@ (:%seg :rcontext) x8664::tcr.save-allocptr))
	819	(movq (:%q car) (:@ x8664::cons.car (:%q x8664::allocptr)))
	820	(movq (:%q cdr) (:@ x8664::cons.cdr (:%q x8664::allocptr)))
	821	(movq (:%q x8664::allocptr) (:%q dest)))
	822
	823	(define-x8664-vinsn unbox-u8 (((dest :u8))
	824	((src :lisp)))
	825	(movq (:$l (lognot (ash #xff x8664::fixnumshift))) (:%q dest))
	826	(andq (:% src) (:% dest))
	827	(je.pt :ok)
	828	(uuo-error-reg-not-type (:%q src) (:$ub arch::error-object-not-unsigned-byte-8))
	829	:ok
	830	(movq (:%q src) (:%q dest))
	831	(shrq (:$ub x8664::fixnumshift) (:%q dest)))
	832
	833	(define-x8664-vinsn %unbox-u8 (((dest :u8))
	834	((src :lisp)))
	835	(movl (:%l src) (:%l dest))
	836	(shrl (:$ub x8664::fixnumshift) (:%l dest))
	837	(movzbl (:%b dest) (:%l dest)))
	838
	839	(define-x8664-vinsn unbox-s8 (((dest :s8))
	840	((src :lisp)))
	841	(movq (:%q src) (:%q dest))
	842	(shlq (:$ub (- x8664::nbits-in-word (+ 8 x8664::fixnumshift))) (:%q dest))
	843	(sarq (:$ub (- x8664::nbits-in-word (+ 8 x8664::fixnumshift))) (:%q dest))
	844	(cmpq (:%q src) (:%q dest))
	845	(jne.pn :bad)
	846	(testb (:$b x8664::fixnummask) (:%b dest))
	847	(jne.pn :bad)
	848	(sarq (:$ub x8664::fixnumshift) (:%q dest))
	849	(jmp :got-it)
	850	:bad
	851	(uuo-error-reg-not-type (:%q src) (:$ub arch::error-object-not-signed-byte-8))
	852	:got-it)
	853
	854	(define-x8664-vinsn unbox-u16 (((dest :u16))
	855	((src :lisp)))
	856	(testq (:$l (lognot (ash #xffff x8664::fixnumshift))) (:% src))
	857	(movq (:%q src) (:%q dest))
	858	(je.pt :ok)
	859	(uuo-error-reg-not-type (:%q src) (:$ub arch::error-object-not-unsigned-byte-16))
	860	:ok
	861	(shrq (:$ub x8664::fixnumshift) (:%q dest)))
	862
	863	(define-x8664-vinsn %unbox-u16 (((dest :u16))
	864	((src :lisp)))
	865	(movq (:%q src) (:%q dest))
	866	(shrq (:$ub x8664::fixnumshift) (:%q dest)))
	867
	868	(define-x8664-vinsn unbox-s16 (((dest :s16))
	869	((src :lisp)))
	870	(movq (:%q src) (:%q dest))
	871	(shlq (:$ub (- x8664::nbits-in-word (+ 16 x8664::fixnumshift))) (:%q dest))
	872	(sarq (:$ub (- x8664::nbits-in-word (+ 16 x8664::fixnumshift))) (:%q dest))
	873	(cmpq (:%q src) (:%q dest))
	874	(jne.pn :bad)
	875	(testb (:$b x8664::fixnummask) (:%b dest))
	876	(je.pt :got-it)
	877	:bad
	878	(uuo-error-reg-not-type (:%q src) (:$ub arch::error-object-not-signed-byte-16))
	879	:got-it
	880	(sarq (:$ub x8664::fixnumshift) (:%q dest)))
	881
	882	(define-x8664-vinsn %unbox-s16 (((dest :s16))
	883	((src :lisp)))
	884	(movq (:%q src) (:%q dest))
	885	(sarq (:$ub x8664::fixnumshift) (:%q dest)))
	886
	887	(define-x8664-vinsn unbox-u32 (((dest :u32))
	888	((src :lisp)))
	889	(movq (:$q (lognot (ash #xffffffff x8664::fixnumshift))) (:%q dest))
	890	(testq (:% src) (:% dest))
	891	(je.pt :ok)
	892	(uuo-error-reg-not-type (:%q src) (:$ub arch::error-object-not-unsigned-byte-32))
	893	:ok
	894	(movq (:%q src) (:%q dest))
	895	(shrq (:$ub x8664::fixnumshift) (:%q dest)))
	896
	897	(define-x8664-vinsn %unbox-u32 (((dest :u32))
	898	((src :lisp)))
	899
	900	(movq (:%q src) (:%q dest))
	901	(shrq (:$ub x8664::fixnumshift) (:%q dest)))
	902
	903	(define-x8664-vinsn unbox-s32 (((dest :s32))
	904	((src :lisp)))
	905	(movq (:%q src) (:%q dest))
	906	(shlq (:$ub (- x8664::nbits-in-word (+ 32 x8664::fixnumshift))) (:%q dest))
	907	(sarq (:$ub (- x8664::nbits-in-word (+ 32 x8664::fixnumshift))) (:%q dest))
	908	(cmpq (:%q src) (:%q dest))
	909	(jne.pn :bad)
	910	(testb (:$b x8664::fixnummask) (:%b dest))
	911	(je.pt :got-it)
	912	:bad
	913	(uuo-error-reg-not-type (:%q src) (:$ub arch::error-object-not-signed-byte-32))
	914	:got-it
	915	(sarq (:$ub x8664::fixnumshift) (:%q dest)))
	916
	917	(define-x8664-vinsn %unbox-s32 (((dest :s32))
	918	((src :lisp)))
	919	(movq (:%q src) (:%q dest))
	920	(sarq (:$ub x8664::fixnumshift) (:%q dest)))
	921
	922
	923	(define-x8664-vinsn unbox-u64 (((dest :u64))
	924	((src :lisp)))
	925	(movq (:$q (lognot (ash x8664::target-most-positive-fixnum x8664::fixnumshift))) (:%q dest))
	926	(testq (:%q dest) (:%q src))
	927	(movq (:%q src) (:%q dest))
	928	(jnz :maybe-bignum)
	929	(sarq (:$ub x8664::fixnumshift) (:%q dest))
	930	(jmp :done)
	931	:maybe-bignum
	932	(andb (:$b x8664::tagmask) (:%b dest))
	933	(cmpb (:$b x8664::tag-misc) (:%b dest))
	934	(jne :have-tag)
	935	(movb (:@ x8664::misc-subtag-offset (:%q src)) (:%b dest))
	936	:have-tag
	937	(cmpb (:$b x8664::subtag-bignum) (:%b dest))
	938	(jne :bad)
	939	(movq (:@ x8664::misc-header-offset (:%q src)) (:%q dest))
	940	(cmpq (:$l x8664::three-digit-bignum-header) (:%q dest))
	941	(je :three)
	942	(cmpq (:$l x8664::two-digit-bignum-header) (:%q dest))
	943	(jne :bad)
	944	(movq (:@ x8664::misc-data-offset (:%q src)) (:%q dest))
	945	(testq (:%q dest) (:%q dest))
	946	(jns :done)
	947	:bad
	948	(uuo-error-reg-not-type (:%q src) (:$ub arch::error-object-not-unsigned-byte-64))
	949	:three
	950	(movl (:@ (+ 8 x8664::misc-data-offset) (:%q src)) (:%l dest))
	951	(testl (:%l dest) (:%l dest))
	952	(movq (:@ x8664::misc-data-offset (:%q src)) (:%q dest))
	953	(jne :bad)
	954	:done)
	955
	956	(define-x8664-vinsn unbox-s64 (((dest :s64))
	957	((src :lisp)))
	958	(movq (:%q src) (:%q dest))
	959	(sarq (:$ub x8664::fixnumshift) (:%q dest))
	960	;; Was it a fixnum ?
	961	(testb (:$b x8664::fixnummask) (:%b src))
	962	(je :done)
	963	;; May be a 2-digit bignum
	964	(movb (:%b src) (:%b dest))
	965	(andb (:$b x8664::tagmask) (:%b dest))
	966	(cmpb (:$b x8664::tag-misc) (:%b dest))
	967	(jne :bad)
	968	(cmpq (:$l x8664::two-digit-bignum-header) (:@ x8664::misc-header-offset (:%q src)))
	969	(movq (:@ x8664::misc-data-offset (:%q src)) (:%q dest))
	970	(je :done)
	971	:bad
	972	(uuo-error-reg-not-type (:%q src) (:$ub arch::error-object-not-signed-byte-64))
	973	:done)
	974
	975	(define-x8664-vinsn sign-extend-s8 (((dest :s64))
	976	((src :s8)))
	977	(movsbq (:%b src) (:%q dest)))
	978
	979	(define-x8664-vinsn sign-extend-s16 (((dest :s64))
	980	((src :s16)))
	981	(movswq (:%w src) (:%q dest)))
	982
	983	(define-x8664-vinsn sign-extend-s32 (((dest :s64))
	984	((src :s32)))
	985	(movslq (:%l src) (:%q dest)))
	986
	987
	988	(define-x8664-vinsn zero-extend-u8 (((dest :s64))
	989	((src :u8)))
	990	(movzbl (:%b src) (:%l dest)))
	991
	992	(define-x8664-vinsn zero-extend-u16 (((dest :s64))
	993	((src :u16)))
	994	(movzwl (:%w src) (:%l dest)))
	995
	996	(define-x8664-vinsn zero-extend-u32 (((dest :s64))
	997	((src :u32)))
	998	(movl (:%l src) (:%l dest)))
	999
	1000	(define-x8664-vinsn (jump-subprim :jumpLR) (()
	1001	((spno :s32const)))
	1002	(jmp (:@ spno)))
	1003
	1004	;;; "call" a subprimitive that manipulates the stack in some way,
	1005	;;; using an lea/jmp calling convention.
	1006	(define-x8664-vinsn (lea-jmp-subprim :call) (()
	1007	((spno :s32const))
	1008	((entry (:label 1))))
	1009	(leaq (:@ (:^ :back) (:%q x8664::fn)) (:%q x8664::ra0))
	1010	(:talign 4)
	1011	(jmp (:@ spno))
	1012	:back
	1013	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	1014
	1015	;;; Call a subprimitive using a tail-aligned CALL instruction.
	1016	(define-x8664-vinsn (call-subprim :call) (()
	1017	((spno :s32const))
	1018	((entry (:label 1))))
	1019	(:talign 4)
	1020	(call (:@ spno))
	1021	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	1022
	1023	(define-x8664-vinsn fixnum-subtract-from (((dest t)
	1024	(y t))
	1025	((y t)
	1026	(x t)))
	1027	(subq (:%q y) (:%q x)))
	1028
	1029	(define-x8664-vinsn %logand-c (((dest t)
	1030	(val t))
	1031	((val t)
	1032	(const :s32const)))
	1033	((:and (:pred >= const -128) (:pred <= const 127))
	1034	(andq (:$b const) (:%q val)))
	1035	((:not (:and (:pred >= const -128) (:pred <= const 127)))
	1036	(andq (:$l const) (:%q val))))
	1037
	1038	(define-x8664-vinsn %logior-c (((dest t)
	1039	(val t))
	1040	((val t)
	1041	(const :s32const)))
	1042	((:and (:pred >= const -128) (:pred <= const 127))
	1043	(orq (:$b const) (:%q val)))
	1044	((:not (:and (:pred >= const -128) (:pred <= const 127)))
	1045	(orq (:$l const) (:%q val))))
	1046
	1047	(define-x8664-vinsn %logxor-c (((dest t)
	1048	(val t))
	1049	((val t)
	1050	(const :s32const)))
	1051	((:and (:pred >= const -128) (:pred <= const 127))
	1052	(xorq (:$b const) (:%q val)))
	1053	((:not (:and (:pred >= const -128) (:pred <= const 127)))
	1054	(xorq (:$l const) (:%q val))))
	1055
	1056	(define-x8664-vinsn character->fixnum (((dest :lisp))
	1057	((src :lisp))
	1058	())
	1059	((:not (:pred =
	1060	(:apply %hard-regspec-value dest)
	1061	(:apply %hard-regspec-value src)))
	1062	(movq (:%q src) (:%q dest)))
	1063	(shrq (:$ub (- x8664::charcode-shift x8664::fixnumshift)) (:%q dest)))
	1064
	1065	(define-x8664-vinsn compare (()
	1066	((x t)
	1067	(y t)))
	1068	(rcmpq (:%q x) (:%q y)))
	1069
	1070	(define-x8664-vinsn negate-fixnum (((val :lisp))
	1071	((val :imm)))
	1072	(negq (:% val)))
	1073
	1074	;;; This handles the 1-bit overflow from addition/subtraction/unary negation
	1075	(define-x8664-vinsn set-bigits-and-header-for-fixnum-overflow
	1076	(()
	1077	((val :lisp)
	1078	(no-overflow
	1079	:label))
	1080	((header (:u64 #.x8664::imm0))
	1081	(scaled-size (:u64 #.x8664::imm1))))
	1082	(jno.pt no-overflow)
	1083	(movq (:%q val) (:%q scaled-size))
	1084	(sarq (:$ub x8664::fixnumshift) (:%q scaled-size))
	1085	(movq (:$q #xe000000000000000) (:%q header))
	1086	(xorq (:%q header) (:%q scaled-size))
	1087	(movd (:%q scaled-size) (:%mmx x8664::mm0))
	1088	(movq (:$l x8664::two-digit-bignum-header) (:%q header))
	1089	(movq (:$l (- 16 x8664::fulltag-misc)) (:%q scaled-size)))
	1090
	1091	(define-x8664-vinsn %set-z-flag-if-s64-fits-in-fixnum (((dest :imm))
	1092	((src :s64))
	1093	((temp :s64)))
	1094	(movq (:%q src) (:%q temp))
	1095	(shlq (:$ub x8664::fixnumshift) (:%q temp))
	1096	(movq (:%q temp) (:%q dest)) ; tagged as a fixnum
	1097	(sarq (:$ub x8664::fixnumshift) (:%q temp))
	1098	(cmpq (:%q src) (:%q temp)))
	1099
	1100	(define-x8664-vinsn %set-z-flag-if-u64-fits-in-fixnum (((dest :imm))
	1101	((src :u64))
	1102	((temp :u64)))
	1103	(movq (:%q src) (:%q temp))
	1104	(shlq (:$ub (1+ x8664::fixnumshift)) (:%q temp))
	1105	(movq (:%q temp) (:%q dest)) ; tagged as an even fixnum
	1106	(shrq (:$ub (1+ x8664::fixnumshift)) (:%q temp))
	1107	(shrq (:%q dest))
	1108	(cmpq (:%q src) (:%q temp))
	1109	:done)
	1110
	1111
	1112	(define-x8664-vinsn setup-bignum-alloc-for-s64-overflow (()
	1113	((src :s64)))
	1114	(movd (:%q src) (:%mmx x8664::mm0))
	1115	(movl (:$l x8664::two-digit-bignum-header) (:%l x8664::imm0.l))
	1116	(movl (:$l (- 16 x8664::fulltag-misc)) (:%l x8664::imm1.l)))
	1117
	1118
	1119	;;; If the sign bit is set in SRC, need to make a 3-digit bignum
	1120	;;; that requires 32 bytes of aligned memory
	1121	(define-x8664-vinsn setup-bignum-alloc-for-u64-overflow (()
	1122	((src :s64)))
	1123	(testq (:%q src) (:%q src))
	1124	(movd (:%q src) (:%mmx x8664::mm0))
	1125	(movl (:$l x8664::two-digit-bignum-header) (:%l x8664::imm0.l))
	1126	(movl (:$l (- 16 x8664::fulltag-misc)) (:%l x8664::imm1.l))
	1127	(jns :done)
	1128	(movl (:$l x8664::three-digit-bignum-header) (:%l x8664::imm0.l))
	1129	(movl (:$l (- 32 x8664::fulltag-misc)) (:%l x8664::imm1.l))
	1130	:done)
	1131
	1132
	1133
	1134	(define-x8664-vinsn %allocate-uvector (((dest :lisp))
	1135	()
	1136	((header (:u64 #.x8664::imm0))
	1137	(freeptr (:lisp #.x8664::allocptr))))
	1138	(subq (:%q x8664::imm1) (:@ (:%seg :rcontext) x8664::tcr.save-allocptr))
	1139	(movq (:@ (:%seg :rcontext) x8664::tcr.save-allocptr) (:%q freeptr))
	1140	(rcmpq (:%q freeptr) (:@ (:%seg :rcontext) x8664::tcr.save-allocbase))
	1141	(:byte #x7f) (:byte #x02) ;(jg :no-trap)
	1142	(uuo-alloc)
	1143	:no-trap
	1144	(movq (:%q header) (:@ x8664::misc-header-offset (:%q freeptr)))
	1145	(andb (:$b (lognot x8664::fulltagmask)) (:@ (:%seg :rcontext) x8664::tcr.save-allocptr))
	1146	((:not (:pred = freeptr
	1147	(:apply %hard-regspec-value dest)))
	1148	(movq (:%q freeptr) (:%q dest))))
	1149
	1150	(define-x8664-vinsn set-bigits-after-fixnum-overflow (()
	1151	((bignum :lisp)))
	1152	(movq (:%mmx x8664::mm0) (:@ x8664::misc-data-offset (:%q bignum))))
	1153
	1154
	1155	(define-x8664-vinsn box-fixnum (((dest :imm))
	1156	((src :s8)))
	1157	(imulq (:$b x8664::fixnumone) (:%q src)(:%q dest)))
	1158
	1159
	1160	(define-x8664-vinsn (fix-fixnum-overflow-ool :call)
	1161	(((val :lisp))
	1162	((val :lisp))
	1163	((unboxed (:s64 #.x8664::imm1))
	1164	(header (:u64 #.x8664::imm0))
	1165	(entry (:label 1))))
	1166	(jno.pt :done)
	1167	((:not (:pred = x8664::arg_z
	1168	(:apply %hard-regspec-value val)))
	1169	(movq (:%q val) (:%q x8664::arg_z)))
	1170	(:talign 4)
	1171	(call (:@ .SPfix-overflow))
	1172	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn))
	1173	((:not (:pred = x8664::arg_z
	1174	(:apply %hard-regspec-value val)))
	1175	(movq (:%q x8664::arg_z) (:%q val)))
	1176	:done)
	1177
	1178	(define-x8664-vinsn (fix-fixnum-overflow-ool-and-branch :call)
	1179	(((val :lisp))
	1180	((val :lisp)
	1181	(lab :label))
	1182	((unboxed (:s64 #.x8664::imm1))
	1183	(header (:u64 #.x8664::imm0))
	1184	(entry (:label 1))))
	1185	(jno.pt lab)
	1186	((:not (:pred = x8664::arg_z
	1187	(:apply %hard-regspec-value val)))
	1188	(movq (:%q val) (:%q x8664::arg_z)))
	1189	(:talign 4)
	1190	(call (:@ .SPfix-overflow))
	1191	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn))
	1192	((:not (:pred = x8664::arg_z
	1193	(:apply %hard-regspec-value val)))
	1194	(movq (:%q x8664::arg_z) (:%q val)))
	1195	(jmp lab))
	1196
	1197	(define-x8664-vinsn add-constant (((dest :imm))
	1198	((dest :imm)
	1199	(const :s32const)))
	1200	((:and (:pred >= const -128) (:pred <= const 127))
	1201	(addq (:$b const) (:%q dest)))
	1202	((:not (:and (:pred >= const -128) (:pred <= const 127)))
	1203	(addq (:$l const) (:%q dest))))
	1204
	1205	(define-x8664-vinsn add-constant3 (((dest :imm))
	1206	((src :imm)
	1207	(const :s32const)))
	1208	((:pred = (:apply %hard-regspec-value dest)
	1209	(:apply %hard-regspec-value src))
	1210	((:and (:pred >= const -128) (:pred <= const 127))
	1211	(addq (:$b const) (:%q dest)))
	1212	((:not (:and (:pred >= const -128) (:pred <= const 127)))
	1213	(addq (:$l const) (:%q dest))))
	1214	((:not (:pred = (:apply %hard-regspec-value dest)
	1215	(:apply %hard-regspec-value src)))
	1216	(leaq (:@ const (:%q src)) (:%q dest))))
	1217
	1218
	1219
	1220	(define-x8664-vinsn fixnum-add2 (((dest :imm))
	1221	((dest :imm)
	1222	(other :imm)))
	1223	(addq (:%q other) (:%q dest)))
	1224
	1225	(define-x8664-vinsn fixnum-sub2 (((dest :imm))
	1226	((x :imm)
	1227	(y :imm))
	1228	((temp :imm)))
	1229	(movq (:%q x) (:%q temp))
	1230	(subq (:%q y) (:%q temp))
	1231	(movq (:%q temp) (:%q dest)))
	1232
	1233
	1234
	1235	(define-x8664-vinsn fixnum-add3 (((dest :imm))
	1236	((x :imm)
	1237	(y :imm)))
	1238
	1239	((:pred =
	1240	(:apply %hard-regspec-value x)
	1241	(:apply %hard-regspec-value dest))
	1242	(addq (:%q y) (:%q dest)))
	1243	((:not (:pred =
	1244	(:apply %hard-regspec-value x)
	1245	(:apply %hard-regspec-value dest)))
	1246	((:pred =
	1247	(:apply %hard-regspec-value y)
	1248	(:apply %hard-regspec-value dest))
	1249	(addq (:%q x) (:%q dest)))
	1250	((:not (:pred =
	1251	(:apply %hard-regspec-value y)
	1252	(:apply %hard-regspec-value dest)))
	1253	(leaq (:@ (:%q x) (:%q y)) (:%q dest)))))
	1254
	1255	(define-x8664-vinsn copy-gpr (((dest t))
	1256	((src t)))
	1257	((:not (:pred =
	1258	(:apply %hard-regspec-value dest)
	1259	(:apply %hard-regspec-value src)))
	1260	(movq (:%q src) (:%q dest))))
	1261
	1262	(define-x8664-vinsn (vpop-register :pop :node :vsp)
	1263	(((dest :lisp))
	1264	())
	1265	(popq (:%q dest)))
	1266
	1267
	1268	(define-x8664-vinsn (push-argregs :push :node :vsp) (()
	1269	())
	1270	(testw (:%w x8664::nargs) (:%w x8664::nargs))
	1271	(jz :done)
	1272	(rcmpw (:%w x8664::nargs) (:$w (* 2 x8664::node-size)))
	1273	(jb :one)
	1274	(je :two)
	1275	(pushq (:%q x8664::arg_x))
	1276	:two
	1277	(pushq (:%q x8664::arg_y))
	1278	:one
	1279	(pushq (:%q x8664::arg_z))
	1280	:done)
	1281
	1282	(define-x8664-vinsn (push-max-argregs :push :node :vsp) (()
	1283	((max :u32const)))
	1284	((:pred >= max 3)
	1285	(testw (:%w x8664::nargs) (:%w x8664::nargs))
	1286	(jz :done)
	1287	(rcmpw (:%w x8664::nargs) (:$w (* 2 x8664::node-size)))
	1288	(jb :one)
	1289	(je :two)
	1290	(pushq (:%q x8664::arg_x))
	1291	:two
	1292	(pushq (:%q x8664::arg_y))
	1293	:one
	1294	(pushq (:%q x8664::arg_z))
	1295	:done)
	1296	((:pred = max 2)
	1297	(rcmpw (:%w x8664::nargs) (:$w (* 1 x8664::node-size)))
	1298	(jb :done)
	1299	(je :one)
	1300	(pushq (:%q x8664::arg_y))
	1301	:one
	1302	(pushq (:%q x8664::arg_z))
	1303	:done)
	1304	((:pred = max 1)
	1305	(testw (:%w x8664::nargs) (:%w x8664::nargs))
	1306	(je :done)
	1307	(pushq (:%q x8664::arg_z))
	1308	:done))
	1309
	1310	(define-x8664-vinsn (call-label :call) (()
	1311	((label :label))
	1312	((entry (:label 1))))
	1313	(:talign 4)
	1314	(call label)
	1315	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	1316
	1317	(define-x8664-vinsn double-float-compare (()
	1318	((arg0 :double-float)
	1319	(arg1 :double-float)))
	1320	(comisd (:%xmm arg1) (:%xmm arg0)))
	1321
	1322	(define-x8664-vinsn single-float-compare (()
	1323	((arg0 :single-float)
	1324	(arg1 :single-float)))
	1325	(comiss (:%xmm arg1) (:%xmm arg0)))
	1326
	1327
	1328	(define-x8664-vinsn double-float+-2 (((result :double-float))
	1329	((x :double-float)
	1330	(y :double-float)))
	1331	((:pred =
	1332	(:apply %hard-regspec-value result)
	1333	(:apply %hard-regspec-value x))
	1334	(addsd (:%xmm y) (:%xmm result)))
	1335	((:and (:not (:pred =
	1336	(:apply %hard-regspec-value result)
	1337	(:apply %hard-regspec-value x)))
	1338	(:pred =
	1339	(:apply %hard-regspec-value result)
	1340	(:apply %hard-regspec-value y)))
	1341	(addsd (:%xmm x) (:%xmm result)))
	1342	((:and (:not (:pred =
	1343	(:apply %hard-regspec-value result)
	1344	(:apply %hard-regspec-value x)))
	1345	(:not (:pred =
	1346	(:apply %hard-regspec-value result)
	1347	(:apply %hard-regspec-value y))))
	1348	(movsd (:%xmm x) (:%xmm result))
	1349	(addsd (:%xmm y) (:%xmm result))))
	1350
	1351	;;; Caller guarantees (not (eq y result))
	1352	(define-x8664-vinsn double-float--2 (((result :double-float))
	1353	((x :double-float)
	1354	(y :double-float)))
	1355	((:not (:pred = (:apply %hard-regspec-value result)
	1356	(:apply %hard-regspec-value x)))
	1357	(movsd (:%xmm x) (:%xmm result)))
	1358	(subsd (:%xmm y) (:%xmm result)))
	1359
	1360	(define-x8664-vinsn double-float*-2 (((result :double-float))
	1361	((x :double-float)
	1362	(y :double-float)))
	1363	((:pred =
	1364	(:apply %hard-regspec-value result)
	1365	(:apply %hard-regspec-value x))
	1366	(mulsd (:%xmm y) (:%xmm result)))
	1367	((:and (:not (:pred =
	1368	(:apply %hard-regspec-value result)
	1369	(:apply %hard-regspec-value x)))
	1370	(:pred =
	1371	(:apply %hard-regspec-value result)
	1372	(:apply %hard-regspec-value y)))
	1373	(mulsd (:%xmm x) (:%xmm result)))
	1374	((:and (:not (:pred =
	1375	(:apply %hard-regspec-value result)
	1376	(:apply %hard-regspec-value x)))
	1377	(:not (:pred =
	1378	(:apply %hard-regspec-value result)
	1379	(:apply %hard-regspec-value y))))
	1380	(movsd (:%xmm x) (:%xmm result))
	1381	(mulsd (:%xmm y) (:%xmm result))))
	1382
	1383	;;; Caller guarantees (not (eq y result))
	1384	(define-x8664-vinsn double-float/-2 (((result :double-float))
	1385	((x :double-float)
	1386	(y :double-float)))
	1387	((:not (:pred = (:apply %hard-regspec-value result)
	1388	(:apply %hard-regspec-value x)))
	1389	(movsd (:%xmm x) (:%xmm result)))
	1390	(divsd (:%xmm y) (:%xmm result)))
	1391
	1392	(define-x8664-vinsn single-float+-2 (((result :single-float))
	1393	((x :single-float)
	1394	(y :single-float)))
	1395	((:pred =
	1396	(:apply %hard-regspec-value result)
	1397	(:apply %hard-regspec-value x))
	1398	(addss (:%xmm y) (:%xmm result)))
	1399	((:and (:not (:pred =
	1400	(:apply %hard-regspec-value result)
	1401	(:apply %hard-regspec-value x)))
	1402	(:pred =
	1403	(:apply %hard-regspec-value result)
	1404	(:apply %hard-regspec-value y)))
	1405	(addss (:%xmm x) (:%xmm result)))
	1406	((:and (:not (:pred =
	1407	(:apply %hard-regspec-value result)
	1408	(:apply %hard-regspec-value x)))
	1409	(:not (:pred =
	1410	(:apply %hard-regspec-value result)
	1411	(:apply %hard-regspec-value y))))
	1412	(movss (:%xmm x) (:%xmm result))
	1413	(addss (:%xmm y) (:%xmm result))))
	1414
	1415	;;; Caller guarantees (not (eq y result))
	1416	(define-x8664-vinsn single-float--2 (((result :single-float))
	1417	((x :single-float)
	1418	(y :single-float)))
	1419	((:not (:pred = (:apply %hard-regspec-value result)
	1420	(:apply %hard-regspec-value x)))
	1421	(movss (:%xmm x) (:%xmm result)))
	1422	(subss (:%xmm y) (:%xmm result)))
	1423
	1424	(define-x8664-vinsn single-float*-2 (((result :single-float))
	1425	((x :single-float)
	1426	(y :single-float)))
	1427	((:pred =
	1428	(:apply %hard-regspec-value result)
	1429	(:apply %hard-regspec-value x))
	1430	(mulss (:%xmm y) (:%xmm result)))
	1431	((:and (:not (:pred =
	1432	(:apply %hard-regspec-value result)
	1433	(:apply %hard-regspec-value x)))
	1434	(:pred =
	1435	(:apply %hard-regspec-value result)
	1436	(:apply %hard-regspec-value y)))
	1437	(mulss (:%xmm x) (:%xmm result)))
	1438	((:and (:not (:pred =
	1439	(:apply %hard-regspec-value result)
	1440	(:apply %hard-regspec-value x)))
	1441	(:not (:pred =
	1442	(:apply %hard-regspec-value result)
	1443	(:apply %hard-regspec-value y))))
	1444	(movss (:%xmm x) (:%xmm result))
	1445	(mulss (:%xmm y) (:%xmm result))))
	1446
	1447	;;; Caller guarantees (not (eq y result))
	1448	(define-x8664-vinsn single-float/-2 (((result :single-float))
	1449	((x :single-float)
	1450	(y :single-float)))
	1451	((:not (:pred = (:apply %hard-regspec-value result)
	1452	(:apply %hard-regspec-value x)))
	1453	(movss (:%xmm x) (:%xmm result)))
	1454	(divss (:%xmm y) (:%xmm result)))
	1455
	1456	(define-x8664-vinsn get-single (((result :single-float))
	1457	((source :lisp)))
	1458	(movd (:%q source) (:%xmm result))
	1459	(psrlq (:$ub 32) (:%xmm result)))
	1460
	1461	(define-x8664-vinsn get-double (((result :double-float))
	1462	((source :lisp)))
	1463	(movsd (:@ x8664::double-float.value (:%q source)) (:%xmm result)))
	1464
	1465	;;; Extract a double-float value, typechecking in the process.
	1466	;;; IWBNI we could simply call the "trap-unless-typecode=" vinsn here,
	1467	;;; instead of replicating it ..
	1468
	1469	(define-x8664-vinsn get-double? (((target :double-float))
	1470	((source :lisp))
	1471	((tag :u8)))
	1472	(movb (:%b source) (:%b tag))
	1473	(andb (:$b x8664::tagmask) (:%b tag))
	1474	(cmpb (:$b x8664::tag-misc) (:%b tag))
	1475	(jne :have-tag)
	1476	(movb (:@ x8664::misc-subtag-offset (:%q source)) (:%b tag))
	1477	:have-tag
	1478	(cmpb (:$b x8664::subtag-double-float) (:%b tag))
	1479	(je.pt :ok)
	1480	(uuo-error-reg-not-tag (:%q source) (:$ub x8664::subtag-double-float))
	1481	:ok
	1482	(movsd (:@ x8664::double-float.value (:%q source)) (:%xmm target)))
	1483
	1484	(define-x8664-vinsn single->node (((result :lisp)
	1485	(source :single-float))
	1486	((source :single-float)))
	1487	(psllq (:$ub 32) (:%xmm source))
	1488	(movd (:%xmm source) (:%q result))
	1489	(movb (:$b x8664::tag-single-float) (:%b result)))
	1490
	1491	(define-x8664-vinsn copy-double-float (((dest :double-float))
	1492	((src :double-float)))
	1493	(movsd (:%xmm src) (:%xmm dest)))
	1494
	1495	(define-x8664-vinsn copy-single-float (((dest :single-float))
	1496	((src :single-float)))
	1497	(movss (:%xmm src) (:%xmm dest)))
	1498
	1499
	1500	(define-x8664-vinsn copy-single-to-double (((dest :double-float))
	1501	((src :single-float)))
	1502	(cvtss2sd (:%xmm src) (:%xmm dest)))
	1503
	1504	(define-x8664-vinsn copy-double-to-single (((dest :single-float))
	1505	((src :double-float)))
	1506	(cvtsd2ss (:%xmm src) (:%xmm dest)))
	1507
	1508	(define-x8664-vinsn u8->fixnum (((result :imm))
	1509	((val :u8))
	1510	())
	1511	(leaq (:@ (:%q val) 8) (:%q result)))
	1512
	1513	(define-x8664-vinsn fitvals (()
	1514	((n :u16const))
	1515	((imm :u16)))
	1516	((:pred = n 0)
	1517	(xorq (:%q imm) (:%q imm)))
	1518	((:not (:pred = n 0))
	1519	(movw (:$w (:apply ash n x8664::fixnumshift)) (:%w imm)))
	1520	(subw (:%w x8664::nargs) (:%w imm))
	1521	(jae :push-more)
	1522	(movswq (:%w imm) (:%q imm))
	1523	(subq (:%q imm) (:%q x8664::rsp))
	1524	(jmp :done)
	1525	:push-loop
	1526	(pushq (:$l x8664::nil-value))
	1527	(addw (:$b x8664::node-size) (:%w x8664::nargs))
	1528	(subw (:$b x8664::node-size) (:%w imm))
	1529	:push-more
	1530	(jne :push-loop)
	1531	:done)
	1532
	1533	(define-x8664-vinsn (nvalret :jumpLR) (()
	1534	())
	1535
	1536	(jmp (:@ .SPnvalret)))
	1537
	1538
	1539	(define-x8664-vinsn lisp-word-ref (((dest t))
	1540	((base t)
	1541	(offset t)))
	1542	(movq (:@ (:%q base) (:%q offset)) (:%q dest)))
	1543
	1544
	1545	(define-x8664-vinsn lisp-word-ref-c (((dest t))
	1546	((base t)
	1547	(offset :s32const)))
	1548	((:pred = offset 0)
	1549	(movq (:@ (:%q base)) (:%q dest)))
	1550	((:not (:pred = offset 0))
	1551	(movq (:@ offset (:%q base)) (:%q dest))))
	1552
	1553
	1554	(define-x8664-vinsn (vpush-label :push :node :vsp) (()
	1555	((label :label)))
	1556	(leaq (:@ (:^ label) (:%q x8664::fn)) (:%q x8664::ra0))
	1557	(pushq (:%q x8664::ra0)))
	1558
	1559	;; ????
	1560	(define-x8664-vinsn emit-aligned-label (()
	1561	((label :label)))
	1562	(:align 3)
	1563	(:long (:^ label)))
	1564
	1565	;;; %ra0 is pointing into %fn, so no need to copy %fn here.
	1566	(define-x8664-vinsn pass-multiple-values-symbol (()
	1567	())
	1568	(pushq (:@ (+ x8664::nil-value (x8664::%kernel-global 'x86::ret1valaddr))))
	1569	(jmp (:@ x8664::symbol.fcell (:% x8664::fname))))
	1570
	1571	;;; It'd be good to have a variant that deals with a known function
	1572	;;; as well as this.
	1573	(define-x8664-vinsn pass-multiple-values (()
	1574	()
	1575	((tag :u8)))
	1576	(movb (:%b x8664::temp0) (:%b tag))
	1577	(andb (:$b x8664::fulltagmask) (:%b tag))
	1578	(cmpb (:$b x8664::fulltag-symbol) (:%b tag))
	1579	(cmovgq (:%q x8664::temp0) (:%q x8664::fn))
	1580	(jl :bad)
	1581	(cmoveq (:@ x8664::symbol.fcell (:%q x8664::fname)) (:%q x8664::fn))
	1582	(pushq (:@ (+ x8664::nil-value (x8664::%kernel-global 'x86::ret1valaddr))))
	1583	(jmp (:%q x8664::fn))
	1584	:bad
	1585	(uuo-error-not-callable)
	1586	;; If we don't do this (and leave %fn as a TRA into itself), reporting
	1587	;; the error is likely a little harder. Tough.
	1588	;; (leaq (@ (:apply - (:^ :bad)) (:%q x8664::rn)) (:%q x8664::fn))
	1589	)
	1590
	1591
	1592
	1593	(define-x8664-vinsn reserve-outgoing-frame (()
	1594	())
	1595	(pushq (:$b x8664::reserved-frame-marker))
	1596	(pushq (:$b x8664::reserved-frame-marker)))
	1597
	1598
	1599	(define-x8664-vinsn (call-known-function :call) (()
	1600	()
	1601	((entry (:label 1))))
	1602	(:talign 4)
	1603	(call (:%q x8664::temp0))
	1604	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	1605
	1606	(define-x8664-vinsn (jump-known-function :jumplr) (()
	1607	())
	1608	(movq (:%q x8664::fn) (:%q x8664::xfn))
	1609	(movq (:%q x8664::temp0) (:%q x8664::fn))
	1610	(jmp (:%q x8664::fn)))
	1611
	1612	(define-x8664-vinsn (list :call) (()
	1613	()
	1614	((entry (:label 1))))
	1615	(leaq (:@ (:^ :back) (:%q x8664::fn)) (:%q x8664::ra0))
	1616	(:talign 4)
	1617	(jmp (:@ .SPconslist))
	1618	:back
	1619	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	1620
	1621
	1622	(define-x8664-vinsn make-tsp-cons (((dest :lisp))
	1623	((car :lisp) (cdr :lisp))
	1624	((temp :imm)))
	1625	(subq (:$b (+ x8664::cons.size x8664::dnode-size)) (:@ (:%seg :rcontext) x8664::tcr.next-tsp))
	1626	(movq (:@ (:%seg :rcontext) x8664::tcr.next-tsp) (:%q temp))
	1627	(movapd (:%xmm x8664::fpzero) (:@ (:%q temp)))
	1628	(movapd (:%xmm x8664::fpzero) (:@ 16 (:%q temp)))
	1629	(movq (:@ (:%seg :rcontext) x8664::tcr.save-tsp) (:%mmx x8664::stack-temp))
	1630	(movq (:%mmx x8664::stack-temp) (:@ (:%q temp)))
	1631	(movq (:%q temp) (:@ (:%seg :rcontext) x8664::tcr.save-tsp))
	1632	(leaq (:@ (+ x8664::dnode-size x8664::fulltag-cons) (:%q temp)) (:%q temp))
	1633	(movq (:%q car) (:@ x8664::cons.car (:%q temp)))
	1634	(movq (:%q cdr) (:@ x8664::cons.cdr (:%q temp)))
	1635	(movq (:%q temp) (:%q dest)))
	1636
	1637	(define-x8664-vinsn make-fixed-stack-gvector (((dest :lisp))
	1638	((aligned-size :u32const)
	1639	(header :s32const))
	1640	((tempa :imm)
	1641	(tempb :imm)))
	1642	((:and (:pred >= (:apply + aligned-size x8664::dnode-size) -128)
	1643	(:pred <= (:apply + aligned-size x8664::dnode-size) 127))
	1644	(subq (:$b (:apply + aligned-size x8664::dnode-size))
	1645	(:@ (:%seg :rcontext) x8664::tcr.next-tsp)))
	1646	((:not (:and (:pred >= (:apply + aligned-size x8664::dnode-size) -128)
	1647	(:pred <= (:apply + aligned-size x8664::dnode-size) 127)))
	1648	(subq (:$l (:apply + aligned-size x8664::dnode-size))
	1649	(:@ (:%seg :rcontext) x8664::tcr.next-tsp)))
	1650	(movq (:@ (:%seg :rcontext) x8664::tcr.save-tsp) (:%q tempb))
	1651	(movq (:@ (:%seg :rcontext) x8664::tcr.next-tsp) (:%q tempa))
	1652	(movd (:%q tempb) (:%mmx x8664::stack-temp))
	1653	:loop
	1654	(movapd (:%xmm x8664::fpzero) (:@ -16 (:%q tempb)))
	1655	(subq (:$b x8664::dnode-size) (:%q tempb))
	1656	(cmpq (:%q tempa) (:%q tempb))
	1657	(jnz :loop)
	1658	(movq (:%mmx x8664::stack-temp) (:@ (:%q tempa)))
	1659	(movq (:%q tempa) (:@ (:%seg :rcontext) x8664::tcr.save-tsp))
	1660	(movl (:$l header) (:@ x8664::dnode-size (:%q tempa)))
	1661	(leaq (:@ (+ x8664::dnode-size x8664::fulltag-misc) (:%q tempa)) (:%q dest)))
	1662
	1663
	1664	(define-x8664-vinsn discard-temp-frame (()
	1665	()
	1666	((temp :imm)))
	1667	(movq (:@ (:%seg :rcontext) x8664::tcr.save-tsp) (:%q temp))
	1668	(movq (:@ (:%q temp)) (:%q temp))
	1669	(movq (:%q temp) (:@ (:%seg :rcontext) x8664::tcr.save-tsp))
	1670	(movq (:%q temp) (:@ (:%seg :rcontext) x8664::tcr.next-tsp))
	1671	)
	1672
	1673	(define-x8664-vinsn discard-c-frame (()
	1674	()
	1675	((temp :imm)))
	1676	(movq (:@ (:%seg :rcontext) x8664::tcr.foreign-sp) (:%q temp))
	1677	(movq (:@ (:%q temp)) (:%q temp))
	1678	(movq (:%q temp) (:@ (:%seg :rcontext) x8664::tcr.foreign-sp)))
	1679
	1680
	1681	(define-x8664-vinsn vstack-discard (()
	1682	((nwords :u32const)))
	1683	((:not (:pred = nwords 0))
	1684	((:pred < nwords 16)
	1685	(addq (:$b (:apply ash nwords x8664::word-shift)) (:%q x8664::rsp)))
	1686	((:not (:pred < nwords 16))
	1687	(addq (:$l (:apply ash nwords x8664::word-shift)) (:%q x8664::rsp)))))
	1688
	1689
	1690	(defmacro define-x8664-subprim-lea-jmp-vinsn ((name &rest other-attrs) spno)
	1691	`(define-x8664-vinsn (,name :call :subprim-call ,@other-attrs) (() () ((entry (:label 1))))
	1692	(leaq (:@ (:^ :back) (:%q x8664::fn)) (:%q x8664::ra0))
	1693	(:talign 4)
	1694	(jmp (:@ ,spno))
	1695	:back
	1696	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn))))
	1697
	1698	(defmacro define-x8664-subprim-call-vinsn ((name &rest other-attrs) spno)
	1699	`(define-x8664-vinsn (,name :call :subprim-call ,@other-attrs) (() () ((entry (:label 1))))
	1700	(:talign 4)
	1701	(call (:@ ,spno))
	1702	:back
	1703	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn))))
	1704
	1705	(defmacro define-x8664-subprim-jump-vinsn ((name &rest other-attrs) spno)
	1706	`(define-x8664-vinsn (,name :jump :jumpLR ,@other-attrs) (() ())
	1707	(jmp (:@ ,spno))))
	1708
	1709	(define-x8664-vinsn (nthrowvalues :call :subprim-call) (()
	1710	((lab :label)))
	1711	(leaq (:@ (:^ lab) (:%q x8664::fn)) (:%q x8664::ra0))
	1712	(jmp (:@ .SPnthrowvalues)))
	1713
	1714	(define-x8664-vinsn (nthrow1value :call :subprim-call) (()
	1715	((lab :label)))
	1716	(leaq (:@ (:^ lab) (:%q x8664::fn)) (:%q x8664::ra0))
	1717	(jmp (:@ .SPnthrow1value)))
	1718
	1719
	1720	(define-x8664-subprim-lea-jmp-vinsn (bind-interrupt-level-0) .SPbind-interrupt-level-0)
	1721
	1722	(define-x8664-vinsn bind-interrupt-level-0-inline (()
	1723	()
	1724	((temp :imm)))
	1725	(movq (:@ (:%seg :rcontext) x8664::tcr.tlb-pointer) (:%q temp))
	1726	(cmpq (:$b 0) (:@ x8664::interrupt-level-binding-index (:%q temp)))
	1727	(pushq (:@ x8664::interrupt-level-binding-index (:%q temp)))
	1728	(pushq (:$b x8664::interrupt-level-binding-index))
	1729	(pushq (:@ (:%seg :rcontext) x8664::tcr.db-link))
	1730	(movq (:$l 0) (:@ x8664::interrupt-level-binding-index (:%q temp)))
	1731	(movq (:%q x8664::rsp) (:@ (:%seg :rcontext) x8664::tcr.db-link))
	1732	(jns.pt :done)
	1733	(btrq (:$ub 63) (:@ (:%seg :rcontext) x8664::tcr.interrupt-pending))
	1734	(jae.pt :done)
	1735	(ud2a)
	1736	(:byte 2)
	1737	:done)
	1738
	1739
	1740
	1741	(define-x8664-subprim-lea-jmp-vinsn (bind-interrupt-level-m1) .SPbind-interrupt-level-m1)
	1742
	1743	(define-x8664-vinsn bind-interrupt-level-m1-inline (()
	1744	()
	1745	((temp :imm)))
	1746	(movq (:@ (:%seg :rcontext) x8664::tcr.tlb-pointer) (:%q temp))
	1747	(pushq (:@ x8664::interrupt-level-binding-index (:%q temp)))
	1748	(pushq (:$b x8664::interrupt-level-binding-index))
	1749	(pushq (:@ (:%seg :rcontext) x8664::tcr.db-link))
	1750	(movq (:$l (ash -1 x8664::fixnumshift)) (:@ x8664::interrupt-level-binding-index (:%q temp)))
	1751	(movq (:%q x8664::rsp) (:@ (:%seg :rcontext) x8664::tcr.db-link)))
	1752
	1753	(define-x8664-subprim-lea-jmp-vinsn (bind-interrupt-level) .SPbind-interrupt-level)
	1754
	1755	(define-x8664-subprim-call-vinsn (unbind-interrupt-level) .SPunbind-interrupt-level)
	1756
	1757	(define-x8664-vinsn unbind-interrupt-level-inline (()
	1758	()
	1759	((link :imm)
	1760	(curval :imm)
	1761	(oldval :imm)
	1762	(tlb :imm)))
	1763	(movq (:@ (:%seg :rcontext) x8664::tcr.tlb-pointer) (:%q tlb))
	1764	(movq (:@ (:%seg :rcontext) x8664::tcr.db-link) (:%q link))
	1765	(movq (:@ x8664::interrupt-level-binding-index (:%q tlb)) (:%q curval))
	1766	(testq (:%q curval) (:%q curval))
	1767	(movq (:@ 16 #\|binding.val\|# (:%q link)) (:%q oldval))
	1768	(movq (:@ #\|binding.link\|# (:%q link)) (:%q link))
	1769	(movq (:%q oldval) (:@ x8664::interrupt-level-binding-index (:%q tlb)))
	1770	(movq (:%q link) (:@ (:%seg :rcontext) x8664::tcr.db-link))
	1771	(jns.pt :done)
	1772	(testq (:%q oldval) (:%q oldval))
	1773	(js.pt :done)
	1774	(btrq (:$ub 63) (:@ (:%seg :rcontext) x8664::tcr.interrupt-pending))
	1775	(jae.pt :done)
	1776	(ud2a)
	1777	(:byte 2)
	1778	:done)
	1779
	1780	(define-x8664-vinsn (jump-return-pc :jumpLR)
	1781	(()
	1782	())
	1783	(ret))
	1784
	1785	(define-x8664-vinsn (nmkcatchmv :call :subprim-call) (()
	1786	((lab :label))
	1787	((entry (:label 1))))
	1788	(leaq (:@ (:^ lab) (:%q x8664::fn)) (:%q x8664::xfn))
	1789	(:talign 4)
	1790	(call (:@ .SPmkcatchmv))
	1791	:back
	1792	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	1793
	1794	(define-x8664-vinsn (nmkcatch1v :call :subprim-call) (()
	1795	((lab :label))
	1796	((entry (:label 1))))
	1797	(leaq (:@ (:^ lab) (:%q x8664::fn)) (:%q x8664::xfn))
	1798	(:talign 4)
	1799	(call (:@ .SPmkcatch1v))
	1800	:back
	1801	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	1802
	1803
	1804	(define-x8664-vinsn (make-simple-unwind :call :subprim-call) (()
	1805	((protform-lab :label)
	1806	(cleanup-lab :label)))
	1807	(leaq (:@ (:^ protform-lab) (:%q x8664::fn)) (:%q x8664::ra0))
	1808	(leaq (:@ (:^ cleanup-lab) (:%q x8664::fn)) (:%q x8664::xfn))
	1809	(jmp (:@ .SPmkunwind)))
	1810
	1811	(define-x8664-vinsn (nmkunwind :call :subprim-call) (()
	1812	((protform-lab :label)
	1813	(cleanup-lab :label)))
	1814	(leaq (:@ (:^ protform-lab) (:%q x8664::fn)) (:%q x8664::ra0))
	1815	(leaq (:@ (:^ cleanup-lab) (:%q x8664::fn)) (:%q x8664::xfn))
	1816	(jmp (:@ .SPnmkunwind)))
	1817
	1818	;;; "old" mkunwind. Used by PROGV, since the binding of interrupt-level
	1819	;;; on entry to the new mkunwind confuses the issue.
	1820
	1821	(define-x8664-vinsn (mkunwind :call :subprim-call) (()
	1822	((protform-lab :label)
	1823	(cleanup-lab :label)))
	1824	(leaq (:@ (:^ protform-lab) (:%q x8664::fn)) (:%q x8664::ra0))
	1825	(leaq (:@ (:^ cleanup-lab) (:%q x8664::fn)) (:%q x8664::xfn))
	1826	(jmp (:@ .SPmkunwind)))
	1827
	1828	(define-x8664-subprim-lea-jmp-vinsn (gvector) .SPgvector)
	1829
	1830	(define-x8664-subprim-call-vinsn (getu64) .SPgetu64)
	1831
	1832	;;; Call something callable and obtain the single value that it
	1833	;;; returns.
	1834	(define-x8664-vinsn funcall (()
	1835	()
	1836	((tag :u8)
	1837	(entry (:label 1))))
	1838	(movb (:%b x8664::temp0) (:%b tag))
	1839	(andb (:$b x8664::fulltagmask) (:%b tag))
	1840	(cmpb (:$b x8664::fulltag-symbol) (:%b tag))
	1841	(cmovgq (:%q x8664::temp0) (:%q x8664::xfn))
	1842	(jge.pt :call)
	1843	(uuo-error-not-callable)
	1844	:call
	1845	(cmoveq (:@ x8664::symbol.fcell (:%q x8664::fname)) (:%q x8664::xfn))
	1846	(:talign 4)
	1847	(call (:%q x8664::xfn))
	1848	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	1849
	1850	(define-x8664-vinsn tail-funcall (()
	1851	()
	1852	((tag :u8)))
	1853	(movb (:%b x8664::temp0) (:%b tag))
	1854	(andb (:$b x8664::fulltagmask) (:%b tag))
	1855	(cmpb (:$b x8664::fulltag-symbol) (:%b tag))
	1856	(cmovgq (:%q x8664::temp0) (:%q x8664::xfn))
	1857	(jl :bad)
	1858	(cmoveq (:@ x8664::symbol.fcell (:%q x8664::fname)) (:%q x8664::xfn))
	1859	(jmp (:%q x8664::xfn))
	1860	:bad
	1861	(uuo-error-not-callable))
	1862
	1863
	1864
	1865	;;; Magic numbers in here include the address of .SPcall-closure.
	1866	(define-x8664-vinsn init-nclosure (()
	1867	((closure :lisp))
	1868	((imm0 :u64)))
	1869	(movq (:$q #x24fffffffff92d8d) (:%q imm0))
	1870	(movb (:$b 4) (:@ x8664::misc-data-offset (:%q closure))) ; code word count
	1871	(movb (:$b #x4c) (:@ (+ x8664::misc-data-offset 7) (:%q closure))) ; 1st byte of lea
	1872	(movq (:%q imm0) (:@ (+ x8664::misc-data-offset 8) (:%q closure))) ; rest of lea, start of jmp
	1873	(movl (:$l #x00516825) (:@ (+ x8664::misc-data-offset 16) (:%q closure)))
	1874	(movb (:$b x8664::function-boundary-marker) (:@ (+ x8664::misc-data-offset 24) (:%q closure))))
	1875
	1876
	1877	(define-x8664-vinsn finalize-closure (((closure :lisp))
	1878	((closure :lisp)))
	1879	(addq (:$b (- x8664::fulltag-function x8664::fulltag-misc)) (:%q closure)))
	1880
	1881
	1882	(define-x8664-vinsn (ref-symbol-value :call :subprim-call)
	1883	(((val :lisp))
	1884	((sym (:lisp (:ne val))))
	1885	((entry (:label 1))))
	1886	(:talign 4)
	1887	(call (:@ .SPspecrefcheck))
	1888	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)) )
	1889
	1890	(define-x8664-vinsn ref-symbol-value-inline (((dest :lisp))
	1891	((src (:lisp (:ne dest))))
	1892	((table :imm)
	1893	(idx :imm)))
	1894	(movq (:@ x8664::symbol.binding-index (:%q src)) (:%q idx))
	1895	(rcmpq (:%q idx) (:@ (:%seg :rcontext) x8664::tcr.tlb-limit))
	1896	(movq (:@ (:%seg :rcontext) x8664::tcr.tlb-pointer) (:%q table))
	1897	(jae :symbol)
	1898	(movq (:@ (:%q table) (:%q idx)) (:%q dest))
	1899	(cmpb (:$b x8664::subtag-no-thread-local-binding) (:%b dest))
	1900	(jne :test)
	1901	:symbol
	1902	(movq (:@ x8664::symbol.vcell (:%q src)) (:%q dest))
	1903	:test
	1904	(cmpb (:$b x8664::unbound-marker) (:%b dest))
	1905	(jne.pt :done)
	1906	(uuo-error-unbound (:%q src))
	1907	:done)
	1908
	1909
	1910	(define-x8664-vinsn (%ref-symbol-value :call :subprim-call)
	1911	(((val :lisp))
	1912	((sym (:lisp (:ne val))))
	1913	((entry (:label 1))))
	1914	(:talign 4)
	1915	(call (:@ .SPspecref))
	1916	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	1917
	1918	(define-x8664-vinsn %ref-symbol-value-inline (((dest :lisp))
	1919	((src (:lisp (:ne dest))))
	1920	((table :imm)
	1921	(idx :imm)))
	1922	(movq (:@ x8664::symbol.binding-index (:%q src)) (:%q idx))
	1923	(rcmpq (:%q idx) (:@ (:%seg :rcontext) x8664::tcr.tlb-limit))
	1924	(movq (:@ (:%seg :rcontext) x8664::tcr.tlb-pointer) (:%q table))
	1925	(jae :symbol)
	1926	(movq (:@ (:%q table) (:%q idx)) (:%q dest))
	1927	(cmpb (:$b x8664::subtag-no-thread-local-binding) (:%b dest))
	1928	(jne :done)
	1929	:symbol
	1930	(movq (:@ x8664::symbol.vcell (:%q src)) (:%q dest))
	1931	:done)
	1932
	1933	(define-x8664-vinsn ref-interrupt-level (((dest :imm))
	1934	()
	1935	((temp :u64)))
	1936	(movq (:@ (:%seg :rcontext) x8664::tcr.tlb-pointer) (:%q temp))
	1937	(movq (:@ x8664::INTERRUPT-LEVEL-BINDING-INDEX (:%q temp)) (:%q dest)))
	1938
	1939
	1940
	1941
	1942	(define-x8664-vinsn setup-double-float-allocation (()
	1943	())
	1944	(movl (:$l (arch::make-vheader x8664::double-float.element-count x8664::subtag-double-float)) (:%l x8664::imm0.l))
	1945	(movl (:$l (- x8664::double-float.size x8664::fulltag-misc)) (:%l x8664::imm1.l)))
	1946
	1947	(define-x8664-vinsn set-double-float-value (()
	1948	((node :lisp)
	1949	(val :double-float)))
	1950	(movsd (:%xmm val) (:@ x8664::double-float.value (:%q node))))
	1951
	1952	(define-x8664-vinsn word-index-and-bitnum-from-index (((word-index :u64)
	1953	(bitnum :u8))
	1954	((index :imm)))
	1955	(movq (:%q index) (:%q word-index))
	1956	(shrq (:$ub x8664::fixnumshift) (:%q word-index))
	1957	(movl (:$l 63) (:%l bitnum))
	1958	(andl (:%l word-index) (:%l bitnum))
	1959	(shrq (:$ub 6) (:%q word-index)))
	1960
	1961	(define-x8664-vinsn ref-bit-vector-fixnum (((dest :imm)
	1962	(bitnum :u8))
	1963	((bitnum :u8)
	1964	(bitvector :lisp)
	1965	(word-index :u64)))
	1966	(btq (:%q bitnum) (:@ x8664::misc-data-offset (:%q bitvector) (:%q word-index) 8))
	1967	(setb (:%b bitnum))
	1968	(negb (:%b bitnum))
	1969	(andl (:$l x8664::fixnumone) (:%l bitnum))
	1970	(movl (:%l bitnum) (:%l dest)))
	1971
	1972
	1973	(define-x8664-vinsn misc-ref-c-bit-fixnum (((dest :imm))
	1974	((src :lisp)
	1975	(idx :u64const))
	1976	((temp :u8)))
	1977	(btq (:$ub (:apply logand 63 idx))
	1978	(:@ (:apply + x8664::misc-data-offset (:apply ash (:apply ash idx -6) x8664::word-shift)) (:%q src)))
	1979	(setb (:%b temp))
	1980	(negb (:%b temp))
	1981	(andl (:$l x8664::fixnumone) (:%l temp))
	1982	(movl (:%l temp) (:%l dest)))
	1983
	1984	(define-x8664-vinsn deref-macptr (((addr :address))
	1985	((src :lisp))
	1986	())
	1987	(movq (:@ x8664::macptr.address (:%q src)) (:%q addr)))
	1988
	1989	(define-x8664-vinsn (temp-push-unboxed-word :push :word :csp)
	1990	(()
	1991	((w :u64)))
	1992	(movq (:@ (:%seg :rcontext) x8664::tcr.foreign-sp) (:%mmx x8664::stack-temp))
	1993	(subq (:$b 16) (:@ (:%seg :rcontext) x8664::tcr.foreign-sp))
	1994	(movq (:@ (:%seg :rcontext) x8664::tcr.foreign-sp) (:%q x8664::ra0))
	1995	(movq (:%mmx x8664::stack-temp) (:@ (:%q x8664::ra0)))
	1996	(movq (:%q w) (:@ 8 (:%q x8664::ra0))))
	1997
	1998
	1999	(define-x8664-vinsn (temp-push-node :push :word :tsp)
	2000	(()
	2001	((w :lisp))
	2002	((temp :imm)))
	2003	(subq (:$b (* 2 x8664::dnode-size)) (:@ (:%seg :rcontext) x8664::tcr.next-tsp))
	2004	(movq (:@ (:%seg :rcontext) x8664::tcr.save-tsp) (:%mmx x8664::stack-temp))
	2005	(movq (:@ (:%seg :rcontext) x8664::tcr.next-tsp) (:%q temp))
	2006	(movapd (:%xmm x8664::fpzero) (:@ (:%q temp)))
	2007	(movapd (:%xmm x8664::fpzero) (:@ 16 (:%q temp)))
	2008	(movq (:%mmx x8664::stack-temp) (:@ (:%q temp)))
	2009	(movq (:%q temp) (:@ (:%seg :rcontext) x8664::tcr.save-tsp))
	2010	(movq (:%q w) (:@ x8664::dnode-size (:%q temp))))
	2011
	2012	(define-x8664-vinsn (temp-push-double-float :push :word :csp)
	2013	(()
	2014	((f :double-float)))
	2015	(movq (:@ (:%seg :rcontext) x8664::tcr.foreign-sp) (:%mmx x8664::stack-temp))
	2016	(subq (:$b 16) (:@ (:%seg :rcontext) x8664::tcr.foreign-sp))
	2017	(movq (:@ (:%seg :rcontext) x8664::tcr.foreign-sp) (:%q x8664::ra0))
	2018	(movq (:%mmx x8664::stack-temp) (:@ (:%q x8664::ra0)))
	2019	(movsd (:%xmm f) (:@ 8 (:%q x8664::ra0))))
	2020
	2021
	2022	(define-x8664-vinsn (vpush-single-float :push :word :vsp)
	2023	(()
	2024	((f :single-float)))
	2025	(pushq (:$b x8664::tag-single-float))
	2026	(movss (:%xmm f) (:@ 4 (:%q x8664::rsp))))
	2027
	2028	(define-x8664-vinsn (vpop-single-float :pop :word :vsp)
	2029	(()
	2030	((f :single-float)))
	2031	(movss (:@ 4 (:%q x8664::rsp)) (:%xmm f))
	2032	(addq (:$b x8664::node-size) (:%q x8664::rsp)))
	2033
	2034	(define-x8664-vinsn (temp-pop-unboxed-word :pop :word :csp)
	2035	(((w :u64))
	2036	())
	2037	(movq (:@ (:%seg :rcontext) x8664::tcr.foreign-sp) (:%q x8664::ra0))
	2038	(movq (:@ 8 (:%q x8664::ra0)) (:%q w))
	2039	(addq (:$b 16) (:@ (:%seg :rcontext) x8664::tcr.foreign-sp)))
	2040
	2041
	2042	(define-x8664-vinsn (temp-pop-node :pop :word :tsp)
	2043	(((w :lisp))
	2044	()
	2045	((temp :imm)))
	2046	(movq (:@ (:%seg :rcontext) x8664::tcr.save-tsp) (:%q temp))
	2047	(movq (:@ x8664::dnode-size (:%q temp)) (:%q w))
	2048	(movq (:@ (:%q temp)) (:%q temp))
	2049	(movq (:%q temp) (:@ (:%seg :rcontext) x8664::tcr.save-tsp))
	2050	(movq (:%q temp) (:@ (:%seg :rcontext) x8664::tcr.next-tsp)))
	2051
	2052	(define-x8664-vinsn (temp-pop-double-float :pop :word :csp)
	2053	(((f :double-float))
	2054	())
	2055	(movq (:@ (:%seg :rcontext) x8664::tcr.foreign-sp) (:%q x8664::ra0))
	2056	(movsd (:@ 8 (:%q x8664::ra0)) (:%xmm f))
	2057	(addq (:$b 16) (:@ (:%seg :rcontext) x8664::tcr.foreign-sp)))
	2058
	2059
	2060
	2061	(define-x8664-vinsn macptr->stack (((dest :lisp))
	2062	((ptr :address)))
	2063	(movq (:@ (:%seg :rcontext) x8664::tcr.foreign-sp) (:%mmx x8664::stack-temp))
	2064	(subq (:$b (+ 16 x8664::macptr.size)) (:@ (:%seg :rcontext) x8664::tcr.foreign-sp))
	2065	(movq (:@ (:%seg :rcontext) x8664::tcr.foreign-sp) (:%q x8664::ra0))
	2066	(movq (:%mmx x8664::stack-temp) (:@ (:%q x8664::ra0)))
	2067	(leaq (:@ (+ 16 x8664::fulltag-misc) (:%q x8664::ra0)) (:%q dest))
	2068	(movq (:$l x8664::macptr-header) (:@ x8664::macptr.header (:%q dest)))
	2069	(movq (:%q ptr) (:@ x8664::macptr.address (:%q dest)))
	2070	(movapd (:%xmm x8664::fpzero) (:@ x8664::macptr.domain (:%q dest))))
	2071
	2072	(define-x8664-vinsn fixnum->signed-natural (((dest :s64))
	2073	((src :imm)))
	2074	(movq (:%q src) (:%q dest))
	2075	(sarq (:$ub x8664::fixnumshift) (:%q dest)))
	2076
	2077	(define-x8664-vinsn mem-set-double-float (()
	2078	((val :double-float)
	2079	(src :address)
	2080	(index :s64)))
	2081	(movsd (:%xmm val) (:@ (:%q src) (:%q index))))
	2082
	2083	(define-x8664-vinsn mem-set-single-float (()
	2084	((val :single-float)
	2085	(src :address)
	2086	(index :s64)))
	2087	(movss (:%xmm val) (:@ (:%q src) (:%q index))))
	2088
	2089
	2090
	2091	(define-x8664-vinsn mem-set-c-doubleword (()
	2092	((val :u64)
	2093	(dest :address)
	2094	(offset :s32const)))
	2095	((:pred = offset 0)
	2096	(movq (:%q val) (:@ (:%q dest))))
	2097	((:not (:pred = offset 0))
	2098	(movq (:%q val) (:@ offset (:%q dest)))))
	2099
	2100	(define-x8664-vinsn mem-set-c-fullword (()
	2101	((val :u32)
	2102	(dest :address)
	2103	(offset :s32const)))
	2104	((:pred = offset 0)
	2105	(movl (:%l val) (:@ (:%q dest))))
	2106	((:not (:pred = offset 0))
	2107	(movl (:%l val) (:@ offset (:%q dest)))))
	2108
	2109	(define-x8664-vinsn mem-set-c-halfword (()
	2110	((val :u16)
	2111	(dest :address)
	2112	(offset :s32const)))
	2113	((:pred = offset 0)
	2114	(movw (:%w val) (:@ (:%q dest))))
	2115	((:not (:pred = offset 0))
	2116	(movw (:%w val) (:@ offset (:%q dest)))))
	2117
	2118	(define-x8664-vinsn mem-set-c-byte (()
	2119	((val :u8)
	2120	(dest :address)
	2121	(offset :s32const)))
	2122	((:pred = offset 0)
	2123	(movb (:%b val) (:@ (:%q dest))))
	2124	((:not (:pred = offset 0))
	2125	(movb (:%b val) (:@ offset (:%q dest)))))
	2126
	2127	(define-x8664-vinsn mem-set-c-constant-doubleword (()
	2128	((val :s32const)
	2129	(dest :address)
	2130	(offset :s32const)))
	2131	((:pred = offset 0)
	2132	(movq (:$l val) (:@ (:%q dest))))
	2133	((:not (:pred = offset 0))
	2134	(movq (:$l val) (:@ offset (:%q dest)))))
	2135
	2136	(define-x8664-vinsn mem-set-c-constant-fullword (()
	2137	((val :s32const)
	2138	(dest :address)
	2139	(offset :s32const)))
	2140	((:pred = offset 0)
	2141	(movl (:$l val) (:@ (:%q dest))))
	2142	((:not (:pred = offset 0))
	2143	(movl (:$l val) (:@ offset (:%q dest)))))
	2144
	2145	(define-x8664-vinsn mem-set-c-constant-halfword (()
	2146	((val :s16const)
	2147	(dest :address)
	2148	(offset :s32const)))
	2149	((:pred = offset 0)
	2150	(movw (:$w val) (:@ (:%q dest))))
	2151	((:not (:pred = offset 0))
	2152	(movw (:$w val) (:@ offset (:%q dest)))))
	2153
	2154	(define-x8664-vinsn mem-set-c-constant-byte (()
	2155	((val :s8const)
	2156	(dest :address)
	2157	(offset :s32const)))
	2158	((:pred = offset 0)
	2159	(movb (:$b val) (:@ (:%q dest))))
	2160	((:not (:pred = offset 0))
	2161	(movb (:$b val) (:@ offset (:%q dest)))))
	2162
	2163
	2164
	2165
	2166
	2167
	2168	(define-x8664-vinsn mem-ref-natural (((dest :u64))
	2169	((src :address)
	2170	(index :s64)))
	2171	(movq (:@ (:%q src) (:%q index)) (:%q dest)))
	2172
	2173	(define-x8664-vinsn setup-macptr-allocation (()
	2174	((src :address)))
	2175	(movd (:%q src) (:%mmx x8664::mm0))
	2176	(movl (:$l x8664::macptr-header) (:%l x8664::imm0.l))
	2177	(movl (:$l (- x8664::macptr.size x8664::fulltag-misc)) (:%l x8664::imm1.l)))
	2178
	2179	(define-x8664-vinsn %set-new-macptr-value (()
	2180	((ptr :lisp)))
	2181	(movq (:%mmx x8664::mm0) (:@ x8664::macptr.address (:%q ptr))))
	2182
	2183	(define-x8664-vinsn mem-ref-c-fullword (((dest :u32))
	2184	((src :address)
	2185	(index :s32const)))
	2186	((:pred = index 0)
	2187	(movl (:@ (:%q src)) (:%l dest)))
	2188	((:not (:pred = index 0))
	2189	(movl (:@ index (:%q src)) (:%l dest))))
	2190
	2191	(define-x8664-vinsn mem-ref-c-signed-fullword (((dest :s32))
	2192	((src :address)
	2193	(index :s32const)))
	2194	((:pred = index 0)
	2195	(movslq (:@ (:%q src)) (:%q dest)))
	2196	((:not (:pred = index 0))
	2197	(movslq (:@ index (:%q src)) (:%q dest))))
	2198
	2199
	2200	(define-x8664-vinsn mem-ref-c-single-float (((dest :single-float))
	2201	((src :address)
	2202	(index :s32const)))
	2203	((:pred = index 0)
	2204	(movss (:@ (:%q src)) (:%xmm dest)))
	2205	((:not (:pred = index 0))
	2206	(movss (:@ index (:%q src)) (:%xmm dest))))
	2207
	2208	(define-x8664-vinsn mem-set-c-single-float (()
	2209	((val :single-float)
	2210	(src :address)
	2211	(index :s16const)))
	2212	((:pred = index 0)
	2213	(movss (:%xmm val) (:@ (:%q src))))
	2214	((:not (:pred = index 0))
	2215	(movss (:%xmm val) (:@ index (:%q src)))))
	2216
	2217	(define-x8664-vinsn mem-ref-c-doubleword (((dest :u64))
	2218	((src :address)
	2219	(index :s32const)))
	2220	((:pred = index 0)
	2221	(movq (:@ (:%q src)) (:%q dest)))
	2222	((:not (:pred = index 0))
	2223	(movq (:@ index (:%q src)) (:%q dest))))
	2224
	2225	(define-x8664-vinsn mem-ref-c-signed-doubleword (((dest :s64))
	2226	((src :address)
	2227	(index :s32const)))
	2228	((:pred = index 0)
	2229	(movq (:@ (:%q src)) (:%q dest)))
	2230	((:not (:pred = index 0))
	2231	(movq (:@ index (:%q src)) (:%q dest))))
	2232
	2233	(define-x8664-vinsn mem-ref-c-natural (((dest :u64))
	2234	((src :address)
	2235	(index :s32const)))
	2236	((:pred = index 0)
	2237	(movq (:@ (:%q src)) (:%q dest)))
	2238	((:not (:pred = index 0))
	2239	(movq (:@ index (:%q src)) (:%q dest))))
	2240
	2241	(define-x8664-vinsn mem-ref-c-double-float (((dest :double-float))
	2242	((src :address)
	2243	(index :s32const)))
	2244	((:pred = index 0)
	2245	(movsd (:@ (:%q src)) (:%xmm dest)))
	2246	((:not (:pred = index 0))
	2247	(movsd (:@ index (:%q src)) (:%xmm dest))))
	2248
	2249	(define-x8664-vinsn mem-set-c-double-float (()
	2250	((val :double-float)
	2251	(src :address)
	2252	(index :s16const)))
	2253	((:pred = index 0)
	2254	(movsd (:%xmm val) (:@ (:%q src))))
	2255	((:not (:pred = index 0))
	2256	(movsd (:%xmm val) (:@ index (:%q src)))))
	2257
	2258	(define-x8664-vinsn mem-ref-fullword (((dest :u32))
	2259	((src :address)
	2260	(index :s64)))
	2261	(movl (:@ (:%q src) (:%q index)) (:%l dest)))
	2262
	2263	(define-x8664-vinsn mem-ref-signed-fullword (((dest :s32))
	2264	((src :address)
	2265	(index :s64)))
	2266	(movslq (:@ (:%q src) (:%q index)) (:%q dest)))
	2267
	2268	(define-x8664-vinsn mem-ref-doubleword (((dest :u64))
	2269	((src :address)
	2270	(index :s64)))
	2271	(movq (:@ (:%q src) (:%q index)) (:%q dest)))
	2272
	2273	(define-x8664-vinsn mem-ref-natural (((dest :u64))
	2274	((src :address)
	2275	(index :s64)))
	2276	(movq (:@ (:%q src) (:%q index)) (:%q dest)))
	2277
	2278	(define-x8664-vinsn mem-ref-signed-doubleword (((dest :s64))
	2279	((src :address)
	2280	(index :s64)))
	2281	(movq (:@ (:%q src) (:%q index)) (:%q dest)))
	2282
	2283	(define-x8664-vinsn mem-ref-c-u16 (((dest :u16))
	2284	((src :address)
	2285	(index :s32const)))
	2286	((:pred = index 0)
	2287	(movzwq (:@ (:%q src)) (:%q dest)))
	2288	((:not (:pred = index 0))
	2289	(movzwq (:@ index (:%q src)) (:%q dest))))
	2290
	2291	(define-x8664-vinsn mem-ref-u16 (((dest :u16))
	2292	((src :address)
	2293	(index :s64)))
	2294	(movzwq (:@ (:%q src) (:%q index)) (:%q dest)))
	2295
	2296
	2297	(define-x8664-vinsn mem-ref-c-s16 (((dest :s16))
	2298	((src :address)
	2299	(index :s32const)))
	2300	((:pred = index 0)
	2301	(movswq (:@ (:%q src)) (:%q dest)))
	2302	((:not (:pred = index 0))
	2303	(movswq (:@ index (:%q src)) (:%q dest))))
	2304
	2305	(define-x8664-vinsn mem-ref-s16 (((dest :s16))
	2306	((src :address)
	2307	(index :s32)))
	2308	(movswq (:@ (:%q src) (:%q index)) (:%q dest)))
	2309
	2310	(define-x8664-vinsn mem-ref-c-u8 (((dest :u8))
	2311	((src :address)
	2312	(index :s16const)))
	2313	((:pred = index 0)
	2314	(movzbq (:@ (:%q src)) (:%q dest)))
	2315	((:not (:pred = index 0))
	2316	(movzbq (:@ index (:%q src)) (:%q dest))))
	2317
	2318	(define-x8664-vinsn mem-ref-u8 (((dest :u8))
	2319	((src :address)
	2320	(index :s32)))
	2321	(movzbq (:@ (:%q src) (:%q index)) (:%q dest)))
	2322
	2323	(define-x8664-vinsn mem-ref-c-s8 (((dest :s8))
	2324	((src :address)
	2325	(index :s16const)))
	2326	((:pred = index 0)
	2327	(movsbq (:@ (:%q src)) (:%q dest)))
	2328	((:not (:pred = index 0))
	2329	(movsbq (:@ index (:%q src)) (:%q dest))))
	2330
	2331	(define-x8664-vinsn misc-set-c-s8 (((val :s8))
	2332	((v :lisp)
	2333	(idx :u32const))
	2334	())
	2335	(movb (:%b val) (:@ (:apply + x8664::misc-data-offset idx) (:%q v))))
	2336
	2337	(define-x8664-vinsn misc-set-s8 (((val :s8))
	2338	((v :lisp)
	2339	(scaled-idx :s64))
	2340	())
	2341	(movb (:%b val) (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx))))
	2342
	2343	(define-x8664-vinsn mem-ref-s8 (((dest :s8))
	2344	((src :address)
	2345	(index :s32)))
	2346	(movsbq (:@ (:%q src) (:%q index)) (:%q dest)))
	2347
	2348	(define-x8664-vinsn mem-set-constant-doubleword (()
	2349	((val :s32const)
	2350	(ptr :address)
	2351	(offset :s64)))
	2352	(movq (:$l val) (:@ (:%q ptr) (:%q offset))))
	2353
	2354	(define-x8664-vinsn mem-set-constant-fullword (()
	2355	((val :s32const)
	2356	(ptr :address)
	2357	(offset :s64)))
	2358	(movl (:$l val) (:@ (:%q ptr) (:%q offset))))
	2359
	2360
	2361	(define-x8664-vinsn mem-set-constant-halfword (()
	2362	((val :s16const)
	2363	(ptr :address)
	2364	(offset :s64)))
	2365	(movw (:$w val) (:@ (:%q ptr) (:%q offset))))
	2366
	2367	(define-x8664-vinsn mem-set-constant-byte (()
	2368	((val :s8const)
	2369	(ptr :address)
	2370	(offset :s64)))
	2371	(movb (:$b val) (:@ (:%q ptr) (:%q offset))))
	2372
	2373	(define-x8664-vinsn misc-set-c-u8 (((val :u8))
	2374	((v :lisp)
	2375	(idx :u32const))
	2376	())
	2377	(movb (:%b val) (:@ (:apply + x8664::misc-data-offset idx) (:%q v))))
	2378
	2379	(define-x8664-vinsn misc-set-u8 (((val :u8))
	2380	((v :lisp)
	2381	(scaled-idx :s64))
	2382	())
	2383	(movb (:%b val) (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx))))
	2384
	2385	(define-x8664-vinsn misc-set-c-u8 (((val :u8))
	2386	((v :lisp)
	2387	(idx :s32const))
	2388	())
	2389	(movb (:%b val) (:@ (:apply + x8664::misc-data-offset idx) (:%q v))))
	2390
	2391	(define-x8664-vinsn misc-set-u8 (()
	2392	((val :u8)
	2393	(v :lisp)
	2394	(scaled-idx :s64))
	2395	())
	2396	(movb (:%b val) (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx))))
	2397
	2398	(define-x8664-vinsn misc-set-c-u16 (()
	2399	((val :u16)
	2400	(v :lisp)
	2401	(idx :s32const))
	2402	())
	2403	(movw (:%w val) (:@ (:apply + x8664::misc-data-offset (:apply * 2 idx)) (:%q v))))
	2404
	2405
	2406	(define-x8664-vinsn misc-set-u16 (()
	2407	((val :u16)
	2408	(v :lisp)
	2409	(scaled-idx :s64))
	2410	())
	2411	(movw (:%w val) (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx))))
	2412
	2413	(define-x8664-vinsn misc-set-c-s16 (()
	2414	((val :s16)
	2415	(v :lisp)
	2416	(idx :s32const))
	2417	())
	2418	(movw (:%w val) (:@ (:apply + x8664::misc-data-offset (:apply * 2 idx)) (:%q v))))
	2419
	2420
	2421	(define-x8664-vinsn misc-set-s16 (()
	2422	((val :s16)
	2423	(v :lisp)
	2424	(scaled-idx :s64))
	2425	())
	2426	(movw (:%w val) (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx))))
	2427
	2428	(define-x8664-vinsn misc-set-c-u32 (()
	2429	((val :u32)
	2430	(v :lisp)
	2431	(idx :u32const)) ; sic
	2432	())
	2433	(movl (:%l val) (:@ (:apply + x8664::misc-data-offset (:apply ash idx 2)) (:%q v))))
	2434
	2435	(define-x8664-vinsn misc-set-u32 (()
	2436	((val :u32)
	2437	(v :lisp)
	2438	(scaled-idx :s64))
	2439	())
	2440	(movl (:%l val) (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx))))
	2441
	2442	(define-x8664-vinsn misc-set-c-s32 (()
	2443	((val :s32)
	2444	(v :lisp)
	2445	(idx :u32const)) ; sic
	2446	())
	2447	(movl (:%l val) (:@ (:apply + x8664::misc-data-offset (:apply ash idx 2)) (:%q v))))
	2448
	2449	(define-x8664-vinsn misc-set-s32 (()
	2450	((val :s32)
	2451	(v :lisp)
	2452	(scaled-idx :s64))
	2453	())
	2454	(movl (:%l val) (:@ x8664::misc-data-offset (:%q v) (:%q scaled-idx))))
	2455
	2456	(define-x8664-vinsn %iasr (((dest :imm))
	2457	((count :imm)
	2458	(src :imm))
	2459	((temp :s64)
	2460	(shiftcount (:s64 #.x8664::rcx))))
	2461	(movq (:%q count) (:%q temp))
	2462	(sarq (:$ub x8664::fixnumshift) (:%q temp))
	2463	(rcmpq (:%q temp) (:$l 63))
	2464	(cmovbw (:%w temp) (:%w shiftcount))
	2465	(movq (:%q src) (:%q temp))
	2466	(jae :shift-max)
	2467	(sarq (:%shift x8664::cl) (:%q temp))
	2468	(jmp :done)
	2469	:shift-max
	2470	(sarq (:$ub 63) (:%q temp))
	2471	:done
	2472	(andb (:$b (lognot x8664::fixnummask)) (:%b temp))
	2473	(movq (:%q temp) (:%q dest)))
	2474
	2475	(define-x8664-vinsn %ilsr (((dest :imm))
	2476	((count :imm)
	2477	(src :imm))
	2478	((temp :s64)
	2479	(shiftcount (:s64 #.x8664::rcx))))
	2480	(movq (:%q count) (:%q temp))
	2481	(sarq (:$ub x8664::fixnumshift) (:%q temp))
	2482	(rcmpq (:%q temp) (:$l 63))
	2483	(cmovbw (:%w temp) (:%w shiftcount))
	2484	(movq (:%q src) (:%q temp))
	2485	(jae :shift-max)
	2486	(shrq (:%shift x8664::cl) (:%q temp))
	2487	(jmp :done)
	2488	:shift-max
	2489	(shrq (:$ub 63) (:%q temp))
	2490	:done
	2491	(andb (:$b (lognot x8664::fixnummask)) (:%b temp))
	2492	(movq (:%q temp) (:%q dest)))
	2493
	2494	(define-x8664-vinsn %iasr-c (((dest :imm))
	2495	((count :u8const)
	2496	(src :imm))
	2497	((temp :s64)))
	2498	(movq (:%q src) (:%q temp))
	2499	(sarq (:$ub count) (:%q temp))
	2500	(andb (:$b (lognot x8664::fixnummask)) (:%b temp))
	2501	(movq (:%q temp) (:%q dest)))
	2502
	2503	(define-x8664-vinsn %ilsr-c (((dest :imm))
	2504	((count :u8const)
	2505	(src :imm))
	2506	((temp :s64)))
	2507	(movq (:%q src) (:%q temp))
	2508	(shrq (:$ub count) (:%q temp))
	2509	(andb (:$b (lognot x8664::fixnummask)) (:%b temp))
	2510	(movq (:%q temp) (:%q dest)))
	2511
	2512	(define-x8664-vinsn %ilsl (((dest :imm))
	2513	((count :imm)
	2514	(src :imm))
	2515	((temp :s64)
	2516	(shiftcount (:s64 #.x8664::rcx))))
	2517	(movq (:%q count) (:%q temp))
	2518	(sarq (:$ub x8664::fixnumshift) (:%q temp))
	2519	(rcmpq (:%q temp) (:$l 63))
	2520	(cmovbw (:%w temp) (:%w shiftcount))
	2521	(movq (:%q src) (:%q temp))
	2522	(jae :shift-max)
	2523	(shlq (:%shift x8664::cl) (:%q temp))
	2524	(jmp :done)
	2525	:shift-max
	2526	(xorq (:%q temp) (:%q temp))
	2527	:done
	2528	(movq (:%q temp) (:%q dest)))
	2529
	2530	(define-x8664-vinsn %ilsl-c (((dest :imm))
	2531	((count :u8const)
	2532	(src :imm)))
	2533	((:not (:pred =
	2534	(:apply %hard-regspec-value src)
	2535	(:apply %hard-regspec-value dest)))
	2536	(movq (:%q src) (:%q dest)))
	2537	(shlq (:$ub count) (:%q dest)))
	2538
	2539	;;; In safe code, something else has ensured that the value is of type
	2540	;;; BIT.
	2541	(define-x8664-vinsn set-variable-bit-to-variable-value (()
	2542	((vec :lisp)
	2543	(word-index :s64)
	2544	(bitnum :u8)
	2545	(value :lisp)))
	2546	(testb (:%b value) (:%b value))
	2547	(je :clr)
	2548	(btsq (:%q bitnum) (:@ x8664::misc-data-offset (:%q vec) (:%q word-index) 8))
	2549	(jmp :done)
	2550	:clr
	2551	(btrq (:%q bitnum) (:@ x8664::misc-data-offset (:%q vec) (:%q word-index) 8))
	2552	:done)
	2553
	2554	(define-x8664-vinsn set-variable-bit-to-zero (()
	2555	((vec :lisp)
	2556	(word-index :s64)
	2557	(bitnum :u8)))
	2558	(btrq (:%q bitnum) (:@ x8664::misc-data-offset (:%q vec) (:%q word-index) 8)))
	2559
	2560	(define-x8664-vinsn set-variable-bit-to-one (()
	2561	((vec :lisp)
	2562	(word-index :s64)
	2563	(bitnum :u8)))
	2564	(btsq (:%q bitnum) (:@ x8664::misc-data-offset (:%q vec) (:%q word-index) 8)))
	2565
	2566	(define-x8664-vinsn set-constant-bit-to-zero (()
	2567	((src :lisp)
	2568	(idx :u64const)))
	2569	(btrq (:$ub (:apply logand 63 idx))
	2570	(:@ (:apply + x8664::misc-data-offset (:apply ash (:apply ash idx -6) x8664::word-shift)) (:%q src))))
	2571
	2572	(define-x8664-vinsn set-constant-bit-to-one (()
	2573	((src :lisp)
	2574	(idx :u64const)))
	2575	(btsq (:$ub (:apply logand 63 idx))
	2576	(:@ (:apply + x8664::misc-data-offset (:apply ash (:apply ash idx -6) x8664::word-shift)) (:%q src))))
	2577
	2578	(define-x8664-vinsn set-constant-bit-to-variable-value (()
	2579	((src :lisp)
	2580	(idx :u64const)
	2581	(value :lisp)))
	2582	(testb (:%b value) (:%b value))
	2583	(je :clr)
	2584	(btsq (:$ub (:apply logand 63 idx))
	2585	(:@ (:apply + x8664::misc-data-offset (:apply ash (:apply ash idx -6) x8664::word-shift)) (:%q src)))
	2586	(jmp :done)
	2587	:clr
	2588	(btrq (:$ub (:apply logand 63 idx))
	2589	(:@ (:apply + x8664::misc-data-offset (:apply ash (:apply ash idx -6) x8664::word-shift)) (:%q src)))
	2590	:done)
	2591
	2592
	2593	(define-x8664-vinsn require-fixnum (()
	2594	((object :lisp)))
	2595	:again
	2596	(testb (:$b x8664::fixnummask) (:%b object))
	2597	(je.pt :got-it)
	2598	(uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-fixnum))
	2599	(jmp :again)
	2600	:got-it)
	2601
	2602	(define-x8664-vinsn require-integer (()
	2603	((object :lisp))
	2604	((tag :u8)))
	2605	:again
	2606	(movb (:%b object) (:%b tag))
	2607	(andb (:$b x8664::fixnummask) (:%b tag))
	2608	(je.pt :got-it)
	2609	(cmpb (:$b x8664::tag-misc) (:%b tag))
	2610	(jne :bad)
	2611	(cmpb (:$b x8664::subtag-bignum) (:@ x8664::misc-subtag-offset (:%q object)))
	2612	(je :got-it)
	2613	:bad
	2614	(uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-integer))
	2615	(jmp :again)
	2616	:got-it)
	2617
	2618	(define-x8664-vinsn require-simple-vector (()
	2619	((object :lisp))
	2620	((tag :u8)))
	2621	:again
	2622	(movb (:%b object) (:%b tag))
	2623	(andb (:$b x8664::fixnummask) (:%b tag))
	2624	(cmpb (:$b x8664::tag-misc) (:%b tag))
	2625	(jne :bad)
	2626	(cmpb (:$b x8664::subtag-simple-vector) (:@ x8664::misc-subtag-offset (:%q object)))
	2627	(je :got-it)
	2628	:bad
	2629	(uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-simple-vector))
	2630	(jmp :again)
	2631	:got-it)
	2632
	2633	(define-x8664-vinsn require-simple-string (()
	2634	((object :lisp))
	2635	((tag :u8)))
	2636	:again
	2637	(movb (:%b object) (:%b tag))
	2638	(andb (:$b x8664::fixnummask) (:%b tag))
	2639	(cmpb (:$b x8664::tag-misc) (:%b tag))
	2640	(jne :bad)
	2641	(cmpb (:$b x8664::subtag-simple-base-string) (:@ x8664::misc-subtag-offset (:%q object)))
	2642	(je :got-it)
	2643	:bad
	2644	(uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-simple-string))
	2645	(jmp :again)
	2646	:got-it)
	2647
	2648	(define-x8664-vinsn require-real (()
	2649	((object :lisp))
	2650	((tag :u8)
	2651	(mask :u64)))
	2652	(movq (:$q (logior (ash 1 x8664::tag-fixnum)
	2653	(ash 1 x8664::tag-single-float)
	2654	(ash 1 x8664::subtag-double-float)
	2655	(ash 1 x8664::subtag-bignum)
	2656	(ash 1 x8664::subtag-ratio)))
	2657	(:%q mask))
	2658	:again
	2659	(movb (:$b x8664::tagmask) (:%b tag))
	2660	(andb (:%b object) (:%b tag))
	2661	(cmpb (:$b x8664::tag-misc) (:%b tag))
	2662	(jne :have-tag)
	2663	(movb (:@ x8664::misc-subtag-offset (:%q object)) (:%b tag))
	2664	:have-tag
	2665	(rcmpb (:%b tag) (:$b 64))
	2666	(jae :bad)
	2667	(btq (:%q tag) (:%q mask))
	2668	(jb.pt :good)
	2669	:bad
	2670	(uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-real))
	2671	(jmp :again)
	2672	:good)
	2673
	2674	(define-x8664-vinsn require-number (()
	2675	((object :lisp))
	2676	((tag :u8)
	2677	(mask :u64)))
	2678	(movq (:$q (logior (ash 1 x8664::tag-fixnum)
	2679	(ash 1 x8664::tag-single-float)
	2680	(ash 1 x8664::subtag-double-float)
	2681	(ash 1 x8664::subtag-bignum)
	2682	(ash 1 x8664::subtag-ratio)
	2683	(ash 1 x8664::subtag-complex)))
	2684	(:%q mask))
	2685	:again
	2686	(movb (:$b x8664::tagmask) (:%b tag))
	2687	(andb (:%b object) (:%b tag))
	2688	(cmpb (:$b x8664::tag-misc) (:%b tag))
	2689	(jne :have-tag)
	2690	(movb (:@ x8664::misc-subtag-offset (:%q object)) (:%b tag))
	2691	:have-tag
	2692	(rcmpb (:%b tag) (:$b 64))
	2693	;;(movzbl (:%b tag) (:%l tag))
	2694	(jae :bad)
	2695	(btq (:%q tag) (:%q mask))
	2696	(jb.pt :good)
	2697	:bad
	2698	(uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-number))
	2699	(jmp :again)
	2700	:good)
	2701
	2702	(define-x8664-vinsn require-list (()
	2703	((object :lisp))
	2704	((tag :u8)))
	2705	:again
	2706	(movb (:%b object) (:%b tag))
	2707	(andb (:$b x8664::tagmask) (:%b tag))
	2708	(cmpb (:$b x8664::tag-list) (:%b tag))
	2709	(je :good)
	2710	(uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-list))
	2711	(jmp :again)
	2712	:good)
	2713
	2714	(define-x8664-vinsn require-symbol (()
	2715	((object :lisp))
	2716	((tag :u8)))
	2717	:again
	2718	(cmpb (:$b x8664::fulltag-nil) (:%b object))
	2719	(je :good)
	2720	(movb (:%b object) (:%b tag))
	2721	(andb (:$b x8664::tagmask) (:%b tag))
	2722	(cmpb (:$b x8664::tag-symbol) (:%b tag))
	2723	(je :good)
	2724	(uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-symbol))
	2725	(jmp :again)
	2726	:good)
	2727
	2728	(define-x8664-vinsn require-character (()
	2729	((object :lisp)))
	2730	:again
	2731	(cmpb (:$b x8664::subtag-character) (:%b object))
	2732	(je.pt :ok)
	2733	(uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-character))
	2734	(jmp :again)
	2735	:ok)
	2736
	2737	(define-x8664-vinsn require-s8 (()
	2738	((object :lisp))
	2739	((tag :u32)))
	2740	:again
	2741	(movq (:%q object) (:%q tag))
	2742	(shlq (:$ub (- x8664::nbits-in-word (+ 8 x8664::fixnumshift))) (:%q tag))
	2743	(sarq (:$ub (- x8664::nbits-in-word 8)) (:%q tag))
	2744	(shlq (:$ub x8664::fixnumshift) (:%q tag))
	2745	(cmpq (:%q object) (:%q tag))
	2746	(je.pt :ok)
	2747	:bad
	2748	(uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-signed-byte-8))
	2749	(jmp :again)
	2750	:ok)
	2751
	2752	(define-x8664-vinsn require-u8 (()
	2753	((object :lisp))
	2754	((tag :u32)))
	2755	:again
	2756	(movq (:$l (lognot (ash #xff x8664::fixnumshift))) (:%q tag))
	2757	(andq (:% object) (:% tag))
	2758	(je.pt :ok)
	2759	(uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-unsigned-byte-8))
	2760	(jmp :again)
	2761	:ok)
	2762
	2763	(define-x8664-vinsn require-s16 (()
	2764	((object :lisp))
	2765	((tag :s64)))
	2766	:again
	2767	(movq (:%q object) (:%q tag))
	2768	(shlq (:$ub (- x8664::nbits-in-word (+ 16 x8664::fixnumshift))) (:%q tag))
	2769	(sarq (:$ub (- x8664::nbits-in-word 16)) (:%q tag))
	2770	(shlq (:$ub x8664::fixnumshift) (:%q tag))
	2771	(cmpq (:%q object) (:%q tag))
	2772	(je.pt :ok)
	2773	:bad
	2774	(uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-signed-byte-16))
	2775	(jmp :again)
	2776	:ok)
	2777
	2778	(define-x8664-vinsn require-u16 (()
	2779	((object :lisp))
	2780	((tag :u32)))
	2781	:again
	2782	(movq (:$l (lognot (ash #xffff x8664::fixnumshift))) (:%q tag))
	2783	(andq (:% object) (:% tag))
	2784	(je.pt :ok)
	2785	(uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-unsigned-byte-16))
	2786	(jmp :again)
	2787	:ok)
	2788
	2789	(define-x8664-vinsn require-s32 (()
	2790	((object :lisp))
	2791	((tag :s64)))
	2792	:again
	2793	(movq (:%q object) (:%q tag))
	2794	(shlq (:$ub (- x8664::nbits-in-word (+ 32 x8664::fixnumshift))) (:%q tag))
	2795	(sarq (:$ub (- x8664::nbits-in-word 32)) (:%q tag))
	2796	(shlq (:$ub x8664::fixnumshift) (:%q tag))
	2797	(cmpq (:%q object) (:%q tag))
	2798	(jne.pn :bad)
	2799	(testb (:$b x8664::fixnummask) (:%b object))
	2800	(je.pt :bad)
	2801	:bad
	2802	(uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-signed-byte-32))
	2803	(jmp :again)
	2804	:ok)
	2805
	2806	(define-x8664-vinsn require-u32 (()
	2807	((object :lisp))
	2808	((tag :u32)))
	2809	:again
	2810	(movq (:$q (lognot (ash #xffffffff x8664::fixnumshift))) (:%q tag))
	2811	(andq (:% object) (:% tag))
	2812	(je.pt :ok)
	2813	(uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-unsigned-byte-32))
	2814	(jmp :again)
	2815	:ok)
	2816
	2817	(define-x8664-vinsn require-s64 (()
	2818	((object :lisp))
	2819	((tag :s64)))
	2820	:again
	2821	(testb (:$b x8664::fixnummask) (:%b object))
	2822	(movq (:%q object) (:%q tag))
	2823	(je.pt :ok)
	2824	(andb (:$b x8664::fulltagmask) (:%b tag))
	2825	(cmpb (:$b x8664::fulltag-misc) (:%b tag))
	2826	(jne.pn :bad)
	2827	(cmpq (:$l x8664::two-digit-bignum-header) (:@ x8664::misc-header-offset (:%q object)))
	2828	(je.pt :ok)
	2829	:bad
	2830	(uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-signed-byte-64))
	2831	(jmp :again)
	2832	:ok)
	2833
	2834	(define-x8664-vinsn require-u64 (()
	2835	((object :lisp))
	2836	((tag :s64)))
	2837	:again
	2838	(testb (:$b x8664::fixnummask) (:%b object))
	2839	(movq (:%q object) (:%q tag))
	2840	(je.pt :ok-if-non-negative)
	2841	(andb (:$b x8664::fulltagmask) (:%b tag))
	2842	(cmpb (:$b x8664::fulltag-misc) (:%b tag))
	2843	(jne.pn :bad)
	2844	(cmpq (:$l x8664::two-digit-bignum-header) (:@ x8664::misc-header-offset (:%q object)))
	2845	(je :two)
	2846	(cmpq (:$l x8664::three-digit-bignum-header) (:@ x8664::misc-header-offset (:%q object)))
	2847	(jne.pn :bad)
	2848	(cmpl (:$b 0) (:@ (+ x8664::misc-data-offset 8) (:%q object)))
	2849	(je :ok)
	2850	:bad
	2851	(uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-unsigned-byte-64))
	2852	(jmp :again)
	2853	:two
	2854	(movq (:@ x8664::misc-data-offset (:%q object)) (:%q tag))
	2855	:ok-if-non-negative
	2856	(testq (:%q tag) (:%q tag))
	2857	(js :bad)
	2858	:ok)
	2859
	2860	(define-x8664-vinsn require-char-code (()
	2861	((object :lisp))
	2862	((tag :u32)))
	2863	:again
	2864	(testb (:$b x8664::fixnummask) (:%b object))
	2865	(jne.pn :bad)
	2866	(cmpq (:$l (ash #x110000 x8664::fixnumshift)) (:%q object))
	2867	(jb.pt :ok)
	2868	:bad
	2869	(uuo-error-reg-not-type (:%q object) (:$ub arch::error-object-not-mod-char-code-limit))
	2870	(jmp :again)
	2871	:ok)
	2872
	2873
	2874
	2875
	2876
	2877	(define-x8664-vinsn mask-base-char (((dest :u8))
	2878	((src :lisp)))
	2879	(movzbl (:%b src) (:%l dest)))
	2880
	2881	(define-x8664-vinsn single-float-bits (((dest :u32))
	2882	((src :lisp)))
	2883	(movq (:%q src) (:%q dest))
	2884	(shrq (:$ub 32) (:%q dest)))
	2885
	2886	(define-x8664-vinsn zero-double-float-register (((dest :double-float))
	2887	())
	2888	(movsd (:%xmm x8664::fpzero) (:%xmm dest)))
	2889
	2890	(define-x8664-vinsn zero-single-float-register (((dest :single-float))
	2891	())
	2892	(movss (:%xmm x8664::fpzero) (:%xmm dest)))
	2893
	2894	(define-x8664-subprim-lea-jmp-vinsn (heap-rest-arg) .SPheap-rest-arg)
	2895	(define-x8664-subprim-lea-jmp-vinsn (stack-rest-arg) .SPstack-rest-arg)
	2896	(define-x8664-subprim-lea-jmp-vinsn (req-stack-rest-arg) .SPreq-stack-rest-arg)
	2897
	2898	(define-x8664-subprim-call-vinsn (stack-misc-alloc) .SPstack-misc-alloc)
	2899
	2900	(define-x8664-vinsn misc-element-count-fixnum (((dest :imm))
	2901	((src :lisp))
	2902	((temp :u64)))
	2903	(movq (:@ x8664::misc-header-offset (:%q src)) (:%q temp))
	2904	(movb (:$b 0) (:%b temp))
	2905	(movq (:%q temp) (:%q dest))
	2906	(shrq (:$ub (- x8664::num-subtag-bits x8664::fixnumshift)) (:%q dest)))
	2907
	2908	(define-x8664-vinsn %logior2 (((dest :imm))
	2909	((x :imm)
	2910	(y :imm)))
	2911	((:pred =
	2912	(:apply %hard-regspec-value x)
	2913	(:apply %hard-regspec-value dest))
	2914	(orq (:%q y) (:%q dest)))
	2915	((:not (:pred =
	2916	(:apply %hard-regspec-value x)
	2917	(:apply %hard-regspec-value dest)))
	2918	((:pred =
	2919	(:apply %hard-regspec-value y)
	2920	(:apply %hard-regspec-value dest))
	2921	(orq (:%q x) (:%q dest)))
	2922	((:not (:pred =
	2923	(:apply %hard-regspec-value y)
	2924	(:apply %hard-regspec-value dest)))
	2925	(movq (:%q x) (:%q dest))
	2926	(orq (:%q y) (:%q dest)))))
	2927
	2928	(define-x8664-vinsn %logand2 (((dest :imm))
	2929	((x :imm)
	2930	(y :imm)))
	2931	((:pred =
	2932	(:apply %hard-regspec-value x)
	2933	(:apply %hard-regspec-value dest))
	2934	(andq (:%q y) (:%q dest)))
	2935	((:not (:pred =
	2936	(:apply %hard-regspec-value x)
	2937	(:apply %hard-regspec-value dest)))
	2938	((:pred =
	2939	(:apply %hard-regspec-value y)
	2940	(:apply %hard-regspec-value dest))
	2941	(andq (:%q x) (:%q dest)))
	2942	((:not (:pred =
	2943	(:apply %hard-regspec-value y)
	2944	(:apply %hard-regspec-value dest)))
	2945	(movq (:%q x) (:%q dest))
	2946	(andq (:%q y) (:%q dest)))))
	2947
	2948	(define-x8664-vinsn %logxor2 (((dest :imm))
	2949	((x :imm)
	2950	(y :imm)))
	2951	((:pred =
	2952	(:apply %hard-regspec-value x)
	2953	(:apply %hard-regspec-value dest))
	2954	(xorq (:%q y) (:%q dest)))
	2955	((:not (:pred =
	2956	(:apply %hard-regspec-value x)
	2957	(:apply %hard-regspec-value dest)))
	2958	((:pred =
	2959	(:apply %hard-regspec-value y)
	2960	(:apply %hard-regspec-value dest))
	2961	(xorq (:%q x) (:%q dest)))
	2962	((:not (:pred =
	2963	(:apply %hard-regspec-value y)
	2964	(:apply %hard-regspec-value dest)))
	2965	(movq (:%q x) (:%q dest))
	2966	(xorq (:%q y) (:%q dest)))))
	2967
	2968	(define-x8664-subprim-call-vinsn (integer-sign) .SPinteger-sign)
	2969
	2970	(define-x8664-vinsn vcell-ref (((dest :lisp))
	2971	((vcell :lisp)))
	2972	(movq (:@ x8664::misc-data-offset (:%q vcell)) (:%q dest)))
	2973
	2974	(define-x8664-vinsn (call-subprim-3 :call :subprim-call) (((dest t))
	2975	((spno :s32const)
	2976	(x t)
	2977	(y t)
	2978	(z t))
	2979	((entry (:label 1))))
	2980	(:talign 4)
	2981	(call (:@ spno))
	2982	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	2983
	2984	(define-x8664-vinsn setup-vcell-allocation (()
	2985	())
	2986	(movl (:$l x8664::value-cell-header) (:%l x8664::imm0))
	2987	(movl (:$l (- x8664::value-cell.size x8664::fulltag-misc)) (:%l x8664::imm1)))
	2988
	2989	(define-x8664-vinsn %init-vcell (()
	2990	((vcell :lisp)
	2991	(closed :lisp)))
	2992	(movq (:%q closed) (:@ x8664::value-cell.value (:%q vcell))))
	2993
	2994	(define-x8664-subprim-call-vinsn (progvsave) .SPprogvsave)
	2995
	2996	(define-x8664-subprim-jump-vinsn (progvrestore) .SPprogvrestore)
	2997
	2998	(define-x8664-subprim-lea-jmp-vinsn (simple-keywords) .SPsimple-keywords)
	2999
	3000	(define-x8664-subprim-lea-jmp-vinsn (keyword-args) .SPkeyword-args)
	3001
	3002	(define-x8664-subprim-lea-jmp-vinsn (keyword-bind) .SPkeyword-bind)
	3003
	3004	(define-x8664-vinsn scale-nargs (()
	3005	((nfixed :s16const)))
	3006	((:pred > nfixed 0)
	3007	(addw (:$w (:apply - (:apply ash nfixed x8664::word-shift))) (:%w x8664::nargs))))
	3008
	3009	(define-x8664-vinsn opt-supplied-p (()
	3010	())
	3011	(xorl (:%l x8664::imm1) (:%l x8664::imm1))
	3012	(movl (:$l x8664::t-value) (:%l x8664::arg_y))
	3013	:loop
	3014	(rcmpw (:%w x8664::imm1) (:%w x8664::nargs))
	3015	(movl (:$l x8664::nil-value) (:%l x8664::arg_z))
	3016	(cmovll (:%l x8664::arg_y) (:%l x8664::arg_z))
	3017	(addl (:$b x8664::node-size) (:%l x8664::imm1))
	3018	(cmpl (:%l x8664::imm1) (:%l x8664::imm0))
	3019	(pushq (:%q x8664::arg_z))
	3020	(jne :loop))
	3021
	3022	(define-x8664-vinsn one-opt-supplied-p (()
	3023	()
	3024	((temp :u64)))
	3025	(testw (:%w x8664::nargs) (:%w x8664::nargs))
	3026	(setne (:%b temp))
	3027	(negb (:%b temp))
	3028	(andl (:$b x8664::t-offset) (:%l temp))
	3029	(addl (:$l x8664::nil-value) (:%l temp))
	3030	(pushq (:%q temp)))
	3031
	3032	(define-x8664-vinsn two-opt-supplied-p (()
	3033	()
	3034	((temp0 :u64)
	3035	(temp1 :u64)))
	3036	(rcmpw (:%w x8664::nargs) (:$w x8664::node-size))
	3037	(setae (:%b temp0))
	3038	(seta (:%b temp1))
	3039	(negb (:%b temp0))
	3040	(negb (:%b temp1))
	3041	(andl (:$b x8664::t-offset) (:%l temp0))
	3042	(andl (:$b x8664::t-offset) (:%l temp1))
	3043	(addl (:$l x8664::nil-value) (:%l temp0))
	3044	(addl (:$l x8664::nil-value) (:%l temp1))
	3045	(pushq (:%q temp0))
	3046	(pushq (:%q temp1)))
	3047
	3048
	3049	(define-x8664-vinsn set-c-flag-if-constant-logbitp (()
	3050	((bit :u8const)
	3051	(int :imm)))
	3052	(btq (:$ub bit) (:%q int)))
	3053
	3054	(define-x8664-vinsn set-c-flag-if-variable-logbitp (()
	3055	((bit :imm)
	3056	(int :imm))
	3057	((temp0 :u8)
	3058	(temp1 :u8)))
	3059	(movl (:$l 63) (:%l temp1))
	3060	(movq (:%q bit) (:%q temp0))
	3061	(sarq (:$ub x8664::fixnumshift) (:%q temp0))
	3062	(addq (:$b x8664::fixnumshift) (:%q temp0))
	3063	(rcmpq (:%q temp0) (:%q temp1))
	3064	(cmoval (:%l temp1) (:%l temp0))
	3065	(btq (:%q temp0) (:%q int)))
	3066
	3067	(define-x8664-vinsn multiply-immediate (((dest :imm))
	3068	((src :imm)
	3069	(const :s32const)))
	3070	((:and (:pred >= const -128) (:pred <= const 127))
	3071	(imulq (:$b const) (:%q src) (:%q dest)))
	3072	((:not (:and (:pred >= const -128) (:pred <= const 127)))
	3073	(imulq (:$l const) (:%q src) (:%q dest))))
	3074
	3075	(define-x8664-vinsn multiply-fixnums (((dest :imm))
	3076	((x :imm)
	3077	(y :imm))
	3078	((unboxed :s64)))
	3079	((:pred =
	3080	(:apply %hard-regspec-value x)
	3081	(:apply %hard-regspec-value dest))
	3082	(movq (:%q y) (:%q unboxed))
	3083	(sarq (:$ub x8664::fixnumshift) (:%q unboxed))
	3084	(imulq (:%q unboxed) (:%q dest)))
	3085	((:and (:not (:pred =
	3086	(:apply %hard-regspec-value x)
	3087	(:apply %hard-regspec-value dest)))
	3088	(:pred =
	3089	(:apply %hard-regspec-value y)
	3090	(:apply %hard-regspec-value dest)))
	3091	(movq (:%q x) (:%q unboxed))
	3092	(sarq (:$ub x8664::fixnumshift) (:%q unboxed))
	3093	(imulq (:%q unboxed) (:%q dest)))
	3094	((:and (:not (:pred =
	3095	(:apply %hard-regspec-value x)
	3096	(:apply %hard-regspec-value dest)))
	3097	(:not (:pred =
	3098	(:apply %hard-regspec-value y)
	3099	(:apply %hard-regspec-value dest))))
	3100	(movq (:%q y) (:%q dest))
	3101	(movq (:%q x) (:%q unboxed))
	3102	(sarq (:$ub x8664::fixnumshift) (:%q unboxed))
	3103	(imulq (:%q unboxed) (:%q dest))))
	3104
	3105
	3106	(define-x8664-vinsn save-lexpr-argregs (()
	3107	((min-fixed :u16const)))
	3108	((:pred >= min-fixed $numx8664argregs)
	3109	(pushq (:%q x8664::arg_x))
	3110	(pushq (:%q x8664::arg_y))
	3111	(pushq (:%q x8664::arg_z)))
	3112	((:pred = min-fixed 2) ; at least 2 args
	3113	(cmpw (:$w (ash 2 x8664::word-shift)) (:%w x8664::nargs))
	3114	(je :yz2) ; skip arg_x if exactly 2
	3115	(pushq (:%q x8664::arg_x))
	3116	:yz2
	3117	(pushq (:%q x8664::arg_y))
	3118	(pushq (:%q x8664::arg_z)))
	3119	((:pred = min-fixed 1) ; at least one arg
	3120	(rcmpw (:%w x8664::nargs) (:$w (ash 2 x8664::word-shift)))
	3121	(jl :z1) ; branch if exactly one
	3122	(je :yz1) ; branch if exactly two
	3123	(pushq (:%q x8664::arg_x))
	3124	:yz1
	3125	(pushq (:%q x8664::arg_y))
	3126	:z1
	3127	(pushq (:%q x8664::arg_z)))
	3128	((:pred = min-fixed 0)
	3129	(testw (:%w x8664::nargs) (:%w x8664::nargs))
	3130	(je :none) ; exactly zero
	3131	(rcmpw (:%w x8664::nargs) (:$w (ash 2 x8664::word-shift)))
	3132	(je :yz0) ; exactly two
	3133	(jl :z0) ; one
	3134	; Three or more ...
	3135	(pushq (:%q x8664::arg_x))
	3136	:yz0
	3137	(pushq (:%q x8664::arg_y))
	3138	:z0
	3139	(pushq (:%q x8664::arg_z))
	3140	:none
	3141	)
	3142	(movzwl (:%w x8664::nargs) (:%l x8664::nargs))
	3143	((:not (:pred = min-fixed 0))
	3144	(leaq (:@ (:apply - (:apply ash min-fixed x8664::word-shift)) (:%q x8664::nargs))
	3145	(:%q x8664::nargs)))
	3146	(pushq (:%q x8664::nargs))
	3147	(movq (:%q x8664::rsp) (:%q x8664::arg_z)))
	3148
	3149
	3150
	3151
	3152	;;; The frame that was built (by SAVE-LISP-CONTEXT-VARIABLE-ARG-COUNT
	3153	;;; and SAVE-LEXPR-ARGREGS) contains an unknown number of arguments
	3154	;;; followed by the count of non-required arguments; the count is on
	3155	;;; top of the stack and its address is in %arg_z. We need to build a
	3156	;;; frame so that the function can address its arguments (copies of
	3157	;;; the required arguments and the lexpr) and locals; when the
	3158	;;; function returns, it should one or more values (depending on how
	3159	;;; it was called) and discard the hidden lexpr frame. At this point,
	3160	;;; %ra0 still contains the "real" return address. If it's not the
	3161	;;; magic multiple-value address, we can make the function return to
	3162	;;; something that does a single-value return (.SPpopj); otherwise, we
	3163	;;; need to make it return multiple values to the real caller. (Unlike
	3164	;;; the PPC, this case only involves creating one frame here, but that
	3165	;;; frame has two return addresses.)
	3166	(define-x8664-vinsn build-lexpr-frame (()
	3167	()
	3168	((temp :imm)))
	3169	(movq (:@ (+ x8664::nil-value (x8664::%kernel-global 'x86::ret1valaddr)))
	3170	(:%q temp))
	3171	(cmpq (:%q temp)
	3172	(:%q x8664::ra0))
	3173	(je :multiple)
	3174	(pushq (:@ (+ x8664::nil-value (x8664::%kernel-global 'x86::lexpr-return1v))))
	3175	(jmp :finish)
	3176	:multiple
	3177	(pushq (:@ (+ x8664::nil-value (x8664::%kernel-global 'x86::lexpr-return))))
	3178	(pushq (:%q temp))
	3179	:finish
	3180	(pushq (:%q x8664::rbp))
	3181	(movq (:%q x8664::rsp) (:%q x8664::rbp)))
	3182
	3183
	3184	(define-x8664-vinsn copy-lexpr-argument (()
	3185	((n :u16const))
	3186	((temp :imm)))
	3187	(movq (:@ (:%q x8664::arg_z)) (:%q temp))
	3188	(pushq (:@ (:apply ash n x8664::word-shift) (:%q x8664::arg_z) (:%q temp))))
	3189
	3190
	3191	(define-x8664-vinsn %current-tcr (((dest :lisp))
	3192	())
	3193	(movq (:@ (:%seg :rcontext) x8664::tcr.linear) (:%q dest)))
	3194
	3195	(define-x8664-vinsn (setq-special :call :subprim-call)
	3196	(()
	3197	((sym :lisp)
	3198	(val :lisp))
	3199	((entry (:label 1))))
	3200	(:talign 4)
	3201	(call (:@ .SPspecset))
	3202	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	3203
	3204	(define-x8664-vinsn set-z-flag-if-istruct-typep (()
	3205	((val :lisp)
	3206	(type :lisp))
	3207	((tag :u8)
	3208	(valtype :lisp)))
	3209	(xorl (:%l valtype) (:%l valtype))
	3210	(movb (:%b val) (:%b tag))
	3211	(andb (:$b x8664::tagmask) (:%b tag))
	3212	(cmpb (:$b x8664::tag-misc) (:%b tag))
	3213	(jne :have-tag)
	3214	(movb (:@ x8664::misc-subtag-offset (:%q val)) (:%b tag))
	3215	:have-tag
	3216	(cmpb (:$b x8664::subtag-istruct) (:%b tag))
	3217	(jne :do-compare)
	3218	(movq (:@ x8664::misc-data-offset (:%q val)) (:%q valtype))
	3219	:do-compare
	3220	(cmpq (:%q valtype) (:%q type)))
	3221
	3222	(define-x8664-subprim-call-vinsn (misc-ref) .SPmisc-ref)
	3223
	3224	(define-x8664-subprim-call-vinsn (ksignalerr) .SPksignalerr)
	3225
	3226	(define-x8664-subprim-call-vinsn (misc-alloc-init) .SPmisc-alloc-init)
	3227
	3228	(define-x8664-subprim-call-vinsn (misc-alloc) .SPmisc-alloc)
	3229
	3230	(define-x8664-subprim-lea-jmp-vinsn (make-stack-gvector) .SPstkgvector)
	3231
	3232	(define-x8664-vinsn load-character-constant (((dest :lisp))
	3233	((code :u32const))
	3234	())
	3235	(movl (:$l (:apply logior (:apply ash code 8) x8664::subtag-character))
	3236	(:%l dest)))
	3237
	3238	(define-x8664-vinsn %scharcode8 (((code :imm))
	3239	((str :lisp)
	3240	(idx :imm))
	3241	((imm :u64)))
	3242	(movq (:%q idx) (:%q imm))
	3243	(sarq (:$ub x8664::fixnumshift) (:%q imm))
	3244	(movzbl (:@ x8664::misc-data-offset (:%q str) (:%q imm)) (:%l imm))
	3245	(imulq (:$b x8664::fixnumone) (:%q imm)(:%q code)))
	3246
	3247	(define-x8664-vinsn %scharcode32 (((code :imm))
	3248	((str :lisp)
	3249	(idx :imm))
	3250	((imm :u64)))
	3251	(movq (:%q idx) (:%q imm))
	3252	(sarq (:$ub 1) (:%q imm))
	3253	(movl (:@ x8664::misc-data-offset (:%q str) (:%q imm)) (:%l imm))
	3254	(imulq (:$b x8664::fixnumone) (:%q imm)(:%q code)))
	3255
	3256	(define-x8664-subprim-jump-vinsn (tail-call-sym-slide) .SPtcallsymslide)
	3257
	3258	(define-x8664-subprim-jump-vinsn (tail-call-sym-vsp) .SPtcallsymvsp)
	3259
	3260
	3261	(define-x8664-vinsn character->code (((dest :u32))
	3262	((src :lisp)))
	3263	(movq (:%q src) (:%q dest))
	3264	(sarq (:$ub x8664::charcode-shift) (:%q dest)))
	3265
	3266	(define-x8664-vinsn adjust-vsp (()
	3267	((amount :s32const)))
	3268	((:and (:pred >= amount -128) (:pred <= amount 127))
	3269	(addq (:$b amount) (:%q x8664::rsp)))
	3270	((:not (:and (:pred >= amount -128) (:pred <= amount 127)))
	3271	(addq (:$l amount) (:%q x8664::rsp))))
	3272
	3273	(define-x8664-vinsn (call-subprim-2 :call :subprim-call) (((dest t))
	3274	((spno :s32const)
	3275	(y t)
	3276	(z t))
	3277	((entry (:label 1))))
	3278	(:talign 4)
	3279	(call (:@ spno))
	3280	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	3281
	3282
	3283
	3284	(define-x8664-vinsn set-macptr-address (()
	3285	((addr :address)
	3286	(src :lisp))
	3287	())
	3288	(movq (:%q addr) (:@ x8664::macptr.address (:%q src))))
	3289
	3290	(define-x8664-vinsn %symbol->symptr (((dest :lisp))
	3291	((src :lisp))
	3292	((tag :u8)))
	3293	(movl (:$l (+ x8664::nil-value x8664::nilsym-offset)) (:%l tag))
	3294	(cmpb (:$b x8664::fulltag-nil) (:%b src))
	3295	(cmoveq (:%q tag) (:%q dest))
	3296	(movb (:%b src) (:%b tag))
	3297	(je :ok)
	3298	(andb (:$b x8664::tagmask) (:%b tag))
	3299	(cmpb (:$b x8664::tag-symbol) (:%b tag))
	3300	(je.pt :no-trap)
	3301	(uuo-error-reg-not-tag (:%q src) (:$ub x8664::fulltag-symbol))
	3302	:no-trap
	3303	((:not (:pred =
	3304	(:apply %hard-regspec-value dest)
	3305	(:apply %hard-regspec-value src)))
	3306	(movq (:% src) (:% dest)))
	3307	:ok)
	3308
	3309	(define-x8664-vinsn symbol-function (((val :lisp))
	3310	((sym (:lisp (:ne val))))
	3311	((tag :u8)))
	3312	(movq (:@ x8664::symbol.fcell (:%q sym)) (:%q val))
	3313	(movb (:%b val) (:%b tag))
	3314	(andb (:$b x8664::tagmask) (:%b tag))
	3315	(cmpb (:$b x8664::tag-function) (:%b tag))
	3316	(je.pt :ok)
	3317	(uuo-error-udf (:%q sym))
	3318	:ok)
	3319
	3320	(define-x8664-subprim-jump-vinsn (tail-call-fn-slide) .SPtcallnfnslide)
	3321
	3322	(define-x8664-vinsn load-double-float-constant (((dest :double-float))
	3323	((lab :label)
	3324	))
	3325	(movsd (:@ (:^ lab) (:%q x8664::fn)) (:%xmm dest)))
	3326
	3327	(define-x8664-vinsn load-single-float-constant (((dest :single-float))
	3328	((lab :label)
	3329	))
	3330	(movss (:@ (:^ lab) (:%q x8664::fn)) (:%xmm dest)))
	3331
	3332	(define-x8664-subprim-call-vinsn (misc-set) .SPmisc-set)
	3333
	3334	(define-x8664-subprim-lea-jmp-vinsn (slide-values) .SPmvslide)
	3335
	3336	(define-x8664-subprim-lea-jmp-vinsn (spread-list) .SPspreadargz)
	3337
	3338	;;; Even though it's implemented by calling a subprim, THROW is really
	3339	;;; a JUMP (to a possibly unknown destination). If the destination's
	3340	;;; really known, it should probably be inlined (stack-cleanup, value
	3341	;;; transfer & jump ...)
	3342	(define-x8664-vinsn (throw :jump :jump-unknown) (()
	3343	()
	3344	((entry (:label 1))))
	3345	(leaq (:@ (:^ :back) (:%q x8664::fn)) (:%q x8664::ra0))
	3346	(:talign 4)
	3347	(jmp (:@ .SPthrow))
	3348	:back
	3349	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	3350
	3351
	3352
	3353	(define-x8664-vinsn unbox-base-char (((dest :u64))
	3354	((src :lisp)))
	3355	(movq (:%q src) (:%q dest))
	3356	(shrq (:$ub x8664::charcode-shift) (:%q dest))
	3357	(cmpb (:$b x8664::subtag-character) (:%b src))
	3358	(je.pt ::got-it)
	3359	(uuo-error-reg-not-tag (:%q src) (:$ub x8664::subtag-character))
	3360	:got-it)
	3361
	3362	(define-x8664-subprim-lea-jmp-vinsn (save-values) .SPsave-values)
	3363
	3364	(define-x8664-subprim-lea-jmp-vinsn (recover-values) .SPrecover-values)
	3365
	3366	(define-x8664-subprim-lea-jmp-vinsn (recover-values-for-mvcall) .SPrecover-values-for-mvcall)
	3367
	3368	(define-x8664-subprim-lea-jmp-vinsn (add-values) .SPadd-values)
	3369
	3370	(define-x8664-subprim-call-vinsn (make-stack-block) .SPmakestackblock)
	3371
	3372	(define-x8664-subprim-call-vinsn (make-stack-block0) .Spmakestackblock0)
	3373
	3374	;;; "dest" is preallocated, presumably on a stack somewhere.
	3375	(define-x8664-vinsn store-double (()
	3376	((dest :lisp)
	3377	(source :double-float))
	3378	())
	3379	(movsd (:%xmm source) (:@ x8664::double-float.value (:%q dest))))
	3380
	3381	(define-x8664-vinsn fixnum->char (((dest :lisp))
	3382	((src :imm))
	3383	((temp :u32)))
	3384	(movl (:%l src) (:%l temp))
	3385	(sarl (:$ub (+ x8664::fixnumshift 11)) (:%l temp))
	3386	(cmpl (:$b (ash #xd800 -11))(:%l temp))
	3387	(movl (:$l x8664::nil-value) (:%l temp))
	3388	(cmovel (:%l temp) (:%l dest))
	3389	(je :done)
	3390	((:not (:pred =
	3391	(:apply %hard-regspec-value dest)
	3392	(:apply %hard-regspec-value src)))
	3393	(movl (:%l src) (:%l dest)))
	3394	(shll (:$ub (- x8664::charcode-shift x8664::fixnumshift)) (:%l dest))
	3395	(addb (:$b x8664::subtag-character) (:%b dest))
	3396	:done)
	3397
	3398
	3399	(define-x8664-vinsn sign-extend-halfword (((dest :imm))
	3400	((src :imm)))
	3401	(movq (:%q src ) (:%q dest))
	3402	(shlq (:$ub (- 48 x8664::fixnumshift)) (:%q dest))
	3403	(sarq (:$ub (- 48 x8664::fixnumshift)) (:%q dest)))
	3404
	3405	(define-x8664-subprim-jump-vinsn (tail-funcall-gen) .SPtfuncallgen)
	3406
	3407	(define-x8664-subprim-call-vinsn (gets64) .SPgets64)
	3408
	3409	(define-x8664-subprim-call-vinsn (getu64) .SPgetu64)
	3410
	3411	(define-x8664-vinsn %init-gvector (()
	3412	((v :lisp)
	3413	(nbytes :u32const))
	3414	((count :imm)))
	3415	(movl (:$l nbytes) (:%l count))
	3416	(jmp :test)
	3417	:loop
	3418	(popq (:@ x8664::misc-data-offset (:%q v) (:%q count)))
	3419	:test
	3420	(subq (:$b x8664::node-size) (:%q count))
	3421	(jge :loop))
	3422
	3423	(define-x8664-subprim-jump-vinsn (tail-funcall-slide) .SPtfuncallslide)
	3424
	3425	(define-x8664-vinsn nth-value (((result :lisp))
	3426	()
	3427	((imm0 :u64)))
	3428	(movzwl (:%w x8664::nargs) (:%l x8664::nargs))
	3429	(leaq (:@ (:%q x8664::rsp) (:%q x8664::nargs)) (:%q imm0))
	3430	(subq (:@ (:%q imm0)) (:%q x8664::nargs))
	3431	(movl (:$l x8664::nil-value) (:%l result))
	3432	(jle :done)
	3433	;; I -think- that a CMOV would be safe here, assuming that N wasn't
	3434	;; extremely large. Don't know if we can assume that.
	3435	(movq (:@ (- x8664::node-size) (:%q x8664::rsp) (:%q x8664::nargs)) (:%q result))
	3436	:done
	3437	(leaq (:@ x8664::node-size (:%q imm0)) (:%q x8664::rsp)))
	3438
	3439
	3440	(define-x8664-subprim-lea-jmp-vinsn (req-heap-rest-arg) .SPreq-heap-rest-arg)
	3441
	3442	(define-x8664-subprim-call-vinsn (stack-misc-alloc-init) .SPstack-misc-alloc-init)
	3443
	3444	(define-x8664-vinsn fixnum->unsigned-natural (((dest :u64))
	3445	((src :imm)))
	3446	(movq (:%q src) (:%q dest))
	3447	(shrq (:$ub x8664::fixnumshift) (:%q dest)))
	3448
	3449	(define-x8664-vinsn %debug-trap (()
	3450	())
	3451	(uuo-error-debug-trap))
	3452
	3453	(define-x8664-vinsn double-to-single (((result :single-float))
	3454	((arg :double-float)))
	3455	(cvtsd2ss (:%xmm arg) (:%xmm result)))
	3456
	3457	(define-x8664-vinsn single-to-double (((result :double-float))
	3458	((arg :single-float)))
	3459	(cvtss2sd (:%xmm arg) (:%xmm result)))
	3460
	3461
	3462	(define-x8664-vinsn alloc-c-frame (()
	3463	((nwords :u32const)))
	3464	(movq (:@ (:%seg :rcontext) x8664::tcr.foreign-sp) (:%mmx x8664::stack-temp))
	3465	((:pred < (:apply ash (:apply logandc2 (:apply + nwords 9) 1) x8664::word-shift) 128)
	3466	(subq (:$b (:apply ash (:apply logandc2 (:apply + nwords 9) 1) x8664::word-shift)) (:@ (:%seg :rcontext) x8664::tcr.foreign-sp)))
	3467	((:not (:pred < (:apply ash (:apply logandc2 (:apply + nwords 9) 1) x8664::word-shift) 128))
	3468	(subq (:$l (:apply ash (:apply logandc2 (:apply + nwords 9) 1) x8664::word-shift)) (:@ (:%seg :rcontext) x8664::tcr.foreign-sp)))
	3469	(movq (:@ (:%seg :rcontext) x8664::tcr.foreign-sp) (:%q x8664::ra0))
	3470	(movq (:%mmx x8664::stack-temp) (:@ (:%q x8664::ra0))))
	3471
	3472	(define-x8664-vinsn alloc-variable-c-frame (()
	3473	((nwords :imm))
	3474	((size :s64)))
	3475	(leaq (:@ (* 9 x8664::node-size) (:%q nwords)) (:%q size))
	3476	(andb (:$b (lognot x8664::fulltagmask)) (:%b size))
	3477
	3478	(movq (:@ (:%seg :rcontext) x8664::tcr.foreign-sp) (:%mmx x8664::stack-temp))
	3479	(subq (:%q size) (:@ (:%seg :rcontext) x8664::tcr.foreign-sp))
	3480	(movq (:@ (:%seg :rcontext) x8664::tcr.foreign-sp) (:%q x8664::ra0))
	3481	(movq (:%mmx x8664::stack-temp) (:@ (:%q x8664::ra0))))
	3482
	3483	(define-x8664-vinsn set-c-arg (()
	3484	((arg :u64)
	3485	(offset :u32const)))
	3486	(movq (:@ (:%seg :rcontext) x8664::tcr.foreign-sp) (:%q x8664::ra0))
	3487	(movq (:%q arg) (:@ (:apply + 16 (:apply ash offset 3)) (:%q x8664::ra0))))
	3488
	3489	(define-x8664-vinsn set-single-c-arg (()
	3490	((arg :single-float)
	3491	(offset :u32const)))
	3492	(movq (:@ (:%seg :rcontext) x8664::tcr.foreign-sp) (:%q x8664::ra0))
	3493	(movss (:%xmm arg) (:@ (:apply + 16 (:apply ash offset 3)) (:%q x8664::ra0))))
	3494
	3495	(define-x8664-vinsn reload-single-c-arg (((arg :single-float))
	3496	((offset :u32const)))
	3497	(movq (:@ (:%seg :rcontext) x8664::tcr.foreign-sp) (:%q x8664::ra0))
	3498	(movss (:@ (:apply + 16 (:apply ash offset 3)) (:%q x8664::ra0)) (:%xmm arg)))
	3499
	3500	(define-x8664-vinsn set-double-c-arg (()
	3501	((arg :double-float)
	3502	(offset :u32const)))
	3503	(movq (:@ (:%seg :rcontext) x8664::tcr.foreign-sp) (:%q x8664::ra0))
	3504	(movsd (:%xmm arg) (:@ (:apply + 16 (:apply ash offset 3)) (:%q x8664::ra0))))
	3505
	3506	(define-x8664-vinsn reload-double-c-arg (((arg :double-float))
	3507	((offset :u32const)))
	3508	(movq (:@ (:%seg :rcontext) x8664::tcr.foreign-sp) (:%q x8664::ra0))
	3509	(movsd (:@ (:apply + 16 (:apply ash offset 3)) (:%q x8664::ra0)) (:%xmm arg)))
	3510
	3511	(define-x8664-subprim-call-vinsn (ff-call) .SPffcall)
	3512
	3513	(define-x8664-subprim-call-vinsn (ff-call-return-registers) .SPffcall-return-registers)
	3514
	3515	(define-x8664-subprim-call-vinsn (syscall) .SPsyscall)
	3516
	3517	(define-x8664-subprim-call-vinsn (setqsym) .SPsetqsym)
	3518
	3519	(define-x8664-vinsn recover-fn-from-rip (()
	3520	())
	3521	(leaq (:@ (:apply - (:^ :disp)) (:%q x8664::rip)) (:%q x8664::fn))
	3522	:disp)
	3523
	3524
	3525
	3526	(define-x8664-subprim-call-vinsn (makeu64) .SPmakeu64)
	3527
	3528	(define-x8664-subprim-call-vinsn (makes64) .SPmakes64)
	3529
	3530	(define-x8664-subprim-lea-jmp-vinsn (stack-cons-list*) .SPstkconslist-star)
	3531
	3532	(define-x8664-subprim-lea-jmp-vinsn (list*) .SPconslist-star)
	3533
	3534	(define-x8664-vinsn make-tsp-vcell (((dest :lisp))
	3535	((closed :lisp))
	3536	((temp :imm)))
	3537	(subq (:$b (+ x8664::value-cell.size x8664::dnode-size)) (:@ (:%seg :rcontext) x8664::tcr.next-tsp))
	3538	(movq (:@ (:%seg :rcontext) x8664::tcr.save-tsp) (:%mmx x8664::stack-temp))
	3539	(movq (:@ (:%seg :rcontext) x8664::tcr.next-tsp) (:%q temp))
	3540	(movapd (:%xmm x8664::fpzero) (:@ (:%q temp)))
	3541	(movapd (:%xmm x8664::fpzero) (:@ x8664::dnode-size (:%q temp)))
	3542	(movq (:%mmx x8664::stack-temp) (:@ (:%q temp)))
	3543	(movq (:%q temp) (:@ (:%seg :rcontext) x8664::tcr.save-tsp))
	3544	(movq (:$l x8664::value-cell-header) (:@ x8664::dnode-size (:%q temp)))
	3545	(movq (:%q closed) (:@ (+ x8664::dnode-size x8664::node-size) (:%q temp)))
	3546	(leaq (:@ (+ x8664::dnode-size x8664::fulltag-misc) (:%q temp)) (:%q dest)))
	3547
	3548	(define-x8664-subprim-lea-jmp-vinsn (bind-nil) .SPbind-nil)
	3549
	3550	(define-x8664-subprim-lea-jmp-vinsn (bind-self) .SPbind-self)
	3551
	3552	(define-x8664-subprim-lea-jmp-vinsn (bind-self-boundp-check) .SPbind-self-boundp-check)
	3553
	3554	(define-x8664-subprim-lea-jmp-vinsn (bind) .SPbind)
	3555
	3556	(define-x8664-vinsn (dpayback :call :subprim-call) (()
	3557	((n :s16const))
	3558	((temp (:u32 #.x8664::imm0))
	3559	(entry (:label 1))))
	3560	((:pred > n 0)
	3561	((:pred > n 1)
	3562	(movl (:$l n) (:%l temp))
	3563	(:talign 4)
	3564	(call (:@ .SPunbind-n)))
	3565	((:pred = n 1)
	3566	(:talign 4)
	3567	(call (:@ .SPunbind)))
	3568	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn))))
	3569
	3570	(define-x8664-subprim-jump-vinsn (tail-call-sym-gen) .SPtcallsymgen)
	3571
	3572	(define-x8664-subprim-call-vinsn (make-stack-list) .Spmakestacklist)
	3573
	3574	(define-x8664-vinsn node-slot-ref (((dest :lisp))
	3575	((node :lisp)
	3576	(cellno :u32const)))
	3577	(movq (:@ (:apply + x8664::misc-data-offset (:apply ash cellno 3))
	3578	(:%q node)) (:%q dest)))
	3579
	3580	(define-x8664-subprim-lea-jmp-vinsn (stack-cons-list) .SPstkconslist)
	3581
	3582
	3583	(define-x8664-vinsn %slot-ref (((dest :lisp))
	3584	((instance (:lisp (:ne dest)))
	3585	(index :lisp)))
	3586	(movq (:@ x8664::misc-data-offset (:%q instance) (:%q index)) (:%q dest))
	3587	(cmpb (:$b x8664::slot-unbound-marker) (:%b dest))
	3588	(jne.pt :ok)
	3589	(uuo-error-slot-unbound (:%q dest) (:%q instance) (:%q index))
	3590	:ok)
	3591
	3592	(define-x8664-vinsn eep.address (((dest t))
	3593	((src (:lisp (:ne dest )))))
	3594	(movq (:@ (+ (ash 1 x8664::word-shift) x8664::misc-data-offset) (:%q src))
	3595	(:%q dest))
	3596	(cmpb (:$b x8664::fulltag-nil) (:%b dest))
	3597	(jne :ok)
	3598	(uuo-error-eep-unresolved (:%q src) (:%q dest))
	3599	:ok)
	3600
	3601	(define-x8664-subprim-lea-jmp-vinsn (heap-cons-rest-arg) .SPheap-cons-rest-arg)
	3602
	3603	(define-x8664-subprim-lea-jmp-vinsn (stack-cons-rest-arg) .SPstack-cons-rest-arg)
	3604
	3605	(define-x8664-subprim-lea-jmp-vinsn (make-stack-vector) .SPmkstackv)
	3606
	3607	(define-x8664-vinsn %current-frame-ptr (((dest :imm))
	3608	())
	3609	(movq (:%q x8664::rbp) (:%q dest)))
	3610
	3611	(define-x8664-vinsn %foreign-stack-pointer (((dest :imm))
	3612	())
	3613	(movq (:@ (:%seg :rcontext) x8664::tcr.foreign-sp) (:%q dest)))
	3614
	3615
	3616	(define-x8664-vinsn %set-scharcode8 (()
	3617	((str :lisp)
	3618	(idx :imm)
	3619	(code :imm))
	3620	((imm :u64)
	3621	(imm1 :u64)))
	3622	(movq (:%q code) (:%q imm1))
	3623	(movq (:%q idx) (:%q imm))
	3624	(shrq (:$ub x8664::fixnumshift) (:%q imm1))
	3625	(shrq (:$ub x8664::word-shift) (:%q imm))
	3626	(movb (:%b imm1) (:@ x8664::misc-data-offset (:%q str) (:%q imm))))
	3627
	3628
	3629	(define-x8664-vinsn %set-scharcode32 (()
	3630	((str :lisp)
	3631	(idx :imm)
	3632	(code :imm))
	3633	((imm :u64)
	3634	(imm1 :u64)))
	3635	(movq (:%q code) (:%q imm1))
	3636	(movq (:%q idx) (:%q imm))
	3637	(shrq (:$ub x8664::fixnumshift) (:%q imm1))
	3638	(shrq (:$ub 1) (:%q imm))
	3639	(movl (:%l imm1) (:@ x8664::misc-data-offset (:%q str) (:%q imm))))
	3640
	3641
	3642
	3643
	3644	(define-x8664-vinsn pop-argument-registers (()
	3645	())
	3646	(testw (:%w x8664::nargs) (:%w x8664::nargs))
	3647	(je :done)
	3648	(rcmpw (:%w x8664::nargs) (:$w (ash 2 x8664::word-shift)))
	3649	(popq (:%q x8664::arg_z))
	3650	(jb :done)
	3651	(popq (:%q x8664::arg_y))
	3652	(je :done)
	3653	(popq (:%q x8664::arg_x))
	3654	:done)
	3655
	3656	(define-x8664-vinsn %symptr->symvector (((target :lisp))
	3657	((target :lisp)))
	3658	(subb (:$b (- x8664::fulltag-symbol x8664::fulltag-misc)) (:%b target)))
	3659
	3660	(define-x8664-vinsn %symvector->symptr (((target :lisp))
	3661	((target :lisp)))
	3662	(addb (:$b (- x8664::fulltag-symbol x8664::fulltag-misc)) (:%b target)))
	3663
	3664
	3665	(define-x8664-subprim-lea-jmp-vinsn (spread-lexpr) .SPspread-lexpr-z)
	3666
	3667	(define-x8664-vinsn mem-ref-double-float (((dest :double-float))
	3668	((src :address)
	3669	(index :s64)))
	3670	(movsd (:@ (:%q src) (:%q index)) (:%xmm dest)))
	3671
	3672	(define-x8664-vinsn mem-ref-single-float (((dest :single-float))
	3673	((src :address)
	3674	(index :s64)))
	3675	(movss (:@ (:%q src) (:%q index)) (:%xmm dest)))
	3676
	3677	(define-x8664-vinsn zero-extend-nargs (()
	3678	())
	3679	(movzwl (:%w x8664::nargs) (:%l x8664::nargs)))
	3680
	3681	(define-x8664-vinsn load-adl (()
	3682	((n :u32const)))
	3683	(movl (:$l n) (:%l x8664::nargs)))
	3684
	3685	(define-x8664-subprim-lea-jmp-vinsn (macro-bind) .SPmacro-bind)
	3686
	3687	(define-x8664-subprim-lea-jmp-vinsn (destructuring-bind-inner) .SPdestructuring-bind-inner)
	3688
	3689	(define-x8664-subprim-lea-jmp-vinsn (destructuring-bind) .SPdestructuring-bind)
	3690
	3691	(define-x8664-vinsn symbol-ref (((dest :lisp))
	3692	((src :lisp)
	3693	(cellno :u32const)))
	3694	(movq (:@ (:apply + (- x8664::node-size x8664::fulltag-symbol)
	3695	(:apply ash cellno 3))
	3696	(:%q src)) (:%q dest)))
	3697
	3698	(define-x8664-vinsn mem-ref-c-bit-fixnum (((dest :lisp))
	3699	((src :address)
	3700	(offset :s32const))
	3701	((temp :u32)))
	3702	((:pred = 0 (:apply ash offset -6))
	3703	(btq (:$ub (:apply logand 63 offset))
	3704	(:@ (:%q src))))
	3705	((:not (:pred = 0 (:apply ash offset -6)))
	3706	(btq (:$ub (:apply logand 63 offset))
	3707	(:@ (:apply ash (:apply ash offset -6) 3) (:%q src))))
	3708	(movl (:$l x8664::fixnumone) (:%l temp))
	3709	(leaq (:@ (- x8664::fixnumone) (:%q temp)) (:%q dest))
	3710	(cmovbl (:%l temp) (:%l dest)))
	3711
	3712	(define-x8664-vinsn mem-ref-c-bit (((dest :lisp))
	3713	((src :address)
	3714	(offset :s32const))
	3715	((temp :u32)))
	3716	((:pred = 0 (:apply ash offset -6))
	3717	(btq (:$ub (:apply logand 63 offset))
	3718	(:@ (:%q src))))
	3719	((:not (:pred = 0 (:apply ash offset -6)))
	3720	(btq (:$ub (:apply logand 63 offset))
	3721	(:@ (:apply ash (:apply ash offset -6) 3) (:%q src))))
	3722	(setb (:%b temp))
	3723	(movzbl (:%b temp) (:%l dest)))
	3724
	3725	(define-x8664-vinsn mem-ref-bit-fixnum (((dest :lisp)
	3726	(src :address))
	3727	((src :address)
	3728	(offset :lisp))
	3729	((temp :u32)))
	3730	(movq (:%q offset) (:%q temp))
	3731	(shrq (:$ub (+ 6 x8664::fixnumshift)) (:%q temp))
	3732	(leaq (:@ (:%q src) (:%q temp) 8) (:%q src))
	3733	(movq (:%q offset) (:%q temp))
	3734	(shrq (:$ub x8664::fixnumshift) (:%q temp))
	3735	(andl (:$l 63) (:%l temp))
	3736	(btq (:%q temp) (:@ (:%q src)))
	3737	(movl (:$l x8664::fixnumone) (:%l temp))
	3738	(leaq (:@ (- x8664::fixnumone) (:%q temp)) (:%q dest))
	3739	(cmovbl (:%l temp) (:%l dest)))
	3740
	3741	(define-x8664-vinsn mem-ref-bit (((dest :lisp)
	3742	(src :address))
	3743	((src :address)
	3744	(offset :lisp))
	3745	((temp :u32)))
	3746	(movq (:%q offset) (:%q temp))
	3747	(shrq (:$ub (+ 6 x8664::fixnumshift)) (:%q temp))
	3748	(leaq (:@ (:%q src) (:%q temp) 8) (:%q src))
	3749	(movq (:%q offset) (:%q temp))
	3750	(shrq (:$ub x8664::fixnumshift) (:%q temp))
	3751	(andl (:$l 63) (:%l temp))
	3752	(btq (:%q temp) (:@ (:%q src)))
	3753	(setb (:%b temp))
	3754	(movzbl (:%b temp) (:%l dest)))
	3755
	3756
	3757	(define-x8664-vinsn mem-set-c-bit-0 (()
	3758	((src :address)
	3759	(offset :s32const)))
	3760
	3761	((:pred = 0 (:apply ash offset -6))
	3762	(btrq (:$ub (:apply logand 63 offset))
	3763	(:@ (:%q src))))
	3764	((:not (:pred = 0 (:apply ash offset -6)))
	3765	(btrq (:$ub (:apply logand 63 offset))
	3766	(:@ (:apply ash (:apply ash offset -6) 3) (:%q src)))))
	3767
	3768	(define-x8664-vinsn mem-set-c-bit-1 (()
	3769	((src :address)
	3770	(offset :s32const)))
	3771
	3772	((:pred = 0 (:apply ash offset -6))
	3773	(btsq (:$ub (:apply logand 63 offset))
	3774	(:@ (:%q src))))
	3775	((:not (:pred = 0 (:apply ash offset -6)))
	3776	(btsq (:$ub (:apply logand 63 offset))
	3777	(:@ (:apply ash (:apply ash offset -6) 3) (:%q src)))))
	3778
	3779	(define-x8664-vinsn mem-set-c-bit-variable-value (()
	3780	((src :address)
	3781	(offset :s32const)
	3782	(value :lisp)))
	3783	(testq (:%q value) (:%q value))
	3784	(jne :set)
	3785	((:pred = 0 (:apply ash offset -6))
	3786	(btrq (:$ub (:apply logand 63 offset))
	3787	(:@ (:%q src))))
	3788	((:not (:pred = 0 (:apply ash offset -6)))
	3789	(btrq (:$ub (:apply logand 63 offset))
	3790	(:@ (:apply ash (:apply ash offset -6) 3) (:%q src))))
	3791	(jmp :done)
	3792	:set
	3793	((:pred = 0 (:apply ash offset -6))
	3794	(btsq (:$ub (:apply logand 63 offset))
	3795	(:@ (:%q src))))
	3796	((:not (:pred = 0 (:apply ash offset -6)))
	3797	(btsq (:$ub (:apply logand 63 offset))
	3798	(:@ (:apply ash (:apply ash offset -6) 3) (:%q src))))
	3799	:done)
	3800
	3801
	3802	(define-x8664-vinsn mem-set-bit-0 (((src :address))
	3803	((src :address)
	3804	(offset :lisp))
	3805	((temp :u32)))
	3806	(movq (:%q offset) (:%q temp))
	3807	(shrq (:$ub (+ 6 x8664::fixnumshift)) (:%q temp))
	3808	(leaq (:@ (:%q src) (:%q temp) 8) (:%q src))
	3809	(movq (:%q offset) (:%q temp))
	3810	(shrq (:$ub x8664::fixnumshift) (:%q temp))
	3811	(andl (:$l 63) (:%l temp))
	3812	(btrq (:%q temp) (:@ (:%q src))))
	3813
	3814	(define-x8664-vinsn mem-set-bit-1 (((src :address))
	3815	((src :address)
	3816	(offset :lisp))
	3817	((temp :u32)))
	3818	(movq (:%q offset) (:%q temp))
	3819	(shrq (:$ub (+ 6 x8664::fixnumshift)) (:%q temp))
	3820	(leaq (:@ (:%q src) (:%q temp) 8) (:%q src))
	3821	(movq (:%q offset) (:%q temp))
	3822	(shrq (:$ub x8664::fixnumshift) (:%q temp))
	3823	(andl (:$l 63) (:%l temp))
	3824	(btsq (:%q temp) (:@ (:%q src))))
	3825
	3826
	3827	(define-x8664-vinsn mem-set-bit-variable-value (((src :address))
	3828	((src :address)
	3829	(offset :lisp)
	3830	(value :lisp))
	3831	((temp :u32)))
	3832	(movq (:%q offset) (:%q temp))
	3833	(shrq (:$ub (+ 6 x8664::fixnumshift)) (:%q temp))
	3834	(leaq (:@ (:%q src) (:%q temp) 8) (:%q src))
	3835	(movq (:%q offset) (:%q temp))
	3836	(shrq (:$ub x8664::fixnumshift) (:%q temp))
	3837	(andl (:$l 63) (:%l temp))
	3838	(testq (:%q value) (:%q value))
	3839	(jne :set)
	3840	(btrq (:%q temp) (:@ (:%q src)))
	3841	(jmp :done)
	3842	:set
	3843	(btsq (:%q temp) (:@ (:%q src)))
	3844	:done)
	3845
	3846	(define-x8664-vinsn %natural+ (((result :u64))
	3847	((result :u64)
	3848	(other :u64)))
	3849	(addq (:%q other) (:%q result)))
	3850
	3851	(define-x8664-vinsn %natural+-c (((result :u64))
	3852	((result :u64)
	3853	(constant :s32const)))
	3854	(addq (:$l constant) (:%q result)))
	3855
	3856	(define-x8664-vinsn %natural- (((result :u64))
	3857	((result :u64)
	3858	(other :u64)))
	3859	(subq (:%q other) (:%q result)))
	3860
	3861	(define-x8664-vinsn %natural--c (((result :u64))
	3862	((result :u64)
	3863	(constant :s32const)))
	3864	(subq (:$l constant) (:%q result)))
	3865
	3866	(define-x8664-vinsn %natural-logior (((result :u64))
	3867	((result :u64)
	3868	(other :u64)))
	3869	(orq (:%q other) (:%q result)))
	3870
	3871	(define-x8664-vinsn %natural-logior-c (((result :u64))
	3872	((result :u64)
	3873	(constant :s32const)))
	3874	(orq (:$l constant) (:%q result)))
	3875
	3876	(define-x8664-vinsn %natural-logand (((result :u64))
	3877	((result :u64)
	3878	(other :u64)))
	3879	(andq (:%q other) (:%q result)))
	3880
	3881	(define-x8664-vinsn %natural-logand-c (((result :u64))
	3882	((result :u64)
	3883	(constant :s32const)))
	3884	(andq (:$l constant) (:%q result)))
	3885
	3886	(define-x8664-vinsn %natural-logxor (((result :u64))
	3887	((result :u64)
	3888	(other :u64)))
	3889	(xorq (:%q other) (:%q result)))
	3890
	3891	(define-x8664-vinsn %natural-logxor-c (((result :u64))
	3892	((result :u64)
	3893	(constant :s32const)))
	3894	(xorq (:$l constant) (:%q result)))
	3895
	3896	(define-x8664-vinsn natural-shift-left (((dest :u64))
	3897	((dest :u64)
	3898	(amt :u8const)))
	3899	(shlq (:$ub amt) (:%q dest)))
	3900
	3901	(define-x8664-vinsn natural-shift-right (((dest :u64))
	3902	((dest :u64)
	3903	(amt :u8const)))
	3904	(shrq (:$ub amt) (:%q dest)))
	3905
	3906	(define-x8664-vinsn trap-unless-simple-array-2 (()
	3907	((object :lisp)
	3908	(expected-flags :u32const)
	3909	(type-error :u8const))
	3910	((tag :u8)))
	3911
	3912	(movb (:%b object) (:%b tag))
	3913	(andb (:$b x8664::tagmask) (:%b tag))
	3914	(cmpb (:$b x8664::tag-misc) (:%b tag))
	3915	(jne :bad)
	3916	(cmpb (:$b x8664::subtag-arrayH) (:@ x8664::misc-subtag-offset (:%q object)))
	3917	(jne :bad)
	3918	(cmpq (:$b (ash 2 x8664::fixnumshift)) (:@ x8664::arrayH.rank (:%q object)))
	3919	(jne :bad)
	3920	(cmpq (:$l (:apply ash expected-flags x8664::fixnumshift)) (:@ x8664::arrayH.flags (:%q object)))
	3921	(je.pt :good)
	3922	:bad
	3923	(uuo-error-reg-not-type (:%q object) (:$ub type-error))
	3924	:good)
	3925
	3926	(define-x8664-vinsn trap-unless-simple-array-3 (()
	3927	((object :lisp)
	3928	(expected-flags :u32const)
	3929	(type-error :u8const))
	3930	((tag :u8)))
	3931
	3932	(movb (:%b object) (:%b tag))
	3933	(andb (:$b x8664::tagmask) (:%b tag))
	3934	(cmpb (:$b x8664::tag-misc) (:%b tag))
	3935	(jne :bad)
	3936	(cmpb (:$b x8664::subtag-arrayH) (:@ x8664::misc-subtag-offset (:%q object)))
	3937	(jne :bad)
	3938	(cmpq (:$b (ash 3 x8664::fixnumshift)) (:@ x8664::arrayH.rank (:%q object)))
	3939	(jne :bad)
	3940	(cmpq (:$l (:apply ash expected-flags x8664::fixnumshift)) (:@ x8664::arrayH.flags (:%q object)))
	3941	(je.pt :good)
	3942	:bad
	3943	(uuo-error-reg-not-type (:%q object) (:$ub type-error))
	3944	:good)
	3945
	3946	(define-x8664-vinsn trap-unless-array-header (()
	3947	((object :lisp))
	3948	((tag :u8)))
	3949	(movb (:%b object) (:%b tag))
	3950	(andb (:$b x8664::tagmask) (:%b tag))
	3951	(cmpb (:$b x8664::tag-misc) (:%b tag))
	3952	(jne :trap)
	3953	(cmpb (:$b x8664::subtag-arrayH) (:@ x8664::misc-subtag-offset (:%q object)))
	3954	(je :ok)
	3955	:trap
	3956	(uuo-error-reg-not-tag (:%q object) (:$ub x8664::subtag-arrayH))
	3957	:ok)
	3958
	3959	(define-x8664-vinsn check-arrayH-rank (()
	3960	((header :lisp)
	3961	(expected :u32const))
	3962	((rank :imm)))
	3963	(movl (:$l (:apply ash expected x8664::fixnumshift)) (:%l rank))
	3964	(cmpq (:@ x8664::arrayH.rank (:%q header)) (:%q rank))
	3965	(je.pt :ok)
	3966	(uuo-error-array-rank (:%q header) (:%q rank))
	3967	:ok)
	3968
	3969	(define-x8664-vinsn check-arrayH-flags (()
	3970	((header :lisp)
	3971	(expected :u32const)
	3972	(type-error :u8const)))
	3973	(cmpq (:$l (:apply ash expected x8664::fixnumshift))
	3974	(:@ x8664::arrayH.flags (:%q header)))
	3975	(je.pt :ok)
	3976	(uuo-error-reg-not-type (:%q header) (:$ub type-error))
	3977	:ok)
	3978
	3979	(define-x8664-vinsn misc-ref-c-u16 (((dest :u16))
	3980	((v :lisp)
	3981	(idx :u32const))
	3982	())
	3983	(movzwl (:@ (:apply + x8664::misc-data-offset (:apply ash idx 1)) (:%q v)) (:%l dest)))
	3984
	3985	(define-x8664-vinsn misc-ref-c-s16 (((dest :s16))
	3986	((v :lisp)
	3987	(idx :u32const))
	3988	())
	3989	(movswq (:@ (:apply + x8664::misc-data-offset (:apply ash idx 1)) (:%q v)) (:%q dest)))
	3990
	3991	(define-x8664-vinsn misc-set-single-float (()
	3992	((val :single-float)
	3993	(v :lisp)
	3994	(scaled-idx :u32)))
	3995	(movss (:%xmm val) (:@ x8664::misc-data-offset (:% v) (:% scaled-idx))))
	3996
	3997	(define-x8664-vinsn u16->u32 (((dest :u32))
	3998	((src :u16)))
	3999	(movzwl (:%w src) (:%l dest)))
	4000
	4001	(define-x8664-vinsn u8->u32 (((dest :u32))
	4002	((src :u8)))
	4003	(movzbl (:%b src) (:%l dest)))
	4004
	4005
	4006	(define-x8664-vinsn s16->s32 (((dest :s32))
	4007	((src :s16)))
	4008	(movswl (:%w src) (:%l dest)))
	4009
	4010	(define-x8664-vinsn s8->s32 (((dest :s32))
	4011	((src :s8)))
	4012	(movsbl (:%b src) (:%l dest)))
	4013
	4014	(define-x8664-subprim-jump-vinsn (tail-call-fn-gen) .SPtcallnfngen)
	4015
	4016	(define-x8664-subprim-jump-vinsn (tail-call-fn-vsp) .SPtcallnfnvsp)
	4017
	4018	(define-x8664-vinsn set-eq-bit (()
	4019	())
	4020	(testb (:%b x8664::arg_z) (:%b x8664::arg_z)))
	4021
	4022	(define-x8664-vinsn %schar8 (((char :imm))
	4023	((str :lisp)
	4024	(idx :imm))
	4025	((imm :u32)))
	4026	(movq (:%q idx) (:%q imm))
	4027	(shrq (:$ub x8664::fixnumshift) (:%q imm))
	4028	(movzbl (:@ x8664::misc-data-offset (:%q str) (:%q imm)) (:%l imm))
	4029	(shll (:$ub x8664::charcode-shift) (:%l imm))
	4030	(leaq (:@ x8664::subtag-character (:%q imm)) (:%q char)))
	4031
	4032	(define-x8664-vinsn %schar32 (((char :imm))
	4033	((str :lisp)
	4034	(idx :imm))
	4035	((imm :u32)))
	4036	(movq (:%q idx) (:%q imm))
	4037	(shrq (:$ub 1) (:%q imm))
	4038	(movl (:@ x8664::misc-data-offset (:%q str) (:%q imm)) (:%l imm))
	4039	(shll (:$ub x8664::charcode-shift) (:%l imm))
	4040	(leaq (:@ x8664::subtag-character (:%q imm)) (:%q char)))
	4041
	4042
	4043	(define-x8664-vinsn %set-schar8 (()
	4044	((str :lisp)
	4045	(idx :imm)
	4046	(char :imm))
	4047	((imm0 :u64)
	4048	(imm1 :u64)))
	4049	(movq (:%q idx) (:%q imm0))
	4050	(movl (:%l char) (:%l imm1))
	4051	(shrq (:$ub x8664::fixnumshift) (:%q imm0))
	4052	(shrl (:$ub x8664::charcode-shift) (:%l imm1))
	4053	(movb (:%b imm1) (:@ x8664::misc-data-offset (:%q str) (:%q imm0))))
	4054
	4055	(define-x8664-vinsn %set-schar32 (()
	4056	((str :lisp)
	4057	(idx :imm)
	4058	(char :imm))
	4059	((imm0 :u64)
	4060	(imm1 :u64)))
	4061	(movq (:%q idx) (:%q imm0))
	4062	(movl (:%l char) (:%l imm1))
	4063	(shrq (:$ub 1) (:%q imm0))
	4064	(shrl (:$ub x8664::charcode-shift) (:%l imm1))
	4065	(movl (:%l imm1) (:@ x8664::misc-data-offset (:%q str) (:%q imm0))))
	4066
	4067	(define-x8664-vinsn misc-set-c-single-float (((val :single-float))
	4068	((v :lisp)
	4069	(idx :u32const)))
	4070	(movsd (:%xmm val) (:@ (:apply + x8664::misc-data-offset (:apply ash idx 2))(:%q v))))
	4071
	4072	(define-x8664-vinsn array-data-vector-ref (((dest :lisp))
	4073	((header :lisp)))
	4074	(movq (:@ x8664::arrayH.data-vector (:%q header)) (:%q dest)))
	4075
	4076	(define-x8664-subprim-call-vinsn (subtag-misc-ref) .SPsubtag-misc-ref)
	4077
	4078	(define-x8664-subprim-call-vinsn (subtag-misc-set) .SPsubtag-misc-set)
	4079
	4080	(define-x8664-vinsn mem-ref-c-absolute-u8 (((dest :u8))
	4081	((addr :s32const)))
	4082	(movzbl (:@ addr) (:%l dest)))
	4083
	4084	(define-x8664-vinsn mem-ref-c-absolute-s8 (((dest :s8))
	4085	((addr :s32const)))
	4086	(movsbq (:@ addr) (:%q dest)))
	4087
	4088	(define-x8664-vinsn mem-ref-c-absolute-u16 (((dest :u16))
	4089	((addr :s32const)))
	4090	(movzwl (:@ addr) (:%l dest)))
	4091
	4092	(define-x8664-vinsn mem-ref-c-absolute-s16 (((dest :s16))
	4093	((addr :s32const)))
	4094	(movswq (:@ addr) (:%q dest)))
	4095
	4096	(define-x8664-vinsn mem-ref-c-absolute-fullword (((dest :u32))
	4097	((addr :s32const)))
	4098	(movl (:@ addr) (:%l dest)))
	4099
	4100	(define-x8664-vinsn mem-ref-c-absolute-signed-fullword (((dest :s32))
	4101	((addr :s32const)))
	4102	(movslq (:@ addr) (:%q dest)))
	4103
	4104	(define-x8664-vinsn mem-ref-c-absolute-doubleword (((dest :s64))
	4105	((addr :s32const)))
	4106	(movq (:@ addr) (:%q dest)))
	4107
	4108	(define-x8664-vinsn mem-ref-c-absolute-signed-doubleword (((dest :s64))
	4109	((addr :s32const)))
	4110	(movq (:@ addr) (:%q dest)))
	4111
	4112	(define-x8664-vinsn mem-ref-c-absolute-natural (((dest :u64))
	4113	((addr :s32const)))
	4114	(movq (:@ addr) (:%q dest)))
	4115
	4116	(define-x8664-vinsn event-poll (()
	4117	())
	4118	(btrq (:$ub 63) (:@ (:%seg :rcontext) x8664::tcr.interrupt-pending))
	4119	(jae :no-interrupt)
	4120	(ud2a)
	4121	(:byte 2)
	4122	:no-interrupt)
	4123
	4124	;;; Return dim1 (unboxed)
	4125	(define-x8664-vinsn check-2d-bound (((dim :u64))
	4126	((i :imm)
	4127	(j :imm)
	4128	(header :lisp)))
	4129	(cmpq (:@ (+ x8664::misc-data-offset (* 8 x8664::arrayH.dim0-cell)) (:%q header)) (:%q i))
	4130	(jb :i-ok)
	4131	(uuo-error-array-bounds (:%q i) (:%q header))
	4132	:i-ok
	4133	(movq (:@ (+ x8664::misc-data-offset (* 8 (1+ x8664::arrayH.dim0-cell))) (:%q header))
	4134	(:%q dim))
	4135	(cmpq (:%q dim) (:%q j))
	4136	(jb :j-ok)
	4137	(uuo-error-array-bounds (:%q j) (:%q header))
	4138	:j-ok
	4139	(sarq (:$ub x8664::fixnumshift) (:%q dim)))
	4140
	4141	;;; Return dim1, dim2 (unboxed)
	4142	(define-x8664-vinsn check-3d-bound (((dim1 :u64)
	4143	(dim2 :u64))
	4144	((i :imm)
	4145	(j :imm)
	4146	(k :imm)
	4147	(header :lisp)))
	4148	(cmpq (:@ (+ x8664::misc-data-offset (* 8 x8664::arrayH.dim0-cell)) (:%q header)) (:%q i))
	4149	(jb :i-ok)
	4150	(uuo-error-array-bounds (:%q i) (:%q header))
	4151	:i-ok
	4152	(movq (:@ (+ x8664::misc-data-offset (* 8 (1+ x8664::arrayH.dim0-cell))) (:%q header)) (:%q dim1))
	4153	(cmpq (:%q dim1) (:%q j))
	4154	(jb :j-ok)
	4155	(uuo-error-array-bounds (:%q j) (:%q header))
	4156	:j-ok
	4157	(sarq (:$ub x8664::fixnumshift) (:%q dim1))
	4158	(movq (:@ (+ x8664::misc-data-offset (* 8 (+ 2 x8664::arrayH.dim0-cell))) (:%q header)) (:%q dim2))
	4159	(cmpq (:%q dim2) (:%q k))
	4160	(jb ::k-ok)
	4161	(uuo-error-array-bounds (:%q k) (:%q header))
	4162	:k-ok
	4163	(sarq (:$ub x8664::fixnumshift) (:%q dim2)))
	4164
	4165
	4166	(define-x8664-vinsn 2d-dim1 (((dest :u64))
	4167	((header :lisp)))
	4168	(movq (:@ (+ x8664::misc-data-offset (* 8 (1+ x8664::arrayH.dim0-cell))) (:%q header))
	4169	(:%q dest))
	4170	(sarq (:$ub x8664::fixnumshift) (:%q dest)))
	4171
	4172
	4173	(define-x8664-vinsn 3d-dims (((dim1 :u64)
	4174	(dim2 :u64))
	4175	((header :lisp)))
	4176	(movq (:@ (+ x8664::misc-data-offset (* 8 (1+ x8664::arrayH.dim0-cell))) (:%q header)) (:%q dim1))
	4177	(movq (:@ (+ x8664::misc-data-offset (* 8 (+ 2 x8664::arrayH.dim0-cell))) (:%q header)) (:%q dim2))
	4178	(sarq (:$ub x8664::fixnumshift) (:%q dim1))
	4179	(sarq (:$ub x8664::fixnumshift) (:%q dim2)))
	4180
	4181	(define-x8664-vinsn 2d-unscaled-index (((dest :imm)
	4182	(dim1 :u64))
	4183	((dim1 :u64)
	4184	(i :imm)
	4185	(j :imm)))
	4186
	4187	(imulq (:%q i) (:%q dim1))
	4188	(leaq (:@ (:%q j) (:%q dim1)) (:%q dest)))
	4189
	4190
	4191	;; dest <- (+ (* i dim1 dim2) (* j dim2) k)
	4192	(define-x8664-vinsn 3d-unscaled-index (((dest :imm)
	4193	(dim1 :u64)
	4194	(dim2 :u64))
	4195	((dim1 :u64)
	4196	(dim2 :u64)
	4197	(i :imm)
	4198	(j :imm)
	4199	(k :imm)))
	4200	(imulq (:%q dim2) (:%q dim1))
	4201	(imulq (:%q j) (:%q dim2))
	4202	(imulq (:%q i) (:%q dim1))
	4203	(addq (:%q dim1) (:%q dim2))
	4204	(leaq (:@ (:%q k) (:%q dim2)) (:%q dest)))
	4205
	4206	(define-x8664-vinsn branch-unless-both-args-fixnums (()
	4207	((a :lisp)
	4208	(b :lisp)
	4209	(dest :label))
	4210	((tag :u8)))
	4211	(movb (:%b a) (:%b tag))
	4212	(orb (:%b b) (:%b tag))
	4213	(testb (:$b x8664::fixnummask) (:%b tag))
	4214	(jne dest))
	4215
	4216	(define-x8664-vinsn branch-unless-arg-fixnum (()
	4217	((a :lisp)
	4218	(dest :label)))
	4219	(testb (:$b x8664::fixnummask) (:%b a))
	4220	(jne dest))
	4221
	4222	(define-x8664-vinsn fixnum->single-float (((f :single-float))
	4223	((arg :lisp))
	4224	((unboxed :s64)))
	4225	(movq (:%q arg) (:%q unboxed))
	4226	(sarq (:$ub x8664::fixnumshift) (:%q unboxed))
	4227	(cvtsi2ssq (:%q unboxed) (:%xmm f)))
	4228
	4229	(define-x8664-vinsn fixnum->double-float (((f :double-float))
	4230	((arg :lisp))
	4231	((unboxed :s64)))
	4232	(movq (:%q arg) (:%q unboxed))
	4233	(sarq (:$ub x8664::fixnumshift) (:%q unboxed))
	4234	(cvtsi2sdq (:%q unboxed) (:%xmm f)))
	4235
	4236
	4237	(define-x8664-vinsn xchg-registers (()
	4238	((a t)
	4239	(b t)))
	4240	(xchgq (:%q a) (:%q b)))
	4241
	4242	(define-x8664-vinsn establish-fn (()
	4243	()
	4244	((entry (:label 1))))
	4245	(leaq (:@ (:^ entry) (:% x8664::rip)) (:%q x8664::fn)))
	4246
	4247	(queue-fixup
	4248	(fixup-x86-vinsn-templates
	4249	x8664-vinsn-templates
	4250	x86::x86-64-opcode-template-lists))
	4251
	4252	(provide "X8664-VINSNS")
	4253

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: branches/working-0710/ccl/compiler/X86/X8664/x8664-vinsns.lisp

Download in other formats: