Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Specialize binary ops for floats #484

Draft
wants to merge 6 commits into
base: trunk
Choose a base branch
from
Draft

Specialize binary ops for floats #484

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7476954
Add BINARY_{ADD,SUB,MUL,POWER}_FLOAT specializations in interpreter
tekknolagi May 14, 2023
8a58ed1
Add asm interpreter implementation of DUP_TOP_TWO
tekknolagi May 14, 2023
9f33d6a
Add asm interpreter implementation of ROT_THREE
tekknolagi May 14, 2023
e4f84b6
Add INPLACE_{ADD,SUB}_FLOAT specializations in interpreter
tekknolagi May 14, 2023
ba5778a
Add asm interpreter implementation of BINARY_MUL_FLOAT
tekknolagi May 14, 2023
bb5cff0
Add the other ops; they are very similar
tekknolagi May 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions runtime/bytecode.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,12 +59,12 @@ namespace py {
V(UNUSED_BYTECODE_39, 39, doInvalidBytecode) \
V(UNUSED_BYTECODE_40, 40, doInvalidBytecode) \
V(UNUSED_BYTECODE_41, 41, doInvalidBytecode) \
V(UNUSED_BYTECODE_42, 42, doInvalidBytecode) \
V(UNUSED_BYTECODE_43, 43, doInvalidBytecode) \
V(UNUSED_BYTECODE_44, 44, doInvalidBytecode) \
V(UNUSED_BYTECODE_45, 45, doInvalidBytecode) \
V(UNUSED_BYTECODE_46, 46, doInvalidBytecode) \
V(UNUSED_BYTECODE_47, 47, doInvalidBytecode) \
V(INPLACE_ADD_FLOAT, 42, doInplaceAddFloat) \
V(INPLACE_SUB_FLOAT, 43, doInplaceSubFloat) \
V(BINARY_ADD_FLOAT, 44, doBinaryAddFloat) \
V(BINARY_SUB_FLOAT, 45, doBinarySubFloat) \
V(BINARY_MUL_FLOAT, 46, doBinaryMulFloat) \
V(BINARY_POWER_FLOAT, 47, doBinaryPowerFloat) \
V(LOAD_BOOL, 48, doLoadBool) \
V(UNUSED_BYTECODE_49, 49, doInvalidBytecode) \
V(GET_AITER, 50, doGetAiter) \
Expand Down
109 changes: 106 additions & 3 deletions runtime/interpreter-gen-x64.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -635,8 +635,9 @@ void emitJumpIfNotHasLayoutId(EmitEnv* env, Register r_obj, LayoutId layout_id,
}

void emitJumpIfNotHeapObjectWithLayoutId(EmitEnv* env, Register r_obj,
LayoutId layout_id, Label* target) {
emitJumpIfImmediate(env, r_obj, target, Assembler::kNearJump);
LayoutId layout_id, Label* target,
bool is_near = Assembler::kNearJump) {
emitJumpIfImmediate(env, r_obj, target, is_near);

// It is a HeapObject.
emitJumpIfNotHasLayoutId(env, r_obj, layout_id, target);
Expand Down Expand Up @@ -733,6 +734,32 @@ void emitPushBoundMethod(EmitEnv* env, Label* slow_path, Register r_self,
__ pushq(r_scratch);
}

// Allocate and push a Float on the stack. If the heap is full and a GC
// is needed, jump to slow_path instead. r_value will be used to populate the
// Float. r_space and r_scratch are used as scratch registers.
//
// Writes to r_space and r_scratch.
void emitPushFloat(EmitEnv* env, Label* slow_path, XmmRegister r_value) {
ScratchReg r_scratch(env);
ScratchReg r_space(env);
__ movq(r_space, Address(env->thread, Thread::runtimeOffset()));
__ movq(r_space,
Address(r_space, Runtime::heapOffset() + Heap::spaceOffset()));

__ movq(r_scratch, Address(r_space, Space::fillOffset()));
__ addq(r_scratch, Immediate(Float::allocationSize()));
__ cmpq(r_scratch, Address(r_space, Space::endOffset()));
__ jcc(GREATER, slow_path, Assembler::kFarJump);
__ xchgq(r_scratch, Address(r_space, Space::fillOffset()));
RawHeader header = Header::from(Float::kSize, /*hash=*/0, LayoutId::kFloat,
ObjectFormat::kData);
__ movq(Address(r_scratch, 0), Immediate(header.raw()));
__ leaq(r_scratch, Address(r_scratch, -RawFloat::kHeaderOffset +
Object::kHeapObjectTag));
__ movsd(Address(r_scratch, heapObjectDisp(RawFloat::kValueOffset)), r_value);
__ pushq(r_scratch);
}

// Given a RawObject in r_obj and its LayoutId (as a SmallInt) in r_layout_id,
// load its overflow RawTuple into r_dst.
//
Expand Down Expand Up @@ -824,6 +851,60 @@ void emitHandler<BINARY_ADD_SMALLINT>(EmitEnv* env) {
emitHandleContinue(env, kGenericHandler);
}

static void emitBinaryOpFloat(EmitEnv* env,
void (Assembler::*asm_op)(XmmRegister left,
XmmRegister right)) {
ScratchReg r_right(env);
ScratchReg r_left(env);
Label slow_path;

__ popq(r_right);
__ popq(r_left);
emitJumpIfNotHeapObjectWithLayoutId(env, r_left, LayoutId::kFloat, &slow_path,
Assembler::kFarJump);
emitJumpIfNotHeapObjectWithLayoutId(env, r_right, LayoutId::kFloat,
&slow_path, Assembler::kNearJump);
__ movsd(XMM0, Address(r_left, heapObjectDisp(Float::kValueOffset)));
__ movsd(XMM1, Address(r_right, heapObjectDisp(Float::kValueOffset)));
(env->as.*asm_op)(XMM0, XMM1);
emitPushFloat(env, &slow_path, XMM0);
emitNextOpcode(env);

__ bind(&slow_path);
__ pushq(r_left);
__ pushq(r_right);
if (env->in_jit) {
emitJumpToDeopt(env);
return;
}
emitJumpToGenericHandler(env);
}

template <>
void emitHandler<BINARY_MUL_FLOAT>(EmitEnv* env) {
emitBinaryOpFloat(env, &Assembler::mulsd);
}

template <>
void emitHandler<BINARY_ADD_FLOAT>(EmitEnv* env) {
emitBinaryOpFloat(env, &Assembler::addsd);
}

template <>
void emitHandler<INPLACE_ADD_FLOAT>(EmitEnv* env) {
emitBinaryOpFloat(env, &Assembler::addsd);
}

template <>
void emitHandler<BINARY_SUB_FLOAT>(EmitEnv* env) {
emitBinaryOpFloat(env, &Assembler::subsd);
}

template <>
void emitHandler<INPLACE_SUB_FLOAT>(EmitEnv* env) {
emitBinaryOpFloat(env, &Assembler::subsd);
}

template <>
void emitHandler<BINARY_AND_SMALLINT>(EmitEnv* env) {
ScratchReg r_right(env);
Expand Down Expand Up @@ -2445,13 +2526,35 @@ void emitHandler<DUP_TOP>(EmitEnv* env) {
emitNextOpcodeFallthrough(env);
}

template <>
void emitHandler<DUP_TOP_TWO>(EmitEnv* env) {
__ pushq(Address(RSP, kPointerSize));
__ pushq(Address(RSP, kPointerSize));
emitNextOpcodeFallthrough(env);
}

template <>
void emitHandler<ROT_TWO>(EmitEnv* env) {
ScratchReg r_scratch(env);

__ popq(r_scratch);
__ pushq(Address(RSP, 0));
__ movq(Address(RSP, 8), r_scratch);
__ movq(Address(RSP, kPointerSize), r_scratch);
emitNextOpcodeFallthrough(env);
}

template <>
void emitHandler<ROT_THREE>(EmitEnv* env) {
ScratchReg r_top(env);
ScratchReg r_second(env);
ScratchReg r_third(env);

__ popq(r_top);
__ popq(r_second);
__ popq(r_third);
__ pushq(r_top);
__ pushq(r_third);
__ pushq(r_second);
emitNextOpcodeFallthrough(env);
}

Expand Down
141 changes: 141 additions & 0 deletions runtime/interpreter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Copyright (c) Facebook, Inc. and its affiliates. (http://www.facebook.com)
#include "interpreter.h"

#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <sstream>
Expand Down Expand Up @@ -5812,6 +5813,23 @@ Continue Interpreter::doInplaceAddSmallInt(Thread* thread, word arg) {
return inplaceOpUpdateCache(thread, arg, cache);
}

HANDLER_INLINE
Continue Interpreter::doInplaceAddFloat(Thread* thread, word arg) {
RawObject left = thread->stackPeek(1);
RawObject right = thread->stackPeek(0);
if (left.isFloat() && right.isFloat()) {
double left_value = Float::cast(left).value();
double right_value = Float::cast(right).value();
double result_value = left_value + right_value;
thread->stackDrop(1);
thread->stackSetTop(thread->runtime()->newFloat(result_value));
return Continue::NEXT;
}
EVENT_CACHE(INPLACE_ADD_FLOAT);
word cache = currentCacheIndex(thread->currentFrame());
return inplaceOpUpdateCache(thread, arg, cache);
}

HANDLER_INLINE
Continue Interpreter::doInplaceSubSmallInt(Thread* thread, word arg) {
RawObject left = thread->stackPeek(1);
Expand All @@ -5831,6 +5849,23 @@ Continue Interpreter::doInplaceSubSmallInt(Thread* thread, word arg) {
return inplaceOpUpdateCache(thread, arg, cache);
}

HANDLER_INLINE
Continue Interpreter::doInplaceSubFloat(Thread* thread, word arg) {
RawObject left = thread->stackPeek(1);
RawObject right = thread->stackPeek(0);
if (left.isFloat() && right.isFloat()) {
double left_value = Float::cast(left).value();
double right_value = Float::cast(right).value();
double result_value = left_value - right_value;
thread->stackDrop(1);
thread->stackSetTop(thread->runtime()->newFloat(result_value));
return Continue::NEXT;
}
EVENT_CACHE(INPLACE_SUB_FLOAT);
word cache = currentCacheIndex(thread->currentFrame());
return inplaceOpUpdateCache(thread, arg, cache);
}

HANDLER_INLINE
Continue Interpreter::doInplaceOpAnamorphic(Thread* thread, word arg) {
Frame* frame = thread->currentFrame();
Expand All @@ -5848,6 +5883,20 @@ Continue Interpreter::doInplaceOpAnamorphic(Thread* thread, word arg) {
}
}
}
if (thread->stackPeek(0).isFloat() && thread->stackPeek(1).isFloat()) {
switch (static_cast<BinaryOp>(arg)) {
case BinaryOp::ADD:
rewriteCurrentBytecode(frame, INPLACE_ADD_FLOAT);
return doInplaceAddFloat(thread, arg);
case BinaryOp::SUB:
rewriteCurrentBytecode(frame, INPLACE_SUB_FLOAT);
return doInplaceSubFloat(thread, arg);
default: {
word cache = currentCacheIndex(frame);
return inplaceOpUpdateCache(thread, arg, cache);
}
}
}
word cache = currentCacheIndex(frame);
return inplaceOpUpdateCache(thread, arg, cache);
}
Expand Down Expand Up @@ -5974,6 +6023,23 @@ Continue Interpreter::doBinaryAddSmallInt(Thread* thread, word arg) {
return binaryOpUpdateCache(thread, arg, cache);
}

HANDLER_INLINE
Continue Interpreter::doBinaryAddFloat(Thread* thread, word arg) {
RawObject left = thread->stackPeek(1);
RawObject right = thread->stackPeek(0);
if (left.isFloat() && right.isFloat()) {
double left_value = Float::cast(left).value();
double right_value = Float::cast(right).value();
double result_value = left_value + right_value;
thread->stackDrop(1);
thread->stackSetTop(thread->runtime()->newFloat(result_value));
return Continue::NEXT;
}
EVENT_CACHE(BINARY_ADD_FLOAT);
word cache = currentCacheIndex(thread->currentFrame());
return binaryOpUpdateCache(thread, arg, cache);
}

HANDLER_INLINE
Continue Interpreter::doBinaryAndSmallInt(Thread* thread, word arg) {
RawObject left = thread->stackPeek(1);
Expand Down Expand Up @@ -6011,6 +6077,23 @@ Continue Interpreter::doBinaryMulSmallInt(Thread* thread, word arg) {
return binaryOpUpdateCache(thread, arg, cache);
}

HANDLER_INLINE
Continue Interpreter::doBinaryMulFloat(Thread* thread, word arg) {
RawObject left = thread->stackPeek(1);
RawObject right = thread->stackPeek(0);
if (left.isFloat() && right.isFloat()) {
double left_value = Float::cast(left).value();
double right_value = Float::cast(right).value();
double result_value = left_value * right_value;
thread->stackDrop(1);
thread->stackSetTop(thread->runtime()->newFloat(result_value));
return Continue::NEXT;
}
EVENT_CACHE(BINARY_MUL_FLOAT);
word cache = currentCacheIndex(thread->currentFrame());
return binaryOpUpdateCache(thread, arg, cache);
}

HANDLER_INLINE
Continue Interpreter::doBinaryFloordivSmallInt(Thread* thread, word arg) {
RawObject left = thread->stackPeek(1);
Expand Down Expand Up @@ -6053,6 +6136,23 @@ Continue Interpreter::doBinarySubSmallInt(Thread* thread, word arg) {
return binaryOpUpdateCache(thread, arg, cache);
}

HANDLER_INLINE
Continue Interpreter::doBinarySubFloat(Thread* thread, word arg) {
RawObject left = thread->stackPeek(1);
RawObject right = thread->stackPeek(0);
if (left.isFloat() && right.isFloat()) {
double left_value = Float::cast(left).value();
double right_value = Float::cast(right).value();
double result_value = left_value - right_value;
thread->stackDrop(1);
thread->stackSetTop(thread->runtime()->newFloat(result_value));
return Continue::NEXT;
}
EVENT_CACHE(BINARY_SUB_FLOAT);
word cache = currentCacheIndex(thread->currentFrame());
return binaryOpUpdateCache(thread, arg, cache);
}

HANDLER_INLINE
Continue Interpreter::doBinaryOrSmallInt(Thread* thread, word arg) {
RawObject left = thread->stackPeek(1);
Expand All @@ -6071,6 +6171,23 @@ Continue Interpreter::doBinaryOrSmallInt(Thread* thread, word arg) {
return binaryOpUpdateCache(thread, arg, cache);
}

HANDLER_INLINE
Continue Interpreter::doBinaryPowerFloat(Thread* thread, word arg) {
RawObject left = thread->stackPeek(1);
RawObject right = thread->stackPeek(0);
if (left.isFloat() && right.isFloat()) {
double left_value = Float::cast(left).value();
double right_value = Float::cast(right).value();
double result_value = std::pow(left_value, right_value);
thread->stackDrop(1);
thread->stackSetTop(thread->runtime()->newFloat(result_value));
return Continue::NEXT;
}
EVENT_CACHE(BINARY_POWER_FLOAT);
word cache = currentCacheIndex(thread->currentFrame());
return binaryOpUpdateCache(thread, arg, cache);
}

HANDLER_INLINE
Continue Interpreter::doBinaryOpAnamorphic(Thread* thread, word arg) {
Frame* frame = thread->currentFrame();
Expand Down Expand Up @@ -6100,6 +6217,30 @@ Continue Interpreter::doBinaryOpAnamorphic(Thread* thread, word arg) {
}
}
}
if (thread->stackPeek(0).isFloat() && thread->stackPeek(1).isFloat()) {
switch (static_cast<BinaryOp>(arg)) {
case BinaryOp::ADD:
rewriteCurrentBytecode(frame, BINARY_ADD_FLOAT);
return doBinaryAddFloat(thread, arg);
break;
case BinaryOp::SUB:
rewriteCurrentBytecode(frame, BINARY_SUB_FLOAT);
return doBinarySubFloat(thread, arg);
break;
case BinaryOp::MUL:
rewriteCurrentBytecode(frame, BINARY_MUL_FLOAT);
return doBinaryMulFloat(thread, arg);
break;
case BinaryOp::POW:
rewriteCurrentBytecode(frame, BINARY_POWER_FLOAT);
return doBinaryPowerFloat(thread, arg);
break;
default: {
word cache = currentCacheIndex(frame);
return binaryOpUpdateCache(thread, arg, cache);
}
}
}
word cache = currentCacheIndex(frame);
return binaryOpUpdateCache(thread, arg, cache);
}
Expand Down
6 changes: 6 additions & 0 deletions runtime/interpreter.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,10 @@ class Interpreter {
static Continue doBinarySubSmallInt(Thread* thread, word arg);
static Continue doBinaryOrSmallInt(Thread* thread, word arg);
static Continue doBinaryOpAnamorphic(Thread* thread, word arg);
static Continue doBinaryAddFloat(Thread* thread, word arg);
static Continue doBinarySubFloat(Thread* thread, word arg);
static Continue doBinaryMulFloat(Thread* thread, word arg);
static Continue doBinaryPowerFloat(Thread* thread, word arg);
static Continue doBinaryOr(Thread* thread, word arg);
static Continue doBinaryPower(Thread* thread, word arg);
static Continue doBinaryRshift(Thread* thread, word arg);
Expand Down Expand Up @@ -433,6 +437,7 @@ class Interpreter {
static Continue doImportName(Thread* thread, word arg);
static Continue doInplaceAdd(Thread* thread, word arg);
static Continue doInplaceAddSmallInt(Thread* thread, word arg);
static Continue doInplaceAddFloat(Thread* thread, word arg);
static Continue doInplaceAnd(Thread* thread, word arg);
static Continue doInplaceFloorDivide(Thread* thread, word arg);
static Continue doInplaceLshift(Thread* thread, word arg);
Expand All @@ -447,6 +452,7 @@ class Interpreter {
static Continue doInplaceRshift(Thread* thread, word arg);
static Continue doInplaceSubtract(Thread* thread, word arg);
static Continue doInplaceSubSmallInt(Thread* thread, word arg);
static Continue doInplaceSubFloat(Thread* thread, word arg);
static Continue doInplaceTrueDivide(Thread* thread, word arg);
static Continue doInplaceXor(Thread* thread, word arg);
static Continue doInvalidBytecode(Thread* thread, word arg);
Expand Down