/* * Copyright (C) 2016-2022 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "config.h" #include "WasmB3IRGenerator.h" #if ENABLE(WEBASSEMBLY_B3JIT) #include "AirCode.h" #include "AllowMacroScratchRegisterUsageIf.h" #include "B3BasicBlockInlines.h" #include "B3CCallValue.h" #include "B3ConstPtrValue.h" #include "B3EstimateStaticExecutionCounts.h" #include "B3FixSSA.h" #include "B3Generate.h" #include "B3InsertionSet.h" #include "B3StackmapGenerationParams.h" #include "B3SwitchValue.h" #include "B3UpsilonValue.h" #include "B3Validate.h" #include "B3ValueInlines.h" #include "B3ValueKey.h" #include "B3Variable.h" #include "B3VariableValue.h" #include "B3WasmAddressValue.h" #include "B3WasmBoundsCheckValue.h" #include "FunctionAllowlist.h" #include "JSCJSValueInlines.h" #include "JSWebAssemblyInstance.h" #include "ProbeContext.h" #include "ScratchRegisterAllocator.h" #include "WasmBranchHints.h" #include "WasmCallingConvention.h" #include "WasmContextInlines.h" #include "WasmExceptionType.h" #include "WasmFunctionParser.h" #include "WasmIRGeneratorHelpers.h" #include "WasmInstance.h" #include "WasmMemory.h" #include "WasmOSREntryData.h" #include "WasmOpcodeOrigin.h" #include "WasmOperations.h" #include "WasmThunks.h" #include "WasmTypeDefinitionInlines.h" #include #include #if !ENABLE(WEBASSEMBLY) #error ENABLE(WEBASSEMBLY_B3JIT) is enabled, but ENABLE(WEBASSEMBLY) is not. #endif void dumpProcedure(void* ptr) { JSC::B3::Procedure* proc = static_cast(ptr); proc->dump(WTF::dataFile()); } namespace JSC { namespace Wasm { using namespace B3; namespace { namespace WasmB3IRGeneratorInternal { static constexpr bool verbose = false; } } class B3IRGenerator { public: using ExpressionType = Variable*; using ResultList = Vector; struct ControlData { ControlData(Procedure& proc, Origin origin, BlockSignature signature, BlockType type, unsigned stackSize, BasicBlock* continuation, BasicBlock* special = nullptr) : controlBlockType(type) , m_signature(signature) , m_stackSize(stackSize) , continuation(continuation) , special(special) { ASSERT(type != BlockType::Try && type != BlockType::Catch); if (type != BlockType::TopLevel) m_stackSize -= signature->as()->argumentCount(); if (type == BlockType::Loop) { for (unsigned i = 0; i < signature->as()->argumentCount(); ++i) phis.append(proc.add(Phi, toB3Type(signature->as()->argumentType(i)), origin)); } else { for (unsigned i = 0; i < signature->as()->returnCount(); ++i) phis.append(proc.add(Phi, toB3Type(signature->as()->returnType(i)), origin)); } } ControlData(Procedure& proc, Origin origin, BlockSignature signature, BlockType type, unsigned stackSize, BasicBlock* continuation, unsigned tryStart, unsigned tryDepth) : controlBlockType(type) , m_signature(signature) , m_stackSize(stackSize) , continuation(continuation) , special(nullptr) , m_tryStart(tryStart) , m_tryCatchDepth(tryDepth) { for (unsigned i = 0; i < signature->as()->returnCount(); ++i) phis.append(proc.add(Phi, toB3Type(signature->as()->returnType(i)), origin)); } ControlData() { } static bool isIf(const ControlData& control) { return control.blockType() == BlockType::If; } static bool isTry(const ControlData& control) { return control.blockType() == BlockType::Try; } static bool isAnyCatch(const ControlData& control) { return control.blockType() == BlockType::Catch; } static bool isTopLevel(const ControlData& control) { return control.blockType() == BlockType::TopLevel; } static bool isLoop(const ControlData& control) { return control.blockType() == BlockType::Loop; } static bool isBlock(const ControlData& control) { return control.blockType() == BlockType::Block; } static bool isCatch(const ControlData& control) { if (control.blockType() != BlockType::Catch) return false; return control.catchKind() == CatchKind::Catch; } void dump(PrintStream& out) const { switch (blockType()) { case BlockType::If: out.print("If: "); break; case BlockType::Block: out.print("Block: "); break; case BlockType::Loop: out.print("Loop: "); break; case BlockType::TopLevel: out.print("TopLevel: "); break; case BlockType::Try: out.print("Try: "); break; case BlockType::Catch: out.print("Catch: "); break; } out.print("Continuation: ", *continuation, ", Special: "); if (special) out.print(*special); else out.print("None"); } BlockType blockType() const { return controlBlockType; } BlockSignature signature() const { return m_signature; } bool hasNonVoidresult() const { return m_signature->as()->returnsVoid(); } BasicBlock* targetBlockForBranch() { if (blockType() == BlockType::Loop) return special; return continuation; } void convertIfToBlock() { ASSERT(blockType() == BlockType::If); controlBlockType = BlockType::Block; special = nullptr; } void convertTryToCatch(unsigned tryEndCallSiteIndex, Variable* exception) { ASSERT(blockType() == BlockType::Try); controlBlockType = BlockType::Catch; m_catchKind = CatchKind::Catch; m_tryEnd = tryEndCallSiteIndex; m_exception = exception; } void convertTryToCatchAll(unsigned tryEndCallSiteIndex, Variable* exception) { ASSERT(blockType() == BlockType::Try); controlBlockType = BlockType::Catch; m_catchKind = CatchKind::CatchAll; m_tryEnd = tryEndCallSiteIndex; m_exception = exception; } FunctionArgCount branchTargetArity() const { if (blockType() == BlockType::Loop) return m_signature->as()->argumentCount(); return m_signature->as()->returnCount(); } Type branchTargetType(unsigned i) const { ASSERT(i < branchTargetArity()); if (blockType() == BlockType::Loop) return m_signature->as()->argumentType(i); return m_signature->as()->returnType(i); } unsigned tryStart() const { ASSERT(controlBlockType == BlockType::Try || controlBlockType == BlockType::Catch); return m_tryStart; } unsigned tryEnd() const { ASSERT(controlBlockType == BlockType::Catch); return m_tryEnd; } unsigned tryDepth() const { ASSERT(controlBlockType == BlockType::Try || controlBlockType == BlockType::Catch); return m_tryCatchDepth; } CatchKind catchKind() const { ASSERT(controlBlockType == BlockType::Catch); return m_catchKind; } Variable* exception() const { ASSERT(controlBlockType == BlockType::Catch); return m_exception; } unsigned stackSize() const { return m_stackSize; } private: // FIXME: Compress B3IRGenerator::ControlData fields using an union // https://bugs.webkit.org/show_bug.cgi?id=231212 friend class B3IRGenerator; BlockType controlBlockType; BlockSignature m_signature; unsigned m_stackSize; BasicBlock* continuation; BasicBlock* special; Vector phis; unsigned m_tryStart; unsigned m_tryEnd; unsigned m_tryCatchDepth; CatchKind m_catchKind; Variable* m_exception; }; using ControlType = ControlData; using ExpressionList = Vector; using ControlEntry = FunctionParser::ControlEntry; using ControlStack = FunctionParser::ControlStack; using Stack = FunctionParser::Stack; using TypedExpression = FunctionParser::TypedExpression; static_assert(std::is_same_v::ResultList>); typedef String ErrorType; typedef Unexpected UnexpectedResult; typedef Expected, ErrorType> Result; typedef Expected PartialResult; static ExpressionType emptyExpression() { return nullptr; }; template NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const { using namespace FailureHelper; // See ADL comment in WasmParser.h. return UnexpectedResult(makeString("WebAssembly.Module failed compiling: "_s, makeString(args)...)); } #define WASM_COMPILE_FAIL_IF(condition, ...) do { \ if (UNLIKELY(condition)) \ return fail(__VA_ARGS__); \ } while (0) B3IRGenerator(const ModuleInformation&, Procedure&, InternalFunction*, Vector&, unsigned& osrEntryScratchBufferSize, MemoryMode, CompilationMode, unsigned functionIndex, unsigned loopIndexForOSREntry, TierUpCount*); PartialResult WARN_UNUSED_RETURN addArguments(const TypeDefinition&); PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t); ExpressionType addConstant(Type, uint64_t); // References PartialResult WARN_UNUSED_RETURN addRefIsNull(ExpressionType value, ExpressionType& result); PartialResult WARN_UNUSED_RETURN addRefFunc(uint32_t index, ExpressionType& result); // Tables PartialResult WARN_UNUSED_RETURN addTableGet(unsigned, ExpressionType index, ExpressionType& result); PartialResult WARN_UNUSED_RETURN addTableSet(unsigned, ExpressionType index, ExpressionType value); PartialResult WARN_UNUSED_RETURN addTableInit(unsigned, unsigned, ExpressionType dstOffset, ExpressionType srcOffset, ExpressionType length); PartialResult WARN_UNUSED_RETURN addElemDrop(unsigned); PartialResult WARN_UNUSED_RETURN addTableSize(unsigned, ExpressionType& result); PartialResult WARN_UNUSED_RETURN addTableGrow(unsigned, ExpressionType fill, ExpressionType delta, ExpressionType& result); PartialResult WARN_UNUSED_RETURN addTableFill(unsigned, ExpressionType offset, ExpressionType fill, ExpressionType count); PartialResult WARN_UNUSED_RETURN addTableCopy(unsigned, unsigned, ExpressionType dstOffset, ExpressionType srcOffset, ExpressionType length); // Locals PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result); PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value); // Globals PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result); PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value); // Memory PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset); PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset); PartialResult WARN_UNUSED_RETURN addGrowMemory(ExpressionType delta, ExpressionType& result); PartialResult WARN_UNUSED_RETURN addCurrentMemory(ExpressionType& result); PartialResult WARN_UNUSED_RETURN addMemoryFill(ExpressionType dstAddress, ExpressionType targetValue, ExpressionType count); PartialResult WARN_UNUSED_RETURN addMemoryCopy(ExpressionType dstAddress, ExpressionType srcAddress, ExpressionType count); PartialResult WARN_UNUSED_RETURN addMemoryInit(unsigned, ExpressionType dstAddress, ExpressionType srcAddress, ExpressionType length); PartialResult WARN_UNUSED_RETURN addDataDrop(unsigned); // Atomics PartialResult WARN_UNUSED_RETURN atomicLoad(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType& result, uint32_t offset); PartialResult WARN_UNUSED_RETURN atomicStore(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType value, uint32_t offset); PartialResult WARN_UNUSED_RETURN atomicBinaryRMW(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType value, ExpressionType& result, uint32_t offset); PartialResult WARN_UNUSED_RETURN atomicCompareExchange(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType expected, ExpressionType value, ExpressionType& result, uint32_t offset); PartialResult WARN_UNUSED_RETURN atomicWait(ExtAtomicOpType, ExpressionType pointer, ExpressionType value, ExpressionType timeout, ExpressionType& result, uint32_t offset); PartialResult WARN_UNUSED_RETURN atomicNotify(ExtAtomicOpType, ExpressionType pointer, ExpressionType value, ExpressionType& result, uint32_t offset); PartialResult WARN_UNUSED_RETURN atomicFence(ExtAtomicOpType, uint8_t flags); // Saturated truncation. PartialResult WARN_UNUSED_RETURN truncSaturated(Ext1OpType, ExpressionType operand, ExpressionType& result, Type returnType, Type operandType); // GC PartialResult WARN_UNUSED_RETURN addRttCanon(uint32_t typeIndex, ExpressionType& result); // Basic operators template PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result); template PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result); PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result); // Control flow ControlData WARN_UNUSED_RETURN addTopLevel(BlockSignature); PartialResult WARN_UNUSED_RETURN addBlock(BlockSignature, Stack& enclosingStack, ControlType& newBlock, Stack& newStack); PartialResult WARN_UNUSED_RETURN addLoop(BlockSignature, Stack& enclosingStack, ControlType& block, Stack& newStack, uint32_t loopIndex); PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, BlockSignature, Stack& enclosingStack, ControlType& result, Stack& newStack); PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const Stack&); PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&); PartialResult WARN_UNUSED_RETURN addTry(BlockSignature, Stack& enclosingStack, ControlType& result, Stack& newStack); PartialResult WARN_UNUSED_RETURN addCatch(unsigned exceptionIndex, const TypeDefinition&, Stack&, ControlType&, ResultList&); PartialResult WARN_UNUSED_RETURN addCatchToUnreachable(unsigned exceptionIndex, const TypeDefinition&, ControlType&, ResultList&); PartialResult WARN_UNUSED_RETURN addCatchAll(Stack&, ControlType&); PartialResult WARN_UNUSED_RETURN addCatchAllToUnreachable(ControlType&); PartialResult WARN_UNUSED_RETURN addDelegate(ControlType&, ControlType&); PartialResult WARN_UNUSED_RETURN addDelegateToUnreachable(ControlType&, ControlType&); PartialResult WARN_UNUSED_RETURN addThrow(unsigned exceptionIndex, Vector& args, Stack&); PartialResult WARN_UNUSED_RETURN addRethrow(unsigned, ControlType&); PartialResult WARN_UNUSED_RETURN addReturn(const ControlData&, const Stack& returnValues); PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const Stack& returnValues); PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector& targets, ControlData& defaultTargets, const Stack& expressionStack); PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, Stack& expressionStack); PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&, const Stack& = { }); PartialResult WARN_UNUSED_RETURN endTopLevel(BlockSignature, const Stack&) { return { }; } // Calls PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const TypeDefinition&, Vector& args, ResultList& results); PartialResult WARN_UNUSED_RETURN addCallIndirect(unsigned tableIndex, const TypeDefinition&, Vector& args, ResultList& results); PartialResult WARN_UNUSED_RETURN addCallRef(const TypeDefinition&, Vector& args, ResultList& results); PartialResult WARN_UNUSED_RETURN addUnreachable(); PartialResult WARN_UNUSED_RETURN emitIndirectCall(Value* calleeInstance, Value* calleeCode, const TypeDefinition&, Vector& args, ResultList&); B3::Value* createCallPatchpoint(BasicBlock*, Origin, const TypeDefinition&, Vector& args, const ScopedLambda)>& patchpointFunctor); void dump(const ControlStack&, const Stack* expressionStack); void setParser(FunctionParser* parser) { m_parser = parser; }; void didFinishParsingLocals() { } void didPopValueFromStack() { --m_stackSize; } Value* constant(B3::Type, uint64_t bits, std::optional = std::nullopt); Value* framePointer(); void insertEntrySwitch(); void insertConstants(); B3::Type toB3ResultType(BlockSignature); void addStackMap(unsigned callSiteIndex, StackMap&& stackmap) { m_stackmaps.add(CallSiteIndex(callSiteIndex), WTFMove(stackmap)); } StackMaps&& takeStackmaps() { return WTFMove(m_stackmaps); } Vector&& takeExceptionHandlers() { return WTFMove(m_exceptionHandlers); } private: void emitExceptionCheck(CCallHelpers&, ExceptionType); void emitEntryTierUpCheck(); void emitLoopTierUpCheck(uint32_t loopIndex, const Stack& enclosingStack, const Stack& newStack); void emitWriteBarrierForJSWrapper(); Value* emitCheckAndPreparePointer(Value* pointer, uint32_t offset, uint32_t sizeOfOp); B3::Kind memoryKind(B3::Opcode memoryOp); Value* emitLoadOp(LoadOpType, Value* pointer, uint32_t offset); void emitStoreOp(StoreOpType, Value* pointer, Value*, uint32_t offset); Value* sanitizeAtomicResult(ExtAtomicOpType, Type, Value* result); Value* emitAtomicLoadOp(ExtAtomicOpType, Type, Value* pointer, uint32_t offset); void emitAtomicStoreOp(ExtAtomicOpType, Type, Value* pointer, Value*, uint32_t offset); Value* emitAtomicBinaryRMWOp(ExtAtomicOpType, Type, Value* pointer, Value*, uint32_t offset); Value* emitAtomicCompareExchange(ExtAtomicOpType, Type, Value* pointer, Value* expected, Value*, uint32_t offset); void unify(Value* phi, const ExpressionType source); void unifyValuesWithBlock(const Stack& resultStack, const ControlData& block); void emitChecksForModOrDiv(B3::Opcode, Value* left, Value* right); int32_t WARN_UNUSED_RETURN fixupPointerPlusOffset(Value*&, uint32_t); Value* WARN_UNUSED_RETURN fixupPointerPlusOffsetForAtomicOps(ExtAtomicOpType, Value*, uint32_t); void restoreWasmContextInstance(Procedure&, BasicBlock*, Value*); enum class RestoreCachedStackLimit { No, Yes }; void restoreWebAssemblyGlobalState(RestoreCachedStackLimit, const MemoryInformation&, Value* instance, Procedure&, BasicBlock*, bool restoreInstance = true); Value* loadFromScratchBuffer(unsigned& indexInBuffer, Value* pointer, B3::Type); void connectControlAtEntrypoint(unsigned& indexInBuffer, Value* pointer, ControlData&, Stack& expressionStack, ControlData& currentData, bool fillLoopPhis = false); Value* emitCatchImpl(CatchKind, ControlType&, unsigned exceptionIndex = 0); PatchpointExceptionHandle preparePatchpointForExceptions(BasicBlock*, PatchpointValue*); Origin origin(); uint32_t outerLoopIndex() const { if (m_outerLoops.isEmpty()) return UINT32_MAX; return m_outerLoops.last(); } ExpressionType push(Type type) { return push(toB3Type(type)); } ExpressionType push(B3::Type type) { ++m_stackSize; if (m_stackSize > m_maxStackSize) { m_maxStackSize = m_stackSize; Variable* var = m_proc.addVariable(type); m_stack.append(var); return var; } Variable* var = m_stack[m_stackSize - 1]; if (var->type() == type) return var; var = m_proc.addVariable(type); m_stack[m_stackSize - 1] = var; return var; } ExpressionType push(Value* value) { Variable* var = push(value->type()); set(var, value); return var; } Value* get(BasicBlock* block, Variable* variable) { return block->appendNew(m_proc, B3::Get, origin(), variable); } Value* get(Variable* variable) { return get(m_currentBlock, variable); } Value* set(BasicBlock* block, Variable* dst, Value* src) { return block->appendNew(m_proc, B3::Set, origin(), dst, src); } Value* set(Variable* dst, Value* src) { return set(m_currentBlock, dst, src); } Value* set(Variable* dst, Variable* src) { return set(dst, get(src)); } bool useSignalingMemory() const { #if ENABLE(WEBASSEMBLY_SIGNALING_MEMORY) return m_mode == MemoryMode::Signaling; #else return false; #endif } FunctionParser* m_parser { nullptr }; const ModuleInformation& m_info; const MemoryMode m_mode { MemoryMode::BoundsChecking }; const CompilationMode m_compilationMode; const unsigned m_functionIndex { UINT_MAX }; const unsigned m_loopIndexForOSREntry { UINT_MAX }; TierUpCount* m_tierUp { nullptr }; Procedure& m_proc; Vector m_rootBlocks; BasicBlock* m_topLevelBlock; BasicBlock* m_currentBlock { nullptr }; Vector m_outerLoops; Vector m_locals; Vector m_stack; Vector& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with. unsigned& m_osrEntryScratchBufferSize; HashMap m_constantPool; HashMap m_tupleMap; InsertionSet m_constantInsertionValues; Value* m_framePointer { nullptr }; GPRReg m_memoryBaseGPR { InvalidGPRReg }; GPRReg m_boundsCheckingSizeGPR { InvalidGPRReg }; GPRReg m_wasmContextInstanceGPR { InvalidGPRReg }; bool m_makesCalls { false }; Value* m_instanceValue { nullptr }; // Always use the accessor below to ensure the instance value is materialized when used. bool m_usesInstanceValue { false }; bool m_hasCatch { false }; Value* instanceValue() { m_usesInstanceValue = true; return m_instanceValue; } uint32_t m_maxNumJSCallArguments { 0 }; unsigned m_numImportFunctions; Checked m_tryCatchDepth { 0 }; Checked m_callSiteIndex { 0 }; Checked m_stackSize { 0 }; Checked m_maxStackSize { 0 }; StackMaps m_stackmaps; Vector m_exceptionHandlers; }; // Memory accesses in WebAssembly have unsigned 32-bit offsets, whereas they have signed 32-bit offsets in B3. int32_t B3IRGenerator::fixupPointerPlusOffset(Value*& ptr, uint32_t offset) { if (static_cast(offset) > static_cast(std::numeric_limits::max())) { ptr = m_currentBlock->appendNew(m_proc, Add, origin(), ptr, m_currentBlock->appendNew(m_proc, origin(), offset)); return 0; } return offset; } void B3IRGenerator::restoreWasmContextInstance(Procedure& proc, BasicBlock* block, Value* arg) { if (Context::useFastTLS()) { PatchpointValue* patchpoint = block->appendNew(proc, B3::Void, Origin()); if (CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister()) patchpoint->clobber(RegisterSet::macroScratchRegisters()); patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister)); patchpoint->setGenerator( [=] (CCallHelpers& jit, const StackmapGenerationParams& params) { AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister()); jit.storeWasmContextInstance(params[0].gpr()); }); return; } // FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side. // This prevents us from using ArgumentReg to this (logically) immutable pinned register. PatchpointValue* patchpoint = block->appendNew(proc, B3::Void, Origin()); Effects effects = Effects::none(); effects.writesPinned = true; effects.reads = B3::HeapRange::top(); patchpoint->effects = effects; patchpoint->clobberLate(RegisterSet(m_wasmContextInstanceGPR)); patchpoint->append(arg, ValueRep::SomeRegister); GPRReg wasmContextInstanceGPR = m_wasmContextInstanceGPR; patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& param) { jit.move(param[0].gpr(), wasmContextInstanceGPR); }); } B3IRGenerator::B3IRGenerator(const ModuleInformation& info, Procedure& procedure, InternalFunction* compilation, Vector& unlinkedWasmToWasmCalls, unsigned& osrEntryScratchBufferSize, MemoryMode mode, CompilationMode compilationMode, unsigned functionIndex, unsigned loopIndexForOSREntry, TierUpCount* tierUp) : m_info(info) , m_mode(mode) , m_compilationMode(compilationMode) , m_functionIndex(functionIndex) , m_loopIndexForOSREntry(loopIndexForOSREntry) , m_tierUp(tierUp) , m_proc(procedure) , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls) , m_osrEntryScratchBufferSize(osrEntryScratchBufferSize) , m_constantInsertionValues(m_proc) , m_numImportFunctions(info.importFunctionCount()) { m_topLevelBlock = m_proc.addBlock(); m_rootBlocks.append(m_proc.addBlock()); m_currentBlock = m_rootBlocks[0]; // FIXME we don't really need to pin registers here if there's no memory. It makes wasm -> wasm thunks simpler for now. https://bugs.webkit.org/show_bug.cgi?id=166623 const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get(); m_memoryBaseGPR = pinnedRegs.baseMemoryPointer; m_proc.pinRegister(m_memoryBaseGPR); m_wasmContextInstanceGPR = pinnedRegs.wasmContextInstancePointer; if (!Context::useFastTLS()) m_proc.pinRegister(m_wasmContextInstanceGPR); if (mode == MemoryMode::BoundsChecking) { m_boundsCheckingSizeGPR = pinnedRegs.boundsCheckingSizeRegister; m_proc.pinRegister(m_boundsCheckingSizeGPR); } if (info.memory) { m_proc.setWasmBoundsCheckGenerator([=, this] (CCallHelpers& jit, GPRReg pinnedGPR) { AllowMacroScratchRegisterUsage allowScratch(jit); switch (m_mode) { case MemoryMode::BoundsChecking: ASSERT_UNUSED(pinnedGPR, m_boundsCheckingSizeGPR == pinnedGPR); break; #if ENABLE(WEBASSEMBLY_SIGNALING_MEMORY) case MemoryMode::Signaling: ASSERT_UNUSED(pinnedGPR, InvalidGPRReg == pinnedGPR); break; #endif } this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess); }); } { B3::PatchpointValue* getInstance = m_topLevelBlock->appendNew(m_proc, pointerType(), Origin()); if (Context::useFastTLS()) getInstance->clobber(RegisterSet::macroScratchRegisters()); else { // FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side. // This prevents us from using ArgumentReg to this (logically) immutable pinned register. getInstance->effects.writesPinned = false; getInstance->effects.readsPinned = true; getInstance->resultConstraints = { ValueRep::reg(m_wasmContextInstanceGPR) }; } getInstance->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) { if (Context::useFastTLS()) { AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::loadWasmContextInstanceNeedsMacroScratchRegister()); jit.loadWasmContextInstance(params[0].gpr()); } }); m_instanceValue = getInstance; } { auto* calleeMoveLocations = &compilation->calleeMoveLocations; static_assert(CallFrameSlot::codeBlock * sizeof(Register) < WasmCallingConvention::headerSizeInBytes, "We rely on this here for now."); static_assert(CallFrameSlot::callee * sizeof(Register) < WasmCallingConvention::headerSizeInBytes, "We rely on this here for now."); B3::PatchpointValue* getCalleePatchpoint = m_currentBlock->appendNew(m_proc, B3::Int64, Origin()); getCalleePatchpoint->resultConstraints = { B3::ValueRep::SomeRegister }; getCalleePatchpoint->effects = B3::Effects::none(); getCalleePatchpoint->setGenerator( [=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) { GPRReg result = params[0].gpr(); MacroAssembler::DataLabelPtr moveLocation = jit.moveWithPatch(MacroAssembler::TrustedImmPtr(nullptr), result); jit.addLinkTask([calleeMoveLocations, moveLocation] (LinkBuffer& linkBuffer) { calleeMoveLocations->append(linkBuffer.locationOf(moveLocation)); }); }); B3::Value* offsetOfCallee = m_currentBlock->appendNew(m_proc, Origin(), CallFrameSlot::callee * sizeof(Register)); m_currentBlock->appendNew(m_proc, B3::Store, Origin(), getCalleePatchpoint, m_currentBlock->appendNew(m_proc, B3::Add, Origin(), framePointer(), offsetOfCallee)); // FIXME: We shouldn't have to store zero into the CodeBlock* spot in the call frame, // but there are places that interpret non-null CodeBlock slot to mean a valid CodeBlock. // When doing unwinding, we'll need to verify that the entire runtime is OK with a non-null // CodeBlock not implying that the CodeBlock is valid. // https://bugs.webkit.org/show_bug.cgi?id=165321 B3::Value* offsetOfCodeBlock = m_currentBlock->appendNew(m_proc, Origin(), CallFrameSlot::codeBlock * sizeof(Register)); m_currentBlock->appendNew(m_proc, B3::Store, Origin(), m_currentBlock->appendNew(m_proc, Origin(), 0), m_currentBlock->appendNew(m_proc, B3::Add, Origin(), framePointer(), offsetOfCodeBlock)); } { B3::PatchpointValue* stackOverflowCheck = m_currentBlock->appendNew(m_proc, Void, Origin()); stackOverflowCheck->appendSomeRegister(instanceValue()); stackOverflowCheck->appendSomeRegister(framePointer()); stackOverflowCheck->clobber(RegisterSet::macroScratchRegisters()); stackOverflowCheck->numGPScratchRegisters = 2; stackOverflowCheck->setGenerator([=, this] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) { const Checked wasmFrameSize = params.proc().frameSize(); const unsigned minimumParentCheckSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), 1024); const unsigned extraFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), std::max( // This allows us to elide stack checks for functions that are terminal nodes in the call // tree, (e.g they don't make any calls) and have a small enough frame size. This works by // having any such terminal node have its parent caller include some extra size in its // own check for it. The goal here is twofold: // 1. Emit less code. // 2. Try to speed things up by skipping stack checks. minimumParentCheckSize, // This allows us to elide stack checks in the Wasm -> Embedder call IC stub. Since these will // spill all arguments to the stack, we ensure that a stack check here covers the // stack that such a stub would use. Checked(m_maxNumJSCallArguments) * sizeof(Register) + JSCallingConvention::headerSizeInBytes )); const int32_t checkSize = m_makesCalls ? (wasmFrameSize + extraFrameSize).value() : wasmFrameSize.value(); bool needUnderflowCheck = static_cast(checkSize) > Options::reservedZoneSize(); bool needsOverflowCheck = m_makesCalls || wasmFrameSize >= static_cast(minimumParentCheckSize) || needUnderflowCheck; // This allows leaf functions to not do stack checks if their frame size is within // certain limits since their caller would have already done the check. if (needsOverflowCheck) { AllowMacroScratchRegisterUsage allowScratch(jit); GPRReg contextInstance = params[0].gpr(); GPRReg fp = params[1].gpr(); GPRReg scratch1 = params.gpScratch(0); GPRReg scratch2 = params.gpScratch(1); jit.loadPtr(CCallHelpers::Address(contextInstance, Instance::offsetOfCachedStackLimit()), scratch2); jit.addPtr(CCallHelpers::TrustedImm32(-checkSize), fp, scratch1); MacroAssembler::JumpList overflow; if (UNLIKELY(needUnderflowCheck)) overflow.append(jit.branchPtr(CCallHelpers::Above, scratch1, fp)); overflow.append(jit.branchPtr(CCallHelpers::Below, scratch1, scratch2)); jit.addLinkTask([overflow] (LinkBuffer& linkBuffer) { linkBuffer.link(overflow, CodeLocationLabel(Thunks::singleton().stub(throwStackOverflowFromWasmThunkGenerator).code())); }); } }); } emitEntryTierUpCheck(); if (isOSREntry(m_compilationMode)) m_currentBlock = m_proc.addBlock(); } void B3IRGenerator::restoreWebAssemblyGlobalState(RestoreCachedStackLimit restoreCachedStackLimit, const MemoryInformation& memory, Value* instance, Procedure& proc, BasicBlock* block, bool restoreInstance) { if (restoreInstance) restoreWasmContextInstance(proc, block, instance); if (restoreCachedStackLimit == RestoreCachedStackLimit::Yes) { // The Instance caches the stack limit, but also knows where its canonical location is. Value* pointerToActualStackLimit = block->appendNew(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast(Instance::offsetOfPointerToActualStackLimit())); Value* actualStackLimit = block->appendNew(m_proc, Load, pointerType(), origin(), pointerToActualStackLimit); block->appendNew(m_proc, Store, origin(), actualStackLimit, instanceValue(), safeCast(Instance::offsetOfCachedStackLimit())); } if (!!memory) { const PinnedRegisterInfo* pinnedRegs = &PinnedRegisterInfo::get(); RegisterSet clobbers; clobbers.set(pinnedRegs->baseMemoryPointer); clobbers.set(pinnedRegs->boundsCheckingSizeRegister); clobbers.set(RegisterSet::macroScratchRegisters()); B3::PatchpointValue* patchpoint = block->appendNew(proc, B3::Void, origin()); Effects effects = Effects::none(); effects.writesPinned = true; effects.reads = B3::HeapRange::top(); patchpoint->effects = effects; patchpoint->clobber(clobbers); patchpoint->numGPScratchRegisters = 1; patchpoint->append(instance, ValueRep::SomeRegister); patchpoint->setGenerator([pinnedRegs] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); GPRReg baseMemory = pinnedRegs->baseMemoryPointer; GPRReg scratch = params.gpScratch(0); jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedBoundsCheckingSize()), pinnedRegs->boundsCheckingSizeRegister); jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemory()), baseMemory); jit.cageConditionallyAndUntag(Gigacage::Primitive, baseMemory, pinnedRegs->boundsCheckingSizeRegister, scratch); }); } } void B3IRGenerator::emitExceptionCheck(CCallHelpers& jit, ExceptionType type) { jit.move(CCallHelpers::TrustedImm32(static_cast(type)), GPRInfo::argumentGPR1); auto jumpToExceptionStub = jit.jump(); jit.addLinkTask([jumpToExceptionStub] (LinkBuffer& linkBuffer) { linkBuffer.link(jumpToExceptionStub, CodeLocationLabel(Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator).code())); }); } Value* B3IRGenerator::constant(B3::Type type, uint64_t bits, std::optional maybeOrigin) { auto result = m_constantPool.ensure(ValueKey(opcodeForConstant(type), type, static_cast(bits)), [&] { Value* result = m_proc.addConstant(maybeOrigin ? *maybeOrigin : origin(), type, bits); m_constantInsertionValues.insertValue(0, result); return result; }); return result.iterator->value; } Value* B3IRGenerator::framePointer() { if (!m_framePointer) { m_framePointer = m_proc.add(B3::FramePointer, Origin()); ASSERT(m_framePointer); m_constantInsertionValues.insertValue(0, m_framePointer); } return m_framePointer; } void B3IRGenerator::insertEntrySwitch() { m_proc.setNumEntrypoints(m_rootBlocks.size()); Ref catchPrologueGenerator = createSharedTask([] (CCallHelpers& jit, B3::Air::Code& code) { AllowMacroScratchRegisterUsage allowScratch(jit); emitCatchPrologueShared(code, jit); }); for (unsigned i = 1; i < m_rootBlocks.size(); ++i) m_proc.code().setPrologueForEntrypoint(i, catchPrologueGenerator.copyRef()); m_currentBlock = m_topLevelBlock; m_currentBlock->appendNew(m_proc, EntrySwitch, Origin()); for (BasicBlock* block : m_rootBlocks) m_currentBlock->appendSuccessor(FrequentedBlock(block)); } void B3IRGenerator::insertConstants() { m_constantInsertionValues.execute(m_proc.at(0)); if (!m_hasCatch) return; Value* jsInstance = m_proc.add(Load, pointerType(), Origin(), instanceValue(), safeCast(Instance::offsetOfOwner())); Value* store = m_proc.add(B3::Store, Origin(), jsInstance, framePointer(), safeCast(CallFrameSlot::thisArgument * sizeof(Register))); BasicBlock* block = m_rootBlocks[0]; m_constantInsertionValues.insertValue(0, jsInstance); m_constantInsertionValues.insertValue(0, store); m_constantInsertionValues.execute(block); } B3::Type B3IRGenerator::toB3ResultType(BlockSignature returnType) { if (returnType->as()->returnsVoid()) return B3::Void; if (returnType->as()->returnCount() == 1) return toB3Type(returnType->as()->returnType(0)); auto result = m_tupleMap.ensure(returnType, [&] { Vector result; for (unsigned i = 0; i < returnType->as()->returnCount(); ++i) result.append(toB3Type(returnType->as()->returnType(i))); return m_proc.addTuple(WTFMove(result)); }); return result.iterator->value; } auto B3IRGenerator::addLocal(Type type, uint32_t count) -> PartialResult { size_t newSize = m_locals.size() + count; ASSERT(!(CheckedUint32(count) + m_locals.size()).hasOverflowed()); ASSERT(newSize <= maxFunctionLocals); WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(newSize), "can't allocate memory for ", newSize, " locals"); for (uint32_t i = 0; i < count; ++i) { Variable* local = m_proc.addVariable(toB3Type(type)); m_locals.uncheckedAppend(local); auto val = isRefType(type) ? JSValue::encode(jsNull()) : 0; m_currentBlock->appendNew(m_proc, Set, Origin(), local, constant(toB3Type(type), val, Origin())); } return { }; } auto B3IRGenerator::addArguments(const TypeDefinition& signature) -> PartialResult { ASSERT(!m_locals.size()); WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(signature.as()->argumentCount()), "can't allocate memory for ", signature.as()->argumentCount(), " arguments"); m_locals.grow(signature.as()->argumentCount()); CallInformation wasmCallInfo = wasmCallingConvention().callInformationFor(signature, CallRole::Callee); for (size_t i = 0; i < signature.as()->argumentCount(); ++i) { B3::Type type = toB3Type(signature.as()->argumentType(i)); B3::Value* argument; auto rep = wasmCallInfo.params[i]; if (rep.isGPR()) { argument = m_currentBlock->appendNew(m_proc, Origin(), rep.jsr().payloadGPR()); if (type == B3::Int32) argument = m_currentBlock->appendNew(m_proc, B3::Trunc, Origin(), argument); } else if (rep.isFPR()) { argument = m_currentBlock->appendNew(m_proc, Origin(), rep.fpr()); if (type == B3::Float) argument = m_currentBlock->appendNew(m_proc, B3::Trunc, Origin(), argument); } else { ASSERT(rep.isStack()); B3::Value* address = m_currentBlock->appendNew(m_proc, B3::Add, Origin(), framePointer(), m_currentBlock->appendNew(m_proc, Origin(), rep.offsetFromFP())); argument = m_currentBlock->appendNew(m_proc, B3::Load, type, Origin(), address); } Variable* argumentVariable = m_proc.addVariable(argument->type()); m_locals[i] = argumentVariable; m_currentBlock->appendNew(m_proc, Set, Origin(), argumentVariable, argument); } return { }; } auto B3IRGenerator::addRefIsNull(ExpressionType value, ExpressionType& result) -> PartialResult { result = push(m_currentBlock->appendNew(m_proc, B3::Equal, origin(), get(value), m_currentBlock->appendNew(m_proc, origin(), JSValue::encode(jsNull())))); return { }; } auto B3IRGenerator::addTableGet(unsigned tableIndex, ExpressionType index, ExpressionType& result) -> PartialResult { // FIXME: Emit this inline . Value* resultValue = m_currentBlock->appendNew(m_proc, toB3Type(Types::Externref), origin(), m_currentBlock->appendNew(m_proc, origin(), tagCFunction(operationGetWasmTableElement)), instanceValue(), m_currentBlock->appendNew(m_proc, origin(), tableIndex), get(index)); { result = push(resultValue); CheckValue* check = m_currentBlock->appendNew(m_proc, Check, origin(), m_currentBlock->appendNew(m_proc, Equal, origin(), resultValue, m_currentBlock->appendNew(m_proc, origin(), 0))); check->setGenerator([=, this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTableAccess); }); } return { }; } auto B3IRGenerator::addTableSet(unsigned tableIndex, ExpressionType index, ExpressionType value) -> PartialResult { // FIXME: Emit this inline . auto shouldThrow = m_currentBlock->appendNew(m_proc, B3::Int32, origin(), m_currentBlock->appendNew(m_proc, origin(), tagCFunction(operationSetWasmTableElement)), instanceValue(), m_currentBlock->appendNew(m_proc, origin(), tableIndex), get(index), get(value)); { CheckValue* check = m_currentBlock->appendNew(m_proc, Check, origin(), m_currentBlock->appendNew(m_proc, Equal, origin(), shouldThrow, m_currentBlock->appendNew(m_proc, origin(), 0))); check->setGenerator([=, this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTableAccess); }); } return { }; } auto B3IRGenerator::addRefFunc(uint32_t index, ExpressionType& result) -> PartialResult { // FIXME: Emit this inline . result = push(m_currentBlock->appendNew(m_proc, B3::Int64, origin(), m_currentBlock->appendNew(m_proc, origin(), tagCFunction(operationWasmRefFunc)), instanceValue(), constant(toB3Type(Types::I32), index))); return { }; } auto B3IRGenerator::addTableInit(unsigned elementIndex, unsigned tableIndex, ExpressionType dstOffset, ExpressionType srcOffset, ExpressionType length) -> PartialResult { Value* resultValue = m_currentBlock->appendNew( m_proc, toB3Type(Types::I32), origin(), m_currentBlock->appendNew(m_proc, origin(), tagCFunction(operationWasmTableInit)), instanceValue(), m_currentBlock->appendNew(m_proc, origin(), elementIndex), m_currentBlock->appendNew(m_proc, origin(), tableIndex), get(dstOffset), get(srcOffset), get(length)); { CheckValue* check = m_currentBlock->appendNew(m_proc, Check, origin(), m_currentBlock->appendNew(m_proc, Equal, origin(), resultValue, m_currentBlock->appendNew(m_proc, origin(), 0))); check->setGenerator([=, this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTableAccess); }); } return { }; } auto B3IRGenerator::addElemDrop(unsigned elementIndex) -> PartialResult { m_currentBlock->appendNew( m_proc, B3::Void, origin(), m_currentBlock->appendNew(m_proc, origin(), tagCFunction(operationWasmElemDrop)), instanceValue(), m_currentBlock->appendNew(m_proc, origin(), elementIndex)); return { }; } auto B3IRGenerator::addTableSize(unsigned tableIndex, ExpressionType& result) -> PartialResult { // FIXME: Emit this inline . result = push(m_currentBlock->appendNew(m_proc, toB3Type(Types::I32), origin(), m_currentBlock->appendNew(m_proc, origin(), tagCFunction(operationGetWasmTableSize)), instanceValue(), m_currentBlock->appendNew(m_proc, origin(), tableIndex))); return { }; } auto B3IRGenerator::addTableGrow(unsigned tableIndex, ExpressionType fill, ExpressionType delta, ExpressionType& result) -> PartialResult { result = push(m_currentBlock->appendNew(m_proc, toB3Type(Types::I32), origin(), m_currentBlock->appendNew(m_proc, origin(), tagCFunction(operationWasmTableGrow)), instanceValue(), m_currentBlock->appendNew(m_proc, origin(), tableIndex), get(fill), get(delta))); return { }; } auto B3IRGenerator::addTableFill(unsigned tableIndex, ExpressionType offset, ExpressionType fill, ExpressionType count) -> PartialResult { Value* resultValue = m_currentBlock->appendNew(m_proc, toB3Type(Types::I32), origin(), m_currentBlock->appendNew(m_proc, origin(), tagCFunction(operationWasmTableFill)), instanceValue(), m_currentBlock->appendNew(m_proc, origin(), tableIndex), get(offset), get(fill), get(count)); { CheckValue* check = m_currentBlock->appendNew(m_proc, Check, origin(), m_currentBlock->appendNew(m_proc, Equal, origin(), resultValue, m_currentBlock->appendNew(m_proc, origin(), 0))); check->setGenerator([=, this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTableAccess); }); } return { }; } auto B3IRGenerator::addTableCopy(unsigned dstTableIndex, unsigned srcTableIndex, ExpressionType dstOffset, ExpressionType srcOffset, ExpressionType length) -> PartialResult { Value* resultValue = m_currentBlock->appendNew( m_proc, toB3Type(Types::I32), origin(), m_currentBlock->appendNew(m_proc, origin(), tagCFunction(operationWasmTableCopy)), instanceValue(), m_currentBlock->appendNew(m_proc, origin(), dstTableIndex), m_currentBlock->appendNew(m_proc, origin(), srcTableIndex), get(dstOffset), get(srcOffset), get(length)); { CheckValue* check = m_currentBlock->appendNew(m_proc, Check, origin(), m_currentBlock->appendNew(m_proc, Equal, origin(), resultValue, m_currentBlock->appendNew(m_proc, origin(), 0))); check->setGenerator([=, this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTableAccess); }); } return { }; } auto B3IRGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult { ASSERT(m_locals[index]); result = push(m_currentBlock->appendNew(m_proc, B3::Get, origin(), m_locals[index])); return { }; } auto B3IRGenerator::addUnreachable() -> PartialResult { B3::PatchpointValue* unreachable = m_currentBlock->appendNew(m_proc, B3::Void, origin()); unreachable->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::Unreachable); }); unreachable->effects.terminal = true; return { }; } auto B3IRGenerator::emitIndirectCall(Value* calleeInstance, Value* calleeCode, const TypeDefinition& signature, Vector& args, ResultList& results) -> PartialResult { // Do a context switch if needed. { BasicBlock* continuation = m_proc.addBlock(); BasicBlock* doContextSwitch = m_proc.addBlock(); Value* isSameContextInstance = m_currentBlock->appendNew(m_proc, Equal, origin(), calleeInstance, instanceValue()); m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(), isSameContextInstance, FrequentedBlock(continuation), FrequentedBlock(doContextSwitch)); PatchpointValue* patchpoint = doContextSwitch->appendNew(m_proc, B3::Void, origin()); patchpoint->effects.writesPinned = true; // We pessimistically assume we're calling something with BoundsChecking memory. // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181 patchpoint->clobber(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking)); patchpoint->clobber(RegisterSet::macroScratchRegisters()); patchpoint->append(calleeInstance, ValueRep::SomeRegister); patchpoint->append(instanceValue(), ValueRep::SomeRegister); patchpoint->numGPScratchRegisters = 1; patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); GPRReg calleeInstance = params[0].gpr(); GPRReg oldContextInstance = params[1].gpr(); const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get(); GPRReg baseMemory = pinnedRegs.baseMemoryPointer; ASSERT(calleeInstance != baseMemory); jit.loadPtr(CCallHelpers::Address(oldContextInstance, Instance::offsetOfCachedStackLimit()), baseMemory); jit.storePtr(baseMemory, CCallHelpers::Address(calleeInstance, Instance::offsetOfCachedStackLimit())); jit.storeWasmContextInstance(calleeInstance); ASSERT(pinnedRegs.boundsCheckingSizeRegister != baseMemory); // FIXME: We should support more than one memory size register // see: https://bugs.webkit.org/show_bug.cgi?id=162952 ASSERT(pinnedRegs.boundsCheckingSizeRegister != calleeInstance); GPRReg scratch = params.gpScratch(0); jit.loadPtr(CCallHelpers::Address(calleeInstance, Instance::offsetOfCachedBoundsCheckingSize()), pinnedRegs.boundsCheckingSizeRegister); // Memory size. jit.loadPtr(CCallHelpers::Address(calleeInstance, Instance::offsetOfCachedMemory()), baseMemory); // Memory::void*. jit.cageConditionallyAndUntag(Gigacage::Primitive, baseMemory, pinnedRegs.boundsCheckingSizeRegister, scratch); }); doContextSwitch->appendNewControlValue(m_proc, Jump, origin(), continuation); m_currentBlock = continuation; } B3::Type returnType = toB3ResultType(&signature); Value* callResult = createCallPatchpoint(m_currentBlock, origin(), signature, args, scopedLambdaRef)>([=, this] (PatchpointValue* patchpoint, Box handle) -> void { patchpoint->effects.writesPinned = true; patchpoint->effects.readsPinned = true; // We need to clobber all potential pinned registers since we might be leaving the instance. // We pessimistically assume we're always calling something that is bounds checking so // because the wasm->wasm thunk unconditionally overrides the size registers. // FIXME: We should not have to do this, but the wasm->wasm stub assumes it can // use all the pinned registers as scratch: https://bugs.webkit.org/show_bug.cgi?id=172181 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking)); patchpoint->append(calleeCode, ValueRep::SomeRegister); patchpoint->setGenerator([=, this] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); handle->generate(jit, params, this); jit.call(params[params.proc().resultCount(returnType)].gpr(), WasmEntryPtrTag); }); })); switch (returnType.kind()) { case B3::Void: { break; } case B3::Tuple: { const Vector& tuple = m_proc.tupleForType(returnType); for (unsigned i = 0; i < signature.as()->returnCount(); ++i) results.append(push(m_currentBlock->appendNew(m_proc, origin(), tuple[i], callResult, i))); break; } default: { results.append(push(callResult)); break; } } // The call could have been to another WebAssembly instance, and / or could have modified our Memory. restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, instanceValue(), m_proc, m_currentBlock); return { }; } auto B3IRGenerator::addGrowMemory(ExpressionType delta, ExpressionType& result) -> PartialResult { result = push(m_currentBlock->appendNew(m_proc, Int32, origin(), m_currentBlock->appendNew(m_proc, origin(), tagCFunction(operationGrowMemory)), framePointer(), instanceValue(), get(delta))); restoreWebAssemblyGlobalState(RestoreCachedStackLimit::No, m_info.memory, instanceValue(), m_proc, m_currentBlock); return { }; } auto B3IRGenerator::addCurrentMemory(ExpressionType& result) -> PartialResult { static_assert(sizeof(std::declval()->size()) == sizeof(uint64_t), "codegen relies on this size"); Value* memory = m_currentBlock->appendNew(m_proc, Load, Int64, origin(), instanceValue(), safeCast(Instance::offsetOfMemory())); Value* handle = m_currentBlock->appendNew(m_proc, Load, Int64, origin(), memory, safeCast(Memory::offsetOfHandle())); Value* size = m_currentBlock->appendNew(m_proc, Load, Int64, origin(), handle, safeCast(MemoryHandle::offsetOfSize())); constexpr uint32_t shiftValue = 16; static_assert(PageCount::pageSize == 1ull << shiftValue, "This must hold for the code below to be correct."); Value* numPages = m_currentBlock->appendNew(m_proc, ZShr, origin(), size, m_currentBlock->appendNew(m_proc, origin(), shiftValue)); result = push(m_currentBlock->appendNew(m_proc, Trunc, origin(), numPages)); return { }; } auto B3IRGenerator::addMemoryFill(ExpressionType dstAddress, ExpressionType targetValue, ExpressionType count) -> PartialResult { Value* resultValue = m_currentBlock->appendNew( m_proc, toB3Type(Types::I32), origin(), m_currentBlock->appendNew(m_proc, origin(), tagCFunction(operationWasmMemoryFill)), instanceValue(), get(dstAddress), get(targetValue), get(count)); { CheckValue* check = m_currentBlock->appendNew(m_proc, Check, origin(), m_currentBlock->appendNew(m_proc, Equal, origin(), resultValue, m_currentBlock->appendNew(m_proc, origin(), 0))); check->setGenerator([=, this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess); }); } return { }; } auto B3IRGenerator::addMemoryInit(unsigned dataSegmentIndex, ExpressionType dstAddress, ExpressionType srcAddress, ExpressionType length) -> PartialResult { Value* resultValue = m_currentBlock->appendNew( m_proc, toB3Type(Types::I32), origin(), m_currentBlock->appendNew(m_proc, origin(), tagCFunction(operationWasmMemoryInit)), instanceValue(), m_currentBlock->appendNew(m_proc, origin(), dataSegmentIndex), get(dstAddress), get(srcAddress), get(length)); { CheckValue* check = m_currentBlock->appendNew(m_proc, Check, origin(), m_currentBlock->appendNew(m_proc, Equal, origin(), resultValue, m_currentBlock->appendNew(m_proc, origin(), 0))); check->setGenerator([=, this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess); }); } return { }; } auto B3IRGenerator::addMemoryCopy(ExpressionType dstAddress, ExpressionType srcAddress, ExpressionType count) -> PartialResult { Value* resultValue = m_currentBlock->appendNew( m_proc, toB3Type(Types::I32), origin(), m_currentBlock->appendNew(m_proc, origin(), tagCFunction(operationWasmMemoryCopy)), instanceValue(), get(dstAddress), get(srcAddress), get(count)); { CheckValue* check = m_currentBlock->appendNew(m_proc, Check, origin(), m_currentBlock->appendNew(m_proc, Equal, origin(), resultValue, m_currentBlock->appendNew(m_proc, origin(), 0))); check->setGenerator([=, this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess); }); } return { }; } auto B3IRGenerator::addDataDrop(unsigned dataSegmentIndex) -> PartialResult { m_currentBlock->appendNew( m_proc, B3::Void, origin(), m_currentBlock->appendNew(m_proc, origin(), tagCFunction(operationWasmDataDrop)), instanceValue(), m_currentBlock->appendNew(m_proc, origin(), dataSegmentIndex)); return { }; } auto B3IRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult { ASSERT(m_locals[index]); m_currentBlock->appendNew(m_proc, B3::Set, origin(), m_locals[index], get(value)); return { }; } auto B3IRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult { const Wasm::GlobalInformation& global = m_info.globals[index]; Value* globalsArray = m_currentBlock->appendNew(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast(Instance::offsetOfGlobals())); switch (global.bindingMode) { case Wasm::GlobalInformation::BindingMode::EmbeddedInInstance: result = push(m_currentBlock->appendNew(m_proc, Load, toB3Type(global.type), origin(), globalsArray, safeCast(index * sizeof(Register)))); break; case Wasm::GlobalInformation::BindingMode::Portable: { ASSERT(global.mutability == Wasm::Mutability::Mutable); Value* pointer = m_currentBlock->appendNew(m_proc, Load, B3::Int64, origin(), globalsArray, safeCast(index * sizeof(Register))); result = push(m_currentBlock->appendNew(m_proc, Load, toB3Type(global.type), origin(), pointer)); break; } } return { }; } auto B3IRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult { const Wasm::GlobalInformation& global = m_info.globals[index]; ASSERT(toB3Type(global.type) == value->type()); Value* globalsArray = m_currentBlock->appendNew(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast(Instance::offsetOfGlobals())); switch (global.bindingMode) { case Wasm::GlobalInformation::BindingMode::EmbeddedInInstance: m_currentBlock->appendNew(m_proc, Store, origin(), get(value), globalsArray, safeCast(index * sizeof(Register))); if (isRefType(global.type)) emitWriteBarrierForJSWrapper(); break; case Wasm::GlobalInformation::BindingMode::Portable: { ASSERT(global.mutability == Wasm::Mutability::Mutable); Value* pointer = m_currentBlock->appendNew(m_proc, Load, B3::Int64, origin(), globalsArray, safeCast(index * sizeof(Register))); m_currentBlock->appendNew(m_proc, Store, origin(), get(value), pointer); // We emit a write-barrier onto JSWebAssemblyGlobal, not JSWebAssemblyInstance. if (isRefType(global.type)) { Value* instance = m_currentBlock->appendNew(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast(Instance::offsetOfOwner())); Value* cell = m_currentBlock->appendNew(m_proc, Load, pointerType(), origin(), pointer, Wasm::Global::offsetOfOwner() - Wasm::Global::offsetOfValue()); Value* cellState = m_currentBlock->appendNew(m_proc, Load8Z, Int32, origin(), cell, safeCast(JSCell::cellStateOffset())); Value* vm = m_currentBlock->appendNew(m_proc, Load, pointerType(), origin(), instance, safeCast(JSWebAssemblyInstance::offsetOfVM())); Value* threshold = m_currentBlock->appendNew(m_proc, Load, Int32, origin(), vm, safeCast(VM::offsetOfHeapBarrierThreshold())); BasicBlock* fenceCheckPath = m_proc.addBlock(); BasicBlock* fencePath = m_proc.addBlock(); BasicBlock* doSlowPath = m_proc.addBlock(); BasicBlock* continuation = m_proc.addBlock(); m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(), m_currentBlock->appendNew(m_proc, Above, origin(), cellState, threshold), FrequentedBlock(continuation), FrequentedBlock(fenceCheckPath, FrequencyClass::Rare)); fenceCheckPath->addPredecessor(m_currentBlock); continuation->addPredecessor(m_currentBlock); m_currentBlock = fenceCheckPath; Value* shouldFence = m_currentBlock->appendNew(m_proc, Load8Z, Int32, origin(), vm, safeCast(VM::offsetOfHeapMutatorShouldBeFenced())); m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(), shouldFence, FrequentedBlock(fencePath), FrequentedBlock(doSlowPath)); fencePath->addPredecessor(m_currentBlock); doSlowPath->addPredecessor(m_currentBlock); m_currentBlock = fencePath; B3::PatchpointValue* doFence = m_currentBlock->appendNew(m_proc, B3::Void, origin()); doFence->setGenerator([] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { jit.memoryFence(); }); Value* cellStateLoadAfterFence = m_currentBlock->appendNew(m_proc, Load8Z, Int32, origin(), cell, safeCast(JSCell::cellStateOffset())); m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(), m_currentBlock->appendNew(m_proc, Above, origin(), cellStateLoadAfterFence, m_currentBlock->appendNew(m_proc, origin(), blackThreshold)), FrequentedBlock(continuation), FrequentedBlock(doSlowPath, FrequencyClass::Rare)); doSlowPath->addPredecessor(m_currentBlock); continuation->addPredecessor(m_currentBlock); m_currentBlock = doSlowPath; Value* writeBarrierAddress = m_currentBlock->appendNew(m_proc, origin(), tagCFunction(operationWasmWriteBarrierSlowPath)); m_currentBlock->appendNew(m_proc, B3::Void, origin(), writeBarrierAddress, cell, vm); m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), continuation); continuation->addPredecessor(m_currentBlock); m_currentBlock = continuation; } break; } } return { }; } inline void B3IRGenerator::emitWriteBarrierForJSWrapper() { Value* cell = m_currentBlock->appendNew(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast(Instance::offsetOfOwner())); Value* cellState = m_currentBlock->appendNew(m_proc, Load8Z, Int32, origin(), cell, safeCast(JSCell::cellStateOffset())); Value* vm = m_currentBlock->appendNew(m_proc, Load, pointerType(), origin(), cell, safeCast(JSWebAssemblyInstance::offsetOfVM())); Value* threshold = m_currentBlock->appendNew(m_proc, Load, Int32, origin(), vm, safeCast(VM::offsetOfHeapBarrierThreshold())); BasicBlock* fenceCheckPath = m_proc.addBlock(); BasicBlock* fencePath = m_proc.addBlock(); BasicBlock* doSlowPath = m_proc.addBlock(); BasicBlock* continuation = m_proc.addBlock(); m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(), m_currentBlock->appendNew(m_proc, Above, origin(), cellState, threshold), FrequentedBlock(continuation), FrequentedBlock(fenceCheckPath, FrequencyClass::Rare)); fenceCheckPath->addPredecessor(m_currentBlock); continuation->addPredecessor(m_currentBlock); m_currentBlock = fenceCheckPath; Value* shouldFence = m_currentBlock->appendNew(m_proc, Load8Z, Int32, origin(), vm, safeCast(VM::offsetOfHeapMutatorShouldBeFenced())); m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(), shouldFence, FrequentedBlock(fencePath), FrequentedBlock(doSlowPath)); fencePath->addPredecessor(m_currentBlock); doSlowPath->addPredecessor(m_currentBlock); m_currentBlock = fencePath; B3::PatchpointValue* doFence = m_currentBlock->appendNew(m_proc, B3::Void, origin()); doFence->setGenerator([] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { jit.memoryFence(); }); Value* cellStateLoadAfterFence = m_currentBlock->appendNew(m_proc, Load8Z, Int32, origin(), cell, safeCast(JSCell::cellStateOffset())); m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(), m_currentBlock->appendNew(m_proc, Above, origin(), cellStateLoadAfterFence, m_currentBlock->appendNew(m_proc, origin(), blackThreshold)), FrequentedBlock(continuation), FrequentedBlock(doSlowPath, FrequencyClass::Rare)); doSlowPath->addPredecessor(m_currentBlock); continuation->addPredecessor(m_currentBlock); m_currentBlock = doSlowPath; Value* writeBarrierAddress = m_currentBlock->appendNew(m_proc, origin(), tagCFunction(operationWasmWriteBarrierSlowPath)); m_currentBlock->appendNew(m_proc, B3::Void, origin(), writeBarrierAddress, cell, vm); m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), continuation); continuation->addPredecessor(m_currentBlock); m_currentBlock = continuation; } inline Value* B3IRGenerator::emitCheckAndPreparePointer(Value* pointer, uint32_t offset, uint32_t sizeOfOperation) { ASSERT(m_memoryBaseGPR); switch (m_mode) { case MemoryMode::BoundsChecking: { // We're not using signal handling only when the memory is not shared. // Regardless of signaling, we must check that no memory access exceeds the current memory size. ASSERT(m_boundsCheckingSizeGPR); ASSERT(sizeOfOperation + offset > offset); m_currentBlock->appendNew(m_proc, origin(), m_boundsCheckingSizeGPR, pointer, sizeOfOperation + offset - 1); break; } #if ENABLE(WEBASSEMBLY_SIGNALING_MEMORY) case MemoryMode::Signaling: { // We've virtually mapped 4GiB+redzone for this memory. Only the user-allocated pages are addressable, contiguously in range [0, current], // and everything above is mapped PROT_NONE. We don't need to perform any explicit bounds check in the 4GiB range because WebAssembly register // memory accesses are 32-bit. However WebAssembly register + offset accesses perform the addition in 64-bit which can push an access above // the 32-bit limit (the offset is unsigned 32-bit). The redzone will catch most small offsets, and we'll explicitly bounds check any // register + large offset access. We don't think this will be generated frequently. // // We could check that register + large offset doesn't exceed 4GiB+redzone since that's technically the limit we need to avoid overflowing the // PROT_NONE region, but it's better if we use a smaller immediate because it can codegens better. We know that anything equal to or greater // than the declared 'maximum' will trap, so we can compare against that number. If there was no declared 'maximum' then we still know that // any access equal to or greater than 4GiB will trap, no need to add the redzone. if (offset >= Memory::fastMappedRedzoneBytes()) { size_t maximum = m_info.memory.maximum() ? m_info.memory.maximum().bytes() : std::numeric_limits::max(); m_currentBlock->appendNew(m_proc, origin(), pointer, sizeOfOperation + offset - 1, maximum); } break; } #endif } pointer = m_currentBlock->appendNew(m_proc, ZExt32, origin(), pointer); return m_currentBlock->appendNew(m_proc, origin(), pointer, m_memoryBaseGPR); } inline uint32_t sizeOfLoadOp(LoadOpType op) { switch (op) { case LoadOpType::I32Load8S: case LoadOpType::I32Load8U: case LoadOpType::I64Load8S: case LoadOpType::I64Load8U: return 1; case LoadOpType::I32Load16S: case LoadOpType::I64Load16S: case LoadOpType::I32Load16U: case LoadOpType::I64Load16U: return 2; case LoadOpType::I32Load: case LoadOpType::I64Load32S: case LoadOpType::I64Load32U: case LoadOpType::F32Load: return 4; case LoadOpType::I64Load: case LoadOpType::F64Load: return 8; } RELEASE_ASSERT_NOT_REACHED(); } inline B3::Kind B3IRGenerator::memoryKind(B3::Opcode memoryOp) { if (useSignalingMemory() || m_info.memory.isShared()) return trapping(memoryOp); return memoryOp; } inline Value* B3IRGenerator::emitLoadOp(LoadOpType op, Value* pointer, uint32_t uoffset) { int32_t offset = fixupPointerPlusOffset(pointer, uoffset); switch (op) { case LoadOpType::I32Load8S: { return m_currentBlock->appendNew(m_proc, memoryKind(Load8S), origin(), pointer, offset); } case LoadOpType::I64Load8S: { Value* value = m_currentBlock->appendNew(m_proc, memoryKind(Load8S), origin(), pointer, offset); return m_currentBlock->appendNew(m_proc, SExt32, origin(), value); } case LoadOpType::I32Load8U: { return m_currentBlock->appendNew(m_proc, memoryKind(Load8Z), origin(), pointer, offset); } case LoadOpType::I64Load8U: { Value* value = m_currentBlock->appendNew(m_proc, memoryKind(Load8Z), origin(), pointer, offset); return m_currentBlock->appendNew(m_proc, ZExt32, origin(), value); } case LoadOpType::I32Load16S: { return m_currentBlock->appendNew(m_proc, memoryKind(Load16S), origin(), pointer, offset); } case LoadOpType::I64Load16S: { Value* value = m_currentBlock->appendNew(m_proc, memoryKind(Load16S), origin(), pointer, offset); return m_currentBlock->appendNew(m_proc, SExt32, origin(), value); } case LoadOpType::I32Load16U: { return m_currentBlock->appendNew(m_proc, memoryKind(Load16Z), origin(), pointer, offset); } case LoadOpType::I64Load16U: { Value* value = m_currentBlock->appendNew(m_proc, memoryKind(Load16Z), origin(), pointer, offset); return m_currentBlock->appendNew(m_proc, ZExt32, origin(), value); } case LoadOpType::I32Load: { return m_currentBlock->appendNew(m_proc, memoryKind(Load), Int32, origin(), pointer, offset); } case LoadOpType::I64Load32U: { Value* value = m_currentBlock->appendNew(m_proc, memoryKind(Load), Int32, origin(), pointer, offset); return m_currentBlock->appendNew(m_proc, ZExt32, origin(), value); } case LoadOpType::I64Load32S: { Value* value = m_currentBlock->appendNew(m_proc, memoryKind(Load), Int32, origin(), pointer, offset); return m_currentBlock->appendNew(m_proc, SExt32, origin(), value); } case LoadOpType::I64Load: { return m_currentBlock->appendNew(m_proc, memoryKind(Load), Int64, origin(), pointer, offset); } case LoadOpType::F32Load: { return m_currentBlock->appendNew(m_proc, memoryKind(Load), Float, origin(), pointer, offset); } case LoadOpType::F64Load: { return m_currentBlock->appendNew(m_proc, memoryKind(Load), Double, origin(), pointer, offset); } } RELEASE_ASSERT_NOT_REACHED(); } auto B3IRGenerator::load(LoadOpType op, ExpressionType pointerVar, ExpressionType& result, uint32_t offset) -> PartialResult { Value* pointer = get(pointerVar); ASSERT(pointer->type() == Int32); if (UNLIKELY(sumOverflows(offset, sizeOfLoadOp(op)))) { // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435 B3::PatchpointValue* throwException = m_currentBlock->appendNew(m_proc, B3::Void, origin()); throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess); }); switch (op) { case LoadOpType::I32Load8S: case LoadOpType::I32Load16S: case LoadOpType::I32Load: case LoadOpType::I32Load16U: case LoadOpType::I32Load8U: result = push(constant(Int32, 0)); break; case LoadOpType::I64Load8S: case LoadOpType::I64Load8U: case LoadOpType::I64Load16S: case LoadOpType::I64Load32U: case LoadOpType::I64Load32S: case LoadOpType::I64Load: case LoadOpType::I64Load16U: result = push(constant(Int64, 0)); break; case LoadOpType::F32Load: result = push(constant(Float, 0)); break; case LoadOpType::F64Load: result = push(constant(Double, 0)); break; } } else result = push(emitLoadOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset)); return { }; } inline uint32_t sizeOfStoreOp(StoreOpType op) { switch (op) { case StoreOpType::I32Store8: case StoreOpType::I64Store8: return 1; case StoreOpType::I32Store16: case StoreOpType::I64Store16: return 2; case StoreOpType::I32Store: case StoreOpType::I64Store32: case StoreOpType::F32Store: return 4; case StoreOpType::I64Store: case StoreOpType::F64Store: return 8; } RELEASE_ASSERT_NOT_REACHED(); } inline void B3IRGenerator::emitStoreOp(StoreOpType op, Value* pointer, Value* value, uint32_t uoffset) { int32_t offset = fixupPointerPlusOffset(pointer, uoffset); switch (op) { case StoreOpType::I64Store8: value = m_currentBlock->appendNew(m_proc, Trunc, origin(), value); FALLTHROUGH; case StoreOpType::I32Store8: m_currentBlock->appendNew(m_proc, memoryKind(Store8), origin(), value, pointer, offset); return; case StoreOpType::I64Store16: value = m_currentBlock->appendNew(m_proc, Trunc, origin(), value); FALLTHROUGH; case StoreOpType::I32Store16: m_currentBlock->appendNew(m_proc, memoryKind(Store16), origin(), value, pointer, offset); return; case StoreOpType::I64Store32: value = m_currentBlock->appendNew(m_proc, Trunc, origin(), value); FALLTHROUGH; case StoreOpType::I64Store: case StoreOpType::I32Store: case StoreOpType::F32Store: case StoreOpType::F64Store: m_currentBlock->appendNew(m_proc, memoryKind(Store), origin(), value, pointer, offset); return; } RELEASE_ASSERT_NOT_REACHED(); } auto B3IRGenerator::store(StoreOpType op, ExpressionType pointerVar, ExpressionType valueVar, uint32_t offset) -> PartialResult { Value* pointer = get(pointerVar); Value* value = get(valueVar); ASSERT(pointer->type() == Int32); if (UNLIKELY(sumOverflows(offset, sizeOfStoreOp(op)))) { // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435 B3::PatchpointValue* throwException = m_currentBlock->appendNew(m_proc, B3::Void, origin()); throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess); }); } else emitStoreOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset); return { }; } inline B3::Width accessWidth(ExtAtomicOpType op) { return static_cast(memoryLog2Alignment(op)); } inline uint32_t sizeOfAtomicOpMemoryAccess(ExtAtomicOpType op) { return bytesForWidth(accessWidth(op)); } inline Value* B3IRGenerator::sanitizeAtomicResult(ExtAtomicOpType op, Type valueType, Value* result) { auto sanitize32 = [&](Value* result) { switch (accessWidth(op)) { case B3::Width8: return m_currentBlock->appendNew(m_proc, BitAnd, origin(), result, constant(Int32, 0xff)); case B3::Width16: return m_currentBlock->appendNew(m_proc, BitAnd, origin(), result, constant(Int32, 0xffff)); default: return result; } }; switch (valueType.kind) { case TypeKind::I64: { if (accessWidth(op) == B3::Width64) return result; return m_currentBlock->appendNew(m_proc, ZExt32, origin(), sanitize32(result)); } case TypeKind::I32: return sanitize32(result); default: RELEASE_ASSERT_NOT_REACHED(); return nullptr; } } Value* B3IRGenerator::fixupPointerPlusOffsetForAtomicOps(ExtAtomicOpType op, Value* ptr, uint32_t offset) { auto pointer = m_currentBlock->appendNew(m_proc, Add, origin(), ptr, m_currentBlock->appendNew(m_proc, origin(), offset)); if (accessWidth(op) != B3::Width8) { CheckValue* check = m_currentBlock->appendNew(m_proc, Check, origin(), m_currentBlock->appendNew(m_proc, BitAnd, origin(), pointer, constant(pointerType(), sizeOfAtomicOpMemoryAccess(op) - 1))); check->setGenerator([=, this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess); }); } return pointer; } inline Value* B3IRGenerator::emitAtomicLoadOp(ExtAtomicOpType op, Type valueType, Value* pointer, uint32_t uoffset) { pointer = fixupPointerPlusOffsetForAtomicOps(op, pointer, uoffset); Value* value = nullptr; switch (accessWidth(op)) { case B3::Width8: case B3::Width16: case B3::Width32: value = constant(Int32, 0); break; case B3::Width64: value = constant(Int64, 0); break; } return sanitizeAtomicResult(op, valueType, m_currentBlock->appendNew(m_proc, memoryKind(AtomicXchgAdd), origin(), accessWidth(op), value, pointer)); } auto B3IRGenerator::atomicLoad(ExtAtomicOpType op, Type valueType, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult { ASSERT(pointer->type() == Int32); if (UNLIKELY(sumOverflows(offset, sizeOfAtomicOpMemoryAccess(op)))) { // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435 B3::PatchpointValue* throwException = m_currentBlock->appendNew(m_proc, B3::Void, origin()); throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess); }); switch (valueType.kind) { case TypeKind::I32: result = push(constant(Int32, 0)); break; case TypeKind::I64: result = push(constant(Int64, 0)); break; default: RELEASE_ASSERT_NOT_REACHED(); break; } } else result = push(emitAtomicLoadOp(op, valueType, emitCheckAndPreparePointer(get(pointer), offset, sizeOfAtomicOpMemoryAccess(op)), offset)); return { }; } inline void B3IRGenerator::emitAtomicStoreOp(ExtAtomicOpType op, Type valueType, Value* pointer, Value* value, uint32_t uoffset) { pointer = fixupPointerPlusOffsetForAtomicOps(op, pointer, uoffset); if (valueType.isI64() && accessWidth(op) != B3::Width64) value = m_currentBlock->appendNew(m_proc, B3::Trunc, Origin(), value); m_currentBlock->appendNew(m_proc, memoryKind(AtomicXchg), origin(), accessWidth(op), value, pointer); } auto B3IRGenerator::atomicStore(ExtAtomicOpType op, Type valueType, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult { ASSERT(pointer->type() == Int32); if (UNLIKELY(sumOverflows(offset, sizeOfAtomicOpMemoryAccess(op)))) { // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435 B3::PatchpointValue* throwException = m_currentBlock->appendNew(m_proc, B3::Void, origin()); throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess); }); } else emitAtomicStoreOp(op, valueType, emitCheckAndPreparePointer(get(pointer), offset, sizeOfAtomicOpMemoryAccess(op)), get(value), offset); return { }; } inline Value* B3IRGenerator::emitAtomicBinaryRMWOp(ExtAtomicOpType op, Type valueType, Value* pointer, Value* value, uint32_t uoffset) { pointer = fixupPointerPlusOffsetForAtomicOps(op, pointer, uoffset); B3::Opcode opcode = B3::Nop; switch (op) { case ExtAtomicOpType::I32AtomicRmw8AddU: case ExtAtomicOpType::I32AtomicRmw16AddU: case ExtAtomicOpType::I32AtomicRmwAdd: case ExtAtomicOpType::I64AtomicRmw8AddU: case ExtAtomicOpType::I64AtomicRmw16AddU: case ExtAtomicOpType::I64AtomicRmw32AddU: case ExtAtomicOpType::I64AtomicRmwAdd: opcode = AtomicXchgAdd; break; case ExtAtomicOpType::I32AtomicRmw8SubU: case ExtAtomicOpType::I32AtomicRmw16SubU: case ExtAtomicOpType::I32AtomicRmwSub: case ExtAtomicOpType::I64AtomicRmw8SubU: case ExtAtomicOpType::I64AtomicRmw16SubU: case ExtAtomicOpType::I64AtomicRmw32SubU: case ExtAtomicOpType::I64AtomicRmwSub: opcode = AtomicXchgSub; break; case ExtAtomicOpType::I32AtomicRmw8AndU: case ExtAtomicOpType::I32AtomicRmw16AndU: case ExtAtomicOpType::I32AtomicRmwAnd: case ExtAtomicOpType::I64AtomicRmw8AndU: case ExtAtomicOpType::I64AtomicRmw16AndU: case ExtAtomicOpType::I64AtomicRmw32AndU: case ExtAtomicOpType::I64AtomicRmwAnd: opcode = AtomicXchgAnd; break; case ExtAtomicOpType::I32AtomicRmw8OrU: case ExtAtomicOpType::I32AtomicRmw16OrU: case ExtAtomicOpType::I32AtomicRmwOr: case ExtAtomicOpType::I64AtomicRmw8OrU: case ExtAtomicOpType::I64AtomicRmw16OrU: case ExtAtomicOpType::I64AtomicRmw32OrU: case ExtAtomicOpType::I64AtomicRmwOr: opcode = AtomicXchgOr; break; case ExtAtomicOpType::I32AtomicRmw8XorU: case ExtAtomicOpType::I32AtomicRmw16XorU: case ExtAtomicOpType::I32AtomicRmwXor: case ExtAtomicOpType::I64AtomicRmw8XorU: case ExtAtomicOpType::I64AtomicRmw16XorU: case ExtAtomicOpType::I64AtomicRmw32XorU: case ExtAtomicOpType::I64AtomicRmwXor: opcode = AtomicXchgXor; break; case ExtAtomicOpType::I32AtomicRmw8XchgU: case ExtAtomicOpType::I32AtomicRmw16XchgU: case ExtAtomicOpType::I32AtomicRmwXchg: case ExtAtomicOpType::I64AtomicRmw8XchgU: case ExtAtomicOpType::I64AtomicRmw16XchgU: case ExtAtomicOpType::I64AtomicRmw32XchgU: case ExtAtomicOpType::I64AtomicRmwXchg: opcode = AtomicXchg; break; default: RELEASE_ASSERT_NOT_REACHED(); break; } if (valueType.isI64() && accessWidth(op) != B3::Width64) value = m_currentBlock->appendNew(m_proc, B3::Trunc, Origin(), value); return sanitizeAtomicResult(op, valueType, m_currentBlock->appendNew(m_proc, memoryKind(opcode), origin(), accessWidth(op), value, pointer)); } auto B3IRGenerator::atomicBinaryRMW(ExtAtomicOpType op, Type valueType, ExpressionType pointer, ExpressionType value, ExpressionType& result, uint32_t offset) -> PartialResult { ASSERT(pointer->type() == Int32); if (UNLIKELY(sumOverflows(offset, sizeOfAtomicOpMemoryAccess(op)))) { // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435 B3::PatchpointValue* throwException = m_currentBlock->appendNew(m_proc, B3::Void, origin()); throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess); }); switch (valueType.kind) { case TypeKind::I32: result = push(constant(Int32, 0)); break; case TypeKind::I64: result = push(constant(Int64, 0)); break; default: RELEASE_ASSERT_NOT_REACHED(); break; } } else result = push(emitAtomicBinaryRMWOp(op, valueType, emitCheckAndPreparePointer(get(pointer), offset, sizeOfAtomicOpMemoryAccess(op)), get(value), offset)); return { }; } Value* B3IRGenerator::emitAtomicCompareExchange(ExtAtomicOpType op, Type valueType, Value* pointer, Value* expected, Value* value, uint32_t uoffset) { pointer = fixupPointerPlusOffsetForAtomicOps(op, pointer, uoffset); B3::Width accessWidth = Wasm::accessWidth(op); if (widthForType(toB3Type(valueType)) == accessWidth) return sanitizeAtomicResult(op, valueType, m_currentBlock->appendNew(m_proc, memoryKind(AtomicStrongCAS), origin(), accessWidth, expected, value, pointer)); Value* maximum = nullptr; switch (valueType.kind) { case TypeKind::I64: { switch (accessWidth) { case B3::Width8: maximum = constant(Int64, UINT8_MAX); break; case B3::Width16: maximum = constant(Int64, UINT16_MAX); break; case B3::Width32: maximum = constant(Int64, UINT32_MAX); break; case B3::Width64: RELEASE_ASSERT_NOT_REACHED(); } break; } case TypeKind::I32: switch (accessWidth) { case B3::Width8: maximum = constant(Int32, UINT8_MAX); break; case B3::Width16: maximum = constant(Int32, UINT16_MAX); break; case B3::Width32: case B3::Width64: RELEASE_ASSERT_NOT_REACHED(); } break; default: RELEASE_ASSERT_NOT_REACHED(); } BasicBlock* failureCase = m_proc.addBlock(); BasicBlock* successCase = m_proc.addBlock(); BasicBlock* continuation = m_proc.addBlock(); auto condition = m_currentBlock->appendNew(m_proc, Above, origin(), expected, maximum); m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(), condition, FrequentedBlock(failureCase, FrequencyClass::Rare), FrequentedBlock(successCase, FrequencyClass::Normal)); failureCase->addPredecessor(m_currentBlock); successCase->addPredecessor(m_currentBlock); m_currentBlock = successCase; B3::UpsilonValue* successValue = nullptr; { auto truncatedExpected = expected; auto truncatedValue = value; if (valueType.isI64()) { truncatedExpected = m_currentBlock->appendNew(m_proc, B3::Trunc, Origin(), expected); truncatedValue = m_currentBlock->appendNew(m_proc, B3::Trunc, Origin(), value); } auto result = m_currentBlock->appendNew(m_proc, memoryKind(AtomicStrongCAS), origin(), accessWidth, truncatedExpected, truncatedValue, pointer); successValue = m_currentBlock->appendNew(m_proc, origin(), result); m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), continuation); continuation->addPredecessor(m_currentBlock); } m_currentBlock = failureCase; B3::UpsilonValue* failureValue = nullptr; { Value* addingValue = nullptr; switch (accessWidth) { case B3::Width8: case B3::Width16: case B3::Width32: addingValue = constant(Int32, 0); break; case B3::Width64: addingValue = constant(Int64, 0); break; } auto result = m_currentBlock->appendNew(m_proc, memoryKind(AtomicXchgAdd), origin(), accessWidth, addingValue, pointer); failureValue = m_currentBlock->appendNew(m_proc, origin(), result); m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), continuation); continuation->addPredecessor(m_currentBlock); } m_currentBlock = continuation; Value* phi = continuation->appendNew(m_proc, Phi, accessWidth == B3::Width64 ? Int64 : Int32, origin()); successValue->setPhi(phi); failureValue->setPhi(phi); return sanitizeAtomicResult(op, valueType, phi); } auto B3IRGenerator::atomicCompareExchange(ExtAtomicOpType op, Type valueType, ExpressionType pointer, ExpressionType expected, ExpressionType value, ExpressionType& result, uint32_t offset) -> PartialResult { ASSERT(pointer->type() == Int32); if (UNLIKELY(sumOverflows(offset, sizeOfAtomicOpMemoryAccess(op)))) { // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435 B3::PatchpointValue* throwException = m_currentBlock->appendNew(m_proc, B3::Void, origin()); throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess); }); switch (valueType.kind) { case TypeKind::I32: result = push(constant(Int32, 0)); break; case TypeKind::I64: result = push(constant(Int64, 0)); break; default: RELEASE_ASSERT_NOT_REACHED(); break; } } else result = push(emitAtomicCompareExchange(op, valueType, emitCheckAndPreparePointer(get(pointer), offset, sizeOfAtomicOpMemoryAccess(op)), get(expected), get(value), offset)); return { }; } auto B3IRGenerator::atomicWait(ExtAtomicOpType op, ExpressionType pointerVar, ExpressionType valueVar, ExpressionType timeoutVar, ExpressionType& result, uint32_t offset) -> PartialResult { Value* pointer = get(pointerVar); Value* value = get(valueVar); Value* timeout = get(timeoutVar); Value* resultValue = nullptr; if (op == ExtAtomicOpType::MemoryAtomicWait32) { resultValue = m_currentBlock->appendNew(m_proc, Int32, origin(), m_currentBlock->appendNew(m_proc, origin(), tagCFunction(operationMemoryAtomicWait32)), instanceValue(), pointer, m_currentBlock->appendNew(m_proc, origin(), offset), value, timeout); } else { resultValue = m_currentBlock->appendNew(m_proc, Int32, origin(), m_currentBlock->appendNew(m_proc, origin(), tagCFunction(operationMemoryAtomicWait64)), instanceValue(), pointer, m_currentBlock->appendNew(m_proc, origin(), offset), value, timeout); } { result = push(resultValue); CheckValue* check = m_currentBlock->appendNew(m_proc, Check, origin(), m_currentBlock->appendNew(m_proc, LessThan, origin(), resultValue, m_currentBlock->appendNew(m_proc, origin(), 0))); check->setGenerator([=, this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess); }); } return { }; } auto B3IRGenerator::atomicNotify(ExtAtomicOpType, ExpressionType pointer, ExpressionType count, ExpressionType& result, uint32_t offset) -> PartialResult { Value* resultValue = m_currentBlock->appendNew(m_proc, Int32, origin(), m_currentBlock->appendNew(m_proc, origin(), tagCFunction(operationMemoryAtomicNotify)), instanceValue(), get(pointer), m_currentBlock->appendNew(m_proc, origin(), offset), get(count)); { result = push(resultValue); CheckValue* check = m_currentBlock->appendNew(m_proc, Check, origin(), m_currentBlock->appendNew(m_proc, LessThan, origin(), resultValue, m_currentBlock->appendNew(m_proc, origin(), 0))); check->setGenerator([=, this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess); }); } return { }; } auto B3IRGenerator::atomicFence(ExtAtomicOpType, uint8_t) -> PartialResult { m_currentBlock->appendNew(m_proc, origin()); return { }; } auto B3IRGenerator::truncSaturated(Ext1OpType op, ExpressionType argVar, ExpressionType& result, Type returnType, Type) -> PartialResult { Value* arg = get(argVar); Value* maxFloat = nullptr; Value* minFloat = nullptr; Value* signBitConstant = nullptr; bool requiresMacroScratchRegisters = false; switch (op) { case Ext1OpType::I32TruncSatF32S: maxFloat = constant(Float, bitwise_cast(-static_cast(std::numeric_limits::min()))); minFloat = constant(Float, bitwise_cast(static_cast(std::numeric_limits::min()))); break; case Ext1OpType::I32TruncSatF32U: maxFloat = constant(Float, bitwise_cast(static_cast(std::numeric_limits::min()) * static_cast(-2.0))); minFloat = constant(Float, bitwise_cast(static_cast(-1.0))); break; case Ext1OpType::I32TruncSatF64S: maxFloat = constant(Double, bitwise_cast(-static_cast(std::numeric_limits::min()))); minFloat = constant(Double, bitwise_cast(static_cast(std::numeric_limits::min()) - 1.0)); break; case Ext1OpType::I32TruncSatF64U: maxFloat = constant(Double, bitwise_cast(static_cast(std::numeric_limits::min()) * -2.0)); minFloat = constant(Double, bitwise_cast(-1.0)); break; case Ext1OpType::I64TruncSatF32S: maxFloat = constant(Float, bitwise_cast(-static_cast(std::numeric_limits::min()))); minFloat = constant(Float, bitwise_cast(static_cast(std::numeric_limits::min()))); break; case Ext1OpType::I64TruncSatF32U: maxFloat = constant(Float, bitwise_cast(static_cast(std::numeric_limits::min()) * static_cast(-2.0))); minFloat = constant(Float, bitwise_cast(static_cast(-1.0))); // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if // the numbers would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it // so we can pool them if needed. if (isX86()) signBitConstant = constant(Float, bitwise_cast(static_cast(std::numeric_limits::max() - std::numeric_limits::max()))); requiresMacroScratchRegisters = true; break; case Ext1OpType::I64TruncSatF64S: maxFloat = constant(Double, bitwise_cast(-static_cast(std::numeric_limits::min()))); minFloat = constant(Double, bitwise_cast(static_cast(std::numeric_limits::min()))); break; case Ext1OpType::I64TruncSatF64U: maxFloat = constant(Double, bitwise_cast(static_cast(std::numeric_limits::min()) * -2.0)); minFloat = constant(Double, bitwise_cast(-1.0)); // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it // so we can pool them if needed. if (isX86()) signBitConstant = constant(Double, bitwise_cast(static_cast(std::numeric_limits::max() - std::numeric_limits::max()))); requiresMacroScratchRegisters = true; break; default: RELEASE_ASSERT_NOT_REACHED(); break; } PatchpointValue* patchpoint = m_currentBlock->appendNew(m_proc, toB3Type(returnType), origin()); patchpoint->append(arg, ValueRep::SomeRegister); if (requiresMacroScratchRegisters) { if (isX86()) { ASSERT(signBitConstant); patchpoint->append(signBitConstant, ValueRep::SomeRegister); patchpoint->numFPScratchRegisters = 1; } patchpoint->clobber(RegisterSet::macroScratchRegisters()); } patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) { switch (op) { case Ext1OpType::I32TruncSatF32S: jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr()); break; case Ext1OpType::I32TruncSatF32U: jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr()); break; case Ext1OpType::I32TruncSatF64S: jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr()); break; case Ext1OpType::I32TruncSatF64U: jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr()); break; case Ext1OpType::I64TruncSatF32S: jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr()); break; case Ext1OpType::I64TruncSatF32U: { AllowMacroScratchRegisterUsage allowScratch(jit); ASSERT(requiresMacroScratchRegisters); FPRReg scratch = InvalidFPRReg; FPRReg constant = InvalidFPRReg; if (isX86()) { scratch = params.fpScratch(0); constant = params[2].fpr(); } jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant); break; } case Ext1OpType::I64TruncSatF64S: jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr()); break; case Ext1OpType::I64TruncSatF64U: { AllowMacroScratchRegisterUsage allowScratch(jit); ASSERT(requiresMacroScratchRegisters); FPRReg scratch = InvalidFPRReg; FPRReg constant = InvalidFPRReg; if (isX86()) { scratch = params.fpScratch(0); constant = params[2].fpr(); } jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant); break; } default: RELEASE_ASSERT_NOT_REACHED(); break; } }); patchpoint->effects = Effects::none(); Value* maxResult = nullptr; Value* minResult = nullptr; Value* zero = nullptr; bool requiresNaNCheck = false; switch (op) { case Ext1OpType::I32TruncSatF32S: case Ext1OpType::I32TruncSatF64S: maxResult = constant(Int32, bitwise_cast(INT32_MAX)); minResult = constant(Int32, bitwise_cast(INT32_MIN)); zero = constant(Int32, 0); requiresNaNCheck = true; break; case Ext1OpType::I32TruncSatF32U: case Ext1OpType::I32TruncSatF64U: maxResult = constant(Int32, bitwise_cast(UINT32_MAX)); minResult = constant(Int32, bitwise_cast(0U)); break; case Ext1OpType::I64TruncSatF32S: case Ext1OpType::I64TruncSatF64S: maxResult = constant(Int64, bitwise_cast(INT64_MAX)); minResult = constant(Int64, bitwise_cast(INT64_MIN)); zero = constant(Int64, 0); requiresNaNCheck = true; break; case Ext1OpType::I64TruncSatF32U: case Ext1OpType::I64TruncSatF64U: maxResult = constant(Int64, bitwise_cast(UINT64_MAX)); minResult = constant(Int64, bitwise_cast(0ULL)); break; default: RELEASE_ASSERT_NOT_REACHED(); break; } result = push(m_currentBlock->appendNew(m_proc, B3::Select, origin(), m_currentBlock->appendNew(m_proc, GreaterThan, origin(), arg, minFloat), m_currentBlock->appendNew(m_proc, B3::Select, origin(), m_currentBlock->appendNew(m_proc, LessThan, origin(), arg, maxFloat), patchpoint, maxResult), requiresNaNCheck ? m_currentBlock->appendNew(m_proc, B3::Select, origin(), m_currentBlock->appendNew(m_proc, Equal, origin(), arg, arg), minResult, zero) : minResult)); return { }; } auto B3IRGenerator::addRttCanon(uint32_t typeIndex, ExpressionType& result) -> PartialResult { result = push(m_currentBlock->appendNew(m_proc, toB3Type(Types::Rtt), origin(), m_currentBlock->appendNew(m_proc, origin(), tagCFunction(operationWasmRttCanon)), instanceValue(), m_currentBlock->appendNew(m_proc, origin(), typeIndex))); return { }; } auto B3IRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult { result = push(m_currentBlock->appendNew(m_proc, B3::Select, origin(), get(condition), get(nonZero), get(zero))); return { }; } B3IRGenerator::ExpressionType B3IRGenerator::addConstant(Type type, uint64_t value) { return push(constant(toB3Type(type), value)); } void B3IRGenerator::emitEntryTierUpCheck() { if (!m_tierUp) return; ASSERT(m_tierUp); Value* countDownLocation = constant(pointerType(), bitwise_cast(&m_tierUp->m_counter), Origin()); PatchpointValue* patch = m_currentBlock->appendNew(m_proc, B3::Void, Origin()); Effects effects = Effects::none(); // FIXME: we should have a more precise heap range for the tier up count. effects.reads = B3::HeapRange::top(); effects.writes = B3::HeapRange::top(); patch->effects = effects; patch->clobber(RegisterSet::macroScratchRegisters()); patch->append(countDownLocation, ValueRep::SomeRegister); patch->setGenerator([=, this] (CCallHelpers& jit, const StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); CCallHelpers::Jump tierUp = jit.branchAdd32(CCallHelpers::PositiveOrZero, CCallHelpers::TrustedImm32(TierUpCount::functionEntryIncrement()), CCallHelpers::Address(params[0].gpr())); CCallHelpers::Label tierUpResume = jit.label(); params.addLatePath([=, this] (CCallHelpers& jit) { tierUp.link(&jit); const unsigned extraPaddingBytes = 0; RegisterSet registersToSpill = { }; registersToSpill.add(GPRInfo::argumentGPR1); unsigned numberOfStackBytesUsedForRegisterPreservation = ScratchRegisterAllocator::preserveRegistersToStackForCall(jit, registersToSpill, extraPaddingBytes); jit.move(MacroAssembler::TrustedImm32(m_functionIndex), GPRInfo::argumentGPR1); MacroAssembler::Call call = jit.nearCall(); ScratchRegisterAllocator::restoreRegistersFromStackForCall(jit, registersToSpill, RegisterSet(), numberOfStackBytesUsedForRegisterPreservation, extraPaddingBytes); jit.jump(tierUpResume); jit.addLinkTask([=] (LinkBuffer& linkBuffer) { MacroAssembler::repatchNearCall(linkBuffer.locationOfNearCall(call), CodeLocationLabel(Thunks::singleton().stub(triggerOMGEntryTierUpThunkGenerator).code())); }); }); }); } void B3IRGenerator::emitLoopTierUpCheck(uint32_t loopIndex, const Stack& enclosingStack, const Stack& newStack) { uint32_t outerLoopIndex = this->outerLoopIndex(); m_outerLoops.append(loopIndex); if (!m_tierUp) return; Origin origin = this->origin(); ASSERT(m_tierUp->osrEntryTriggers().size() == loopIndex); m_tierUp->osrEntryTriggers().append(TierUpCount::TriggerReason::DontTrigger); m_tierUp->outerLoops().append(outerLoopIndex); Value* countDownLocation = constant(pointerType(), bitwise_cast(&m_tierUp->m_counter), origin); Vector stackmap; for (auto& local : m_locals) stackmap.append(get(local)); for (unsigned controlIndex = 0; controlIndex < m_parser->controlStack().size(); ++controlIndex) { auto& data = m_parser->controlStack()[controlIndex].controlData; auto& expressionStack = m_parser->controlStack()[controlIndex].enclosedExpressionStack; for (TypedExpression value : expressionStack) stackmap.append(get(value)); if (ControlType::isAnyCatch(data)) stackmap.append(get(data.exception())); } for (TypedExpression value : enclosingStack) stackmap.append(get(value)); for (TypedExpression value : newStack) stackmap.append(get(value)); PatchpointValue* patch = m_currentBlock->appendNew(m_proc, B3::Void, origin); Effects effects = Effects::none(); // FIXME: we should have a more precise heap range for the tier up count. effects.reads = B3::HeapRange::top(); effects.writes = B3::HeapRange::top(); effects.exitsSideways = true; patch->effects = effects; patch->clobber(RegisterSet::macroScratchRegisters()); RegisterSet clobberLate; clobberLate.add(GPRInfo::argumentGPR0); patch->clobberLate(clobberLate); patch->append(countDownLocation, ValueRep::SomeRegister); patch->appendVectorWithRep(stackmap, ValueRep::ColdAny); TierUpCount::TriggerReason* forceEntryTrigger = &(m_tierUp->osrEntryTriggers().last()); static_assert(!static_cast(TierUpCount::TriggerReason::DontTrigger), "the JIT code assumes non-zero means 'enter'"); static_assert(sizeof(TierUpCount::TriggerReason) == 1, "branchTest8 assumes this size"); patch->setGenerator([=, this] (CCallHelpers& jit, const StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); CCallHelpers::Jump forceOSREntry = jit.branchTest8(CCallHelpers::NonZero, CCallHelpers::AbsoluteAddress(forceEntryTrigger)); CCallHelpers::Jump tierUp = jit.branchAdd32(CCallHelpers::PositiveOrZero, CCallHelpers::TrustedImm32(TierUpCount::loopIncrement()), CCallHelpers::Address(params[0].gpr())); MacroAssembler::Label tierUpResume = jit.label(); // First argument is the countdown location. ASSERT(params.value()->numChildren() >= 1); StackMap values(params.value()->numChildren() - 1); for (unsigned i = 1; i < params.value()->numChildren(); ++i) values[i - 1] = OSREntryValue(params[i], params.value()->child(i)->type()); OSREntryData& osrEntryData = m_tierUp->addOSREntryData(m_functionIndex, loopIndex, WTFMove(values)); OSREntryData* osrEntryDataPtr = &osrEntryData; params.addLatePath([=] (CCallHelpers& jit) { AllowMacroScratchRegisterUsage allowScratch(jit); forceOSREntry.link(&jit); tierUp.link(&jit); jit.probe(tagCFunction(operationWasmTriggerOSREntryNow), osrEntryDataPtr); jit.branchTestPtr(CCallHelpers::Zero, GPRInfo::argumentGPR0).linkTo(tierUpResume, &jit); jit.farJump(GPRInfo::argumentGPR1, WasmEntryPtrTag); }); }); } Value* B3IRGenerator::loadFromScratchBuffer(unsigned& indexInBuffer, Value* pointer, B3::Type type) { size_t offset = sizeof(uint64_t) * indexInBuffer++; RELEASE_ASSERT(type.isNumeric()); return m_currentBlock->appendNew(m_proc, Load, type, origin(), pointer, offset); } void B3IRGenerator::connectControlAtEntrypoint(unsigned& indexInBuffer, Value* pointer, ControlData& data, Stack& expressionStack, ControlData& currentData, bool fillLoopPhis) { for (unsigned i = 0; i < expressionStack.size(); i++) { TypedExpression value = expressionStack[i]; auto* load = loadFromScratchBuffer(indexInBuffer, pointer, value->type()); if (fillLoopPhis) m_currentBlock->appendNew(m_proc, origin(), load, data.phis[i]); else m_currentBlock->appendNew(m_proc, Set, origin(), value.value(), load); } if (ControlType::isAnyCatch(data) && &data != ¤tData) { auto* load = loadFromScratchBuffer(indexInBuffer, pointer, pointerType()); m_currentBlock->appendNew(m_proc, Set, origin(), data.exception(), load); } }; auto B3IRGenerator::addLoop(BlockSignature signature, Stack& enclosingStack, ControlType& block, Stack& newStack, uint32_t loopIndex) -> PartialResult { BasicBlock* body = m_proc.addBlock(); BasicBlock* continuation = m_proc.addBlock(); block = ControlData(m_proc, origin(), signature, BlockType::Loop, m_stackSize, continuation, body); unsigned offset = enclosingStack.size() - signature->as()->argumentCount(); for (unsigned i = 0; i < signature->as()->argumentCount(); ++i) { TypedExpression value = enclosingStack.at(offset + i); Value* phi = block.phis[i]; m_currentBlock->appendNew(m_proc, origin(), get(value), phi); body->append(phi); set(body, value, phi); newStack.append(value); } enclosingStack.shrink(offset); m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), body); if (loopIndex == m_loopIndexForOSREntry) { dataLogLnIf(WasmB3IRGeneratorInternal::verbose, "Setting up for OSR entry"); m_currentBlock = m_rootBlocks[0]; Value* pointer = m_rootBlocks[0]->appendNew(m_proc, Origin(), GPRInfo::argumentGPR0); unsigned indexInBuffer = 0; for (auto& local : m_locals) m_currentBlock->appendNew(m_proc, Set, Origin(), local, loadFromScratchBuffer(indexInBuffer, pointer, local->type())); for (unsigned controlIndex = 0; controlIndex < m_parser->controlStack().size(); ++controlIndex) { auto& data = m_parser->controlStack()[controlIndex].controlData; auto& expressionStack = m_parser->controlStack()[controlIndex].enclosedExpressionStack; connectControlAtEntrypoint(indexInBuffer, pointer, data, expressionStack, block); } connectControlAtEntrypoint(indexInBuffer, pointer, block, enclosingStack, block); connectControlAtEntrypoint(indexInBuffer, pointer, block, newStack, block, true); m_osrEntryScratchBufferSize = indexInBuffer; m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), body); body->addPredecessor(m_currentBlock); } m_currentBlock = body; emitLoopTierUpCheck(loopIndex, enclosingStack, newStack); return { }; } B3IRGenerator::ControlData B3IRGenerator::addTopLevel(BlockSignature signature) { return ControlData(m_proc, Origin(), signature, BlockType::TopLevel, m_stackSize, m_proc.addBlock()); } auto B3IRGenerator::addBlock(BlockSignature signature, Stack& enclosingStack, ControlType& newBlock, Stack& newStack) -> PartialResult { BasicBlock* continuation = m_proc.addBlock(); splitStack(signature, enclosingStack, newStack); newBlock = ControlData(m_proc, origin(), signature, BlockType::Block, m_stackSize, continuation); return { }; } auto B3IRGenerator::addIf(ExpressionType condition, BlockSignature signature, Stack& enclosingStack, ControlType& result, Stack& newStack) -> PartialResult { // FIXME: This needs to do some kind of stack passing. BasicBlock* taken = m_proc.addBlock(); BasicBlock* notTaken = m_proc.addBlock(); BasicBlock* continuation = m_proc.addBlock(); FrequencyClass takenFrequency = FrequencyClass::Normal; FrequencyClass notTakenFrequency = FrequencyClass::Normal; if (Options::useWebAssemblyBranchHints()) { BranchHint hint = m_info.getBranchHint(m_functionIndex, m_parser->currentOpcodeStartingOffset()); switch (hint) { case BranchHint::Unlikely: takenFrequency = FrequencyClass::Rare; break; case BranchHint::Likely: notTakenFrequency = FrequencyClass::Rare; break; case BranchHint::Invalid: break; } } m_currentBlock->appendNew(m_proc, B3::Branch, origin(), get(condition)); m_currentBlock->setSuccessors(FrequentedBlock(taken, takenFrequency), FrequentedBlock(notTaken, notTakenFrequency)); taken->addPredecessor(m_currentBlock); notTaken->addPredecessor(m_currentBlock); m_currentBlock = taken; splitStack(signature, enclosingStack, newStack); result = ControlData(m_proc, origin(), signature, BlockType::If, m_stackSize, continuation, notTaken); return { }; } auto B3IRGenerator::addElse(ControlData& data, const Stack& currentStack) -> PartialResult { unifyValuesWithBlock(currentStack, data); m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation); return addElseToUnreachable(data); } auto B3IRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult { ASSERT(data.blockType() == BlockType::If); m_stackSize = data.stackSize() + data.m_signature->as()->argumentCount(); m_currentBlock = data.special; data.convertIfToBlock(); return { }; } auto B3IRGenerator::addTry(BlockSignature signature, Stack& enclosingStack, ControlType& result, Stack& newStack) -> PartialResult { ++m_tryCatchDepth; BasicBlock* continuation = m_proc.addBlock(); splitStack(signature, enclosingStack, newStack); result = ControlData(m_proc, origin(), signature, BlockType::Try, m_stackSize, continuation, ++m_callSiteIndex, m_tryCatchDepth); return { }; } auto B3IRGenerator::addCatch(unsigned exceptionIndex, const TypeDefinition& signature, Stack& currentStack, ControlType& data, ResultList& results) -> PartialResult { unifyValuesWithBlock(currentStack, data); m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation); return addCatchToUnreachable(exceptionIndex, signature, data, results); } PatchpointExceptionHandle B3IRGenerator::preparePatchpointForExceptions(BasicBlock* block, PatchpointValue* patch) { ++m_callSiteIndex; if (!m_tryCatchDepth) return { }; Vector liveValues; Origin origin = this->origin(); for (Variable* local : m_locals) { Value* result = block->appendNew(m_proc, B3::Get, origin, local); liveValues.append(result); } for (unsigned controlIndex = 0; controlIndex < m_parser->controlStack().size(); ++controlIndex) { ControlData& data = m_parser->controlStack()[controlIndex].controlData; Stack& expressionStack = m_parser->controlStack()[controlIndex].enclosedExpressionStack; for (Variable* value : expressionStack) liveValues.append(get(block, value)); if (ControlType::isAnyCatch(data)) liveValues.append(get(block, data.exception())); } patch->effects.exitsSideways = true; patch->appendVectorWithRep(liveValues, ValueRep::LateColdAny); return PatchpointExceptionHandle { m_callSiteIndex, static_cast(liveValues.size()) }; } auto B3IRGenerator::addCatchToUnreachable(unsigned exceptionIndex, const TypeDefinition& signature, ControlType& data, ResultList& results) -> PartialResult { Value* operationResult = emitCatchImpl(CatchKind::Catch, data, exceptionIndex); Value* payload = m_currentBlock->appendNew(m_proc, origin(), pointerType(), operationResult, 1); for (unsigned i = 0; i < signature.as()->argumentCount(); ++i) { Type type = signature.as()->argumentType(i); Value* value = m_currentBlock->appendNew(m_proc, Load, toB3Type(type), origin(), payload, i * sizeof(uint64_t)); results.append(push(value)); } return { }; } auto B3IRGenerator::addCatchAll(Stack& currentStack, ControlType& data) -> PartialResult { unifyValuesWithBlock(currentStack, data); m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation); return addCatchAllToUnreachable(data); } auto B3IRGenerator::addCatchAllToUnreachable(ControlType& data) -> PartialResult { emitCatchImpl(CatchKind::CatchAll, data); return { }; } Value* B3IRGenerator::emitCatchImpl(CatchKind kind, ControlType& data, unsigned exceptionIndex) { m_hasCatch = true; m_currentBlock = m_proc.addBlock(); m_rootBlocks.append(m_currentBlock); m_stackSize = data.stackSize(); if (ControlType::isTry(data)) { if (kind == CatchKind::Catch) data.convertTryToCatch(++m_callSiteIndex, m_proc.addVariable(pointerType())); else data.convertTryToCatchAll(++m_callSiteIndex, m_proc.addVariable(pointerType())); } // We convert from "try" to "catch" ControlType above. This doesn't // happen if ControlType is already a "catch". This can happen when // we have multiple catches like "try {} catch(A){} catch(B){}...CatchAll(E){}" ASSERT(ControlType::isAnyCatch(data)); HandlerType handlerType = kind == CatchKind::Catch ? HandlerType::Catch : HandlerType::CatchAll; m_exceptionHandlers.append({ handlerType, data.tryStart(), data.tryEnd(), 0, m_tryCatchDepth, exceptionIndex }); restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, instanceValue(), m_proc, m_currentBlock, false); Value* pointer = m_currentBlock->appendNew(m_proc, Origin(), GPRInfo::argumentGPR0); unsigned indexInBuffer = 0; for (auto& local : m_locals) m_currentBlock->appendNew(m_proc, Set, Origin(), local, loadFromScratchBuffer(indexInBuffer, pointer, local->type())); for (unsigned controlIndex = 0; controlIndex < m_parser->controlStack().size(); ++controlIndex) { auto& controlData = m_parser->controlStack()[controlIndex].controlData; auto& expressionStack = m_parser->controlStack()[controlIndex].enclosedExpressionStack; connectControlAtEntrypoint(indexInBuffer, pointer, controlData, expressionStack, data); } PatchpointValue* result = m_currentBlock->appendNew(m_proc, m_proc.addTuple({ pointerType(), pointerType() }), origin()); result->effects.exitsSideways = true; result->clobber(RegisterSet::macroScratchRegisters()); RegisterSet clobberLate = RegisterSet::volatileRegistersForJSCall(); clobberLate.add(GPRInfo::argumentGPR0); result->clobberLate(clobberLate); result->append(instanceValue(), ValueRep::SomeRegister); result->resultConstraints.append(ValueRep::reg(GPRInfo::returnValueGPR)); result->resultConstraints.append(ValueRep::reg(GPRInfo::returnValueGPR2)); result->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); jit.move(params[2].gpr(), GPRInfo::argumentGPR0); CCallHelpers::Call call = jit.call(OperationPtrTag); jit.addLinkTask([call] (LinkBuffer& linkBuffer) { linkBuffer.link(call, FunctionPtr(operationWasmRetrieveAndClearExceptionIfCatchable)); }); }); Value* exception = m_currentBlock->appendNew(m_proc, origin(), pointerType(), result, 0); set(data.exception(), exception); return result; } auto B3IRGenerator::addDelegate(ControlType& target, ControlType& data) -> PartialResult { return addDelegateToUnreachable(target, data); } auto B3IRGenerator::addDelegateToUnreachable(ControlType& target, ControlType& data) -> PartialResult { unsigned targetDepth = 0; if (ControlType::isTry(target)) targetDepth = target.tryDepth(); m_exceptionHandlers.append({ HandlerType::Delegate, data.tryStart(), ++m_callSiteIndex, 0, m_tryCatchDepth, targetDepth }); return { }; } auto B3IRGenerator::addThrow(unsigned exceptionIndex, Vector& args, Stack&) -> PartialResult { PatchpointValue* patch = m_proc.add(B3::Void, origin()); patch->effects.terminal = true; patch->append(instanceValue(), ValueRep::reg(GPRInfo::argumentGPR0)); patch->append(framePointer(), ValueRep::reg(GPRInfo::argumentGPR1)); for (unsigned i = 0; i < args.size(); ++i) patch->append(get(args[i]), ValueRep::stackArgument(i * sizeof(EncodedJSValue))); patch->clobber(RegisterSet::volatileRegistersForJSCall()); PatchpointExceptionHandle handle = preparePatchpointForExceptions(m_currentBlock, patch); patch->setGenerator([this, exceptionIndex, handle] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); handle.generate(jit, params, this); emitThrowImpl(jit, exceptionIndex); }); m_currentBlock->append(patch); return { }; } auto B3IRGenerator::addRethrow(unsigned, ControlType& data) -> PartialResult { PatchpointValue* patch = m_proc.add(B3::Void, origin()); patch->clobber(RegisterSet::volatileRegistersForJSCall()); patch->effects.terminal = true; patch->append(instanceValue(), ValueRep::reg(GPRInfo::argumentGPR0)); patch->append(framePointer(), ValueRep::reg(GPRInfo::argumentGPR1)); patch->append(get(data.exception()), ValueRep::reg(GPRInfo::argumentGPR2)); PatchpointExceptionHandle handle = preparePatchpointForExceptions(m_currentBlock, patch); patch->setGenerator([this, handle] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); handle.generate(jit, params, this); emitRethrowImpl(jit); }); m_currentBlock->append(patch); return { }; } auto B3IRGenerator::addReturn(const ControlData&, const Stack& returnValues) -> PartialResult { CallInformation wasmCallInfo = wasmCallingConvention().callInformationFor(m_parser->signature(), CallRole::Callee); PatchpointValue* patch = m_proc.add(B3::Void, origin()); patch->setGenerator([] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) { auto calleeSaves = params.code().calleeSaveRegisterAtOffsetList(); jit.emitRestore(calleeSaves); jit.emitFunctionEpilogue(); jit.ret(); }); patch->effects.terminal = true; RELEASE_ASSERT(returnValues.size() >= wasmCallInfo.results.size()); unsigned offset = returnValues.size() - wasmCallInfo.results.size(); for (unsigned i = 0; i < wasmCallInfo.results.size(); ++i) { B3::ValueRep rep = wasmCallInfo.results[i]; if (rep.isStack()) { B3::Value* address = m_currentBlock->appendNew(m_proc, B3::Add, Origin(), framePointer(), constant(pointerType(), rep.offsetFromFP())); m_currentBlock->appendNew(m_proc, B3::Store, Origin(), get(returnValues[offset + i]), address); } else { ASSERT(rep.isReg()); patch->append(get(returnValues[offset + i]), rep); } } m_currentBlock->append(patch); return { }; } auto B3IRGenerator::addBranch(ControlData& data, ExpressionType condition, const Stack& returnValues) -> PartialResult { unifyValuesWithBlock(returnValues, data); BasicBlock* target = data.targetBlockForBranch(); FrequencyClass targetFrequency = FrequencyClass::Normal; FrequencyClass continuationFrequency = FrequencyClass::Normal; if (Options::useWebAssemblyBranchHints()) { BranchHint hint = m_info.getBranchHint(m_functionIndex, m_parser->currentOpcodeStartingOffset()); switch (hint) { case BranchHint::Unlikely: targetFrequency = FrequencyClass::Rare; break; case BranchHint::Likely: continuationFrequency = FrequencyClass::Rare; break; case BranchHint::Invalid: break; } } if (condition) { BasicBlock* continuation = m_proc.addBlock(); m_currentBlock->appendNew(m_proc, B3::Branch, origin(), get(condition)); m_currentBlock->setSuccessors(FrequentedBlock(target, targetFrequency), FrequentedBlock(continuation, continuationFrequency)); target->addPredecessor(m_currentBlock); continuation->addPredecessor(m_currentBlock); m_currentBlock = continuation; } else { m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), FrequentedBlock(target, targetFrequency)); target->addPredecessor(m_currentBlock); } return { }; } auto B3IRGenerator::addSwitch(ExpressionType condition, const Vector& targets, ControlData& defaultTarget, const Stack& expressionStack) -> PartialResult { UNUSED_PARAM(expressionStack); for (size_t i = 0; i < targets.size(); ++i) unifyValuesWithBlock(expressionStack, *targets[i]); unifyValuesWithBlock(expressionStack, defaultTarget); SwitchValue* switchValue = m_currentBlock->appendNew(m_proc, origin(), get(condition)); switchValue->setFallThrough(FrequentedBlock(defaultTarget.targetBlockForBranch())); for (size_t i = 0; i < targets.size(); ++i) switchValue->appendCase(SwitchCase(i, FrequentedBlock(targets[i]->targetBlockForBranch()))); return { }; } auto B3IRGenerator::endBlock(ControlEntry& entry, Stack& expressionStack) -> PartialResult { ControlData& data = entry.controlData; ASSERT(expressionStack.size() == data.signature()->as()->returnCount()); if (data.blockType() != BlockType::Loop) unifyValuesWithBlock(expressionStack, data); m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation); data.continuation->addPredecessor(m_currentBlock); return addEndToUnreachable(entry, expressionStack); } auto B3IRGenerator::addEndToUnreachable(ControlEntry& entry, const Stack& expressionStack) -> PartialResult { ControlData& data = entry.controlData; m_currentBlock = data.continuation; m_stackSize = data.stackSize(); if (data.blockType() == BlockType::If) { data.special->appendNewControlValue(m_proc, Jump, origin(), m_currentBlock); m_currentBlock->addPredecessor(data.special); } else if (data.blockType() == BlockType::Try || data.blockType() == BlockType::Catch) --m_tryCatchDepth; if (data.blockType() != BlockType::Loop) { for (unsigned i = 0; i < data.signature()->as()->returnCount(); ++i) { Value* result = data.phis[i]; m_currentBlock->append(result); entry.enclosedExpressionStack.constructAndAppend(data.signature()->as()->returnType(i), push(result)); } } else { m_outerLoops.removeLast(); for (unsigned i = 0; i < data.signature()->as()->returnCount(); ++i) { if (i < expressionStack.size()) { ++m_stackSize; entry.enclosedExpressionStack.append(expressionStack[i]); } else { Type returnType = data.signature()->as()->returnType(i); entry.enclosedExpressionStack.constructAndAppend(returnType, push(constant(toB3Type(returnType), 0xbbadbeef))); } } } // TopLevel does not have any code after this so we need to make sure we emit a return here. if (data.blockType() == BlockType::TopLevel) return addReturn(entry.controlData, entry.enclosedExpressionStack); return { }; } B3::Value* B3IRGenerator::createCallPatchpoint(BasicBlock* block, Origin origin, const TypeDefinition& signature, Vector& args, const ScopedLambda)>& patchpointFunctor) { Vector constrainedArguments; CallInformation wasmCallInfo = wasmCallingConvention().callInformationFor(signature); for (unsigned i = 0; i < args.size(); ++i) constrainedArguments.append(B3::ConstrainedValue(get(block, args[i]), wasmCallInfo.params[i])); m_proc.requestCallArgAreaSizeInBytes(WTF::roundUpToMultipleOf(stackAlignmentBytes(), wasmCallInfo.headerAndArgumentStackSizeInBytes)); Box exceptionHandle = Box::create(); B3::Type returnType = toB3ResultType(&signature); PatchpointValue* patchpoint = m_proc.add(returnType, origin); patchpoint->clobberEarly(RegisterSet::macroScratchRegisters()); patchpoint->clobberLate(RegisterSet::volatileRegistersForJSCall()); patchpointFunctor(patchpoint, exceptionHandle); patchpoint->appendVector(constrainedArguments); *exceptionHandle = preparePatchpointForExceptions(block, patchpoint); if (returnType != B3::Void) { Vector resultConstraints; for (auto valueLocation : wasmCallInfo.results) resultConstraints.append(B3::ValueRep(valueLocation)); patchpoint->resultConstraints = WTFMove(resultConstraints); } block->append(patchpoint); return patchpoint; } auto B3IRGenerator::addCall(uint32_t functionIndex, const TypeDefinition& signature, Vector& args, ResultList& results) -> PartialResult { ASSERT(signature.as()->argumentCount() == args.size()); m_makesCalls = true; B3::Type returnType = toB3ResultType(&signature); auto fillResults = [&] (Value* callResult) { ASSERT(returnType == callResult->type()); switch (returnType.kind()) { case B3::Void: { break; } case B3::Tuple: { const Vector& tuple = m_proc.tupleForType(returnType); ASSERT(signature.as()->returnCount() == tuple.size()); for (unsigned i = 0; i < signature.as()->returnCount(); ++i) results.append(push(m_currentBlock->appendNew(m_proc, origin(), tuple[i], callResult, i))); break; } default: { results.append(push(callResult)); break; } } }; Vector* unlinkedWasmToWasmCalls = &m_unlinkedWasmToWasmCalls; if (m_info.isImportedFunctionFromFunctionIndexSpace(functionIndex)) { m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast(args.size())); // FIXME: imports can be linked here, instead of generating a patchpoint, because all import stubs are generated before B3 compilation starts. https://bugs.webkit.org/show_bug.cgi?id=166462 Value* targetInstance = m_currentBlock->appendNew(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast(Instance::offsetOfTargetInstance(functionIndex))); // The target instance is 0 unless the call is wasm->wasm. Value* isWasmCall = m_currentBlock->appendNew(m_proc, NotEqual, origin(), targetInstance, m_currentBlock->appendNew(m_proc, origin(), 0)); BasicBlock* isWasmBlock = m_proc.addBlock(); BasicBlock* isEmbedderBlock = m_proc.addBlock(); BasicBlock* continuation = m_proc.addBlock(); m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(), isWasmCall, FrequentedBlock(isWasmBlock), FrequentedBlock(isEmbedderBlock)); Value* wasmCallResult = createCallPatchpoint(isWasmBlock, origin(), signature, args, scopedLambdaRef)>([=, this] (PatchpointValue* patchpoint, Box handle) -> void { patchpoint->effects.writesPinned = true; patchpoint->effects.readsPinned = true; // We need to clobber all potential pinned registers since we might be leaving the instance. // We pessimistically assume we could be calling to something that is bounds checking. // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking)); patchpoint->setGenerator([this, handle, unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); handle->generate(jit, params, this); CCallHelpers::Call call = jit.threadSafePatchableNearCall(); jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) { unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall(call), functionIndex }); }); }); })); UpsilonValue* wasmCallResultUpsilon = returnType == B3::Void ? nullptr : isWasmBlock->appendNew(m_proc, origin(), wasmCallResult); isWasmBlock->appendNewControlValue(m_proc, Jump, origin(), continuation); // FIXME: Let's remove this indirection by creating a PIC friendly IC // for calls out to the embedder. This shouldn't be that hard to do. We could probably // implement the IC to be over Context*. // https://bugs.webkit.org/show_bug.cgi?id=170375 Value* jumpDestination = isEmbedderBlock->appendNew(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast(Instance::offsetOfWasmToEmbedderStub(functionIndex))); Value* embedderCallResult = createCallPatchpoint(isEmbedderBlock, origin(), signature, args, scopedLambdaRef)>([=, this] (PatchpointValue* patchpoint, Box handle) -> void { patchpoint->effects.writesPinned = true; patchpoint->effects.readsPinned = true; patchpoint->append(jumpDestination, ValueRep::SomeRegister); // We need to clobber all potential pinned registers since we might be leaving the instance. // We pessimistically assume we could be calling to something that is bounds checking. // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking)); patchpoint->setGenerator([this, handle, returnType] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); handle->generate(jit, params, this); jit.call(params[params.proc().resultCount(returnType)].gpr(), WasmEntryPtrTag); }); })); UpsilonValue* embedderCallResultUpsilon = returnType == B3::Void ? nullptr : isEmbedderBlock->appendNew(m_proc, origin(), embedderCallResult); isEmbedderBlock->appendNewControlValue(m_proc, Jump, origin(), continuation); m_currentBlock = continuation; if (returnType != B3::Void) { Value* phi = continuation->appendNew(m_proc, Phi, returnType, origin()); wasmCallResultUpsilon->setPhi(phi); embedderCallResultUpsilon->setPhi(phi); fillResults(phi); } // The call could have been to another WebAssembly instance, and / or could have modified our Memory. restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, instanceValue(), m_proc, continuation); } else { Value* patch = createCallPatchpoint(m_currentBlock, origin(), signature, args, scopedLambdaRef)>([=, this] (PatchpointValue* patchpoint, Box handle) -> void { patchpoint->effects.writesPinned = true; patchpoint->effects.readsPinned = true; // We need to clobber the size register since the LLInt always bounds checks if (useSignalingMemory() || m_info.memory.isShared()) patchpoint->clobberLate(RegisterSet { PinnedRegisterInfo::get().boundsCheckingSizeRegister }); patchpoint->setGenerator([this, handle, unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); handle->generate(jit, params, this); CCallHelpers::Call call = jit.threadSafePatchableNearCall(); jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) { unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall(call), functionIndex }); }); }); })); fillResults(patch); } return { }; } auto B3IRGenerator::addCallIndirect(unsigned tableIndex, const TypeDefinition& signature, Vector& args, ResultList& results) -> PartialResult { Value* calleeIndex = get(args.takeLast()); ASSERT(signature.as()->argumentCount() == args.size()); m_makesCalls = true; // Note: call indirect can call either WebAssemblyFunction or WebAssemblyWrapperFunction. Because // WebAssemblyWrapperFunction is like calling into the embedder, we conservatively assume all call indirects // can be to the embedder for our stack check calculation. m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast(args.size())); Value* callableFunctionBuffer; Value* instancesBuffer; Value* callableFunctionBufferLength; { Value* table = m_currentBlock->appendNew(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast(Instance::offsetOfTablePtr(m_numImportFunctions, tableIndex))); callableFunctionBuffer = m_currentBlock->appendNew(m_proc, Load, pointerType(), origin(), table, safeCast(FuncRefTable::offsetOfFunctions())); instancesBuffer = m_currentBlock->appendNew(m_proc, Load, pointerType(), origin(), table, safeCast(FuncRefTable::offsetOfInstances())); callableFunctionBufferLength = m_currentBlock->appendNew(m_proc, Load, Int32, origin(), table, safeCast(Table::offsetOfLength())); } // Check the index we are looking for is valid. { CheckValue* check = m_currentBlock->appendNew(m_proc, Check, origin(), m_currentBlock->appendNew(m_proc, AboveEqual, origin(), calleeIndex, callableFunctionBufferLength)); check->setGenerator([=, this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsCallIndirect); }); } calleeIndex = m_currentBlock->appendNew(m_proc, ZExt32, origin(), calleeIndex); Value* callableFunction; { // Compute the offset in the table index space we are looking for. Value* offset = m_currentBlock->appendNew(m_proc, Mul, origin(), calleeIndex, constant(pointerType(), sizeof(WasmToWasmImportableFunction))); callableFunction = m_currentBlock->appendNew(m_proc, Add, origin(), callableFunctionBuffer, offset); // Check that the WasmToWasmImportableFunction is initialized. We trap if it isn't. An "invalid" SignatureIndex indicates it's not initialized. // FIXME: when we have trap handlers, we can just let the call fail because Signature::invalidIndex is 0. https://bugs.webkit.org/show_bug.cgi?id=177210 static_assert(sizeof(WasmToWasmImportableFunction::typeIndex) == sizeof(uint64_t), "Load codegen assumes i64"); Value* calleeSignatureIndex = m_currentBlock->appendNew(m_proc, Load, Int64, origin(), callableFunction, safeCast(WasmToWasmImportableFunction::offsetOfSignatureIndex())); { CheckValue* check = m_currentBlock->appendNew(m_proc, Check, origin(), m_currentBlock->appendNew(m_proc, Equal, origin(), calleeSignatureIndex, m_currentBlock->appendNew(m_proc, origin(), TypeDefinition::invalidIndex))); check->setGenerator([=, this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::NullTableEntry); }); } // Check the signature matches the value we expect. { Value* expectedSignatureIndex = m_currentBlock->appendNew(m_proc, origin(), TypeInformation::get(signature)); CheckValue* check = m_currentBlock->appendNew(m_proc, Check, origin(), m_currentBlock->appendNew(m_proc, NotEqual, origin(), calleeSignatureIndex, expectedSignatureIndex)); check->setGenerator([=, this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::BadSignature); }); } } Value* offset = m_currentBlock->appendNew(m_proc, Mul, origin(), calleeIndex, constant(pointerType(), sizeof(Instance*))); Value* calleeInstance = m_currentBlock->appendNew(m_proc, Load, pointerType(), origin(), m_currentBlock->appendNew(m_proc, Add, origin(), instancesBuffer, offset)); Value* calleeCode = m_currentBlock->appendNew(m_proc, Load, pointerType(), origin(), m_currentBlock->appendNew(m_proc, Load, pointerType(), origin(), callableFunction, safeCast(WasmToWasmImportableFunction::offsetOfEntrypointLoadLocation()))); return emitIndirectCall(calleeInstance, calleeCode, signature, args, results); } auto B3IRGenerator::addCallRef(const TypeDefinition& signature, Vector& args, ResultList& results) -> PartialResult { Value* callee = get(args.takeLast()); ASSERT(signature.as()->argumentCount() == args.size()); m_makesCalls = true; // Note: call ref can call either WebAssemblyFunction or WebAssemblyWrapperFunction. Because // WebAssemblyWrapperFunction is like calling into the embedder, we conservatively assume all call indirects // can be to the embedder for our stack check calculation. m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast(args.size())); // Check the target reference for null. { CheckValue* check = m_currentBlock->appendNew(m_proc, Check, origin(), m_currentBlock->appendNew(m_proc, Equal, origin(), callee, m_currentBlock->appendNew(m_proc, origin(), JSValue::encode(jsNull())))); check->setGenerator([=, this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::NullReference); }); } Value* jsInstanceOffset = constant(pointerType(), safeCast(WebAssemblyFunctionBase::offsetOfInstance())); Value* jsCalleeInstance = m_currentBlock->appendNew(m_proc, Load, pointerType(), origin(), m_currentBlock->appendNew(m_proc, Add, origin(), callee, jsInstanceOffset)); Value* instanceOffset = constant(pointerType(), safeCast(JSWebAssemblyInstance::offsetOfInstance())); Value* calleeInstance = m_currentBlock->appendNew(m_proc, Load, pointerType(), origin(), m_currentBlock->appendNew(m_proc, Add, origin(), jsCalleeInstance, instanceOffset)); Value* calleeCode = m_currentBlock->appendNew(m_proc, Load, pointerType(), origin(), m_currentBlock->appendNew(m_proc, Load, pointerType(), origin(), callee, safeCast(WebAssemblyFunctionBase::offsetOfEntrypointLoadLocation()))); return emitIndirectCall(calleeInstance, calleeCode, signature, args, results); } void B3IRGenerator::unify(Value* phi, const ExpressionType source) { m_currentBlock->appendNew(m_proc, origin(), get(source), phi); } void B3IRGenerator::unifyValuesWithBlock(const Stack& resultStack, const ControlData& block) { const Vector& phis = block.phis; size_t resultSize = phis.size(); ASSERT(resultSize <= resultStack.size()); for (size_t i = 0; i < resultSize; ++i) unify(phis[resultSize - 1 - i], resultStack.at(resultStack.size() - 1 - i)); } static void dumpExpressionStack(const CommaPrinter& comma, const B3IRGenerator::Stack& expressionStack) { dataLog(comma, "ExpressionStack:"); for (const auto& expression : expressionStack) dataLog(comma, *expression); } void B3IRGenerator::dump(const ControlStack& controlStack, const Stack* expressionStack) { dataLogLn("Constants:"); for (const auto& constant : m_constantPool) dataLogLn(deepDump(m_proc, constant.value)); dataLogLn("Processing Graph:"); dataLog(m_proc); dataLogLn("With current block:", *m_currentBlock); dataLogLn("Control stack:"); ASSERT(controlStack.size()); for (size_t i = controlStack.size(); i--;) { dataLog(" ", controlStack[i].controlData, ": "); CommaPrinter comma(", ", ""); dumpExpressionStack(comma, *expressionStack); expressionStack = &controlStack[i].enclosedExpressionStack; dataLogLn(); } dataLogLn(); } auto B3IRGenerator::origin() -> Origin { OpcodeOrigin origin(m_parser->currentOpcode(), m_parser->currentOpcodeStartingOffset()); ASSERT(isValidOpType(static_cast(origin.opcode()))); return bitwise_cast(origin); } static bool shouldDumpIRFor(uint32_t functionIndex) { static LazyNeverDestroyed dumpAllowlist; static std::once_flag initializeAllowlistFlag; std::call_once(initializeAllowlistFlag, [] { const char* functionAllowlistFile = Options::wasmB3FunctionsToDump(); dumpAllowlist.construct(functionAllowlistFile); }); return dumpAllowlist->shouldDumpWasmFunction(functionIndex); } Expected, String> parseAndCompileB3(CompilationContext& compilationContext, const FunctionData& function, const TypeDefinition& signature, Vector& unlinkedWasmToWasmCalls, const ModuleInformation& info, MemoryMode mode, CompilationMode compilationMode, uint32_t functionIndex, uint32_t loopIndexForOSREntry, TierUpCount* tierUp) { auto result = makeUnique(); compilationContext.wasmEntrypointJIT = makeUnique(); compilationContext.procedure = makeUnique(); Procedure& procedure = *compilationContext.procedure; if (shouldDumpIRFor(functionIndex + info.importFunctionCount())) procedure.setShouldDumpIR(); if (Options::useSamplingProfiler()) { // FIXME: We should do this based on VM relevant info. // But this is good enough for our own profiling for now. // When we start to show this data in web inspector, we'll // need other hooks into this besides the JSC option. procedure.setNeedsPCToOriginMap(); } procedure.setOriginPrinter([] (PrintStream& out, Origin origin) { if (origin.data()) out.print("Wasm: ", OpcodeOrigin(origin)); }); // This means we cannot use either StackmapGenerationParams::usedRegisters() or // StackmapGenerationParams::unavailableRegisters(). In exchange for this concession, we // don't strictly need to run Air::reportUsedRegisters(), which saves a bit of CPU time at // optLevel=1. procedure.setNeedsUsedRegisters(false); procedure.setOptLevel(isAnyBBQ(compilationMode) ? Options::webAssemblyBBQB3OptimizationLevel() : Options::webAssemblyOMGOptimizationLevel()); procedure.code().setForceIRCRegisterAllocation(); B3IRGenerator irGenerator(info, procedure, result.get(), unlinkedWasmToWasmCalls, result->osrEntryScratchBufferSize, mode, compilationMode, functionIndex, loopIndexForOSREntry, tierUp); FunctionParser parser(irGenerator, function.data.data(), function.data.size(), signature, info); WASM_FAIL_IF_HELPER_FAILS(parser.parse()); irGenerator.insertEntrySwitch(); irGenerator.insertConstants(); procedure.resetReachability(); if (ASSERT_ENABLED) validate(procedure, "After parsing:\n"); estimateStaticExecutionCounts(procedure); dataLogIf(WasmB3IRGeneratorInternal::verbose, "Pre SSA: ", procedure); fixSSA(procedure); dataLogIf(WasmB3IRGeneratorInternal::verbose, "Post SSA: ", procedure); { if (shouldDumpDisassemblyFor(compilationMode)) procedure.code().setDisassembler(makeUnique()); B3::prepareForGeneration(procedure); B3::generate(procedure, *compilationContext.wasmEntrypointJIT); compilationContext.wasmEntrypointByproducts = procedure.releaseByproducts(); result->entrypoint.calleeSaveRegisters = procedure.calleeSaveRegisterAtOffsetList(); } result->stackmaps = irGenerator.takeStackmaps(); result->exceptionHandlers = irGenerator.takeExceptionHandlers(); return result; } void computePCToCodeOriginMap(CompilationContext& context, LinkBuffer& linkBuffer) { if (context.procedure && context.procedure->needsPCToOriginMap()) { B3::PCToOriginMap originMap = context.procedure->releasePCToOriginMap(); context.pcToCodeOriginMap = Box::create(PCToCodeOriginMapBuilder(PCToCodeOriginMapBuilder::WasmCodeOriginMap, WTFMove(originMap)), linkBuffer); } } // Custom wasm ops. These are the ones too messy to do in wasm.json. void B3IRGenerator::emitChecksForModOrDiv(B3::Opcode operation, Value* left, Value* right) { ASSERT(operation == Div || operation == Mod || operation == UDiv || operation == UMod); const B3::Type type = left->type(); { CheckValue* check = m_currentBlock->appendNew(m_proc, Check, origin(), m_currentBlock->appendNew(m_proc, Equal, origin(), right, constant(type, 0))); check->setGenerator([=, this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::DivisionByZero); }); } if (operation == Div) { int64_t min = type == Int32 ? std::numeric_limits::min() : std::numeric_limits::min(); CheckValue* check = m_currentBlock->appendNew(m_proc, Check, origin(), m_currentBlock->appendNew(m_proc, BitAnd, origin(), m_currentBlock->appendNew(m_proc, Equal, origin(), left, constant(type, min)), m_currentBlock->appendNew(m_proc, Equal, origin(), right, constant(type, -1)))); check->setGenerator([=, this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::IntegerOverflow); }); } } template<> auto B3IRGenerator::addOp(ExpressionType leftVar, ExpressionType rightVar, ExpressionType& result) -> PartialResult { const B3::Opcode op = Div; Value* left = get(leftVar); Value* right = get(rightVar); emitChecksForModOrDiv(op, left, right); result = push(m_currentBlock->appendNew(m_proc, op, origin(), left, right)); return { }; } template<> auto B3IRGenerator::addOp(ExpressionType leftVar, ExpressionType rightVar, ExpressionType& result) -> PartialResult { const B3::Opcode op = Mod; Value* left = get(leftVar); Value* right = get(rightVar); emitChecksForModOrDiv(op, left, right); result = push(m_currentBlock->appendNew(m_proc, chill(op), origin(), left, right)); return { }; } template<> auto B3IRGenerator::addOp(ExpressionType leftVar, ExpressionType rightVar, ExpressionType& result) -> PartialResult { const B3::Opcode op = UDiv; Value* left = get(leftVar); Value* right = get(rightVar); emitChecksForModOrDiv(op, left, right); result = push(m_currentBlock->appendNew(m_proc, op, origin(), left, right)); return { }; } template<> auto B3IRGenerator::addOp(ExpressionType leftVar, ExpressionType rightVar, ExpressionType& result) -> PartialResult { const B3::Opcode op = UMod; Value* left = get(leftVar); Value* right = get(rightVar); emitChecksForModOrDiv(op, left, right); result = push(m_currentBlock->appendNew(m_proc, op, origin(), left, right)); return { }; } template<> auto B3IRGenerator::addOp(ExpressionType leftVar, ExpressionType rightVar, ExpressionType& result) -> PartialResult { const B3::Opcode op = Div; Value* left = get(leftVar); Value* right = get(rightVar); emitChecksForModOrDiv(op, left, right); result = push(m_currentBlock->appendNew(m_proc, op, origin(), left, right)); return { }; } template<> auto B3IRGenerator::addOp(ExpressionType leftVar, ExpressionType rightVar, ExpressionType& result) -> PartialResult { const B3::Opcode op = Mod; Value* left = get(leftVar); Value* right = get(rightVar); emitChecksForModOrDiv(op, left, right); result = push(m_currentBlock->appendNew(m_proc, chill(op), origin(), left, right)); return { }; } template<> auto B3IRGenerator::addOp(ExpressionType leftVar, ExpressionType rightVar, ExpressionType& result) -> PartialResult { const B3::Opcode op = UDiv; Value* left = get(leftVar); Value* right = get(rightVar); emitChecksForModOrDiv(op, left, right); result = push(m_currentBlock->appendNew(m_proc, op, origin(), left, right)); return { }; } template<> auto B3IRGenerator::addOp(ExpressionType leftVar, ExpressionType rightVar, ExpressionType& result) -> PartialResult { const B3::Opcode op = UMod; Value* left = get(leftVar); Value* right = get(rightVar); emitChecksForModOrDiv(op, left, right); result = push(m_currentBlock->appendNew(m_proc, op, origin(), left, right)); return { }; } template<> auto B3IRGenerator::addOp(ExpressionType argVar, ExpressionType& result) -> PartialResult { Value* arg = get(argVar); PatchpointValue* patchpoint = m_currentBlock->appendNew(m_proc, Int32, origin()); patchpoint->append(arg, ValueRep::SomeRegister); patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) { jit.countTrailingZeros32(params[1].gpr(), params[0].gpr()); }); patchpoint->effects = Effects::none(); result = push(patchpoint); return { }; } template<> auto B3IRGenerator::addOp(ExpressionType argVar, ExpressionType& result) -> PartialResult { Value* arg = get(argVar); PatchpointValue* patchpoint = m_currentBlock->appendNew(m_proc, Int64, origin()); patchpoint->append(arg, ValueRep::SomeRegister); patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) { jit.countTrailingZeros64(params[1].gpr(), params[0].gpr()); }); patchpoint->effects = Effects::none(); result = push(patchpoint); return { }; } template<> auto B3IRGenerator::addOp(ExpressionType argVar, ExpressionType& result) -> PartialResult { Value* arg = get(argVar); #if CPU(X86_64) if (MacroAssembler::supportsCountPopulation()) { PatchpointValue* patchpoint = m_currentBlock->appendNew(m_proc, Int32, origin()); patchpoint->append(arg, ValueRep::SomeRegister); patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) { jit.countPopulation32(params[1].gpr(), params[0].gpr()); }); patchpoint->effects = Effects::none(); result = push(patchpoint); return { }; } #endif Value* funcAddress = m_currentBlock->appendNew(m_proc, origin(), tagCFunction(operationPopcount32)); result = push(m_currentBlock->appendNew(m_proc, Int32, origin(), Effects::none(), funcAddress, arg)); return { }; } template<> auto B3IRGenerator::addOp(ExpressionType argVar, ExpressionType& result) -> PartialResult { Value* arg = get(argVar); #if CPU(X86_64) if (MacroAssembler::supportsCountPopulation()) { PatchpointValue* patchpoint = m_currentBlock->appendNew(m_proc, Int64, origin()); patchpoint->append(arg, ValueRep::SomeRegister); patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) { jit.countPopulation64(params[1].gpr(), params[0].gpr()); }); patchpoint->effects = Effects::none(); result = push(patchpoint); return { }; } #endif Value* funcAddress = m_currentBlock->appendNew(m_proc, origin(), tagCFunction(operationPopcount64)); result = push(m_currentBlock->appendNew(m_proc, Int64, origin(), Effects::none(), funcAddress, arg)); return { }; } template<> auto B3IRGenerator::addOp(ExpressionType argVar, ExpressionType& result) -> PartialResult { Value* arg = get(argVar); PatchpointValue* patchpoint = m_currentBlock->appendNew(m_proc, Double, origin()); if (isX86()) patchpoint->numGPScratchRegisters = 1; patchpoint->clobber(RegisterSet::macroScratchRegisters()); patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister)); patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); #if CPU(X86_64) jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0)); #else jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr()); #endif }); patchpoint->effects = Effects::none(); result = push(patchpoint); return { }; } template<> auto B3IRGenerator::addOp(ExpressionType argVar, ExpressionType& result) -> PartialResult { Value* arg = get(argVar); PatchpointValue* patchpoint = m_currentBlock->appendNew(m_proc, Float, origin()); if (isX86()) patchpoint->numGPScratchRegisters = 1; patchpoint->clobber(RegisterSet::macroScratchRegisters()); patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister)); patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); #if CPU(X86_64) jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0)); #else jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr()); #endif }); patchpoint->effects = Effects::none(); result = push(patchpoint); return { }; } template<> auto B3IRGenerator::addOp(ExpressionType argVar, ExpressionType& result) -> PartialResult { Value* arg = get(argVar); PatchpointValue* patchpoint = m_currentBlock->appendNew(m_proc, Double, origin()); patchpoint->append(arg, ValueRep::SomeRegister); patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) { jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr()); }); patchpoint->effects = Effects::none(); result = push(patchpoint); return { }; } template<> auto B3IRGenerator::addOp(ExpressionType argVar, ExpressionType& result) -> PartialResult { Value* arg = get(argVar); PatchpointValue* patchpoint = m_currentBlock->appendNew(m_proc, Float, origin()); patchpoint->append(arg, ValueRep::SomeRegister); patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) { jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr()); }); patchpoint->effects = Effects::none(); result = push(patchpoint); return { }; } template<> auto B3IRGenerator::addOp(ExpressionType argVar, ExpressionType& result) -> PartialResult { Value* arg = get(argVar); PatchpointValue* patchpoint = m_currentBlock->appendNew(m_proc, Double, origin()); patchpoint->append(arg, ValueRep::SomeRegister); patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) { jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr()); }); patchpoint->effects = Effects::none(); result = push(patchpoint); return { }; } template<> auto B3IRGenerator::addOp(ExpressionType argVar, ExpressionType& result) -> PartialResult { Value* arg = get(argVar); PatchpointValue* patchpoint = m_currentBlock->appendNew(m_proc, Float, origin()); patchpoint->append(arg, ValueRep::SomeRegister); patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) { jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr()); }); patchpoint->effects = Effects::none(); result = push(patchpoint); return { }; } template<> auto B3IRGenerator::addOp(ExpressionType argVar, ExpressionType& result) -> PartialResult { Value* arg = get(argVar); Value* max = constant(Double, bitwise_cast(-static_cast(std::numeric_limits::min()))); Value* min = constant(Double, bitwise_cast(static_cast(std::numeric_limits::min()) - 1.0)); Value* outOfBounds = m_currentBlock->appendNew(m_proc, BitAnd, origin(), m_currentBlock->appendNew(m_proc, LessThan, origin(), arg, max), m_currentBlock->appendNew(m_proc, GreaterThan, origin(), arg, min)); outOfBounds = m_currentBlock->appendNew(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0)); CheckValue* trap = m_currentBlock->appendNew(m_proc, Check, origin(), outOfBounds); trap->setGenerator([=, this] (CCallHelpers& jit, const StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc); }); PatchpointValue* patchpoint = m_currentBlock->appendNew(m_proc, Int32, origin()); patchpoint->append(arg, ValueRep::SomeRegister); patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) { jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr()); }); patchpoint->effects = Effects::none(); result = push(patchpoint); return { }; } template<> auto B3IRGenerator::addOp(ExpressionType argVar, ExpressionType& result) -> PartialResult { Value* arg = get(argVar); Value* max = constant(Float, bitwise_cast(-static_cast(std::numeric_limits::min()))); Value* min = constant(Float, bitwise_cast(static_cast(std::numeric_limits::min()))); Value* outOfBounds = m_currentBlock->appendNew(m_proc, BitAnd, origin(), m_currentBlock->appendNew(m_proc, LessThan, origin(), arg, max), m_currentBlock->appendNew(m_proc, GreaterEqual, origin(), arg, min)); outOfBounds = m_currentBlock->appendNew(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0)); CheckValue* trap = m_currentBlock->appendNew(m_proc, Check, origin(), outOfBounds); trap->setGenerator([=, this] (CCallHelpers& jit, const StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc); }); PatchpointValue* patchpoint = m_currentBlock->appendNew(m_proc, Int32, origin()); patchpoint->append(arg, ValueRep::SomeRegister); patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) { jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr()); }); patchpoint->effects = Effects::none(); result = push(patchpoint); return { }; } template<> auto B3IRGenerator::addOp(ExpressionType argVar, ExpressionType& result) -> PartialResult { Value* arg = get(argVar); Value* max = constant(Double, bitwise_cast(static_cast(std::numeric_limits::min()) * -2.0)); Value* min = constant(Double, bitwise_cast(-1.0)); Value* outOfBounds = m_currentBlock->appendNew(m_proc, BitAnd, origin(), m_currentBlock->appendNew(m_proc, LessThan, origin(), arg, max), m_currentBlock->appendNew(m_proc, GreaterThan, origin(), arg, min)); outOfBounds = m_currentBlock->appendNew(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0)); CheckValue* trap = m_currentBlock->appendNew(m_proc, Check, origin(), outOfBounds); trap->setGenerator([=, this] (CCallHelpers& jit, const StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc); }); PatchpointValue* patchpoint = m_currentBlock->appendNew(m_proc, Int32, origin()); patchpoint->append(arg, ValueRep::SomeRegister); patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) { jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr()); }); patchpoint->effects = Effects::none(); result = push(patchpoint); return { }; } template<> auto B3IRGenerator::addOp(ExpressionType argVar, ExpressionType& result) -> PartialResult { Value* arg = get(argVar); Value* max = constant(Float, bitwise_cast(static_cast(std::numeric_limits::min()) * static_cast(-2.0))); Value* min = constant(Float, bitwise_cast(static_cast(-1.0))); Value* outOfBounds = m_currentBlock->appendNew(m_proc, BitAnd, origin(), m_currentBlock->appendNew(m_proc, LessThan, origin(), arg, max), m_currentBlock->appendNew(m_proc, GreaterThan, origin(), arg, min)); outOfBounds = m_currentBlock->appendNew(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0)); CheckValue* trap = m_currentBlock->appendNew(m_proc, Check, origin(), outOfBounds); trap->setGenerator([=, this] (CCallHelpers& jit, const StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc); }); PatchpointValue* patchpoint = m_currentBlock->appendNew(m_proc, Int32, origin()); patchpoint->append(arg, ValueRep::SomeRegister); patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) { jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr()); }); patchpoint->effects = Effects::none(); result = push(patchpoint); return { }; } template<> auto B3IRGenerator::addOp(ExpressionType argVar, ExpressionType& result) -> PartialResult { Value* arg = get(argVar); Value* max = constant(Double, bitwise_cast(-static_cast(std::numeric_limits::min()))); Value* min = constant(Double, bitwise_cast(static_cast(std::numeric_limits::min()))); Value* outOfBounds = m_currentBlock->appendNew(m_proc, BitAnd, origin(), m_currentBlock->appendNew(m_proc, LessThan, origin(), arg, max), m_currentBlock->appendNew(m_proc, GreaterEqual, origin(), arg, min)); outOfBounds = m_currentBlock->appendNew(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0)); CheckValue* trap = m_currentBlock->appendNew(m_proc, Check, origin(), outOfBounds); trap->setGenerator([=, this] (CCallHelpers& jit, const StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc); }); PatchpointValue* patchpoint = m_currentBlock->appendNew(m_proc, Int64, origin()); patchpoint->append(arg, ValueRep::SomeRegister); patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) { jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr()); }); patchpoint->effects = Effects::none(); result = push(patchpoint); return { }; } template<> auto B3IRGenerator::addOp(ExpressionType argVar, ExpressionType& result) -> PartialResult { Value* arg = get(argVar); Value* max = constant(Double, bitwise_cast(static_cast(std::numeric_limits::min()) * -2.0)); Value* min = constant(Double, bitwise_cast(-1.0)); Value* outOfBounds = m_currentBlock->appendNew(m_proc, BitAnd, origin(), m_currentBlock->appendNew(m_proc, LessThan, origin(), arg, max), m_currentBlock->appendNew(m_proc, GreaterThan, origin(), arg, min)); outOfBounds = m_currentBlock->appendNew(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0)); CheckValue* trap = m_currentBlock->appendNew(m_proc, Check, origin(), outOfBounds); trap->setGenerator([=, this] (CCallHelpers& jit, const StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc); }); Value* signBitConstant; if (isX86()) { // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it // so we can pool them if needed. signBitConstant = constant(Double, bitwise_cast(static_cast(std::numeric_limits::max() - std::numeric_limits::max()))); } PatchpointValue* patchpoint = m_currentBlock->appendNew(m_proc, Int64, origin()); patchpoint->append(arg, ValueRep::SomeRegister); if (isX86()) { patchpoint->append(signBitConstant, ValueRep::SomeRegister); patchpoint->numFPScratchRegisters = 1; } patchpoint->clobber(RegisterSet::macroScratchRegisters()); patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); FPRReg scratch = InvalidFPRReg; FPRReg constant = InvalidFPRReg; if (isX86()) { scratch = params.fpScratch(0); constant = params[2].fpr(); } jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant); }); patchpoint->effects = Effects::none(); result = push(patchpoint); return { }; } template<> auto B3IRGenerator::addOp(ExpressionType argVar, ExpressionType& result) -> PartialResult { Value* arg = get(argVar); Value* max = constant(Float, bitwise_cast(-static_cast(std::numeric_limits::min()))); Value* min = constant(Float, bitwise_cast(static_cast(std::numeric_limits::min()))); Value* outOfBounds = m_currentBlock->appendNew(m_proc, BitAnd, origin(), m_currentBlock->appendNew(m_proc, LessThan, origin(), arg, max), m_currentBlock->appendNew(m_proc, GreaterEqual, origin(), arg, min)); outOfBounds = m_currentBlock->appendNew(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0)); CheckValue* trap = m_currentBlock->appendNew(m_proc, Check, origin(), outOfBounds); trap->setGenerator([=, this] (CCallHelpers& jit, const StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc); }); PatchpointValue* patchpoint = m_currentBlock->appendNew(m_proc, Int64, origin()); patchpoint->append(arg, ValueRep::SomeRegister); patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) { jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr()); }); patchpoint->effects = Effects::none(); result = push(patchpoint); return { }; } template<> auto B3IRGenerator::addOp(ExpressionType argVar, ExpressionType& result) -> PartialResult { Value* arg = get(argVar); Value* max = constant(Float, bitwise_cast(static_cast(std::numeric_limits::min()) * static_cast(-2.0))); Value* min = constant(Float, bitwise_cast(static_cast(-1.0))); Value* outOfBounds = m_currentBlock->appendNew(m_proc, BitAnd, origin(), m_currentBlock->appendNew(m_proc, LessThan, origin(), arg, max), m_currentBlock->appendNew(m_proc, GreaterThan, origin(), arg, min)); outOfBounds = m_currentBlock->appendNew(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0)); CheckValue* trap = m_currentBlock->appendNew(m_proc, Check, origin(), outOfBounds); trap->setGenerator([=, this] (CCallHelpers& jit, const StackmapGenerationParams&) { this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc); }); Value* signBitConstant; if (isX86()) { // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if // the numbers would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it // so we can pool them if needed. signBitConstant = constant(Float, bitwise_cast(static_cast(std::numeric_limits::max() - std::numeric_limits::max()))); } PatchpointValue* patchpoint = m_currentBlock->appendNew(m_proc, Int64, origin()); patchpoint->append(arg, ValueRep::SomeRegister); if (isX86()) { patchpoint->append(signBitConstant, ValueRep::SomeRegister); patchpoint->numFPScratchRegisters = 1; } patchpoint->clobber(RegisterSet::macroScratchRegisters()); patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); FPRReg scratch = InvalidFPRReg; FPRReg constant = InvalidFPRReg; if (isX86()) { scratch = params.fpScratch(0); constant = params[2].fpr(); } jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant); }); patchpoint->effects = Effects::none(); result = push(patchpoint); return { }; } } } // namespace JSC::Wasm #include "WasmB3IRGeneratorInlines.h" #endif // ENABLE(WEBASSEMBLY_B3JIT)