/*

* Copyright (C) 2011, 2012, 2013, 2014 Apple Inc. All rights reserved.

*

* Redistribution and use in source and binary forms, with or without

* modification, are permitted provided that the following conditions

* are met:

* 1. Redistributions of source code must retain the above copyright

* notice, this list of conditions and the following disclaimer.

* 2. Redistributions in binary form must reproduce the above copyright

* notice, this list of conditions and the following disclaimer in the

* documentation and/or other materials provided with the distribution.

*

* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY

* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR

* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,

* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,

* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR

* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY

* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

*/

#include "config.h"

#include "Repatch.h"

#if ENABLE(JIT)

#include "AccessorCallJITStubRoutine.h"

#include "CCallHelpers.h"

#include "DFGOperations.h"

#include "DFGSpeculativeJIT.h"

#include "FTLThunks.h"

#include "GCAwareJITStubRoutine.h"

#include "GetterSetter.h"

#include "JIT.h"

#include "JITInlines.h"

#include "LinkBuffer.h"

#include "JSCInlines.h"

#include "PolymorphicGetByIdList.h"

#include "PolymorphicPutByIdList.h"

#include "RegExpMatchesArray.h"

#include "RepatchBuffer.h"

#include "ScratchRegisterAllocator.h"

#include "StackAlignment.h"

#include "StructureRareDataInlines.h"

#include "StructureStubClearingWatchpoint.h"

#include "ThunkGenerators.h"

#include <wtf/StringPrintStream.h>

namespace JSC {

// Beware: in this code, it is not safe to assume anything about the following registers

// that would ordinarily have well-known values:

// - tagTypeNumberRegister

// - tagMaskRegister

static FunctionPtr readCallTarget(RepatchBuffer& repatchBuffer, CodeLocationCall call)

{

FunctionPtr result = MacroAssembler::readCallTarget(call);

#if ENABLE(FTL_JIT)

CodeBlock* codeBlock = repatchBuffer.codeBlock();

if (codeBlock->jitType() == JITCode::FTLJIT) {

return FunctionPtr(codeBlock->vm()->ftlThunks->keyForSlowPathCallThunk(

MacroAssemblerCodePtr::createFromExecutableAddress(

result.executableAddress())).callTarget());

}

#else

UNUSED_PARAM(repatchBuffer);

#endif // ENABLE(FTL_JIT)

return result;

}

static void repatchCall(RepatchBuffer& repatchBuffer, CodeLocationCall call, FunctionPtr newCalleeFunction)

{

#if ENABLE(FTL_JIT)

CodeBlock* codeBlock = repatchBuffer.codeBlock();

if (codeBlock->jitType() == JITCode::FTLJIT) {

VM& vm = *codeBlock->vm();

FTL::Thunks& thunks = *vm.ftlThunks;

FTL::SlowPathCallKey key = thunks.keyForSlowPathCallThunk(

MacroAssemblerCodePtr::createFromExecutableAddress(

MacroAssembler::readCallTarget(call).executableAddress()));

key = key.withCallTarget(newCalleeFunction.executableAddress());

newCalleeFunction = FunctionPtr(

thunks.getSlowPathCallThunk(vm, key).code().executableAddress());

}

#endif // ENABLE(FTL_JIT)

repatchBuffer.relink(call, newCalleeFunction);

}

static void repatchCall(CodeBlock* codeblock, CodeLocationCall call, FunctionPtr newCalleeFunction)

{

RepatchBuffer repatchBuffer(codeblock);

repatchCall(repatchBuffer, call, newCalleeFunction);

}

static void repatchByIdSelfAccess(VM& vm, CodeBlock* codeBlock, StructureStubInfo& stubInfo, Structure* structure, const Identifier& propertyName, PropertyOffset offset,

const FunctionPtr &slowPathFunction, bool compact)

{

if (structure->typeInfo().newImpurePropertyFiresWatchpoints())

vm.registerWatchpointForImpureProperty(propertyName, stubInfo.addWatchpoint(codeBlock));

RepatchBuffer repatchBuffer(codeBlock);

// Only optimize once!

repatchCall(repatchBuffer, stubInfo.callReturnLocation, slowPathFunction);

// Patch the structure check & the offset of the load.

repatchBuffer.repatch(stubInfo.callReturnLocation.dataLabel32AtOffset(-(intptr_t)stubInfo.patch.deltaCheckImmToCall), bitwise_cast<int32_t>(structure->id()));

repatchBuffer.setLoadInstructionIsActive(stubInfo.callReturnLocation.convertibleLoadAtOffset(stubInfo.patch.deltaCallToStorageLoad), isOutOfLineOffset(offset));

#if USE(JSVALUE64)

if (compact)

repatchBuffer.repatch(stubInfo.callReturnLocation.dataLabelCompactAtOffset(stubInfo.patch.deltaCallToLoadOrStore), offsetRelativeToPatchedStorage(offset));

else

repatchBuffer.repatch(stubInfo.callReturnLocation.dataLabel32AtOffset(stubInfo.patch.deltaCallToLoadOrStore), offsetRelativeToPatchedStorage(offset));

#elif USE(JSVALUE32_64)

if (compact) {

repatchBuffer.repatch(stubInfo.callReturnLocation.dataLabelCompactAtOffset(stubInfo.patch.deltaCallToTagLoadOrStore), offsetRelativeToPatchedStorage(offset) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag));

repatchBuffer.repatch(stubInfo.callReturnLocation.dataLabelCompactAtOffset(stubInfo.patch.deltaCallToPayloadLoadOrStore), offsetRelativeToPatchedStorage(offset) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload));

} else {

repatchBuffer.repatch(stubInfo.callReturnLocation.dataLabel32AtOffset(stubInfo.patch.deltaCallToTagLoadOrStore), offsetRelativeToPatchedStorage(offset) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag));

repatchBuffer.repatch(stubInfo.callReturnLocation.dataLabel32AtOffset(stubInfo.patch.deltaCallToPayloadLoadOrStore), offsetRelativeToPatchedStorage(offset) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload));

}

#endif

}

static void addStructureTransitionCheck(

JSCell* object, Structure* structure, CodeBlock* codeBlock, StructureStubInfo& stubInfo,

MacroAssembler& jit, MacroAssembler::JumpList& failureCases, GPRReg scratchGPR)

{

if (object->structure() == structure && structure->transitionWatchpointSetIsStillValid()) {

structure->addTransitionWatchpoint(stubInfo.addWatchpoint(codeBlock));

if (!ASSERT_DISABLED) {

// If we execute this code, the object must have the structure we expect. Assert

// this in debug modes.

jit.move(MacroAssembler::TrustedImmPtr(object), scratchGPR);

MacroAssembler::Jump ok = branchStructure(

jit,

MacroAssembler::Equal,

MacroAssembler::Address(scratchGPR, JSCell::structureIDOffset()),

structure);

jit.abortWithReason(RepatchIneffectiveWatchpoint);

ok.link(&jit);

}

return;

}

jit.move(MacroAssembler::TrustedImmPtr(object), scratchGPR);

failureCases.append(

branchStructure(jit,

MacroAssembler::NotEqual,

MacroAssembler::Address(scratchGPR, JSCell::structureIDOffset()),

structure));

}

static void addStructureTransitionCheck(

JSValue prototype, CodeBlock* codeBlock, StructureStubInfo& stubInfo,

MacroAssembler& jit, MacroAssembler::JumpList& failureCases, GPRReg scratchGPR)

{

if (prototype.isNull())

return;

ASSERT(prototype.isCell());

addStructureTransitionCheck(

prototype.asCell(), prototype.asCell()->structure(), codeBlock, stubInfo, jit,

failureCases, scratchGPR);

}

static void replaceWithJump(RepatchBuffer& repatchBuffer, StructureStubInfo& stubInfo, const MacroAssemblerCodePtr target)

{

if (MacroAssembler::canJumpReplacePatchableBranch32WithPatch()) {

repatchBuffer.replaceWithJump(

RepatchBuffer::startOfPatchableBranch32WithPatchOnAddress(

stubInfo.callReturnLocation.dataLabel32AtOffset(

-(intptr_t)stubInfo.patch.deltaCheckImmToCall)),

CodeLocationLabel(target));

return;

}

repatchBuffer.relink(

stubInfo.callReturnLocation.jumpAtOffset(

stubInfo.patch.deltaCallToJump),

CodeLocationLabel(target));

}

static void emitRestoreScratch(MacroAssembler& stubJit, bool needToRestoreScratch, GPRReg scratchGPR, MacroAssembler::Jump& success, MacroAssembler::Jump& fail, MacroAssembler::JumpList failureCases)

{

if (needToRestoreScratch) {

stubJit.popToRestore(scratchGPR);

success = stubJit.jump();

// link failure cases here, so we can pop scratchGPR, and then jump back.

failureCases.link(&stubJit);

stubJit.popToRestore(scratchGPR);

fail = stubJit.jump();

return;

}

success = stubJit.jump();

}

static void linkRestoreScratch(LinkBuffer& patchBuffer, bool needToRestoreScratch, MacroAssembler::Jump success, MacroAssembler::Jump fail, MacroAssembler::JumpList failureCases, CodeLocationLabel successLabel, CodeLocationLabel slowCaseBegin)

{

patchBuffer.link(success, successLabel);

if (needToRestoreScratch) {

patchBuffer.link(fail, slowCaseBegin);

return;

}

// link failure cases directly back to normal path

patchBuffer.link(failureCases, slowCaseBegin);

}

static void linkRestoreScratch(LinkBuffer& patchBuffer, bool needToRestoreScratch, StructureStubInfo& stubInfo, MacroAssembler::Jump success, MacroAssembler::Jump fail, MacroAssembler::JumpList failureCases)

{

linkRestoreScratch(patchBuffer, needToRestoreScratch, success, fail, failureCases, stubInfo.callReturnLocation.labelAtOffset(stubInfo.patch.deltaCallToDone), stubInfo.callReturnLocation.labelAtOffset(stubInfo.patch.deltaCallToSlowCase));

}

enum ByIdStubKind {

GetValue,

CallGetter,

CallCustomGetter,

CallSetter,

CallCustomSetter

};

static const char* toString(ByIdStubKind kind)

{

switch (kind) {

case GetValue:

return "GetValue";

case CallGetter:

return "CallGetter";

case CallCustomGetter:

return "CallCustomGetter";

case CallSetter:

return "CallSetter";

case CallCustomSetter:

return "CallCustomSetter";

default:

RELEASE_ASSERT_NOT_REACHED();

return nullptr;

}

}

static ByIdStubKind kindFor(const PropertySlot& slot)

{

if (slot.isCacheableValue())

return GetValue;

if (slot.isCacheableCustom())

return CallCustomGetter;

RELEASE_ASSERT(slot.isCacheableGetter());

return CallGetter;

}

static FunctionPtr customFor(const PropertySlot& slot)

{

if (!slot.isCacheableCustom())

return FunctionPtr();

return FunctionPtr(slot.customGetter());

}

static ByIdStubKind kindFor(const PutPropertySlot& slot)

{

RELEASE_ASSERT(!slot.isCacheablePut());

if (slot.isCacheableSetter())

return CallSetter;

RELEASE_ASSERT(slot.isCacheableCustom());

return CallCustomSetter;

}

static FunctionPtr customFor(const PutPropertySlot& slot)

{

if (!slot.isCacheableCustom())

return FunctionPtr();

return FunctionPtr(slot.customSetter());

}

static void generateByIdStub(

ExecState* exec, ByIdStubKind kind, const Identifier& propertyName,

FunctionPtr custom, StructureStubInfo& stubInfo, StructureChain* chain, size_t count,

PropertyOffset offset, Structure* structure, bool loadTargetFromProxy, WatchpointSet* watchpointSet,

CodeLocationLabel successLabel, CodeLocationLabel slowCaseLabel, RefPtr<JITStubRoutine>& stubRoutine)

{

VM* vm = &exec->vm();

GPRReg baseGPR = static_cast<GPRReg>(stubInfo.patch.baseGPR);

JSValueRegs valueRegs = JSValueRegs(

#if USE(JSVALUE32_64)

static_cast<GPRReg>(stubInfo.patch.valueTagGPR),

#endif

static_cast<GPRReg>(stubInfo.patch.valueGPR));

GPRReg scratchGPR = TempRegisterSet(stubInfo.patch.usedRegisters).getFreeGPR();

bool needToRestoreScratch = scratchGPR == InvalidGPRReg;

RELEASE_ASSERT(!needToRestoreScratch || kind == GetValue);

CCallHelpers stubJit(&exec->vm(), exec->codeBlock());

if (needToRestoreScratch) {

scratchGPR = AssemblyHelpers::selectScratchGPR(

baseGPR, valueRegs.tagGPR(), valueRegs.payloadGPR());

stubJit.pushToSave(scratchGPR);

needToRestoreScratch = true;

}

MacroAssembler::JumpList failureCases;

GPRReg baseForGetGPR;

if (loadTargetFromProxy) {

baseForGetGPR = valueRegs.payloadGPR();

failureCases.append(stubJit.branch8(

MacroAssembler::NotEqual,

MacroAssembler::Address(baseGPR, JSCell::typeInfoTypeOffset()),

MacroAssembler::TrustedImm32(PureForwardingProxyType)));

stubJit.loadPtr(MacroAssembler::Address(baseGPR, JSProxy::targetOffset()), scratchGPR);

failureCases.append(branchStructure(stubJit,

MacroAssembler::NotEqual,

MacroAssembler::Address(scratchGPR, JSCell::structureIDOffset()),

structure));

} else {

baseForGetGPR = baseGPR;

failureCases.append(branchStructure(stubJit,

MacroAssembler::NotEqual,

MacroAssembler::Address(baseForGetGPR, JSCell::structureIDOffset()),

structure));

}

CodeBlock* codeBlock = exec->codeBlock();

if (structure->typeInfo().newImpurePropertyFiresWatchpoints())

vm->registerWatchpointForImpureProperty(propertyName, stubInfo.addWatchpoint(codeBlock));

if (watchpointSet)

watchpointSet->add(stubInfo.addWatchpoint(codeBlock));

Structure* currStructure = structure;

JSObject* protoObject = 0;

if (chain) {

WriteBarrier<Structure>* it = chain->head();

for (unsigned i = 0; i < count; ++i, ++it) {

protoObject = asObject(currStructure->prototypeForLookup(exec));

Structure* protoStructure = protoObject->structure();

if (protoStructure->typeInfo().newImpurePropertyFiresWatchpoints())

vm->registerWatchpointForImpureProperty(propertyName, stubInfo.addWatchpoint(codeBlock));

addStructureTransitionCheck(

protoObject, protoStructure, codeBlock, stubInfo, stubJit,

failureCases, scratchGPR);

currStructure = it->get();

}

}

GPRReg baseForAccessGPR;

if (chain) {

// We could have clobbered scratchGPR earlier, so we have to reload from baseGPR to get the target.

if (loadTargetFromProxy)

stubJit.loadPtr(MacroAssembler::Address(baseGPR, JSProxy::targetOffset()), baseForGetGPR);

stubJit.move(MacroAssembler::TrustedImmPtr(protoObject), scratchGPR);

baseForAccessGPR = scratchGPR;

} else {

// For proxy objects, we need to do all the Structure checks before moving the baseGPR into

// baseForGetGPR because if we fail any of the checks then we would have the wrong value in baseGPR

// on the slow path.

if (loadTargetFromProxy)

stubJit.move(scratchGPR, baseForGetGPR);

baseForAccessGPR = baseForGetGPR;

}

GPRReg loadedValueGPR = InvalidGPRReg;

if (kind != CallCustomGetter && kind != CallCustomSetter) {

if (kind == GetValue)

loadedValueGPR = valueRegs.payloadGPR();

else

loadedValueGPR = scratchGPR;

GPRReg storageGPR;

if (isInlineOffset(offset))

storageGPR = baseForAccessGPR;

else {

stubJit.loadPtr(MacroAssembler::Address(baseForAccessGPR, JSObject::butterflyOffset()), loadedValueGPR);

storageGPR = loadedValueGPR;

}

#if USE(JSVALUE64)

stubJit.load64(MacroAssembler::Address(storageGPR, offsetRelativeToBase(offset)), loadedValueGPR);

#else

if (kind == GetValue)

stubJit.load32(MacroAssembler::Address(storageGPR, offsetRelativeToBase(offset) + TagOffset), valueRegs.tagGPR());

stubJit.load32(MacroAssembler::Address(storageGPR, offsetRelativeToBase(offset) + PayloadOffset), loadedValueGPR);

#endif

}

// Stuff for custom getters.

MacroAssembler::Call operationCall;

MacroAssembler::Call handlerCall;

// Stuff for JS getters.

MacroAssembler::DataLabelPtr addressOfLinkFunctionCheck;

MacroAssembler::Call fastPathCall;

MacroAssembler::Call slowPathCall;

std::unique_ptr<CallLinkInfo> callLinkInfo;

MacroAssembler::Jump success, fail;

if (kind != GetValue) {

// Need to make sure that whenever this call is made in the future, we remember the

// place that we made it from. It just so happens to be the place that we are at

// right now!

stubJit.store32(MacroAssembler::TrustedImm32(exec->locationAsRawBits()),

CCallHelpers::tagFor(static_cast<VirtualRegister>(JSStack::ArgumentCount)));

if (kind == CallGetter || kind == CallSetter) {

// Create a JS call using a JS call inline cache. Assume that:

//

// - SP is aligned and represents the extent of the calling compiler's stack usage.

//

// - FP is set correctly (i.e. it points to the caller's call frame header).

//

// - SP - FP is an aligned difference.

//

// - Any byte between FP (exclusive) and SP (inclusive) could be live in the calling

// code.

//

// Therefore, we temporarily grow the stack for the purpose of the call and then

// shrink it after.

callLinkInfo = std::make_unique<CallLinkInfo>();

callLinkInfo->callType = CallLinkInfo::Call;

callLinkInfo->codeOrigin = stubInfo.codeOrigin;

callLinkInfo->calleeGPR = loadedValueGPR;

MacroAssembler::JumpList done;

// There is a 'this' argument but nothing else.

unsigned numberOfParameters = 1;

// ... unless we're calling a setter.

if (kind == CallSetter)

numberOfParameters++;

// Get the accessor; if there ain't one then the result is jsUndefined().

if (kind == CallSetter) {

stubJit.loadPtr(

MacroAssembler::Address(loadedValueGPR, GetterSetter::offsetOfSetter()),

loadedValueGPR);

} else {

stubJit.loadPtr(

MacroAssembler::Address(loadedValueGPR, GetterSetter::offsetOfGetter()),

loadedValueGPR);

}

MacroAssembler::Jump returnUndefined = stubJit.branchTestPtr(

MacroAssembler::Zero, loadedValueGPR);

unsigned numberOfRegsForCall =

JSStack::CallFrameHeaderSize + numberOfParameters;

unsigned numberOfBytesForCall =

numberOfRegsForCall * sizeof(Register) - sizeof(CallerFrameAndPC);

unsigned alignedNumberOfBytesForCall =

WTF::roundUpToMultipleOf(stackAlignmentBytes(), numberOfBytesForCall);

stubJit.subPtr(

MacroAssembler::TrustedImm32(alignedNumberOfBytesForCall),

MacroAssembler::stackPointerRegister);

MacroAssembler::Address calleeFrame = MacroAssembler::Address(

MacroAssembler::stackPointerRegister,

-static_cast<ptrdiff_t>(sizeof(CallerFrameAndPC)));

stubJit.store32(

MacroAssembler::TrustedImm32(numberOfParameters),

calleeFrame.withOffset(

JSStack::ArgumentCount * sizeof(Register) + PayloadOffset));

stubJit.storeCell(

loadedValueGPR, calleeFrame.withOffset(JSStack::Callee * sizeof(Register)));

stubJit.storeCell(

baseForGetGPR,

calleeFrame.withOffset(

virtualRegisterForArgument(0).offset() * sizeof(Register)));

if (kind == CallSetter) {

stubJit.storeValue(

valueRegs,

calleeFrame.withOffset(

virtualRegisterForArgument(1).offset() * sizeof(Register)));

}

MacroAssembler::Jump slowCase = stubJit.branchPtrWithPatch(

MacroAssembler::NotEqual, loadedValueGPR, addressOfLinkFunctionCheck,

MacroAssembler::TrustedImmPtr(0));

// loadedValueGPR is already burned. We can reuse it. From here on we assume that

// any volatile register will be clobbered anyway.

stubJit.loadPtr(

MacroAssembler::Address(loadedValueGPR, JSFunction::offsetOfScopeChain()),

loadedValueGPR);

stubJit.storeCell(

loadedValueGPR, calleeFrame.withOffset(JSStack::ScopeChain * sizeof(Register)));

fastPathCall = stubJit.nearCall();

stubJit.addPtr(

MacroAssembler::TrustedImm32(alignedNumberOfBytesForCall),

MacroAssembler::stackPointerRegister);

if (kind == CallGetter)

stubJit.setupResults(valueRegs);

done.append(stubJit.jump());

slowCase.link(&stubJit);

stubJit.move(loadedValueGPR, GPRInfo::regT0);

#if USE(JSVALUE32_64)

stubJit.move(MacroAssembler::TrustedImm32(JSValue::CellTag), GPRInfo::regT1);

#endif

stubJit.move(MacroAssembler::TrustedImmPtr(callLinkInfo.get()), GPRInfo::regT2);

slowPathCall = stubJit.nearCall();

stubJit.addPtr(

MacroAssembler::TrustedImm32(alignedNumberOfBytesForCall),

MacroAssembler::stackPointerRegister);

if (kind == CallGetter)

stubJit.setupResults(valueRegs);

done.append(stubJit.jump());

returnUndefined.link(&stubJit);

if (kind == CallGetter)

stubJit.moveTrustedValue(jsUndefined(), valueRegs);

done.link(&stubJit);

} else {

// getter: EncodedJSValue (*GetValueFunc)(ExecState*, JSObject* slotBase, EncodedJSValue thisValue, PropertyName);

// setter: void (*PutValueFunc)(ExecState*, JSObject* base, EncodedJSValue thisObject, EncodedJSValue value);

#if USE(JSVALUE64)

if (kind == CallCustomGetter)

stubJit.setupArgumentsWithExecState(baseForAccessGPR, baseForGetGPR, MacroAssembler::TrustedImmPtr(propertyName.impl()));

else

stubJit.setupArgumentsWithExecState(baseForAccessGPR, baseForGetGPR, valueRegs.gpr());

#else

if (kind == CallCustomGetter)

stubJit.setupArgumentsWithExecState(baseForAccessGPR, baseForGetGPR, MacroAssembler::TrustedImm32(JSValue::CellTag), MacroAssembler::TrustedImmPtr(propertyName.impl()));

else

stubJit.setupArgumentsWithExecState(baseForAccessGPR, baseForGetGPR, MacroAssembler::TrustedImm32(JSValue::CellTag), valueRegs.payloadGPR(), valueRegs.tagGPR());

#endif

stubJit.storePtr(GPRInfo::callFrameRegister, &vm->topCallFrame);

operationCall = stubJit.call();

if (kind == CallCustomGetter)

stubJit.setupResults(valueRegs);

MacroAssembler::Jump noException = stubJit.emitExceptionCheck(CCallHelpers::InvertedExceptionCheck);

stubJit.setupArguments(CCallHelpers::TrustedImmPtr(vm), GPRInfo::callFrameRegister);

handlerCall = stubJit.call();

stubJit.jumpToExceptionHandler();

noException.link(&stubJit);

}

}

emitRestoreScratch(stubJit, needToRestoreScratch, scratchGPR, success, fail, failureCases);

LinkBuffer patchBuffer(*vm, stubJit, exec->codeBlock());

linkRestoreScratch(patchBuffer, needToRestoreScratch, success, fail, failureCases, successLabel, slowCaseLabel);

if (kind == CallCustomGetter || kind == CallCustomSetter) {

patchBuffer.link(operationCall, custom);

patchBuffer.link(handlerCall, lookupExceptionHandler);

} else if (kind == CallGetter || kind == CallSetter) {

callLinkInfo->hotPathOther = patchBuffer.locationOfNearCall(fastPathCall);

callLinkInfo->hotPathBegin = patchBuffer.locationOf(addressOfLinkFunctionCheck);

callLinkInfo->callReturnLocation = patchBuffer.locationOfNearCall(slowPathCall);

ThunkGenerator generator = linkThunkGeneratorFor(

CodeForCall, RegisterPreservationNotRequired);

patchBuffer.link(

slowPathCall, CodeLocationLabel(vm->getCTIStub(generator).code()));

}

MacroAssemblerCodeRef code = FINALIZE_CODE_FOR(

exec->codeBlock(), patchBuffer,

("%s access stub for %s, return point %p",

toString(kind), toCString(*exec->codeBlock()).data(),

successLabel.executableAddress()));

if (kind == CallGetter || kind == CallSetter)

stubRoutine = adoptRef(new AccessorCallJITStubRoutine(code, *vm, WTF::move(callLinkInfo)));

else

stubRoutine = createJITStubRoutine(code, *vm, codeBlock->ownerExecutable(), true);

}

enum InlineCacheAction {

GiveUpOnCache,

RetryCacheLater,

AttemptToCache

};

static InlineCacheAction actionForCell(VM& vm, JSCell* cell)

{

Structure* structure = cell->structure(vm);

TypeInfo typeInfo = structure->typeInfo();

if (typeInfo.prohibitsPropertyCaching())

return GiveUpOnCache;

if (structure->isUncacheableDictionary()) {

if (structure->hasBeenFlattenedBefore())

return GiveUpOnCache;

// Flattening could have changed the offset, so return early for another try.

asObject(cell)->flattenDictionaryObject(vm);

return RetryCacheLater;

}

ASSERT(!structure->isUncacheableDictionary());

if (typeInfo.hasImpureGetOwnPropertySlot() && !typeInfo.newImpurePropertyFiresWatchpoints())

return GiveUpOnCache;

return AttemptToCache;

}

static InlineCacheAction tryCacheGetByID(ExecState* exec, JSValue baseValue, const Identifier& propertyName, const PropertySlot& slot, StructureStubInfo& stubInfo)

{

if (Options::forceICFailure())

return GiveUpOnCache;

// FIXME: Write a test that proves we need to check for recursion here just

// like the interpreter does, then add a check for recursion.

CodeBlock* codeBlock = exec->codeBlock();

VM* vm = &exec->vm();

if ((isJSArray(baseValue) || isRegExpMatchesArray(baseValue) || isJSString(baseValue)) && propertyName == exec->propertyNames().length) {

GPRReg baseGPR = static_cast<GPRReg>(stubInfo.patch.baseGPR);

#if USE(JSVALUE32_64)

GPRReg resultTagGPR = static_cast<GPRReg>(stubInfo.patch.valueTagGPR);

#endif

GPRReg resultGPR = static_cast<GPRReg>(stubInfo.patch.valueGPR);

MacroAssembler stubJit;

if (isJSArray(baseValue) || isRegExpMatchesArray(baseValue)) {

GPRReg scratchGPR = TempRegisterSet(stubInfo.patch.usedRegisters).getFreeGPR();

bool needToRestoreScratch = false;

if (scratchGPR == InvalidGPRReg) {

#if USE(JSVALUE64)

scratchGPR = AssemblyHelpers::selectScratchGPR(baseGPR, resultGPR);

#else

scratchGPR = AssemblyHelpers::selectScratchGPR(baseGPR, resultGPR, resultTagGPR);

#endif

stubJit.pushToSave(scratchGPR);

needToRestoreScratch = true;

}

MacroAssembler::JumpList failureCases;

stubJit.load8(MacroAssembler::Address(baseGPR, JSCell::indexingTypeOffset()), scratchGPR);

failureCases.append(stubJit.branchTest32(MacroAssembler::Zero, scratchGPR, MacroAssembler::TrustedImm32(IsArray)));

failureCases.append(stubJit.branchTest32(MacroAssembler::Zero, scratchGPR, MacroAssembler::TrustedImm32(IndexingShapeMask)));

stubJit.loadPtr(MacroAssembler::Address(baseGPR, JSObject::butterflyOffset()), scratchGPR);

stubJit.load32(MacroAssembler::Address(scratchGPR, ArrayStorage::lengthOffset()), scratchGPR);

failureCases.append(stubJit.branch32(MacroAssembler::LessThan, scratchGPR, MacroAssembler::TrustedImm32(0)));

stubJit.move(scratchGPR, resultGPR);

#if USE(JSVALUE64)

stubJit.or64(AssemblyHelpers::TrustedImm64(TagTypeNumber), resultGPR);

#elif USE(JSVALUE32_64)

stubJit.move(AssemblyHelpers::TrustedImm32(0xffffffff), resultTagGPR); // JSValue::Int32Tag

#endif

MacroAssembler::Jump success, fail;

emitRestoreScratch(stubJit, needToRestoreScratch, scratchGPR, success, fail, failureCases);

LinkBuffer patchBuffer(*vm, stubJit, codeBlock);

linkRestoreScratch(patchBuffer, needToRestoreScratch, stubInfo, success, fail, failureCases);

stubInfo.stubRoutine = FINALIZE_CODE_FOR_STUB(

exec->codeBlock(), patchBuffer,

("GetById array length stub for %s, return point %p",

toCString(*exec->codeBlock()).data(), stubInfo.callReturnLocation.labelAtOffset(

stubInfo.patch.deltaCallToDone).executableAddress()));

RepatchBuffer repatchBuffer(codeBlock);

replaceWithJump(repatchBuffer, stubInfo, stubInfo.stubRoutine->code().code());

repatchCall(repatchBuffer, stubInfo.callReturnLocation, operationGetById);

return RetryCacheLater;

}

// String.length case

MacroAssembler::Jump failure = stubJit.branch8(MacroAssembler::NotEqual, MacroAssembler::Address(baseGPR, JSCell::typeInfoTypeOffset()), MacroAssembler::TrustedImm32(StringType));

stubJit.load32(MacroAssembler::Address(baseGPR, JSString::offsetOfLength()), resultGPR);

#if USE(JSVALUE64)

stubJit.or64(AssemblyHelpers::TrustedImm64(TagTypeNumber), resultGPR);

#elif USE(JSVALUE32_64)

stubJit.move(AssemblyHelpers::TrustedImm32(0xffffffff), resultTagGPR); // JSValue::Int32Tag

#endif

MacroAssembler::Jump success = stubJit.jump();

LinkBuffer patchBuffer(*vm, stubJit, codeBlock);

patchBuffer.link(success, stubInfo.callReturnLocation.labelAtOffset(stubInfo.patch.deltaCallToDone));

patchBuffer.link(failure, stubInfo.callReturnLocation.labelAtOffset(stubInfo.patch.deltaCallToSlowCase));

stubInfo.stubRoutine = FINALIZE_CODE_FOR_STUB(

exec->codeBlock(), patchBuffer,

("GetById string length stub for %s, return point %p",

toCString(*exec->codeBlock()).data(), stubInfo.callReturnLocation.labelAtOffset(

stubInfo.patch.deltaCallToDone).executableAddress()));

RepatchBuffer repatchBuffer(codeBlock);

replaceWithJump(repatchBuffer, stubInfo, stubInfo.stubRoutine->code().code());

repatchCall(repatchBuffer, stubInfo.callReturnLocation, operationGetById);

return RetryCacheLater;

}

// FIXME: Cache property access for immediates.

if (!baseValue.isCell())

return GiveUpOnCache;

JSCell* baseCell = baseValue.asCell();

Structure* structure = baseCell->structure();

if (!slot.isCacheable())

return GiveUpOnCache;

InlineCacheAction action = actionForCell(*vm, baseCell);

if (action != AttemptToCache)

return action;

// Optimize self access.

if (slot.slotBase() == baseValue

&& slot.isCacheableValue()

&& !slot.watchpointSet()

&& MacroAssembler::isCompactPtrAlignedAddressOffset(maxOffsetRelativeToPatchedStorage(slot.cachedOffset()))) {

repatchByIdSelfAccess(*vm, codeBlock, stubInfo, structure, propertyName, slot.cachedOffset(), operationGetByIdBuildList, true);

stubInfo.initGetByIdSelf(*vm, codeBlock->ownerExecutable(), structure);

return RetryCacheLater;

}

repatchCall(codeBlock, stubInfo.callReturnLocation, operationGetByIdBuildList);

return RetryCacheLater;

}

void repatchGetByID(ExecState* exec, JSValue baseValue, const Identifier& propertyName, const PropertySlot& slot, StructureStubInfo& stubInfo)

{

GCSafeConcurrentJITLocker locker(exec->codeBlock()->m_lock, exec->vm().heap);

if (tryCacheGetByID(exec, baseValue, propertyName, slot, stubInfo) == GiveUpOnCache)

repatchCall(exec->codeBlock(), stubInfo.callReturnLocation, operationGetById);

}

static void patchJumpToGetByIdStub(CodeBlock* codeBlock, StructureStubInfo& stubInfo, JITStubRoutine* stubRoutine)

{

RELEASE_ASSERT(stubInfo.accessType == access_get_by_id_list);

RepatchBuffer repatchBuffer(codeBlock);

if (stubInfo.u.getByIdList.list->didSelfPatching()) {

repatchBuffer.relink(

stubInfo.callReturnLocation.jumpAtOffset(

stubInfo.patch.deltaCallToJump),

CodeLocationLabel(stubRoutine->code().code()));

return;

}

replaceWithJump(repatchBuffer, stubInfo, stubRoutine->code().code());

}

static InlineCacheAction tryBuildGetByIDList(ExecState* exec, JSValue baseValue, const Identifier& ident, const PropertySlot& slot, StructureStubInfo& stubInfo)

{

if (!baseValue.isCell()

|| !slot.isCacheable())

return GiveUpOnCache;

JSCell* baseCell = baseValue.asCell();

bool loadTargetFromProxy = false;

if (baseCell->type() == PureForwardingProxyType) {

baseValue = jsCast<JSProxy*>(baseCell)->target();

baseCell = baseValue.asCell();

loadTargetFromProxy = true;

}

VM* vm = &exec->vm();

CodeBlock* codeBlock = exec->codeBlock();

InlineCacheAction action = actionForCell(*vm, baseCell);

if (action != AttemptToCache)

return action;

Structure* structure = baseCell->structure(*vm);

TypeInfo typeInfo = structure->typeInfo();

if (stubInfo.patch.spillMode == NeedToSpill) {

// We cannot do as much inline caching if the registers were not flushed prior to this GetById. In particular,

// non-Value cached properties require planting calls, which requires registers to have been flushed. Thus,

// if registers were not flushed, don't do non-Value caching.

if (!slot.isCacheableValue())

return GiveUpOnCache;

}

PropertyOffset offset = slot.cachedOffset();

StructureChain* prototypeChain = 0;

size_t count = 0;

if (slot.slotBase() != baseValue) {

if (typeInfo.prohibitsPropertyCaching() || structure->isDictionary())

return GiveUpOnCache;

count = normalizePrototypeChainForChainAccess(

exec, baseValue, slot.slotBase(), ident, offset);

if (count == InvalidPrototypeChain)

return GiveUpOnCache;

prototypeChain = structure->prototypeChain(exec);

}

PolymorphicGetByIdList* list = PolymorphicGetByIdList::from(stubInfo);

if (list->isFull()) {

// We need this extra check because of recursion.

return GiveUpOnCache;

}

RefPtr<JITStubRoutine> stubRoutine;

generateByIdStub(

exec, kindFor(slot), ident, customFor(slot), stubInfo, prototypeChain, count, offset,

structure, loadTargetFromProxy, slot.watchpointSet(),

stubInfo.callReturnLocation.labelAtOffset(stubInfo.patch.deltaCallToDone),

CodeLocationLabel(list->currentSlowPathTarget(stubInfo)), stubRoutine);

GetByIdAccess::AccessType accessType;

if (slot.isCacheableValue())

accessType = slot.watchpointSet() ? GetByIdAccess::WatchedStub : GetByIdAccess::SimpleStub;

else if (slot.isCacheableGetter())

accessType = GetByIdAccess::Getter;

else

accessType = GetByIdAccess::CustomGetter;

list->addAccess(GetByIdAccess(

*vm, codeBlock->ownerExecutable(), accessType, stubRoutine, structure,

prototypeChain, count));

patchJumpToGetByIdStub(codeBlock, stubInfo, stubRoutine.get());

return list->isFull() ? GiveUpOnCache : RetryCacheLater;

}

void buildGetByIDList(ExecState* exec, JSValue baseValue, const Identifier& propertyName, const PropertySlot& slot, StructureStubInfo& stubInfo)

{

GCSafeConcurrentJITLocker locker(exec->codeBlock()->m_lock, exec->vm().heap);

if (tryBuildGetByIDList(exec, baseValue, propertyName, slot, stubInfo) == GiveUpOnCache)

repatchCall(exec->codeBlock(), stubInfo.callReturnLocation, operationGetById);

}

static V_JITOperation_ESsiJJI appropriateGenericPutByIdFunction(const PutPropertySlot &slot, PutKind putKind)

{

if (slot.isStrictMode()) {

if (putKind == Direct)

return operationPutByIdDirectStrict;

return operationPutByIdStrict;

}

if (putKind == Direct)

return operationPutByIdDirectNonStrict;

return operationPutByIdNonStrict;

}

static V_JITOperation_ESsiJJI appropriateListBuildingPutByIdFunction(const PutPropertySlot &slot, PutKind putKind)

{

if (slot.isStrictMode()) {

if (putKind == Direct)

return operationPutByIdDirectStrictBuildList;

return operationPutByIdStrictBuildList;

}

if (putKind == Direct)

return operationPutByIdDirectNonStrictBuildList;

return operationPutByIdNonStrictBuildList;

}

static void emitPutReplaceStub(

ExecState* exec,

JSValue,

const Identifier&,

const PutPropertySlot& slot,

StructureStubInfo& stubInfo,

PutKind,

Structure* structure,

CodeLocationLabel failureLabel,

RefPtr<JITStubRoutine>& stubRoutine)

{

VM* vm = &exec->vm();

GPRReg baseGPR = static_cast<GPRReg>(stubInfo.patch.baseGPR);

#if USE(JSVALUE32_64)

GPRReg valueTagGPR = static_cast<GPRReg>(stubInfo.patch.valueTagGPR);

#endif

GPRReg valueGPR = static_cast<GPRReg>(stubInfo.patch.valueGPR);

ScratchRegisterAllocator allocator(stubInfo.patch.usedRegisters);

allocator.lock(baseGPR);

#if USE(JSVALUE32_64)

allocator.lock(valueTagGPR);

#endif

allocator.lock(valueGPR);

GPRReg scratchGPR1 = allocator.allocateScratchGPR();

CCallHelpers stubJit(vm, exec->codeBlock());

allocator.preserveReusedRegistersByPushing(stubJit);

MacroAssembler::Jump badStructure = branchStructure(stubJit,

MacroAssembler::NotEqual,

MacroAssembler::Address(baseGPR, JSCell::structureIDOffset()),

structure);

#if USE(JSVALUE64)

if (isInlineOffset(slot.cachedOffset()))

stubJit.store64(valueGPR, MacroAssembler::Address(baseGPR, JSObject::offsetOfInlineStorage() + offsetInInlineStorage(slot.cachedOffset()) * sizeof(JSValue)));

else {

stubJit.loadPtr(MacroAssembler::Address(baseGPR, JSObject::butterflyOffset()), scratchGPR1);

stubJit.store64(valueGPR, MacroAssembler::Address(scratchGPR1, offsetInButterfly(slot.cachedOffset()) * sizeof(JSValue)));

}

#elif USE(JSVALUE32_64)

if (isInlineOffset(slot.cachedOffset())) {

stubJit.store32(valueGPR, MacroAssembler::Address(baseGPR, JSObject::offsetOfInlineStorage() + offsetInInlineStorage(slot.cachedOffset()) * sizeof(JSValue) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)));

stubJit.store32(valueTagGPR, MacroAssembler::Address(baseGPR, JSObject::offsetOfInlineStorage() + offsetInInlineStorage(slot.cachedOffset()) * sizeof(JSValue) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)));

} else {

stubJit.loadPtr(MacroAssembler::Address(baseGPR, JSObject::butterflyOffset()), scratchGPR1);

stubJit.store32(valueGPR, MacroAssembler::Address(scratchGPR1, offsetInButterfly(slot.cachedOffset()) * sizeof(JSValue) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)));

stubJit.store32(valueTagGPR, MacroAssembler::Address(scratchGPR1, offsetInButterfly(slot.cachedOffset()) * sizeof(JSValue) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)));

}

#endif

MacroAssembler::Jump success;

MacroAssembler::Jump failure;

if (allocator.didReuseRegisters()) {

allocator.restoreReusedRegistersByPopping(stubJit);

success = stubJit.jump();

badStructure.link(&stubJit);

allocator.restoreReusedRegistersByPopping(stubJit);

failure = stubJit.jump();

} else {

success = stubJit.jump();

failure = badStructure;

}

LinkBuffer patchBuffer(*vm, stubJit, exec->codeBlock());

patchBuffer.link(success, stubInfo.callReturnLocation.labelAtOffset(stubInfo.patch.deltaCallToDone));

patchBuffer.link(failure, failureLabel);

stubRoutine = FINALIZE_CODE_FOR_STUB(

exec->codeBlock(), patchBuffer,

("PutById replace stub for %s, return point %p",

toCString(*exec->codeBlock()).data(), stubInfo.callReturnLocation.labelAtOffset(

stubInfo.patch.deltaCallToDone).executableAddress()));

}

static void emitPutTransitionStub(

ExecState* exec,

JSValue,

const Identifier&,

const PutPropertySlot& slot,

StructureStubInfo& stubInfo,

PutKind putKind,

Structure* structure,

Structure* oldStructure,

StructureChain* prototypeChain,

CodeLocationLabel failureLabel,

RefPtr<JITStubRoutine>& stubRoutine)

{

VM* vm = &exec->vm();

GPRReg baseGPR = static_cast<GPRReg>(stubInfo.patch.baseGPR);

#if USE(JSVALUE32_64)

GPRReg valueTagGPR = static_cast<GPRReg>(stubInfo.patch.valueTagGPR);

#endif

GPRReg valueGPR = static_cast<GPRReg>(stubInfo.patch.valueGPR);

ScratchRegisterAllocator allocator(stubInfo.patch.usedRegisters);

allocator.lock(baseGPR);

#if USE(JSVALUE32_64)

allocator.lock(valueTagGPR);

#endif

allocator.lock(valueGPR);

CCallHelpers stubJit(vm);

bool needThirdScratch = false;

if (structure->outOfLineCapacity() != oldStructure->outOfLineCapacity()

&& oldStructure->outOfLineCapacity()) {

needThirdScratch = true;