| // Copyright 2015 the V8 project authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "src/compiler/effect-control-linearizer.h" |
| |
| #include "include/v8-fast-api-calls.h" |
| #include "src/base/bits.h" |
| #include "src/codegen/code-factory.h" |
| #include "src/codegen/machine-type.h" |
| #include "src/common/ptr-compr-inl.h" |
| #include "src/compiler/access-builder.h" |
| #include "src/compiler/compiler-source-position-table.h" |
| #include "src/compiler/feedback-source.h" |
| #include "src/compiler/graph-assembler.h" |
| #include "src/compiler/js-graph.h" |
| #include "src/compiler/js-heap-broker.h" |
| #include "src/compiler/linkage.h" |
| #include "src/compiler/node-matchers.h" |
| #include "src/compiler/node-origin-table.h" |
| #include "src/compiler/node-properties.h" |
| #include "src/compiler/node.h" |
| #include "src/compiler/schedule.h" |
| #include "src/execution/frames.h" |
| #include "src/heap/factory-inl.h" |
| #include "src/objects/heap-number.h" |
| #include "src/objects/oddball.h" |
| #include "src/objects/ordered-hash-table.h" |
| |
| namespace v8 { |
| namespace internal { |
| namespace compiler { |
| |
| class EffectControlLinearizer { |
| public: |
| EffectControlLinearizer(JSGraph* js_graph, Schedule* schedule, |
| Zone* temp_zone, |
| SourcePositionTable* source_positions, |
| NodeOriginTable* node_origins, |
| MaskArrayIndexEnable mask_array_index, |
| MaintainSchedule maintain_schedule, |
| JSHeapBroker* broker) |
| : js_graph_(js_graph), |
| schedule_(schedule), |
| temp_zone_(temp_zone), |
| mask_array_index_(mask_array_index), |
| maintain_schedule_(maintain_schedule), |
| source_positions_(source_positions), |
| node_origins_(node_origins), |
| broker_(broker), |
| graph_assembler_(js_graph, temp_zone, base::nullopt, |
| should_maintain_schedule() ? schedule : nullptr), |
| frame_state_zapper_(nullptr), |
| fast_api_call_stack_slot_(nullptr) {} |
| |
| void Run(); |
| |
| private: |
| void UpdateEffectControlForNode(Node* node); |
| void ProcessNode(Node* node, Node** frame_state); |
| |
| bool TryWireInStateEffect(Node* node, Node* frame_state); |
| Node* LowerChangeBitToTagged(Node* node); |
| Node* LowerChangeInt31ToTaggedSigned(Node* node); |
| Node* LowerChangeInt32ToTagged(Node* node); |
| Node* LowerChangeInt64ToTagged(Node* node); |
| Node* LowerChangeUint32ToTagged(Node* node); |
| Node* LowerChangeUint64ToTagged(Node* node); |
| Node* LowerChangeFloat64ToTagged(Node* node); |
| Node* LowerChangeFloat64ToTaggedPointer(Node* node); |
| Node* LowerChangeTaggedSignedToInt32(Node* node); |
| Node* LowerChangeTaggedSignedToInt64(Node* node); |
| Node* LowerChangeTaggedToBit(Node* node); |
| Node* LowerChangeTaggedToInt32(Node* node); |
| Node* LowerChangeTaggedToUint32(Node* node); |
| Node* LowerChangeTaggedToInt64(Node* node); |
| Node* LowerChangeTaggedToTaggedSigned(Node* node); |
| Node* LowerPoisonIndex(Node* node); |
| Node* LowerCheckInternalizedString(Node* node, Node* frame_state); |
| void LowerCheckMaps(Node* node, Node* frame_state); |
| void LowerDynamicCheckMaps(Node* node, Node* frame_state); |
| Node* LowerCompareMaps(Node* node); |
| Node* LowerCheckNumber(Node* node, Node* frame_state); |
| Node* LowerCheckClosure(Node* node, Node* frame_state); |
| Node* LowerCheckReceiver(Node* node, Node* frame_state); |
| Node* LowerCheckReceiverOrNullOrUndefined(Node* node, Node* frame_state); |
| Node* LowerCheckString(Node* node, Node* frame_state); |
| Node* LowerCheckBigInt(Node* node, Node* frame_state); |
| Node* LowerCheckSymbol(Node* node, Node* frame_state); |
| void LowerCheckIf(Node* node, Node* frame_state); |
| Node* LowerCheckedInt32Add(Node* node, Node* frame_state); |
| Node* LowerCheckedInt32Sub(Node* node, Node* frame_state); |
| Node* LowerCheckedInt32Div(Node* node, Node* frame_state); |
| Node* LowerCheckedInt32Mod(Node* node, Node* frame_state); |
| Node* LowerCheckedUint32Div(Node* node, Node* frame_state); |
| Node* LowerCheckedUint32Mod(Node* node, Node* frame_state); |
| Node* LowerCheckedInt32Mul(Node* node, Node* frame_state); |
| Node* LowerCheckedInt32ToTaggedSigned(Node* node, Node* frame_state); |
| Node* LowerCheckedInt64ToInt32(Node* node, Node* frame_state); |
| Node* LowerCheckedInt64ToTaggedSigned(Node* node, Node* frame_state); |
| Node* LowerCheckedUint32Bounds(Node* node, Node* frame_state); |
| Node* LowerCheckedUint32ToInt32(Node* node, Node* frame_state); |
| Node* LowerCheckedUint32ToTaggedSigned(Node* node, Node* frame_state); |
| Node* LowerCheckedUint64Bounds(Node* node, Node* frame_state); |
| Node* LowerCheckedUint64ToInt32(Node* node, Node* frame_state); |
| Node* LowerCheckedUint64ToTaggedSigned(Node* node, Node* frame_state); |
| Node* LowerCheckedFloat64ToInt32(Node* node, Node* frame_state); |
| Node* LowerCheckedFloat64ToInt64(Node* node, Node* frame_state); |
| Node* LowerCheckedTaggedSignedToInt32(Node* node, Node* frame_state); |
| Node* LowerCheckedTaggedToArrayIndex(Node* node, Node* frame_state); |
| Node* LowerCheckedTaggedToInt32(Node* node, Node* frame_state); |
| Node* LowerCheckedTaggedToInt64(Node* node, Node* frame_state); |
| Node* LowerCheckedTaggedToFloat64(Node* node, Node* frame_state); |
| Node* LowerCheckedTaggedToTaggedSigned(Node* node, Node* frame_state); |
| Node* LowerCheckedTaggedToTaggedPointer(Node* node, Node* frame_state); |
| Node* LowerBigIntAsUintN(Node* node, Node* frame_state); |
| Node* LowerChangeUint64ToBigInt(Node* node); |
| Node* LowerTruncateBigIntToUint64(Node* node); |
| Node* LowerChangeTaggedToFloat64(Node* node); |
| void TruncateTaggedPointerToBit(Node* node, GraphAssemblerLabel<1>* done); |
| Node* LowerTruncateTaggedToBit(Node* node); |
| Node* LowerTruncateTaggedPointerToBit(Node* node); |
| Node* LowerTruncateTaggedToFloat64(Node* node); |
| Node* LowerTruncateTaggedToWord32(Node* node); |
| Node* LowerCheckedTruncateTaggedToWord32(Node* node, Node* frame_state); |
| Node* LowerAllocate(Node* node); |
| Node* LowerNumberToString(Node* node); |
| Node* LowerObjectIsArrayBufferView(Node* node); |
| Node* LowerObjectIsBigInt(Node* node); |
| Node* LowerObjectIsCallable(Node* node); |
| Node* LowerObjectIsConstructor(Node* node); |
| Node* LowerObjectIsDetectableCallable(Node* node); |
| Node* LowerObjectIsMinusZero(Node* node); |
| Node* LowerNumberIsMinusZero(Node* node); |
| Node* LowerObjectIsNaN(Node* node); |
| Node* LowerNumberIsNaN(Node* node); |
| Node* LowerObjectIsNonCallable(Node* node); |
| Node* LowerObjectIsNumber(Node* node); |
| Node* LowerObjectIsReceiver(Node* node); |
| Node* LowerObjectIsSmi(Node* node); |
| Node* LowerObjectIsString(Node* node); |
| Node* LowerObjectIsSymbol(Node* node); |
| Node* LowerObjectIsUndetectable(Node* node); |
| Node* LowerNumberIsFloat64Hole(Node* node); |
| Node* LowerNumberIsFinite(Node* node); |
| Node* LowerObjectIsFiniteNumber(Node* node); |
| Node* LowerNumberIsInteger(Node* node); |
| Node* LowerObjectIsInteger(Node* node); |
| Node* LowerNumberIsSafeInteger(Node* node); |
| Node* LowerObjectIsSafeInteger(Node* node); |
| Node* LowerArgumentsFrame(Node* node); |
| Node* LowerArgumentsLength(Node* node); |
| Node* LowerRestLength(Node* node); |
| Node* LowerNewDoubleElements(Node* node); |
| Node* LowerNewSmiOrObjectElements(Node* node); |
| Node* LowerNewArgumentsElements(Node* node); |
| Node* LowerNewConsString(Node* node); |
| Node* LowerSameValue(Node* node); |
| Node* LowerSameValueNumbersOnly(Node* node); |
| Node* LowerNumberSameValue(Node* node); |
| Node* LowerDeadValue(Node* node); |
| Node* LowerStringConcat(Node* node); |
| Node* LowerStringToNumber(Node* node); |
| Node* LowerStringCharCodeAt(Node* node); |
| Node* LowerStringCodePointAt(Node* node); |
| Node* LowerStringToLowerCaseIntl(Node* node); |
| Node* LowerStringToUpperCaseIntl(Node* node); |
| Node* LowerStringFromSingleCharCode(Node* node); |
| Node* LowerStringFromSingleCodePoint(Node* node); |
| Node* LowerStringIndexOf(Node* node); |
| Node* LowerStringSubstring(Node* node); |
| Node* LowerStringFromCodePointAt(Node* node); |
| Node* LowerStringLength(Node* node); |
| Node* LowerStringEqual(Node* node); |
| Node* LowerStringLessThan(Node* node); |
| Node* LowerStringLessThanOrEqual(Node* node); |
| Node* LowerBigIntAdd(Node* node, Node* frame_state); |
| Node* LowerBigIntSubtract(Node* node, Node* frame_state); |
| Node* LowerBigIntNegate(Node* node); |
| Node* LowerCheckFloat64Hole(Node* node, Node* frame_state); |
| Node* LowerCheckNotTaggedHole(Node* node, Node* frame_state); |
| Node* LowerConvertTaggedHoleToUndefined(Node* node); |
| void LowerCheckEqualsInternalizedString(Node* node, Node* frame_state); |
| void LowerCheckEqualsSymbol(Node* node, Node* frame_state); |
| Node* LowerTypeOf(Node* node); |
| void LowerTierUpCheck(Node* node); |
| void LowerUpdateInterruptBudget(Node* node); |
| Node* LowerToBoolean(Node* node); |
| Node* LowerPlainPrimitiveToNumber(Node* node); |
| Node* LowerPlainPrimitiveToWord32(Node* node); |
| Node* LowerPlainPrimitiveToFloat64(Node* node); |
| Node* LowerEnsureWritableFastElements(Node* node); |
| Node* LowerMaybeGrowFastElements(Node* node, Node* frame_state); |
| void LowerTransitionElementsKind(Node* node); |
| Node* LowerLoadFieldByIndex(Node* node); |
| Node* LowerLoadMessage(Node* node); |
| Node* LowerFastApiCall(Node* node); |
| Node* LowerLoadTypedElement(Node* node); |
| Node* LowerLoadDataViewElement(Node* node); |
| Node* LowerLoadStackArgument(Node* node); |
| void LowerStoreMessage(Node* node); |
| void LowerStoreTypedElement(Node* node); |
| void LowerStoreDataViewElement(Node* node); |
| void LowerStoreSignedSmallElement(Node* node); |
| Node* LowerFindOrderedHashMapEntry(Node* node); |
| Node* LowerFindOrderedHashMapEntryForInt32Key(Node* node); |
| void LowerTransitionAndStoreElement(Node* node); |
| void LowerTransitionAndStoreNumberElement(Node* node); |
| void LowerTransitionAndStoreNonNumberElement(Node* node); |
| void LowerRuntimeAbort(Node* node); |
| Node* LowerAssertType(Node* node); |
| Node* LowerFoldConstant(Node* node); |
| Node* LowerConvertReceiver(Node* node); |
| Node* LowerDateNow(Node* node); |
| |
| // Lowering of optional operators. |
| Maybe<Node*> LowerFloat64RoundUp(Node* node); |
| Maybe<Node*> LowerFloat64RoundDown(Node* node); |
| Maybe<Node*> LowerFloat64RoundTiesEven(Node* node); |
| Maybe<Node*> LowerFloat64RoundTruncate(Node* node); |
| |
| Node* AllocateHeapNumberWithValue(Node* node); |
| Node* BuildCheckedFloat64ToInt32(CheckForMinusZeroMode mode, |
| const FeedbackSource& feedback, Node* value, |
| Node* frame_state); |
| Node* BuildCheckedFloat64ToInt64(CheckForMinusZeroMode mode, |
| const FeedbackSource& feedback, Node* value, |
| Node* frame_state); |
| Node* BuildCheckedFloat64ToIndex(const FeedbackSource& feedback, Node* value, |
| Node* frame_state); |
| Node* BuildCheckedHeapNumberOrOddballToFloat64(CheckTaggedInputMode mode, |
| const FeedbackSource& feedback, |
| Node* value, |
| Node* frame_state); |
| Node* BuildReverseBytes(ExternalArrayType type, Node* value); |
| Node* BuildFloat64RoundDown(Node* value); |
| Node* BuildFloat64RoundTruncate(Node* input); |
| template <size_t VarCount, size_t VarCount2> |
| void SmiTagOrOverflow(Node* value, GraphAssemblerLabel<VarCount>* if_overflow, |
| GraphAssemblerLabel<VarCount2>* done); |
| Node* SmiTagOrDeopt(Node* value, const CheckParameters& params, |
| Node* frame_state); |
| Node* BuildUint32Mod(Node* lhs, Node* rhs); |
| Node* ComputeUnseededHash(Node* value); |
| Node* LowerStringComparison(Callable const& callable, Node* node); |
| Node* IsElementsKindGreaterThan(Node* kind, ElementsKind reference_kind); |
| |
| Node* BuildTypedArrayDataPointer(Node* base, Node* external); |
| |
| template <typename... Args> |
| Node* CallBuiltin(Builtins::Name builtin, Operator::Properties properties, |
| Args...); |
| |
| Node* ChangeInt32ToSmi(Node* value); |
| // In pointer compression, we smi-corrupt. This means the upper bits of a Smi |
| // are not important. ChangeTaggedInt32ToSmi has a known tagged int32 as input |
| // and takes advantage of the smi corruption by emitting a Bitcast node |
| // instead of a Change node in order to save instructions. |
| // In non pointer compression, it behaves like ChangeInt32ToSmi. |
| Node* ChangeTaggedInt32ToSmi(Node* value); |
| Node* ChangeInt32ToIntPtr(Node* value); |
| Node* ChangeInt64ToSmi(Node* value); |
| Node* ChangeIntPtrToInt32(Node* value); |
| Node* ChangeIntPtrToSmi(Node* value); |
| Node* ChangeUint32ToUintPtr(Node* value); |
| Node* ChangeUint32ToSmi(Node* value); |
| Node* ChangeSmiToIntPtr(Node* value); |
| Node* ChangeSmiToInt32(Node* value); |
| Node* ChangeSmiToInt64(Node* value); |
| Node* ObjectIsSmi(Node* value); |
| Node* LoadFromSeqString(Node* receiver, Node* position, Node* is_one_byte); |
| Node* TruncateWordToInt32(Node* value); |
| Node* MakeWeakForComparison(Node* heap_object); |
| Node* BuildIsWeakReferenceTo(Node* maybe_object, Node* value); |
| Node* BuildIsClearedWeakReference(Node* maybe_object); |
| Node* BuildIsStrongReference(Node* value); |
| Node* BuildStrongReferenceFromWeakReference(Node* value); |
| Node* SmiMaxValueConstant(); |
| Node* SmiShiftBitsConstant(); |
| void TransitionElementsTo(Node* node, Node* array, ElementsKind from, |
| ElementsKind to); |
| |
| // This function tries to migrate |value| if its map |value_map| is |
| // deprecated. It deopts, if either |value_map| isn't deprecated or migration |
| // fails. |
| void MigrateInstanceOrDeopt(Node* value, Node* value_map, Node* frame_state, |
| FeedbackSource const& feedback_source, |
| DeoptimizeReason reason); |
| |
| // Helper functions used in LowerDynamicCheckMaps |
| void BuildCallDynamicMapChecksBuiltin(Node* actual_value, |
| Node* actual_handler, |
| int feedback_slot_index, |
| GraphAssemblerLabel<0>* done, |
| Node* frame_state); |
| bool should_maintain_schedule() const { |
| return maintain_schedule_ == MaintainSchedule::kMaintain; |
| } |
| |
| Factory* factory() const { return isolate()->factory(); } |
| Isolate* isolate() const { return jsgraph()->isolate(); } |
| JSGraph* jsgraph() const { return js_graph_; } |
| Graph* graph() const { return js_graph_->graph(); } |
| Schedule* schedule() const { return schedule_; } |
| Zone* temp_zone() const { return temp_zone_; } |
| CommonOperatorBuilder* common() const { return js_graph_->common(); } |
| SimplifiedOperatorBuilder* simplified() const { |
| return js_graph_->simplified(); |
| } |
| MachineOperatorBuilder* machine() const { return js_graph_->machine(); } |
| JSGraphAssembler* gasm() { return &graph_assembler_; } |
| JSHeapBroker* broker() const { return broker_; } |
| |
| JSGraph* js_graph_; |
| Schedule* schedule_; |
| Zone* temp_zone_; |
| MaskArrayIndexEnable mask_array_index_; |
| MaintainSchedule maintain_schedule_; |
| RegionObservability region_observability_ = RegionObservability::kObservable; |
| SourcePositionTable* source_positions_; |
| NodeOriginTable* node_origins_; |
| JSHeapBroker* broker_; |
| JSGraphAssembler graph_assembler_; |
| Node* frame_state_zapper_; // For tracking down compiler::Node::New crashes. |
| Node* fast_api_call_stack_slot_; // For caching the stack slot allocated for |
| // fast API calls. |
| }; |
| |
| namespace { |
| |
| struct BlockEffectControlData { |
| Node* current_effect = nullptr; // New effect. |
| Node* current_control = nullptr; // New control. |
| Node* current_frame_state = nullptr; // New frame state. |
| }; |
| |
| class BlockEffectControlMap { |
| public: |
| explicit BlockEffectControlMap(Zone* temp_zone) : map_(temp_zone) {} |
| |
| BlockEffectControlData& For(BasicBlock* from, BasicBlock* to) { |
| return map_[std::make_pair(from->id().ToInt(), to->id().ToInt())]; |
| } |
| |
| const BlockEffectControlData& For(BasicBlock* from, BasicBlock* to) const { |
| return map_.at(std::make_pair(from->id().ToInt(), to->id().ToInt())); |
| } |
| |
| private: |
| using Key = std::pair<int32_t, int32_t>; |
| using Map = ZoneMap<Key, BlockEffectControlData>; |
| |
| Map map_; |
| }; |
| |
| // Effect phis that need to be updated after the first pass. |
| struct PendingEffectPhi { |
| Node* effect_phi; |
| BasicBlock* block; |
| |
| PendingEffectPhi(Node* effect_phi, BasicBlock* block) |
| : effect_phi(effect_phi), block(block) {} |
| }; |
| |
| void UpdateEffectPhi(Node* node, BasicBlock* block, |
| BlockEffectControlMap* block_effects) { |
| // Update all inputs to an effect phi with the effects from the given |
| // block->effect map. |
| DCHECK_EQ(IrOpcode::kEffectPhi, node->opcode()); |
| DCHECK_EQ(static_cast<size_t>(node->op()->EffectInputCount()), |
| block->PredecessorCount()); |
| for (int i = 0; i < node->op()->EffectInputCount(); i++) { |
| Node* input = node->InputAt(i); |
| BasicBlock* predecessor = block->PredecessorAt(static_cast<size_t>(i)); |
| const BlockEffectControlData& block_effect = |
| block_effects->For(predecessor, block); |
| Node* effect = block_effect.current_effect; |
| if (input != effect) { |
| node->ReplaceInput(i, effect); |
| } |
| } |
| } |
| |
| void UpdateBlockControl(BasicBlock* block, |
| BlockEffectControlMap* block_effects) { |
| Node* control = block->NodeAt(0); |
| DCHECK(NodeProperties::IsControl(control)); |
| |
| // Do not rewire the end node. |
| if (control->opcode() == IrOpcode::kEnd) return; |
| |
| // Update all inputs to the given control node with the correct control. |
| DCHECK(control->opcode() == IrOpcode::kMerge || |
| static_cast<size_t>(control->op()->ControlInputCount()) == |
| block->PredecessorCount()); |
| if (static_cast<size_t>(control->op()->ControlInputCount()) != |
| block->PredecessorCount()) { |
| return; // We already re-wired the control inputs of this node. |
| } |
| for (int i = 0; i < control->op()->ControlInputCount(); i++) { |
| Node* input = NodeProperties::GetControlInput(control, i); |
| BasicBlock* predecessor = block->PredecessorAt(static_cast<size_t>(i)); |
| const BlockEffectControlData& block_effect = |
| block_effects->For(predecessor, block); |
| if (input != block_effect.current_control) { |
| NodeProperties::ReplaceControlInput(control, block_effect.current_control, |
| i); |
| } |
| } |
| } |
| |
| void RemoveRenameNode(Node* node) { |
| DCHECK(IrOpcode::kFinishRegion == node->opcode() || |
| IrOpcode::kBeginRegion == node->opcode() || |
| IrOpcode::kTypeGuard == node->opcode()); |
| // Update the value/context uses to the value input of the finish node and |
| // the effect uses to the effect input. |
| for (Edge edge : node->use_edges()) { |
| DCHECK(!edge.from()->IsDead()); |
| if (NodeProperties::IsEffectEdge(edge)) { |
| edge.UpdateTo(NodeProperties::GetEffectInput(node)); |
| } else { |
| DCHECK(!NodeProperties::IsControlEdge(edge)); |
| DCHECK(!NodeProperties::IsFrameStateEdge(edge)); |
| edge.UpdateTo(node->InputAt(0)); |
| } |
| } |
| node->Kill(); |
| } |
| |
| void TryCloneBranch(Node* node, BasicBlock* block, Zone* temp_zone, |
| Graph* graph, CommonOperatorBuilder* common, |
| BlockEffectControlMap* block_effects, |
| SourcePositionTable* source_positions, |
| NodeOriginTable* node_origins) { |
| DCHECK_EQ(IrOpcode::kBranch, node->opcode()); |
| |
| // This optimization is a special case of (super)block cloning. It takes an |
| // input graph as shown below and clones the Branch node for every predecessor |
| // to the Merge, essentially removing the Merge completely. This avoids |
| // materializing the bit for the Phi and may offer potential for further |
| // branch folding optimizations (i.e. because one or more inputs to the Phi is |
| // a constant). Note that there may be more Phi nodes hanging off the Merge, |
| // but we can only a certain subset of them currently (actually only Phi and |
| // EffectPhi nodes whose uses have either the IfTrue or IfFalse as control |
| // input). |
| |
| // Control1 ... ControlN |
| // ^ ^ |
| // | | Cond1 ... CondN |
| // +----+ +----+ ^ ^ |
| // | | | | |
| // | | +----+ | |
| // Merge<--+ | +------------+ |
| // ^ \|/ |
| // | Phi |
| // | | |
| // Branch----+ |
| // ^ |
| // | |
| // +-----+-----+ |
| // | | |
| // IfTrue IfFalse |
| // ^ ^ |
| // | | |
| |
| // The resulting graph (modulo the Phi and EffectPhi nodes) looks like this: |
| |
| // Control1 Cond1 ... ControlN CondN |
| // ^ ^ ^ ^ |
| // \ / \ / |
| // Branch ... Branch |
| // ^ ^ |
| // | | |
| // +---+---+ +---+----+ |
| // | | | | |
| // IfTrue IfFalse ... IfTrue IfFalse |
| // ^ ^ ^ ^ |
| // | | | | |
| // +--+ +-------------+ | |
| // | | +--------------+ +--+ |
| // | | | | |
| // Merge Merge |
| // ^ ^ |
| // | | |
| |
| SourcePositionTable::Scope scope(source_positions, |
| source_positions->GetSourcePosition(node)); |
| NodeOriginTable::Scope origin_scope(node_origins, "clone branch", node); |
| Node* branch = node; |
| Node* cond = NodeProperties::GetValueInput(branch, 0); |
| if (!cond->OwnedBy(branch) || cond->opcode() != IrOpcode::kPhi) return; |
| Node* merge = NodeProperties::GetControlInput(branch); |
| if (merge->opcode() != IrOpcode::kMerge || |
| NodeProperties::GetControlInput(cond) != merge) { |
| return; |
| } |
| // Grab the IfTrue/IfFalse projections of the Branch. |
| BranchMatcher matcher(branch); |
| // Check/collect other Phi/EffectPhi nodes hanging off the Merge. |
| NodeVector phis(temp_zone); |
| for (Node* const use : merge->uses()) { |
| if (use == branch || use == cond) continue; |
| // We cannot currently deal with non-Phi/EffectPhi nodes hanging off the |
| // Merge. Ideally, we would just clone the nodes (and everything that |
| // depends on it to some distant join point), but that requires knowledge |
| // about dominance/post-dominance. |
| if (!NodeProperties::IsPhi(use)) return; |
| for (Edge edge : use->use_edges()) { |
| // Right now we can only handle Phi/EffectPhi nodes whose uses are |
| // directly control-dependend on either the IfTrue or the IfFalse |
| // successor, because we know exactly how to update those uses. |
| if (edge.from()->op()->ControlInputCount() != 1) return; |
| Node* control = NodeProperties::GetControlInput(edge.from()); |
| if (NodeProperties::IsPhi(edge.from())) { |
| control = NodeProperties::GetControlInput(control, edge.index()); |
| } |
| if (control != matcher.IfTrue() && control != matcher.IfFalse()) return; |
| } |
| phis.push_back(use); |
| } |
| BranchHint const hint = BranchHintOf(branch->op()); |
| int const input_count = merge->op()->ControlInputCount(); |
| DCHECK_LE(1, input_count); |
| Node** const inputs = graph->zone()->NewArray<Node*>(2 * input_count); |
| Node** const merge_true_inputs = &inputs[0]; |
| Node** const merge_false_inputs = &inputs[input_count]; |
| for (int index = 0; index < input_count; ++index) { |
| Node* cond1 = NodeProperties::GetValueInput(cond, index); |
| Node* control1 = NodeProperties::GetControlInput(merge, index); |
| Node* branch1 = graph->NewNode(common->Branch(hint), cond1, control1); |
| merge_true_inputs[index] = graph->NewNode(common->IfTrue(), branch1); |
| merge_false_inputs[index] = graph->NewNode(common->IfFalse(), branch1); |
| } |
| Node* const merge_true = matcher.IfTrue(); |
| Node* const merge_false = matcher.IfFalse(); |
| merge_true->TrimInputCount(0); |
| merge_false->TrimInputCount(0); |
| for (int i = 0; i < input_count; ++i) { |
| merge_true->AppendInput(graph->zone(), merge_true_inputs[i]); |
| merge_false->AppendInput(graph->zone(), merge_false_inputs[i]); |
| } |
| DCHECK_EQ(2u, block->SuccessorCount()); |
| NodeProperties::ChangeOp(matcher.IfTrue(), common->Merge(input_count)); |
| NodeProperties::ChangeOp(matcher.IfFalse(), common->Merge(input_count)); |
| int const true_index = |
| block->SuccessorAt(0)->NodeAt(0) == matcher.IfTrue() ? 0 : 1; |
| BlockEffectControlData* true_block_data = |
| &block_effects->For(block, block->SuccessorAt(true_index)); |
| BlockEffectControlData* false_block_data = |
| &block_effects->For(block, block->SuccessorAt(true_index ^ 1)); |
| for (Node* const phi : phis) { |
| for (int index = 0; index < input_count; ++index) { |
| inputs[index] = phi->InputAt(index); |
| } |
| inputs[input_count] = merge_true; |
| Node* phi_true = graph->NewNode(phi->op(), input_count + 1, inputs); |
| inputs[input_count] = merge_false; |
| Node* phi_false = graph->NewNode(phi->op(), input_count + 1, inputs); |
| if (phi->UseCount() == 0) { |
| DCHECK_EQ(phi->opcode(), IrOpcode::kEffectPhi); |
| } else { |
| for (Edge edge : phi->use_edges()) { |
| Node* control = NodeProperties::GetControlInput(edge.from()); |
| if (NodeProperties::IsPhi(edge.from())) { |
| control = NodeProperties::GetControlInput(control, edge.index()); |
| } |
| DCHECK(control == matcher.IfTrue() || control == matcher.IfFalse()); |
| edge.UpdateTo((control == matcher.IfTrue()) ? phi_true : phi_false); |
| } |
| } |
| if (phi->opcode() == IrOpcode::kEffectPhi) { |
| true_block_data->current_effect = phi_true; |
| false_block_data->current_effect = phi_false; |
| } |
| phi->Kill(); |
| } |
| // Fix up IfTrue and IfFalse and kill all dead nodes. |
| if (branch == block->control_input()) { |
| true_block_data->current_control = merge_true; |
| false_block_data->current_control = merge_false; |
| } |
| branch->Kill(); |
| cond->Kill(); |
| merge->Kill(); |
| } |
| |
| } // namespace |
| |
| void EffectControlLinearizer::Run() { |
| BlockEffectControlMap block_effects(temp_zone()); |
| ZoneVector<PendingEffectPhi> pending_effect_phis(temp_zone()); |
| ZoneVector<BasicBlock*> pending_block_controls(temp_zone()); |
| NodeVector inputs_buffer(temp_zone()); |
| |
| // TODO(rmcilroy) We should not depend on having rpo_order on schedule, and |
| // instead just do our own RPO walk here. |
| for (BasicBlock* block : *(schedule()->rpo_order())) { |
| if (block != schedule()->start() && block->PredecessorCount() == 0) { |
| // Block has been removed from the schedule by a preceeding unreachable |
| // node, just skip it. |
| continue; |
| } |
| |
| gasm()->Reset(block); |
| |
| BasicBlock::iterator instr = block->begin(); |
| BasicBlock::iterator end_instr = block->end(); |
| |
| // The control node should be the first. |
| Node* control = *instr; |
| gasm()->AddNode(control); |
| |
| DCHECK(NodeProperties::IsControl(control)); |
| bool has_incoming_backedge = IrOpcode::kLoop == control->opcode(); |
| // Update the control inputs. |
| if (has_incoming_backedge) { |
| // If there are back edges, we need to update later because we have not |
| // computed the control yet. |
| pending_block_controls.push_back(block); |
| } else { |
| // If there are no back edges, we can update now. |
| UpdateBlockControl(block, &block_effects); |
| } |
| instr++; |
| |
| // Iterate over the phis and update the effect phis. |
| Node* effect_phi = nullptr; |
| Node* terminate = nullptr; |
| for (; instr != end_instr; instr++) { |
| Node* node = *instr; |
| // Only go through the phis and effect phis. |
| if (node->opcode() == IrOpcode::kEffectPhi) { |
| // There should be at most one effect phi in a block. |
| DCHECK_NULL(effect_phi); |
| // IfException blocks should not have effect phis. |
| DCHECK_NE(IrOpcode::kIfException, control->opcode()); |
| effect_phi = node; |
| } else if (node->opcode() == IrOpcode::kPhi) { |
| // Just skip phis. |
| } else if (node->opcode() == IrOpcode::kTerminate) { |
| DCHECK_NULL(terminate); |
| terminate = node; |
| } else { |
| break; |
| } |
| gasm()->AddNode(node); |
| } |
| |
| if (effect_phi) { |
| // Make sure we update the inputs to the incoming blocks' effects. |
| if (has_incoming_backedge) { |
| // In case of loops, we do not update the effect phi immediately |
| // because the back predecessor has not been handled yet. We just |
| // record the effect phi for later processing. |
| pending_effect_phis.push_back(PendingEffectPhi(effect_phi, block)); |
| } else { |
| UpdateEffectPhi(effect_phi, block, &block_effects); |
| } |
| } |
| |
| Node* effect = effect_phi; |
| if (effect == nullptr) { |
| // There was no effect phi. |
| if (block == schedule()->start()) { |
| // Start block => effect is start. |
| DCHECK_EQ(graph()->start(), control); |
| effect = graph()->start(); |
| } else if (control->opcode() == IrOpcode::kEnd) { |
| // End block is just a dummy, no effect needed. |
| DCHECK_EQ(BasicBlock::kNone, block->control()); |
| DCHECK_EQ(1u, block->size()); |
| effect = nullptr; |
| } else { |
| // If all the predecessors have the same effect, we can use it as our |
| // current effect. |
| for (size_t i = 0; i < block->PredecessorCount(); ++i) { |
| const BlockEffectControlData& data = |
| block_effects.For(block->PredecessorAt(i), block); |
| if (!effect) effect = data.current_effect; |
| if (data.current_effect != effect) { |
| effect = nullptr; |
| break; |
| } |
| } |
| if (effect == nullptr) { |
| DCHECK_NE(IrOpcode::kIfException, control->opcode()); |
| // The input blocks do not have the same effect. We have |
| // to create an effect phi node. |
| inputs_buffer.clear(); |
| inputs_buffer.resize(block->PredecessorCount(), jsgraph()->Dead()); |
| inputs_buffer.push_back(control); |
| effect = graph()->NewNode( |
| common()->EffectPhi(static_cast<int>(block->PredecessorCount())), |
| static_cast<int>(inputs_buffer.size()), &(inputs_buffer.front())); |
| gasm()->AddNode(effect); |
| // For loops, we update the effect phi node later to break cycles. |
| if (control->opcode() == IrOpcode::kLoop) { |
| pending_effect_phis.push_back(PendingEffectPhi(effect, block)); |
| } else { |
| UpdateEffectPhi(effect, block, &block_effects); |
| } |
| } else if (control->opcode() == IrOpcode::kIfException) { |
| // The IfException is connected into the effect chain, so we need |
| // to update the effect here. |
| NodeProperties::ReplaceEffectInput(control, effect); |
| effect = control; |
| } |
| } |
| } |
| |
| // Fixup the Terminate node. |
| if (terminate != nullptr) { |
| NodeProperties::ReplaceEffectInput(terminate, effect); |
| } |
| |
| // The frame state at block entry is determined by the frame states leaving |
| // all predecessors. In case there is no frame state dominating this block, |
| // we can rely on a checkpoint being present before the next deoptimization. |
| Node* frame_state = nullptr; |
| if (block != schedule()->start()) { |
| // If all the predecessors have the same effect, we can use it |
| // as our current effect. |
| frame_state = |
| block_effects.For(block->PredecessorAt(0), block).current_frame_state; |
| for (size_t i = 1; i < block->PredecessorCount(); i++) { |
| if (block_effects.For(block->PredecessorAt(i), block) |
| .current_frame_state != frame_state) { |
| frame_state = nullptr; |
| frame_state_zapper_ = graph()->end(); |
| break; |
| } |
| } |
| } |
| |
| gasm()->InitializeEffectControl(effect, control); |
| |
| // Process the ordinary instructions. |
| for (; instr != end_instr; instr++) { |
| Node* node = *instr; |
| ProcessNode(node, &frame_state); |
| } |
| |
| block = gasm()->FinalizeCurrentBlock(block); |
| |
| switch (block->control()) { |
| case BasicBlock::kGoto: |
| case BasicBlock::kNone: |
| break; |
| case BasicBlock::kCall: |
| case BasicBlock::kTailCall: |
| case BasicBlock::kSwitch: |
| case BasicBlock::kReturn: |
| case BasicBlock::kDeoptimize: |
| case BasicBlock::kThrow: |
| case BasicBlock::kBranch: |
| UpdateEffectControlForNode(block->control_input()); |
| gasm()->UpdateEffectControlWith(block->control_input()); |
| break; |
| } |
| |
| if (!should_maintain_schedule() && |
| block->control() == BasicBlock::kBranch) { |
| TryCloneBranch(block->control_input(), block, temp_zone(), graph(), |
| common(), &block_effects, source_positions_, |
| node_origins_); |
| } |
| |
| // Store the effect, control and frame state for later use. |
| for (BasicBlock* successor : block->successors()) { |
| BlockEffectControlData* data = &block_effects.For(block, successor); |
| if (data->current_effect == nullptr) { |
| data->current_effect = gasm()->effect(); |
| } |
| if (data->current_control == nullptr) { |
| data->current_control = gasm()->control(); |
| } |
| data->current_frame_state = frame_state; |
| } |
| } |
| |
| for (BasicBlock* pending_block_control : pending_block_controls) { |
| UpdateBlockControl(pending_block_control, &block_effects); |
| } |
| // Update the incoming edges of the effect phis that could not be processed |
| // during the first pass (because they could have incoming back edges). |
| for (const PendingEffectPhi& pending_effect_phi : pending_effect_phis) { |
| UpdateEffectPhi(pending_effect_phi.effect_phi, pending_effect_phi.block, |
| &block_effects); |
| } |
| |
| schedule_->rpo_order()->clear(); |
| } |
| |
| void EffectControlLinearizer::UpdateEffectControlForNode(Node* node) { |
| // If the node takes an effect, replace with the current one. |
| if (node->op()->EffectInputCount() > 0) { |
| DCHECK_EQ(1, node->op()->EffectInputCount()); |
| NodeProperties::ReplaceEffectInput(node, gasm()->effect()); |
| } else { |
| // New effect chain is only started with a Start or ValueEffect node. |
| DCHECK(node->op()->EffectOutputCount() == 0 || |
| node->opcode() == IrOpcode::kStart); |
| } |
| |
| // Rewire control inputs. |
| for (int i = 0; i < node->op()->ControlInputCount(); i++) { |
| NodeProperties::ReplaceControlInput(node, gasm()->control(), i); |
| } |
| } |
| |
| void EffectControlLinearizer::ProcessNode(Node* node, Node** frame_state) { |
| SourcePositionTable::Scope scope(source_positions_, |
| source_positions_->GetSourcePosition(node)); |
| NodeOriginTable::Scope origin_scope(node_origins_, "process node", node); |
| |
| // If basic block is unreachable after this point, update the node's effect |
| // and control inputs to mark it as dead, but don't process further. |
| if (gasm()->effect() == jsgraph()->Dead()) { |
| UpdateEffectControlForNode(node); |
| return; |
| } |
| |
| // If the node needs to be wired into the effect/control chain, do this |
| // here. Pass current frame state for lowering to eager deoptimization. |
| if (TryWireInStateEffect(node, *frame_state)) { |
| return; |
| } |
| |
| // If the node has a visible effect, then there must be a checkpoint in the |
| // effect chain before we are allowed to place another eager deoptimization |
| // point. We zap the frame state to ensure this invariant is maintained. |
| if (region_observability_ == RegionObservability::kObservable && |
| !node->op()->HasProperty(Operator::kNoWrite)) { |
| *frame_state = nullptr; |
| frame_state_zapper_ = node; |
| } |
| |
| // Remove the end markers of 'atomic' allocation region because the |
| // region should be wired-in now. |
| if (node->opcode() == IrOpcode::kFinishRegion) { |
| // Reset the current region observability. |
| region_observability_ = RegionObservability::kObservable; |
| // Update the value uses to the value input of the finish node and |
| // the effect uses to the effect input. |
| return RemoveRenameNode(node); |
| } |
| if (node->opcode() == IrOpcode::kBeginRegion) { |
| // Determine the observability for this region and use that for all |
| // nodes inside the region (i.e. ignore the absence of kNoWrite on |
| // StoreField and other operators). |
| DCHECK_NE(RegionObservability::kNotObservable, region_observability_); |
| region_observability_ = RegionObservabilityOf(node->op()); |
| // Update the value uses to the value input of the finish node and |
| // the effect uses to the effect input. |
| return RemoveRenameNode(node); |
| } |
| if (node->opcode() == IrOpcode::kTypeGuard) { |
| return RemoveRenameNode(node); |
| } |
| |
| // Special treatment for checkpoint nodes. |
| if (node->opcode() == IrOpcode::kCheckpoint) { |
| // Unlink the check point; effect uses will be updated to the incoming |
| // effect that is passed. The frame state is preserved for lowering. |
| DCHECK_EQ(RegionObservability::kObservable, region_observability_); |
| *frame_state = NodeProperties::GetFrameStateInput(node); |
| return; |
| } |
| |
| // The IfSuccess nodes should always start a basic block (and basic block |
| // start nodes are not handled in the ProcessNode method). |
| DCHECK_NE(IrOpcode::kIfSuccess, node->opcode()); |
| |
| UpdateEffectControlForNode(node); |
| |
| gasm()->AddNode(node); |
| |
| if (node->opcode() == IrOpcode::kUnreachable) { |
| // Break the effect chain on {Unreachable} and reconnect to the graph end. |
| // Mark the following code for deletion by connecting to the {Dead} node. |
| gasm()->ConnectUnreachableToEnd(); |
| } |
| } |
| |
| bool EffectControlLinearizer::TryWireInStateEffect(Node* node, |
| Node* frame_state) { |
| Node* result = nullptr; |
| switch (node->opcode()) { |
| case IrOpcode::kChangeBitToTagged: |
| result = LowerChangeBitToTagged(node); |
| break; |
| case IrOpcode::kChangeInt31ToTaggedSigned: |
| result = LowerChangeInt31ToTaggedSigned(node); |
| break; |
| case IrOpcode::kChangeInt32ToTagged: |
| result = LowerChangeInt32ToTagged(node); |
| break; |
| case IrOpcode::kChangeInt64ToTagged: |
| result = LowerChangeInt64ToTagged(node); |
| break; |
| case IrOpcode::kChangeUint32ToTagged: |
| result = LowerChangeUint32ToTagged(node); |
| break; |
| case IrOpcode::kChangeUint64ToTagged: |
| result = LowerChangeUint64ToTagged(node); |
| break; |
| case IrOpcode::kChangeFloat64ToTagged: |
| result = LowerChangeFloat64ToTagged(node); |
| break; |
| case IrOpcode::kChangeFloat64ToTaggedPointer: |
| result = LowerChangeFloat64ToTaggedPointer(node); |
| break; |
| case IrOpcode::kChangeTaggedSignedToInt32: |
| result = LowerChangeTaggedSignedToInt32(node); |
| break; |
| case IrOpcode::kChangeTaggedSignedToInt64: |
| result = LowerChangeTaggedSignedToInt64(node); |
| break; |
| case IrOpcode::kChangeTaggedToBit: |
| result = LowerChangeTaggedToBit(node); |
| break; |
| case IrOpcode::kChangeTaggedToInt32: |
| result = LowerChangeTaggedToInt32(node); |
| break; |
| case IrOpcode::kChangeTaggedToUint32: |
| result = LowerChangeTaggedToUint32(node); |
| break; |
| case IrOpcode::kChangeTaggedToInt64: |
| result = LowerChangeTaggedToInt64(node); |
| break; |
| case IrOpcode::kChangeTaggedToFloat64: |
| result = LowerChangeTaggedToFloat64(node); |
| break; |
| case IrOpcode::kChangeTaggedToTaggedSigned: |
| result = LowerChangeTaggedToTaggedSigned(node); |
| break; |
| case IrOpcode::kTruncateTaggedToBit: |
| result = LowerTruncateTaggedToBit(node); |
| break; |
| case IrOpcode::kTruncateTaggedPointerToBit: |
| result = LowerTruncateTaggedPointerToBit(node); |
| break; |
| case IrOpcode::kTruncateTaggedToFloat64: |
| result = LowerTruncateTaggedToFloat64(node); |
| break; |
| case IrOpcode::kPoisonIndex: |
| result = LowerPoisonIndex(node); |
| break; |
| case IrOpcode::kCheckClosure: |
| result = LowerCheckClosure(node, frame_state); |
| break; |
| case IrOpcode::kCheckMaps: |
| LowerCheckMaps(node, frame_state); |
| break; |
| case IrOpcode::kDynamicCheckMaps: |
| LowerDynamicCheckMaps(node, frame_state); |
| break; |
| case IrOpcode::kCompareMaps: |
| result = LowerCompareMaps(node); |
| break; |
| case IrOpcode::kCheckNumber: |
| result = LowerCheckNumber(node, frame_state); |
| break; |
| case IrOpcode::kCheckReceiver: |
| result = LowerCheckReceiver(node, frame_state); |
| break; |
| case IrOpcode::kCheckReceiverOrNullOrUndefined: |
| result = LowerCheckReceiverOrNullOrUndefined(node, frame_state); |
| break; |
| case IrOpcode::kCheckSymbol: |
| result = LowerCheckSymbol(node, frame_state); |
| break; |
| case IrOpcode::kCheckString: |
| result = LowerCheckString(node, frame_state); |
| break; |
| case IrOpcode::kCheckBigInt: |
| result = LowerCheckBigInt(node, frame_state); |
| break; |
| case IrOpcode::kCheckInternalizedString: |
| result = LowerCheckInternalizedString(node, frame_state); |
| break; |
| case IrOpcode::kCheckIf: |
| LowerCheckIf(node, frame_state); |
| break; |
| case IrOpcode::kCheckedInt32Add: |
| result = LowerCheckedInt32Add(node, frame_state); |
| break; |
| case IrOpcode::kCheckedInt32Sub: |
| result = LowerCheckedInt32Sub(node, frame_state); |
| break; |
| case IrOpcode::kCheckedInt32Div: |
| result = LowerCheckedInt32Div(node, frame_state); |
| break; |
| case IrOpcode::kCheckedInt32Mod: |
| result = LowerCheckedInt32Mod(node, frame_state); |
| break; |
| case IrOpcode::kCheckedUint32Div: |
| result = LowerCheckedUint32Div(node, frame_state); |
| break; |
| case IrOpcode::kCheckedUint32Mod: |
| result = LowerCheckedUint32Mod(node, frame_state); |
| break; |
| case IrOpcode::kCheckedInt32Mul: |
| result = LowerCheckedInt32Mul(node, frame_state); |
| break; |
| case IrOpcode::kCheckedInt32ToTaggedSigned: |
| result = LowerCheckedInt32ToTaggedSigned(node, frame_state); |
| break; |
| case IrOpcode::kCheckedInt64ToInt32: |
| result = LowerCheckedInt64ToInt32(node, frame_state); |
| break; |
| case IrOpcode::kCheckedInt64ToTaggedSigned: |
| result = LowerCheckedInt64ToTaggedSigned(node, frame_state); |
| break; |
| case IrOpcode::kCheckedUint32Bounds: |
| result = LowerCheckedUint32Bounds(node, frame_state); |
| break; |
| case IrOpcode::kCheckedUint32ToInt32: |
| result = LowerCheckedUint32ToInt32(node, frame_state); |
| break; |
| case IrOpcode::kCheckedUint32ToTaggedSigned: |
| result = LowerCheckedUint32ToTaggedSigned(node, frame_state); |
| break; |
| case IrOpcode::kCheckedUint64Bounds: |
| result = LowerCheckedUint64Bounds(node, frame_state); |
| break; |
| case IrOpcode::kCheckedUint64ToInt32: |
| result = LowerCheckedUint64ToInt32(node, frame_state); |
| break; |
| case IrOpcode::kCheckedUint64ToTaggedSigned: |
| result = LowerCheckedUint64ToTaggedSigned(node, frame_state); |
| break; |
| case IrOpcode::kCheckedFloat64ToInt32: |
| result = LowerCheckedFloat64ToInt32(node, frame_state); |
| break; |
| case IrOpcode::kCheckedFloat64ToInt64: |
| result = LowerCheckedFloat64ToInt64(node, frame_state); |
| break; |
| case IrOpcode::kCheckedTaggedSignedToInt32: |
| if (frame_state == nullptr) { |
| FATAL("No frame state (zapped by #%d: %s)", frame_state_zapper_->id(), |
| frame_state_zapper_->op()->mnemonic()); |
| } |
| result = LowerCheckedTaggedSignedToInt32(node, frame_state); |
| break; |
| case IrOpcode::kCheckedTaggedToArrayIndex: |
| result = LowerCheckedTaggedToArrayIndex(node, frame_state); |
| break; |
| case IrOpcode::kCheckedTaggedToInt32: |
| result = LowerCheckedTaggedToInt32(node, frame_state); |
| break; |
| case IrOpcode::kCheckedTaggedToInt64: |
| result = LowerCheckedTaggedToInt64(node, frame_state); |
| break; |
| case IrOpcode::kCheckedTaggedToFloat64: |
| result = LowerCheckedTaggedToFloat64(node, frame_state); |
| break; |
| case IrOpcode::kCheckedTaggedToTaggedSigned: |
| result = LowerCheckedTaggedToTaggedSigned(node, frame_state); |
| break; |
| case IrOpcode::kCheckedTaggedToTaggedPointer: |
| result = LowerCheckedTaggedToTaggedPointer(node, frame_state); |
| break; |
| case IrOpcode::kBigIntAsUintN: |
| result = LowerBigIntAsUintN(node, frame_state); |
| break; |
| case IrOpcode::kChangeUint64ToBigInt: |
| result = LowerChangeUint64ToBigInt(node); |
| break; |
| case IrOpcode::kTruncateBigIntToUint64: |
| result = LowerTruncateBigIntToUint64(node); |
| break; |
| case IrOpcode::kTruncateTaggedToWord32: |
| result = LowerTruncateTaggedToWord32(node); |
| break; |
| case IrOpcode::kCheckedTruncateTaggedToWord32: |
| result = LowerCheckedTruncateTaggedToWord32(node, frame_state); |
| break; |
| case IrOpcode::kNumberToString: |
| result = LowerNumberToString(node); |
| break; |
| case IrOpcode::kObjectIsArrayBufferView: |
| result = LowerObjectIsArrayBufferView(node); |
| break; |
| case IrOpcode::kObjectIsBigInt: |
| result = LowerObjectIsBigInt(node); |
| break; |
| case IrOpcode::kObjectIsCallable: |
| result = LowerObjectIsCallable(node); |
| break; |
| case IrOpcode::kObjectIsConstructor: |
| result = LowerObjectIsConstructor(node); |
| break; |
| case IrOpcode::kObjectIsDetectableCallable: |
| result = LowerObjectIsDetectableCallable(node); |
| break; |
| case IrOpcode::kObjectIsMinusZero: |
| result = LowerObjectIsMinusZero(node); |
| break; |
| case IrOpcode::kNumberIsMinusZero: |
| result = LowerNumberIsMinusZero(node); |
| break; |
| case IrOpcode::kObjectIsNaN: |
| result = LowerObjectIsNaN(node); |
| break; |
| case IrOpcode::kNumberIsNaN: |
| result = LowerNumberIsNaN(node); |
| break; |
| case IrOpcode::kObjectIsNonCallable: |
| result = LowerObjectIsNonCallable(node); |
| break; |
| case IrOpcode::kObjectIsNumber: |
| result = LowerObjectIsNumber(node); |
| break; |
| case IrOpcode::kObjectIsReceiver: |
| result = LowerObjectIsReceiver(node); |
| break; |
| case IrOpcode::kObjectIsSmi: |
| result = LowerObjectIsSmi(node); |
| break; |
| case IrOpcode::kObjectIsString: |
| result = LowerObjectIsString(node); |
| break; |
| case IrOpcode::kObjectIsSymbol: |
| result = LowerObjectIsSymbol(node); |
| break; |
| case IrOpcode::kObjectIsUndetectable: |
| result = LowerObjectIsUndetectable(node); |
| break; |
| case IrOpcode::kArgumentsFrame: |
| result = LowerArgumentsFrame(node); |
| break; |
| case IrOpcode::kArgumentsLength: |
| result = LowerArgumentsLength(node); |
| break; |
| case IrOpcode::kRestLength: |
| result = LowerRestLength(node); |
| break; |
| case IrOpcode::kToBoolean: |
| result = LowerToBoolean(node); |
| break; |
| case IrOpcode::kTypeOf: |
| result = LowerTypeOf(node); |
| break; |
| case IrOpcode::kTierUpCheck: |
| LowerTierUpCheck(node); |
| break; |
| case IrOpcode::kUpdateInterruptBudget: |
| LowerUpdateInterruptBudget(node); |
| break; |
| case IrOpcode::kNewDoubleElements: |
| result = LowerNewDoubleElements(node); |
| break; |
| case IrOpcode::kNewSmiOrObjectElements: |
| result = LowerNewSmiOrObjectElements(node); |
| break; |
| case IrOpcode::kNewArgumentsElements: |
| result = LowerNewArgumentsElements(node); |
| break; |
| case IrOpcode::kNewConsString: |
| result = LowerNewConsString(node); |
| break; |
| case IrOpcode::kSameValue: |
| result = LowerSameValue(node); |
| break; |
| case IrOpcode::kSameValueNumbersOnly: |
| result = LowerSameValueNumbersOnly(node); |
| break; |
| case IrOpcode::kNumberSameValue: |
| result = LowerNumberSameValue(node); |
| break; |
| case IrOpcode::kDeadValue: |
| result = LowerDeadValue(node); |
| break; |
| case IrOpcode::kStringConcat: |
| result = LowerStringConcat(node); |
| break; |
| case IrOpcode::kStringFromSingleCharCode: |
| result = LowerStringFromSingleCharCode(node); |
| break; |
| case IrOpcode::kStringFromSingleCodePoint: |
| result = LowerStringFromSingleCodePoint(node); |
| break; |
| case IrOpcode::kStringIndexOf: |
| result = LowerStringIndexOf(node); |
| break; |
| case IrOpcode::kStringFromCodePointAt: |
| result = LowerStringFromCodePointAt(node); |
| break; |
| case IrOpcode::kStringLength: |
| result = LowerStringLength(node); |
| break; |
| case IrOpcode::kStringToNumber: |
| result = LowerStringToNumber(node); |
| break; |
| case IrOpcode::kStringCharCodeAt: |
| result = LowerStringCharCodeAt(node); |
| break; |
| case IrOpcode::kStringCodePointAt: |
| result = LowerStringCodePointAt(node); |
| break; |
| case IrOpcode::kStringToLowerCaseIntl: |
| result = LowerStringToLowerCaseIntl(node); |
| break; |
| case IrOpcode::kStringToUpperCaseIntl: |
| result = LowerStringToUpperCaseIntl(node); |
| break; |
| case IrOpcode::kStringSubstring: |
| result = LowerStringSubstring(node); |
| break; |
| case IrOpcode::kStringEqual: |
| result = LowerStringEqual(node); |
| break; |
| case IrOpcode::kStringLessThan: |
| result = LowerStringLessThan(node); |
| break; |
| case IrOpcode::kStringLessThanOrEqual: |
| result = LowerStringLessThanOrEqual(node); |
| break; |
| case IrOpcode::kBigIntAdd: |
| result = LowerBigIntAdd(node, frame_state); |
| break; |
| case IrOpcode::kBigIntSubtract: |
| result = LowerBigIntSubtract(node, frame_state); |
| break; |
| case IrOpcode::kBigIntNegate: |
| result = LowerBigIntNegate(node); |
| break; |
| case IrOpcode::kNumberIsFloat64Hole: |
| result = LowerNumberIsFloat64Hole(node); |
| break; |
| case IrOpcode::kNumberIsFinite: |
| result = LowerNumberIsFinite(node); |
| break; |
| case IrOpcode::kObjectIsFiniteNumber: |
| result = LowerObjectIsFiniteNumber(node); |
| break; |
| case IrOpcode::kNumberIsInteger: |
| result = LowerNumberIsInteger(node); |
| break; |
| case IrOpcode::kObjectIsInteger: |
| result = LowerObjectIsInteger(node); |
| break; |
| case IrOpcode::kNumberIsSafeInteger: |
| result = LowerNumberIsSafeInteger(node); |
| break; |
| case IrOpcode::kObjectIsSafeInteger: |
| result = LowerObjectIsSafeInteger(node); |
| break; |
| case IrOpcode::kCheckFloat64Hole: |
| result = LowerCheckFloat64Hole(node, frame_state); |
| break; |
| case IrOpcode::kCheckNotTaggedHole: |
| result = LowerCheckNotTaggedHole(node, frame_state); |
| break; |
| case IrOpcode::kConvertTaggedHoleToUndefined: |
| result = LowerConvertTaggedHoleToUndefined(node); |
| break; |
| case IrOpcode::kCheckEqualsInternalizedString: |
| LowerCheckEqualsInternalizedString(node, frame_state); |
| break; |
| case IrOpcode::kAllocate: |
| result = LowerAllocate(node); |
| break; |
| case IrOpcode::kCheckEqualsSymbol: |
| LowerCheckEqualsSymbol(node, frame_state); |
| break; |
| case IrOpcode::kPlainPrimitiveToNumber: |
| result = LowerPlainPrimitiveToNumber(node); |
| break; |
| case IrOpcode::kPlainPrimitiveToWord32: |
| result = LowerPlainPrimitiveToWord32(node); |
| break; |
| case IrOpcode::kPlainPrimitiveToFloat64: |
| result = LowerPlainPrimitiveToFloat64(node); |
| break; |
| case IrOpcode::kEnsureWritableFastElements: |
| result = LowerEnsureWritableFastElements(node); |
| break; |
| case IrOpcode::kMaybeGrowFastElements: |
| result = LowerMaybeGrowFastElements(node, frame_state); |
| break; |
| case IrOpcode::kTransitionElementsKind: |
| LowerTransitionElementsKind(node); |
| break; |
| case IrOpcode::kLoadMessage: |
| result = LowerLoadMessage(node); |
| break; |
| case IrOpcode::kStoreMessage: |
| LowerStoreMessage(node); |
| break; |
| case IrOpcode::kFastApiCall: |
| result = LowerFastApiCall(node); |
| break; |
| case IrOpcode::kLoadFieldByIndex: |
| result = LowerLoadFieldByIndex(node); |
| break; |
| case IrOpcode::kLoadTypedElement: |
| result = LowerLoadTypedElement(node); |
| break; |
| case IrOpcode::kLoadDataViewElement: |
| result = LowerLoadDataViewElement(node); |
| break; |
| case IrOpcode::kLoadStackArgument: |
| result = LowerLoadStackArgument(node); |
| break; |
| case IrOpcode::kStoreTypedElement: |
| LowerStoreTypedElement(node); |
| break; |
| case IrOpcode::kStoreDataViewElement: |
| LowerStoreDataViewElement(node); |
| break; |
| case IrOpcode::kStoreSignedSmallElement: |
| LowerStoreSignedSmallElement(node); |
| break; |
| case IrOpcode::kFindOrderedHashMapEntry: |
| result = LowerFindOrderedHashMapEntry(node); |
| break; |
| case IrOpcode::kFindOrderedHashMapEntryForInt32Key: |
| result = LowerFindOrderedHashMapEntryForInt32Key(node); |
| break; |
| case IrOpcode::kTransitionAndStoreNumberElement: |
| LowerTransitionAndStoreNumberElement(node); |
| break; |
| case IrOpcode::kTransitionAndStoreNonNumberElement: |
| LowerTransitionAndStoreNonNumberElement(node); |
| break; |
| case IrOpcode::kTransitionAndStoreElement: |
| LowerTransitionAndStoreElement(node); |
| break; |
| case IrOpcode::kRuntimeAbort: |
| LowerRuntimeAbort(node); |
| break; |
| case IrOpcode::kAssertType: |
| result = LowerAssertType(node); |
| break; |
| case IrOpcode::kConvertReceiver: |
| result = LowerConvertReceiver(node); |
| break; |
| case IrOpcode::kFloat64RoundUp: |
| if (!LowerFloat64RoundUp(node).To(&result)) { |
| return false; |
| } |
| break; |
| case IrOpcode::kFloat64RoundDown: |
| if (!LowerFloat64RoundDown(node).To(&result)) { |
| return false; |
| } |
| break; |
| case IrOpcode::kFloat64RoundTruncate: |
| if (!LowerFloat64RoundTruncate(node).To(&result)) { |
| return false; |
| } |
| break; |
| case IrOpcode::kFloat64RoundTiesEven: |
| if (!LowerFloat64RoundTiesEven(node).To(&result)) { |
| return false; |
| } |
| break; |
| case IrOpcode::kDateNow: |
| result = LowerDateNow(node); |
| break; |
| case IrOpcode::kFoldConstant: |
| result = LowerFoldConstant(node); |
| break; |
| default: |
| return false; |
| } |
| |
| if ((result ? 1 : 0) != node->op()->ValueOutputCount()) { |
| FATAL( |
| "Effect control linearizer lowering of '%s':" |
| " value output count does not agree.", |
| node->op()->mnemonic()); |
| } |
| |
| NodeProperties::ReplaceUses(node, result, gasm()->effect(), |
| gasm()->control()); |
| return true; |
| } |
| |
| #define __ gasm()-> |
| |
| Node* EffectControlLinearizer::LowerChangeFloat64ToTagged(Node* node) { |
| CheckForMinusZeroMode mode = CheckMinusZeroModeOf(node->op()); |
| Node* value = node->InputAt(0); |
| |
| auto done = __ MakeLabel(MachineRepresentation::kTagged); |
| auto if_heapnumber = __ MakeDeferredLabel(); |
| auto if_int32 = __ MakeLabel(); |
| |
| Node* value32 = __ RoundFloat64ToInt32(value); |
| __ GotoIf(__ Float64Equal(value, __ ChangeInt32ToFloat64(value32)), |
| &if_int32); |
| __ Goto(&if_heapnumber); |
| |
| __ Bind(&if_int32); |
| { |
| if (mode == CheckForMinusZeroMode::kCheckForMinusZero) { |
| Node* zero = __ Int32Constant(0); |
| auto if_zero = __ MakeDeferredLabel(); |
| auto if_smi = __ MakeLabel(); |
| |
| __ GotoIf(__ Word32Equal(value32, zero), &if_zero); |
| __ Goto(&if_smi); |
| |
| __ Bind(&if_zero); |
| { |
| // In case of 0, we need to check the high bits for the IEEE -0 pattern. |
| __ GotoIf(__ Int32LessThan(__ Float64ExtractHighWord32(value), zero), |
| &if_heapnumber); |
| __ Goto(&if_smi); |
| } |
| |
| __ Bind(&if_smi); |
| } |
| |
| if (SmiValuesAre32Bits()) { |
| Node* value_smi = ChangeInt32ToSmi(value32); |
| __ Goto(&done, value_smi); |
| } else { |
| SmiTagOrOverflow(value32, &if_heapnumber, &done); |
| } |
| } |
| |
| __ Bind(&if_heapnumber); |
| { |
| Node* value_number = AllocateHeapNumberWithValue(value); |
| __ Goto(&done, value_number); |
| } |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerChangeFloat64ToTaggedPointer(Node* node) { |
| Node* value = node->InputAt(0); |
| return AllocateHeapNumberWithValue(value); |
| } |
| |
| Node* EffectControlLinearizer::LowerChangeBitToTagged(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| auto if_true = __ MakeLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kTagged); |
| |
| __ GotoIf(value, &if_true); |
| __ Goto(&done, __ FalseConstant()); |
| |
| __ Bind(&if_true); |
| __ Goto(&done, __ TrueConstant()); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerChangeInt31ToTaggedSigned(Node* node) { |
| Node* value = node->InputAt(0); |
| return ChangeInt32ToSmi(value); |
| } |
| |
| Node* EffectControlLinearizer::LowerChangeInt32ToTagged(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| if (SmiValuesAre32Bits()) { |
| return ChangeInt32ToSmi(value); |
| } |
| DCHECK(SmiValuesAre31Bits()); |
| |
| auto if_overflow = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kTagged); |
| |
| SmiTagOrOverflow(value, &if_overflow, &done); |
| |
| __ Bind(&if_overflow); |
| Node* number = AllocateHeapNumberWithValue(__ ChangeInt32ToFloat64(value)); |
| __ Goto(&done, number); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerChangeInt64ToTagged(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| auto if_not_in_smi_range = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kTagged); |
| |
| Node* value32 = __ TruncateInt64ToInt32(value); |
| __ GotoIfNot(__ Word64Equal(__ ChangeInt32ToInt64(value32), value), |
| &if_not_in_smi_range); |
| |
| if (SmiValuesAre32Bits()) { |
| Node* value_smi = ChangeInt64ToSmi(value); |
| __ Goto(&done, value_smi); |
| } else { |
| SmiTagOrOverflow(value32, &if_not_in_smi_range, &done); |
| } |
| |
| __ Bind(&if_not_in_smi_range); |
| Node* number = AllocateHeapNumberWithValue(__ ChangeInt64ToFloat64(value)); |
| __ Goto(&done, number); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerChangeUint32ToTagged(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| auto if_not_in_smi_range = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kTagged); |
| |
| Node* check = __ Uint32LessThanOrEqual(value, SmiMaxValueConstant()); |
| __ GotoIfNot(check, &if_not_in_smi_range); |
| __ Goto(&done, ChangeUint32ToSmi(value)); |
| |
| __ Bind(&if_not_in_smi_range); |
| Node* number = AllocateHeapNumberWithValue(__ ChangeUint32ToFloat64(value)); |
| |
| __ Goto(&done, number); |
| __ Bind(&done); |
| |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerChangeUint64ToTagged(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| auto if_not_in_smi_range = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kTagged); |
| |
| Node* check = |
| __ Uint64LessThanOrEqual(value, __ Int64Constant(Smi::kMaxValue)); |
| __ GotoIfNot(check, &if_not_in_smi_range); |
| __ Goto(&done, ChangeInt64ToSmi(value)); |
| |
| __ Bind(&if_not_in_smi_range); |
| Node* number = AllocateHeapNumberWithValue(__ ChangeInt64ToFloat64(value)); |
| |
| __ Goto(&done, number); |
| __ Bind(&done); |
| |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerChangeTaggedSignedToInt32(Node* node) { |
| Node* value = node->InputAt(0); |
| return ChangeSmiToInt32(value); |
| } |
| |
| Node* EffectControlLinearizer::LowerChangeTaggedSignedToInt64(Node* node) { |
| Node* value = node->InputAt(0); |
| return ChangeSmiToInt64(value); |
| } |
| |
| Node* EffectControlLinearizer::LowerChangeTaggedToBit(Node* node) { |
| Node* value = node->InputAt(0); |
| return __ TaggedEqual(value, __ TrueConstant()); |
| } |
| |
| void EffectControlLinearizer::TruncateTaggedPointerToBit( |
| Node* node, GraphAssemblerLabel<1>* done) { |
| Node* value = node->InputAt(0); |
| |
| auto if_heapnumber = __ MakeDeferredLabel(); |
| auto if_bigint = __ MakeDeferredLabel(); |
| |
| Node* zero = __ Int32Constant(0); |
| Node* fzero = __ Float64Constant(0.0); |
| |
| // Check if {value} is false. |
| __ GotoIf(__ TaggedEqual(value, __ FalseConstant()), done, zero); |
| |
| // Check if {value} is the empty string. |
| __ GotoIf(__ TaggedEqual(value, __ EmptyStringConstant()), done, zero); |
| |
| // Load the map of {value}. |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| |
| // Check if the {value} is undetectable and immediately return false. |
| // This includes undefined and null. |
| Node* value_map_bitfield = |
| __ LoadField(AccessBuilder::ForMapBitField(), value_map); |
| __ GotoIfNot( |
| __ Word32Equal( |
| __ Word32And(value_map_bitfield, |
| __ Int32Constant(Map::Bits1::IsUndetectableBit::kMask)), |
| zero), |
| done, zero); |
| |
| // Check if {value} is a HeapNumber. |
| __ GotoIf(__ TaggedEqual(value_map, __ HeapNumberMapConstant()), |
| &if_heapnumber); |
| |
| // Check if {value} is a BigInt. |
| __ GotoIf(__ TaggedEqual(value_map, __ BigIntMapConstant()), &if_bigint); |
| |
| // All other values that reach here are true. |
| __ Goto(done, __ Int32Constant(1)); |
| |
| __ Bind(&if_heapnumber); |
| { |
| // For HeapNumber {value}, just check that its value is not 0.0, -0.0 or |
| // NaN. |
| Node* value_value = |
| __ LoadField(AccessBuilder::ForHeapNumberValue(), value); |
| __ Goto(done, __ Float64LessThan(fzero, __ Float64Abs(value_value))); |
| } |
| |
| __ Bind(&if_bigint); |
| { |
| Node* bitfield = __ LoadField(AccessBuilder::ForBigIntBitfield(), value); |
| Node* length_is_zero = __ Word32Equal( |
| __ Word32And(bitfield, __ Int32Constant(BigInt::LengthBits::kMask)), |
| __ Int32Constant(0)); |
| __ Goto(done, __ Word32Equal(length_is_zero, zero)); |
| } |
| } |
| |
| Node* EffectControlLinearizer::LowerTruncateTaggedToBit(Node* node) { |
| auto done = __ MakeLabel(MachineRepresentation::kBit); |
| auto if_smi = __ MakeDeferredLabel(); |
| |
| Node* value = node->InputAt(0); |
| __ GotoIf(ObjectIsSmi(value), &if_smi); |
| |
| TruncateTaggedPointerToBit(node, &done); |
| |
| __ Bind(&if_smi); |
| { |
| // If {value} is a Smi, then we only need to check that it's not zero. |
| __ Goto(&done, __ Word32Equal(__ TaggedEqual(value, __ SmiConstant(0)), |
| __ Int32Constant(0))); |
| } |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerTruncateTaggedPointerToBit(Node* node) { |
| auto done = __ MakeLabel(MachineRepresentation::kBit); |
| |
| TruncateTaggedPointerToBit(node, &done); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerChangeTaggedToInt32(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| auto if_not_smi = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kWord32); |
| |
| Node* check = ObjectIsSmi(value); |
| __ GotoIfNot(check, &if_not_smi); |
| __ Goto(&done, ChangeSmiToInt32(value)); |
| |
| __ Bind(&if_not_smi); |
| STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset, |
| Oddball::kToNumberRawOffset); |
| Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value); |
| vfalse = __ ChangeFloat64ToInt32(vfalse); |
| __ Goto(&done, vfalse); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerChangeTaggedToUint32(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| auto if_not_smi = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kWord32); |
| |
| Node* check = ObjectIsSmi(value); |
| __ GotoIfNot(check, &if_not_smi); |
| __ Goto(&done, ChangeSmiToInt32(value)); |
| |
| __ Bind(&if_not_smi); |
| STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset, |
| Oddball::kToNumberRawOffset); |
| Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value); |
| vfalse = __ ChangeFloat64ToUint32(vfalse); |
| __ Goto(&done, vfalse); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerChangeTaggedToInt64(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| auto if_not_smi = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kWord64); |
| |
| Node* check = ObjectIsSmi(value); |
| __ GotoIfNot(check, &if_not_smi); |
| __ Goto(&done, ChangeSmiToInt64(value)); |
| |
| __ Bind(&if_not_smi); |
| STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset, |
| Oddball::kToNumberRawOffset); |
| Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value); |
| vfalse = __ ChangeFloat64ToInt64(vfalse); |
| __ Goto(&done, vfalse); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerChangeTaggedToFloat64(Node* node) { |
| return LowerTruncateTaggedToFloat64(node); |
| } |
| |
| Node* EffectControlLinearizer::LowerChangeTaggedToTaggedSigned(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| auto if_not_smi = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kWord32); |
| |
| Node* check = ObjectIsSmi(value); |
| __ GotoIfNot(check, &if_not_smi); |
| __ Goto(&done, value); |
| |
| __ Bind(&if_not_smi); |
| STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset, |
| Oddball::kToNumberRawOffset); |
| Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value); |
| vfalse = __ ChangeFloat64ToInt32(vfalse); |
| vfalse = ChangeInt32ToSmi(vfalse); |
| __ Goto(&done, vfalse); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerTruncateTaggedToFloat64(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| auto if_not_smi = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kFloat64); |
| |
| Node* check = ObjectIsSmi(value); |
| __ GotoIfNot(check, &if_not_smi); |
| Node* vtrue = ChangeSmiToInt32(value); |
| vtrue = __ ChangeInt32ToFloat64(vtrue); |
| __ Goto(&done, vtrue); |
| |
| __ Bind(&if_not_smi); |
| STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset, |
| Oddball::kToNumberRawOffset); |
| Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value); |
| __ Goto(&done, vfalse); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerPoisonIndex(Node* node) { |
| Node* index = node->InputAt(0); |
| if (mask_array_index_ == MaskArrayIndexEnable::kMaskArrayIndex) { |
| index = __ Word32PoisonOnSpeculation(index); |
| } |
| return index; |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckClosure(Node* node, |
| Node* frame_state) { |
| Handle<FeedbackCell> feedback_cell = FeedbackCellOf(node->op()); |
| Node* value = node->InputAt(0); |
| |
| // Check that {value} is actually a JSFunction. |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| Node* value_instance_type = |
| __ LoadField(AccessBuilder::ForMapInstanceType(), value_map); |
| Node* check_instance_type = |
| __ Word32Equal(value_instance_type, __ Int32Constant(JS_FUNCTION_TYPE)); |
| __ DeoptimizeIfNot(DeoptimizeReason::kWrongCallTarget, FeedbackSource(), |
| check_instance_type, frame_state); |
| |
| // Check that the {value}s feedback vector cell matches the one |
| // we recorded before. |
| Node* value_cell = |
| __ LoadField(AccessBuilder::ForJSFunctionFeedbackCell(), value); |
| Node* check_cell = __ WordEqual(value_cell, __ HeapConstant(feedback_cell)); |
| __ DeoptimizeIfNot(DeoptimizeReason::kWrongFeedbackCell, FeedbackSource(), |
| check_cell, frame_state); |
| return value; |
| } |
| |
| void EffectControlLinearizer::MigrateInstanceOrDeopt( |
| Node* value, Node* value_map, Node* frame_state, |
| FeedbackSource const& feedback_source, DeoptimizeReason reason) { |
| // If map is not deprecated the migration attempt does not make sense. |
| Node* bitfield3 = __ LoadField(AccessBuilder::ForMapBitField3(), value_map); |
| Node* is_not_deprecated = __ Word32Equal( |
| __ Word32And(bitfield3, |
| __ Int32Constant(Map::Bits3::IsDeprecatedBit::kMask)), |
| __ Int32Constant(0)); |
| __ DeoptimizeIf(reason, feedback_source, is_not_deprecated, frame_state, |
| IsSafetyCheck::kCriticalSafetyCheck); |
| Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow; |
| Runtime::FunctionId id = Runtime::kTryMigrateInstance; |
| auto call_descriptor = Linkage::GetRuntimeCallDescriptor( |
| graph()->zone(), id, 1, properties, CallDescriptor::kNoFlags); |
| Node* result = __ Call(call_descriptor, __ CEntryStubConstant(1), value, |
| __ ExternalConstant(ExternalReference::Create(id)), |
| __ Int32Constant(1), __ NoContextConstant()); |
| Node* check = ObjectIsSmi(result); |
| __ DeoptimizeIf(DeoptimizeReason::kInstanceMigrationFailed, feedback_source, |
| check, frame_state, IsSafetyCheck::kCriticalSafetyCheck); |
| } |
| |
| void EffectControlLinearizer::LowerCheckMaps(Node* node, Node* frame_state) { |
| CheckMapsParameters const& p = CheckMapsParametersOf(node->op()); |
| Node* value = node->InputAt(0); |
| |
| ZoneHandleSet<Map> const& maps = p.maps(); |
| size_t const map_count = maps.size(); |
| |
| if (p.flags() & CheckMapsFlag::kTryMigrateInstance) { |
| auto done = __ MakeLabel(); |
| auto migrate = __ MakeDeferredLabel(); |
| |
| // Load the current map of the {value}. |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| |
| // Perform the map checks. |
| for (size_t i = 0; i < map_count; ++i) { |
| Node* map = __ HeapConstant(maps[i]); |
| Node* check = __ TaggedEqual(value_map, map); |
| if (i == map_count - 1) { |
| __ BranchWithCriticalSafetyCheck(check, &done, &migrate); |
| } else { |
| auto next_map = __ MakeLabel(); |
| __ BranchWithCriticalSafetyCheck(check, &done, &next_map); |
| __ Bind(&next_map); |
| } |
| } |
| |
| // Perform the (deferred) instance migration. |
| __ Bind(&migrate); |
| MigrateInstanceOrDeopt(value, value_map, frame_state, p.feedback(), |
| DeoptimizeReason::kWrongMap); |
| |
| // Reload the current map of the {value}. |
| value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| |
| // Perform the map checks again. |
| for (size_t i = 0; i < map_count; ++i) { |
| Node* map = __ HeapConstant(maps[i]); |
| Node* check = __ TaggedEqual(value_map, map); |
| if (i == map_count - 1) { |
| __ DeoptimizeIfNot(DeoptimizeReason::kWrongMap, p.feedback(), check, |
| frame_state, IsSafetyCheck::kCriticalSafetyCheck); |
| } else { |
| auto next_map = __ MakeLabel(); |
| __ BranchWithCriticalSafetyCheck(check, &done, &next_map); |
| __ Bind(&next_map); |
| } |
| } |
| |
| __ Goto(&done); |
| __ Bind(&done); |
| } else { |
| auto done = __ MakeLabel(); |
| |
| // Load the current map of the {value}. |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| |
| for (size_t i = 0; i < map_count; ++i) { |
| Node* map = __ HeapConstant(maps[i]); |
| Node* check = __ TaggedEqual(value_map, map); |
| |
| if (i == map_count - 1) { |
| __ DeoptimizeIfNot(DeoptimizeReason::kWrongMap, p.feedback(), check, |
| frame_state, IsSafetyCheck::kCriticalSafetyCheck); |
| } else { |
| auto next_map = __ MakeLabel(); |
| __ BranchWithCriticalSafetyCheck(check, &done, &next_map); |
| __ Bind(&next_map); |
| } |
| } |
| __ Goto(&done); |
| __ Bind(&done); |
| } |
| } |
| |
| void EffectControlLinearizer::BuildCallDynamicMapChecksBuiltin( |
| Node* actual_value, Node* actual_handler, int feedback_slot_index, |
| GraphAssemblerLabel<0>* done, Node* frame_state) { |
| Node* slot_index = __ IntPtrConstant(feedback_slot_index); |
| Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow; |
| auto builtin = Builtins::kDynamicMapChecks; |
| Node* result = CallBuiltin(builtin, properties, slot_index, actual_value, |
| actual_handler); |
| __ GotoIf(__ WordEqual(result, __ IntPtrConstant(static_cast<int>( |
| DynamicMapChecksStatus::kSuccess))), |
| done); |
| __ DeoptimizeIf(DeoptimizeKind::kBailout, DeoptimizeReason::kMissingMap, |
| FeedbackSource(), |
| __ WordEqual(result, __ IntPtrConstant(static_cast<int>( |
| DynamicMapChecksStatus::kBailout))), |
| frame_state, IsSafetyCheck::kCriticalSafetyCheck); |
| __ DeoptimizeIf(DeoptimizeReason::kWrongHandler, FeedbackSource(), |
| __ WordEqual(result, __ IntPtrConstant(static_cast<int>( |
| DynamicMapChecksStatus::kDeopt))), |
| frame_state, IsSafetyCheck::kCriticalSafetyCheck); |
| __ Unreachable(done); |
| } |
| |
| void EffectControlLinearizer::LowerDynamicCheckMaps(Node* node, |
| Node* frame_state) { |
| DynamicCheckMapsParameters const& p = |
| DynamicCheckMapsParametersOf(node->op()); |
| Node* actual_value = node->InputAt(0); |
| |
| FeedbackSource const& feedback = p.feedback(); |
| Node* actual_value_map = __ LoadField(AccessBuilder::ForMap(), actual_value); |
| Node* actual_handler = |
| p.handler()->IsSmi() |
| ? __ SmiConstant(Smi::ToInt(*p.handler())) |
| : __ HeapConstant(Handle<HeapObject>::cast(p.handler())); |
| |
| auto done = __ MakeLabel(); |
| auto call_builtin = __ MakeDeferredLabel(); |
| |
| ZoneHandleSet<Map> maps = p.maps(); |
| size_t const map_count = maps.size(); |
| for (size_t i = 0; i < map_count; ++i) { |
| Node* map = __ HeapConstant(maps[i]); |
| Node* check = __ TaggedEqual(actual_value_map, map); |
| if (i == map_count - 1) { |
| __ BranchWithCriticalSafetyCheck(check, &done, &call_builtin); |
| } else { |
| auto next_map = __ MakeLabel(); |
| __ BranchWithCriticalSafetyCheck(check, &done, &next_map); |
| __ Bind(&next_map); |
| } |
| } |
| |
| __ Bind(&call_builtin); |
| { |
| BuildCallDynamicMapChecksBuiltin(actual_value, actual_handler, |
| feedback.index(), &done, frame_state); |
| } |
| |
| __ Bind(&done); |
| } |
| |
| Node* EffectControlLinearizer::LowerCompareMaps(Node* node) { |
| ZoneHandleSet<Map> const& maps = CompareMapsParametersOf(node->op()); |
| size_t const map_count = maps.size(); |
| Node* value = node->InputAt(0); |
| |
| auto done = __ MakeLabel(MachineRepresentation::kBit); |
| |
| // Load the current map of the {value}. |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| |
| for (size_t i = 0; i < map_count; ++i) { |
| Node* map = __ HeapConstant(maps[i]); |
| Node* check = __ TaggedEqual(value_map, map); |
| |
| auto next_map = __ MakeLabel(); |
| auto passed = __ MakeLabel(); |
| __ BranchWithCriticalSafetyCheck(check, &passed, &next_map); |
| |
| __ Bind(&passed); |
| __ Goto(&done, __ Int32Constant(1)); |
| |
| __ Bind(&next_map); |
| } |
| __ Goto(&done, __ Int32Constant(0)); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckNumber(Node* node, Node* frame_state) { |
| Node* value = node->InputAt(0); |
| const CheckParameters& params = CheckParametersOf(node->op()); |
| |
| auto if_not_smi = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(); |
| |
| Node* check0 = ObjectIsSmi(value); |
| __ GotoIfNot(check0, &if_not_smi); |
| __ Goto(&done); |
| |
| __ Bind(&if_not_smi); |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| Node* check1 = __ TaggedEqual(value_map, __ HeapNumberMapConstant()); |
| __ DeoptimizeIfNot(DeoptimizeReason::kNotAHeapNumber, params.feedback(), |
| check1, frame_state); |
| __ Goto(&done); |
| |
| __ Bind(&done); |
| return value; |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckReceiver(Node* node, |
| Node* frame_state) { |
| Node* value = node->InputAt(0); |
| |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| Node* value_instance_type = |
| __ LoadField(AccessBuilder::ForMapInstanceType(), value_map); |
| |
| STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE); |
| Node* check = __ Uint32LessThanOrEqual( |
| __ Uint32Constant(FIRST_JS_RECEIVER_TYPE), value_instance_type); |
| __ DeoptimizeIfNot(DeoptimizeReason::kNotAJavaScriptObject, FeedbackSource(), |
| check, frame_state); |
| return value; |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckReceiverOrNullOrUndefined( |
| Node* node, Node* frame_state) { |
| Node* value = node->InputAt(0); |
| |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| Node* value_instance_type = |
| __ LoadField(AccessBuilder::ForMapInstanceType(), value_map); |
| |
| // Rule out all primitives except oddballs (true, false, undefined, null). |
| STATIC_ASSERT(LAST_PRIMITIVE_HEAP_OBJECT_TYPE == ODDBALL_TYPE); |
| STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE); |
| Node* check0 = __ Uint32LessThanOrEqual(__ Uint32Constant(ODDBALL_TYPE), |
| value_instance_type); |
| __ DeoptimizeIfNot(DeoptimizeReason::kNotAJavaScriptObjectOrNullOrUndefined, |
| FeedbackSource(), check0, frame_state); |
| |
| // Rule out booleans. |
| Node* check1 = __ TaggedEqual(value_map, __ BooleanMapConstant()); |
| __ DeoptimizeIf(DeoptimizeReason::kNotAJavaScriptObjectOrNullOrUndefined, |
| FeedbackSource(), check1, frame_state); |
| return value; |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckSymbol(Node* node, Node* frame_state) { |
| Node* value = node->InputAt(0); |
| |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| |
| Node* check = |
| __ TaggedEqual(value_map, __ HeapConstant(factory()->symbol_map())); |
| __ DeoptimizeIfNot(DeoptimizeReason::kNotASymbol, FeedbackSource(), check, |
| frame_state); |
| return value; |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckString(Node* node, Node* frame_state) { |
| Node* value = node->InputAt(0); |
| const CheckParameters& params = CheckParametersOf(node->op()); |
| |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| Node* value_instance_type = |
| __ LoadField(AccessBuilder::ForMapInstanceType(), value_map); |
| |
| Node* check = __ Uint32LessThan(value_instance_type, |
| __ Uint32Constant(FIRST_NONSTRING_TYPE)); |
| __ DeoptimizeIfNot(DeoptimizeReason::kNotAString, params.feedback(), check, |
| frame_state); |
| return value; |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckInternalizedString(Node* node, |
| Node* frame_state) { |
| Node* value = node->InputAt(0); |
| |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| Node* value_instance_type = |
| __ LoadField(AccessBuilder::ForMapInstanceType(), value_map); |
| |
| Node* check = __ Word32Equal( |
| __ Word32And(value_instance_type, |
| __ Int32Constant(kIsNotStringMask | kIsNotInternalizedMask)), |
| __ Int32Constant(kInternalizedTag)); |
| __ DeoptimizeIfNot(DeoptimizeReason::kWrongInstanceType, FeedbackSource(), |
| check, frame_state); |
| |
| return value; |
| } |
| |
| void EffectControlLinearizer::LowerCheckIf(Node* node, Node* frame_state) { |
| Node* value = node->InputAt(0); |
| const CheckIfParameters& p = CheckIfParametersOf(node->op()); |
| __ DeoptimizeIfNot(p.reason(), p.feedback(), value, frame_state); |
| } |
| |
| Node* EffectControlLinearizer::LowerStringConcat(Node* node) { |
| Node* lhs = node->InputAt(1); |
| Node* rhs = node->InputAt(2); |
| |
| Callable const callable = |
| CodeFactory::StringAdd(isolate(), STRING_ADD_CHECK_NONE); |
| auto call_descriptor = Linkage::GetStubCallDescriptor( |
| graph()->zone(), callable.descriptor(), |
| callable.descriptor().GetStackParameterCount(), CallDescriptor::kNoFlags, |
| Operator::kNoDeopt | Operator::kNoWrite | Operator::kNoThrow); |
| |
| Node* value = __ Call(call_descriptor, __ HeapConstant(callable.code()), lhs, |
| rhs, __ NoContextConstant()); |
| |
| return value; |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedInt32Add(Node* node, |
| Node* frame_state) { |
| Node* lhs = node->InputAt(0); |
| Node* rhs = node->InputAt(1); |
| |
| Node* value = __ Int32AddWithOverflow(lhs, rhs); |
| Node* check = __ Projection(1, value); |
| __ DeoptimizeIf(DeoptimizeReason::kOverflow, FeedbackSource(), check, |
| frame_state); |
| return __ Projection(0, value); |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedInt32Sub(Node* node, |
| Node* frame_state) { |
| Node* lhs = node->InputAt(0); |
| Node* rhs = node->InputAt(1); |
| |
| Node* value = __ Int32SubWithOverflow(lhs, rhs); |
| Node* check = __ Projection(1, value); |
| __ DeoptimizeIf(DeoptimizeReason::kOverflow, FeedbackSource(), check, |
| frame_state); |
| return __ Projection(0, value); |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedInt32Div(Node* node, |
| Node* frame_state) { |
| Node* lhs = node->InputAt(0); |
| Node* rhs = node->InputAt(1); |
| Node* zero = __ Int32Constant(0); |
| |
| // Check if the {rhs} is a known power of two. |
| Int32Matcher m(rhs); |
| if (m.IsPowerOf2()) { |
| // Since we know that {rhs} is a power of two, we can perform a fast |
| // check to see if the relevant least significant bits of the {lhs} |
| // are all zero, and if so we know that we can perform a division |
| // safely (and fast by doing an arithmetic - aka sign preserving - |
| // right shift on {lhs}). |
| int32_t divisor = m.ResolvedValue(); |
| Node* mask = __ Int32Constant(divisor - 1); |
| Node* shift = __ Int32Constant(base::bits::WhichPowerOfTwo(divisor)); |
| Node* check = __ Word32Equal(__ Word32And(lhs, mask), zero); |
| __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, FeedbackSource(), |
| check, frame_state); |
| return __ Word32Sar(lhs, shift); |
| } else { |
| auto if_rhs_positive = __ MakeLabel(); |
| auto if_rhs_negative = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kWord32); |
| |
| // Check if {rhs} is positive (and not zero). |
| Node* check_rhs_positive = __ Int32LessThan(zero, rhs); |
| __ Branch(check_rhs_positive, &if_rhs_positive, &if_rhs_negative); |
| |
| __ Bind(&if_rhs_positive); |
| { |
| // Fast case, no additional checking required. |
| __ Goto(&done, __ Int32Div(lhs, rhs)); |
| } |
| |
| __ Bind(&if_rhs_negative); |
| { |
| auto if_lhs_minint = __ MakeDeferredLabel(); |
| auto if_lhs_notminint = __ MakeLabel(); |
| |
| // Check if {rhs} is zero. |
| Node* check_rhs_zero = __ Word32Equal(rhs, zero); |
| __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, FeedbackSource(), |
| check_rhs_zero, frame_state); |
| |
| // Check if {lhs} is zero, as that would produce minus zero. |
| Node* check_lhs_zero = __ Word32Equal(lhs, zero); |
| __ DeoptimizeIf(DeoptimizeReason::kMinusZero, FeedbackSource(), |
| check_lhs_zero, frame_state); |
| |
| // Check if {lhs} is kMinInt and {rhs} is -1, in which case we'd have |
| // to return -kMinInt, which is not representable as Word32. |
| Node* check_lhs_minint = __ Word32Equal(lhs, __ Int32Constant(kMinInt)); |
| __ Branch(check_lhs_minint, &if_lhs_minint, &if_lhs_notminint); |
| |
| __ Bind(&if_lhs_minint); |
| { |
| // Check that {rhs} is not -1, otherwise result would be -kMinInt. |
| Node* check_rhs_minusone = __ Word32Equal(rhs, __ Int32Constant(-1)); |
| __ DeoptimizeIf(DeoptimizeReason::kOverflow, FeedbackSource(), |
| check_rhs_minusone, frame_state); |
| |
| // Perform the actual integer division. |
| __ Goto(&done, __ Int32Div(lhs, rhs)); |
| } |
| |
| __ Bind(&if_lhs_notminint); |
| { |
| // Perform the actual integer division. |
| __ Goto(&done, __ Int32Div(lhs, rhs)); |
| } |
| } |
| |
| __ Bind(&done); |
| Node* value = done.PhiAt(0); |
| |
| // Check if the remainder is non-zero. |
| Node* check = __ Word32Equal(lhs, __ Int32Mul(value, rhs)); |
| __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, FeedbackSource(), |
| check, frame_state); |
| |
| return value; |
| } |
| } |
| |
| template <size_t VarCount, size_t VarCount2> |
| void EffectControlLinearizer::SmiTagOrOverflow( |
| Node* value, GraphAssemblerLabel<VarCount>* if_overflow, |
| GraphAssemblerLabel<VarCount2>* done) { |
| DCHECK(SmiValuesAre31Bits()); |
| // Check for overflow at the same time that we are smi tagging. |
| // Since smi tagging shifts left by one, it's the same as adding value twice. |
| Node* add = __ Int32AddWithOverflow(value, value); |
| Node* ovf = __ Projection(1, add); |
| __ GotoIf(ovf, if_overflow); |
| Node* value_smi = __ Projection(0, add); |
| value_smi = ChangeTaggedInt32ToSmi(value_smi); |
| __ Goto(done, value_smi); |
| } |
| |
| Node* EffectControlLinearizer::SmiTagOrDeopt(Node* value, |
| const CheckParameters& params, |
| Node* frame_state) { |
| DCHECK(SmiValuesAre31Bits()); |
| // Check for the lost precision at the same time that we are smi tagging. |
| // Since smi tagging shifts left by one, it's the same as adding value twice. |
| Node* add = __ Int32AddWithOverflow(value, value); |
| Node* check = __ Projection(1, add); |
| __ DeoptimizeIf(DeoptimizeReason::kLostPrecision, params.feedback(), check, |
| frame_state); |
| Node* result = __ Projection(0, add); |
| return ChangeTaggedInt32ToSmi(result); |
| } |
| |
| Node* EffectControlLinearizer::BuildUint32Mod(Node* lhs, Node* rhs) { |
| auto if_rhs_power_of_two = __ MakeLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kWord32); |
| |
| // Compute the mask for the {rhs}. |
| Node* one = __ Int32Constant(1); |
| Node* msk = __ Int32Sub(rhs, one); |
| |
| // Check if the {rhs} is a power of two. |
| __ GotoIf(__ Word32Equal(__ Word32And(rhs, msk), __ Int32Constant(0)), |
| &if_rhs_power_of_two); |
| { |
| // The {rhs} is not a power of two, do a generic Uint32Mod. |
| __ Goto(&done, __ Uint32Mod(lhs, rhs)); |
| } |
| |
| __ Bind(&if_rhs_power_of_two); |
| { |
| // The {rhs} is a power of two, just do a fast bit masking. |
| __ Goto(&done, __ Word32And(lhs, msk)); |
| } |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedInt32Mod(Node* node, |
| Node* frame_state) { |
| // General case for signed integer modulus, with optimization for (unknown) |
| // power of 2 right hand side. |
| // |
| // if rhs <= 0 then |
| // rhs = -rhs |
| // deopt if rhs == 0 |
| // let msk = rhs - 1 in |
| // if lhs < 0 then |
| // let lhs_abs = -lsh in |
| // let res = if rhs & msk == 0 then |
| // lhs_abs & msk |
| // else |
| // lhs_abs % rhs in |
| // if lhs < 0 then |
| // deopt if res == 0 |
| // -res |
| // else |
| // res |
| // else |
| // if rhs & msk == 0 then |
| // lhs & msk |
| // else |
| // lhs % rhs |
| // |
| Node* lhs = node->InputAt(0); |
| Node* rhs = node->InputAt(1); |
| |
| auto if_rhs_not_positive = __ MakeDeferredLabel(); |
| auto if_lhs_negative = __ MakeDeferredLabel(); |
| auto if_rhs_power_of_two = __ MakeLabel(); |
| auto rhs_checked = __ MakeLabel(MachineRepresentation::kWord32); |
| auto done = __ MakeLabel(MachineRepresentation::kWord32); |
| |
| Node* zero = __ Int32Constant(0); |
| |
| // Check if {rhs} is not strictly positive. |
| Node* check0 = __ Int32LessThanOrEqual(rhs, zero); |
| __ GotoIf(check0, &if_rhs_not_positive); |
| __ Goto(&rhs_checked, rhs); |
| |
| __ Bind(&if_rhs_not_positive); |
| { |
| // Negate {rhs}, might still produce a negative result in case of |
| // -2^31, but that is handled safely below. |
| Node* vtrue0 = __ Int32Sub(zero, rhs); |
| |
| // Ensure that {rhs} is not zero, otherwise we'd have to return NaN. |
| __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, FeedbackSource(), |
| __ Word32Equal(vtrue0, zero), frame_state); |
| __ Goto(&rhs_checked, vtrue0); |
| } |
| |
| __ Bind(&rhs_checked); |
| rhs = rhs_checked.PhiAt(0); |
| |
| __ GotoIf(__ Int32LessThan(lhs, zero), &if_lhs_negative); |
| { |
| // The {lhs} is a non-negative integer. |
| __ Goto(&done, BuildUint32Mod(lhs, rhs)); |
| } |
| |
| __ Bind(&if_lhs_negative); |
| { |
| // The {lhs} is a negative integer. This is very unlikely and |
| // we intentionally don't use the BuildUint32Mod() here, which |
| // would try to figure out whether {rhs} is a power of two, |
| // since this is intended to be a slow-path. |
| Node* res = __ Uint32Mod(__ Int32Sub(zero, lhs), rhs); |
| |
| // Check if we would have to return -0. |
| __ DeoptimizeIf(DeoptimizeReason::kMinusZero, FeedbackSource(), |
| __ Word32Equal(res, zero), frame_state); |
| __ Goto(&done, __ Int32Sub(zero, res)); |
| } |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedUint32Div(Node* node, |
| Node* frame_state) { |
| Node* lhs = node->InputAt(0); |
| Node* rhs = node->InputAt(1); |
| Node* zero = __ Int32Constant(0); |
| |
| // Check if the {rhs} is a known power of two. |
| Uint32Matcher m(rhs); |
| if (m.IsPowerOf2()) { |
| // Since we know that {rhs} is a power of two, we can perform a fast |
| // check to see if the relevant least significant bits of the {lhs} |
| // are all zero, and if so we know that we can perform a division |
| // safely (and fast by doing a logical - aka zero extending - right |
| // shift on {lhs}). |
| uint32_t divisor = m.ResolvedValue(); |
| Node* mask = __ Uint32Constant(divisor - 1); |
| Node* shift = __ Uint32Constant(base::bits::WhichPowerOfTwo(divisor)); |
| Node* check = __ Word32Equal(__ Word32And(lhs, mask), zero); |
| __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, FeedbackSource(), |
| check, frame_state); |
| return __ Word32Shr(lhs, shift); |
| } else { |
| // Ensure that {rhs} is not zero, otherwise we'd have to return NaN. |
| Node* check = __ Word32Equal(rhs, zero); |
| __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, FeedbackSource(), check, |
| frame_state); |
| |
| // Perform the actual unsigned integer division. |
| Node* value = __ Uint32Div(lhs, rhs); |
| |
| // Check if the remainder is non-zero. |
| check = __ Word32Equal(lhs, __ Int32Mul(rhs, value)); |
| __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, FeedbackSource(), |
| check, frame_state); |
| return value; |
| } |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedUint32Mod(Node* node, |
| Node* frame_state) { |
| Node* lhs = node->InputAt(0); |
| Node* rhs = node->InputAt(1); |
| |
| Node* zero = __ Int32Constant(0); |
| |
| // Ensure that {rhs} is not zero, otherwise we'd have to return NaN. |
| Node* check = __ Word32Equal(rhs, zero); |
| __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, FeedbackSource(), check, |
| frame_state); |
| |
| // Perform the actual unsigned integer modulus. |
| return BuildUint32Mod(lhs, rhs); |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedInt32Mul(Node* node, |
| Node* frame_state) { |
| CheckForMinusZeroMode mode = CheckMinusZeroModeOf(node->op()); |
| Node* lhs = node->InputAt(0); |
| Node* rhs = node->InputAt(1); |
| |
| Node* projection = __ Int32MulWithOverflow(lhs, rhs); |
| Node* check = __ Projection(1, projection); |
| __ DeoptimizeIf(DeoptimizeReason::kOverflow, FeedbackSource(), check, |
| frame_state); |
| |
| Node* value = __ Projection(0, projection); |
| |
| if (mode == CheckForMinusZeroMode::kCheckForMinusZero) { |
| auto if_zero = __ MakeDeferredLabel(); |
| auto check_done = __ MakeLabel(); |
| Node* zero = __ Int32Constant(0); |
| Node* check_zero = __ Word32Equal(value, zero); |
| __ GotoIf(check_zero, &if_zero); |
| __ Goto(&check_done); |
| |
| __ Bind(&if_zero); |
| // We may need to return negative zero. |
| Node* check_or = __ Int32LessThan(__ Word32Or(lhs, rhs), zero); |
| __ DeoptimizeIf(DeoptimizeReason::kMinusZero, FeedbackSource(), check_or, |
| frame_state); |
| __ Goto(&check_done); |
| |
| __ Bind(&check_done); |
| } |
| |
| return value; |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedInt32ToTaggedSigned( |
| Node* node, Node* frame_state) { |
| DCHECK(SmiValuesAre31Bits()); |
| Node* value = node->InputAt(0); |
| const CheckParameters& params = CheckParametersOf(node->op()); |
| return SmiTagOrDeopt(value, params, frame_state); |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedInt64ToInt32(Node* node, |
| Node* frame_state) { |
| Node* value = node->InputAt(0); |
| const CheckParameters& params = CheckParametersOf(node->op()); |
| |
| Node* value32 = __ TruncateInt64ToInt32(value); |
| Node* check = __ Word64Equal(__ ChangeInt32ToInt64(value32), value); |
| __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, params.feedback(), check, |
| frame_state); |
| return value32; |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedInt64ToTaggedSigned( |
| Node* node, Node* frame_state) { |
| Node* value = node->InputAt(0); |
| const CheckParameters& params = CheckParametersOf(node->op()); |
| |
| Node* value32 = __ TruncateInt64ToInt32(value); |
| Node* check = __ Word64Equal(__ ChangeInt32ToInt64(value32), value); |
| __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, params.feedback(), check, |
| frame_state); |
| |
| if (SmiValuesAre32Bits()) { |
| return ChangeInt64ToSmi(value); |
| } else { |
| return SmiTagOrDeopt(value32, params, frame_state); |
| } |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedUint32Bounds(Node* node, |
| Node* frame_state) { |
| Node* index = node->InputAt(0); |
| Node* limit = node->InputAt(1); |
| const CheckBoundsParameters& params = CheckBoundsParametersOf(node->op()); |
| |
| Node* check = __ Uint32LessThan(index, limit); |
| if (!(params.flags() & CheckBoundsFlag::kAbortOnOutOfBounds)) { |
| __ DeoptimizeIfNot(DeoptimizeReason::kOutOfBounds, |
| params.check_parameters().feedback(), check, frame_state, |
| IsSafetyCheck::kCriticalSafetyCheck); |
| } else { |
| auto if_abort = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(); |
| |
| __ Branch(check, &done, &if_abort); |
| |
| __ Bind(&if_abort); |
| __ Unreachable(&done); |
| |
| __ Bind(&done); |
| } |
| |
| return index; |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedUint32ToInt32(Node* node, |
| Node* frame_state) { |
| Node* value = node->InputAt(0); |
| const CheckParameters& params = CheckParametersOf(node->op()); |
| Node* unsafe = __ Int32LessThan(value, __ Int32Constant(0)); |
| __ DeoptimizeIf(DeoptimizeReason::kLostPrecision, params.feedback(), unsafe, |
| frame_state); |
| return value; |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedUint32ToTaggedSigned( |
| Node* node, Node* frame_state) { |
| Node* value = node->InputAt(0); |
| const CheckParameters& params = CheckParametersOf(node->op()); |
| Node* check = __ Uint32LessThanOrEqual(value, SmiMaxValueConstant()); |
| __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, params.feedback(), check, |
| frame_state); |
| return ChangeUint32ToSmi(value); |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedUint64Bounds(Node* node, |
| Node* frame_state) { |
| Node* const index = node->InputAt(0); |
| Node* const limit = node->InputAt(1); |
| const CheckBoundsParameters& params = CheckBoundsParametersOf(node->op()); |
| |
| Node* check = __ Uint64LessThan(index, limit); |
| if (!(params.flags() & CheckBoundsFlag::kAbortOnOutOfBounds)) { |
| __ DeoptimizeIfNot(DeoptimizeReason::kOutOfBounds, |
| params.check_parameters().feedback(), check, frame_state, |
| IsSafetyCheck::kCriticalSafetyCheck); |
| } else { |
| auto if_abort = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(); |
| |
| __ Branch(check, &done, &if_abort); |
| |
| __ Bind(&if_abort); |
| __ Unreachable(&done); |
| |
| __ Bind(&done); |
| } |
| return index; |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedUint64ToInt32(Node* node, |
| Node* frame_state) { |
| Node* value = node->InputAt(0); |
| const CheckParameters& params = CheckParametersOf(node->op()); |
| |
| Node* check = __ Uint64LessThanOrEqual(value, __ Int64Constant(kMaxInt)); |
| __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, params.feedback(), check, |
| frame_state); |
| return __ TruncateInt64ToInt32(value); |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedUint64ToTaggedSigned( |
| Node* node, Node* frame_state) { |
| Node* value = node->InputAt(0); |
| const CheckParameters& params = CheckParametersOf(node->op()); |
| |
| Node* check = |
| __ Uint64LessThanOrEqual(value, __ Int64Constant(Smi::kMaxValue)); |
| __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, params.feedback(), check, |
| frame_state); |
| return ChangeInt64ToSmi(value); |
| } |
| |
| Node* EffectControlLinearizer::BuildCheckedFloat64ToInt32( |
| CheckForMinusZeroMode mode, const FeedbackSource& feedback, Node* value, |
| Node* frame_state) { |
| Node* value32 = __ RoundFloat64ToInt32(value); |
| Node* check_same = __ Float64Equal(value, __ ChangeInt32ToFloat64(value32)); |
| __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecisionOrNaN, feedback, |
| check_same, frame_state); |
| |
| if (mode == CheckForMinusZeroMode::kCheckForMinusZero) { |
| // Check if {value} is -0. |
| auto if_zero = __ MakeDeferredLabel(); |
| auto check_done = __ MakeLabel(); |
| |
| Node* check_zero = __ Word32Equal(value32, __ Int32Constant(0)); |
| __ GotoIf(check_zero, &if_zero); |
| __ Goto(&check_done); |
| |
| __ Bind(&if_zero); |
| // In case of 0, we need to check the high bits for the IEEE -0 pattern. |
| Node* check_negative = __ Int32LessThan(__ Float64ExtractHighWord32(value), |
| __ Int32Constant(0)); |
| __ DeoptimizeIf(DeoptimizeReason::kMinusZero, feedback, check_negative, |
| frame_state); |
| __ Goto(&check_done); |
| |
| __ Bind(&check_done); |
| } |
| return value32; |
| } |
| |
| Node* EffectControlLinearizer::BuildCheckedFloat64ToIndex( |
| const FeedbackSource& feedback, Node* value, Node* frame_state) { |
| if (machine()->Is64()) { |
| Node* value64 = __ TruncateFloat64ToInt64(value); |
| Node* check_same = __ Float64Equal(value, __ ChangeInt64ToFloat64(value64)); |
| __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecisionOrNaN, feedback, |
| check_same, frame_state); |
| Node* check_max = |
| __ IntLessThan(value64, __ Int64Constant(kMaxSafeInteger)); |
| __ DeoptimizeIfNot(DeoptimizeReason::kNotAnArrayIndex, feedback, check_max, |
| frame_state); |
| Node* check_min = |
| __ IntLessThan(__ Int64Constant(-kMaxSafeInteger), value64); |
| __ DeoptimizeIfNot(DeoptimizeReason::kNotAnArrayIndex, feedback, check_min, |
| frame_state); |
| return value64; |
| } else { |
| Node* value32 = __ RoundFloat64ToInt32(value); |
| Node* check_same = __ Float64Equal(value, __ ChangeInt32ToFloat64(value32)); |
| __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecisionOrNaN, feedback, |
| check_same, frame_state); |
| return value32; |
| } |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedFloat64ToInt32(Node* node, |
| Node* frame_state) { |
| const CheckMinusZeroParameters& params = |
| CheckMinusZeroParametersOf(node->op()); |
| Node* value = node->InputAt(0); |
| return BuildCheckedFloat64ToInt32(params.mode(), params.feedback(), value, |
| frame_state); |
| } |
| |
| Node* EffectControlLinearizer::BuildCheckedFloat64ToInt64( |
| CheckForMinusZeroMode mode, const FeedbackSource& feedback, Node* value, |
| Node* frame_state) { |
| Node* value64 = __ TruncateFloat64ToInt64(value); |
| Node* check_same = __ Float64Equal(value, __ ChangeInt64ToFloat64(value64)); |
| __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecisionOrNaN, feedback, |
| check_same, frame_state); |
| |
| if (mode == CheckForMinusZeroMode::kCheckForMinusZero) { |
| // Check if {value} is -0. |
| auto if_zero = __ MakeDeferredLabel(); |
| auto check_done = __ MakeLabel(); |
| |
| Node* check_zero = __ Word64Equal(value64, __ Int64Constant(0)); |
| __ GotoIf(check_zero, &if_zero); |
| __ Goto(&check_done); |
| |
| __ Bind(&if_zero); |
| // In case of 0, we need to check the high bits for the IEEE -0 pattern. |
| Node* check_negative = __ Int32LessThan(__ Float64ExtractHighWord32(value), |
| __ Int32Constant(0)); |
| __ DeoptimizeIf(DeoptimizeReason::kMinusZero, feedback, check_negative, |
| frame_state); |
| __ Goto(&check_done); |
| |
| __ Bind(&check_done); |
| } |
| return value64; |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedFloat64ToInt64(Node* node, |
| Node* frame_state) { |
| const CheckMinusZeroParameters& params = |
| CheckMinusZeroParametersOf(node->op()); |
| Node* value = node->InputAt(0); |
| return BuildCheckedFloat64ToInt64(params.mode(), params.feedback(), value, |
| frame_state); |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedTaggedSignedToInt32( |
| Node* node, Node* frame_state) { |
| Node* value = node->InputAt(0); |
| const CheckParameters& params = CheckParametersOf(node->op()); |
| Node* check = ObjectIsSmi(value); |
| __ DeoptimizeIfNot(DeoptimizeReason::kNotASmi, params.feedback(), check, |
| frame_state); |
| return ChangeSmiToInt32(value); |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedTaggedToArrayIndex( |
| Node* node, Node* frame_state) { |
| CheckParameters const& params = CheckParametersOf(node->op()); |
| Node* value = node->InputAt(0); |
| |
| auto if_not_smi = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineType::PointerRepresentation()); |
| |
| __ GotoIfNot(ObjectIsSmi(value), &if_not_smi); |
| // In the Smi case, just convert to intptr_t. |
| __ Goto(&done, ChangeSmiToIntPtr(value)); |
| |
| // In the non-Smi case, check the heap numberness, load the number and convert |
| // to integer. |
| __ Bind(&if_not_smi); |
| auto if_not_heap_number = __ MakeDeferredLabel(); |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| Node* is_heap_number = __ TaggedEqual(value_map, __ HeapNumberMapConstant()); |
| __ GotoIfNot(is_heap_number, &if_not_heap_number); |
| |
| Node* number = __ LoadField(AccessBuilder::ForHeapNumberValue(), value); |
| number = BuildCheckedFloat64ToIndex(params.feedback(), number, frame_state); |
| __ Goto(&done, number); |
| |
| __ Bind(&if_not_heap_number); |
| auto calculate_index = __ MakeDeferredLabel(); |
| Node* value_instance_type = |
| __ LoadField(AccessBuilder::ForMapInstanceType(), value_map); |
| Node* is_string = __ Uint32LessThan(value_instance_type, |
| __ Uint32Constant(FIRST_NONSTRING_TYPE)); |
| __ DeoptimizeIfNot(DeoptimizeReason::kNotAString, params.feedback(), |
| is_string, frame_state); |
| |
| MachineSignature::Builder builder(graph()->zone(), 1, 1); |
| builder.AddReturn(MachineType::IntPtr()); |
| builder.AddParam(MachineType::TaggedPointer()); |
| Node* string_to_array_index_function = |
| __ ExternalConstant(ExternalReference::string_to_array_index_function()); |
| auto call_descriptor = |
| Linkage::GetSimplifiedCDescriptor(graph()->zone(), builder.Build()); |
| Node* index = __ Call(common()->Call(call_descriptor), |
| string_to_array_index_function, value); |
| |
| __ DeoptimizeIf(DeoptimizeReason::kNotAnArrayIndex, params.feedback(), |
| __ Word32Equal(index, __ Int32Constant(-1)), frame_state); |
| |
| __ Goto(&done, index); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedTaggedToInt32(Node* node, |
| Node* frame_state) { |
| const CheckMinusZeroParameters& params = |
| CheckMinusZeroParametersOf(node->op()); |
| Node* value = node->InputAt(0); |
| |
| auto if_not_smi = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kWord32); |
| Node* check = ObjectIsSmi(value); |
| __ GotoIfNot(check, &if_not_smi); |
| // In the Smi case, just convert to int32. |
| __ Goto(&done, ChangeSmiToInt32(value)); |
| |
| // In the non-Smi case, check the heap numberness, load the number and convert |
| // to int32. |
| __ Bind(&if_not_smi); |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| Node* check_map = __ TaggedEqual(value_map, __ HeapNumberMapConstant()); |
| __ DeoptimizeIfNot(DeoptimizeReason::kNotAHeapNumber, params.feedback(), |
| check_map, frame_state); |
| Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value); |
| vfalse = BuildCheckedFloat64ToInt32(params.mode(), params.feedback(), vfalse, |
| frame_state); |
| __ Goto(&done, vfalse); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedTaggedToInt64(Node* node, |
| Node* frame_state) { |
| const CheckMinusZeroParameters& params = |
| CheckMinusZeroParametersOf(node->op()); |
| Node* value = node->InputAt(0); |
| |
| auto if_not_smi = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kWord64); |
| |
| Node* check = ObjectIsSmi(value); |
| __ GotoIfNot(check, &if_not_smi); |
| // In the Smi case, just convert to int64. |
| __ Goto(&done, ChangeSmiToInt64(value)); |
| |
| // In the non-Smi case, check the heap numberness, load the number and convert |
| // to int64. |
| __ Bind(&if_not_smi); |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| Node* check_map = __ TaggedEqual(value_map, __ HeapNumberMapConstant()); |
| __ DeoptimizeIfNot(DeoptimizeReason::kNotAHeapNumber, params.feedback(), |
| check_map, frame_state); |
| Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value); |
| vfalse = BuildCheckedFloat64ToInt64(params.mode(), params.feedback(), vfalse, |
| frame_state); |
| __ Goto(&done, vfalse); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::BuildCheckedHeapNumberOrOddballToFloat64( |
| CheckTaggedInputMode mode, const FeedbackSource& feedback, Node* value, |
| Node* frame_state) { |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| Node* check_number = __ TaggedEqual(value_map, __ HeapNumberMapConstant()); |
| switch (mode) { |
| case CheckTaggedInputMode::kNumber: { |
| __ DeoptimizeIfNot(DeoptimizeReason::kNotAHeapNumber, feedback, |
| check_number, frame_state); |
| break; |
| } |
| case CheckTaggedInputMode::kNumberOrBoolean: { |
| auto check_done = __ MakeLabel(); |
| |
| __ GotoIf(check_number, &check_done); |
| __ DeoptimizeIfNot(DeoptimizeReason::kNotANumberOrBoolean, feedback, |
| __ TaggedEqual(value_map, __ BooleanMapConstant()), |
| frame_state); |
| STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset, |
| Oddball::kToNumberRawOffset); |
| __ Goto(&check_done); |
| |
| __ Bind(&check_done); |
| break; |
| } |
| case CheckTaggedInputMode::kNumberOrOddball: { |
| auto check_done = __ MakeLabel(); |
| |
| __ GotoIf(check_number, &check_done); |
| // For oddballs also contain the numeric value, let us just check that |
| // we have an oddball here. |
| Node* instance_type = |
| __ LoadField(AccessBuilder::ForMapInstanceType(), value_map); |
| Node* check_oddball = |
| __ Word32Equal(instance_type, __ Int32Constant(ODDBALL_TYPE)); |
| __ DeoptimizeIfNot(DeoptimizeReason::kNotANumberOrOddball, feedback, |
| check_oddball, frame_state); |
| STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset, |
| Oddball::kToNumberRawOffset); |
| __ Goto(&check_done); |
| |
| __ Bind(&check_done); |
| break; |
| } |
| } |
| return __ LoadField(AccessBuilder::ForHeapNumberValue(), value); |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedTaggedToFloat64(Node* node, |
| Node* frame_state) { |
| CheckTaggedInputParameters const& p = |
| CheckTaggedInputParametersOf(node->op()); |
| Node* value = node->InputAt(0); |
| |
| auto if_smi = __ MakeLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kFloat64); |
| |
| Node* check = ObjectIsSmi(value); |
| __ GotoIf(check, &if_smi); |
| |
| // In the Smi case, just convert to int32 and then float64. |
| // Otherwise, check heap numberness and load the number. |
| Node* number = BuildCheckedHeapNumberOrOddballToFloat64( |
| p.mode(), p.feedback(), value, frame_state); |
| __ Goto(&done, number); |
| |
| __ Bind(&if_smi); |
| Node* from_smi = ChangeSmiToInt32(value); |
| from_smi = __ ChangeInt32ToFloat64(from_smi); |
| __ Goto(&done, from_smi); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedTaggedToTaggedSigned( |
| Node* node, Node* frame_state) { |
| Node* value = node->InputAt(0); |
| const CheckParameters& params = CheckParametersOf(node->op()); |
| |
| Node* check = ObjectIsSmi(value); |
| __ DeoptimizeIfNot(DeoptimizeReason::kNotASmi, params.feedback(), check, |
| frame_state); |
| |
| return value; |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedTaggedToTaggedPointer( |
| Node* node, Node* frame_state) { |
| Node* value = node->InputAt(0); |
| const CheckParameters& params = CheckParametersOf(node->op()); |
| |
| Node* check = ObjectIsSmi(value); |
| __ DeoptimizeIf(DeoptimizeReason::kSmi, params.feedback(), check, |
| frame_state); |
| return value; |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckBigInt(Node* node, Node* frame_state) { |
| Node* value = node->InputAt(0); |
| const CheckParameters& params = CheckParametersOf(node->op()); |
| |
| // Check for Smi. |
| Node* smi_check = ObjectIsSmi(value); |
| __ DeoptimizeIf(DeoptimizeReason::kSmi, params.feedback(), smi_check, |
| frame_state); |
| |
| // Check for BigInt. |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| Node* bi_check = __ TaggedEqual(value_map, __ BigIntMapConstant()); |
| __ DeoptimizeIfNot(DeoptimizeReason::kWrongInstanceType, params.feedback(), |
| bi_check, frame_state); |
| |
| return value; |
| } |
| |
| Node* EffectControlLinearizer::LowerBigIntAsUintN(Node* node, |
| Node* frame_state) { |
| DCHECK(machine()->Is64()); |
| |
| const int bits = OpParameter<int>(node->op()); |
| DCHECK(0 <= bits && bits <= 64); |
| |
| if (bits == 64) { |
| // Reduce to nop. |
| return node->InputAt(0); |
| } else { |
| const uint64_t msk = (1ULL << bits) - 1ULL; |
| return __ Word64And(node->InputAt(0), __ Int64Constant(msk)); |
| } |
| } |
| |
| Node* EffectControlLinearizer::LowerChangeUint64ToBigInt(Node* node) { |
| DCHECK(machine()->Is64()); |
| |
| Node* value = node->InputAt(0); |
| Node* map = __ HeapConstant(factory()->bigint_map()); |
| // BigInts with value 0 must be of size 0 (canonical form). |
| auto if_zerodigits = __ MakeLabel(); |
| auto if_onedigit = __ MakeLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kTagged); |
| |
| __ GotoIf(__ Word64Equal(value, __ IntPtrConstant(0)), &if_zerodigits); |
| __ Goto(&if_onedigit); |
| |
| __ Bind(&if_onedigit); |
| { |
| Node* result = __ Allocate(AllocationType::kYoung, |
| __ IntPtrConstant(BigInt::SizeFor(1))); |
| const auto bitfield = BigInt::LengthBits::update(0, 1); |
| __ StoreField(AccessBuilder::ForMap(), result, map); |
| __ StoreField(AccessBuilder::ForBigIntBitfield(), result, |
| __ IntPtrConstant(bitfield)); |
| // BigInts have no padding on 64 bit architectures with pointer compression. |
| if (BigInt::HasOptionalPadding()) { |
| __ StoreField(AccessBuilder::ForBigIntOptionalPadding(), result, |
| __ IntPtrConstant(0)); |
| } |
| __ StoreField(AccessBuilder::ForBigIntLeastSignificantDigit64(), result, |
| value); |
| __ Goto(&done, result); |
| } |
| |
| __ Bind(&if_zerodigits); |
| { |
| Node* result = __ Allocate(AllocationType::kYoung, |
| __ IntPtrConstant(BigInt::SizeFor(0))); |
| const auto bitfield = BigInt::LengthBits::update(0, 0); |
| __ StoreField(AccessBuilder::ForMap(), result, map); |
| __ StoreField(AccessBuilder::ForBigIntBitfield(), result, |
| __ IntPtrConstant(bitfield)); |
| // BigInts have no padding on 64 bit architectures with pointer compression. |
| if (BigInt::HasOptionalPadding()) { |
| __ StoreField(AccessBuilder::ForBigIntOptionalPadding(), result, |
| __ IntPtrConstant(0)); |
| } |
| __ Goto(&done, result); |
| } |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerTruncateBigIntToUint64(Node* node) { |
| DCHECK(machine()->Is64()); |
| |
| auto done = __ MakeLabel(MachineRepresentation::kWord64); |
| auto if_neg = __ MakeLabel(); |
| auto if_not_zero = __ MakeLabel(); |
| |
| Node* value = node->InputAt(0); |
| |
| Node* bitfield = __ LoadField(AccessBuilder::ForBigIntBitfield(), value); |
| __ GotoIfNot(__ Word32Equal(bitfield, __ Int32Constant(0)), &if_not_zero); |
| __ Goto(&done, __ Int64Constant(0)); |
| |
| __ Bind(&if_not_zero); |
| { |
| Node* lsd = |
| __ LoadField(AccessBuilder::ForBigIntLeastSignificantDigit64(), value); |
| Node* sign = |
| __ Word32And(bitfield, __ Int32Constant(BigInt::SignBits::kMask)); |
| __ GotoIf(__ Word32Equal(sign, __ Int32Constant(1)), &if_neg); |
| __ Goto(&done, lsd); |
| |
| __ Bind(&if_neg); |
| __ Goto(&done, __ Int64Sub(__ Int64Constant(0), lsd)); |
| } |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerTruncateTaggedToWord32(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| auto if_not_smi = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kWord32); |
| |
| Node* check = ObjectIsSmi(value); |
| __ GotoIfNot(check, &if_not_smi); |
| __ Goto(&done, ChangeSmiToInt32(value)); |
| |
| __ Bind(&if_not_smi); |
| STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset, |
| Oddball::kToNumberRawOffset); |
| Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value); |
| vfalse = __ TruncateFloat64ToWord32(vfalse); |
| __ Goto(&done, vfalse); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckedTruncateTaggedToWord32( |
| Node* node, Node* frame_state) { |
| const CheckTaggedInputParameters& params = |
| CheckTaggedInputParametersOf(node->op()); |
| Node* value = node->InputAt(0); |
| |
| auto if_not_smi = __ MakeLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kWord32); |
| |
| Node* check = ObjectIsSmi(value); |
| __ GotoIfNot(check, &if_not_smi); |
| // In the Smi case, just convert to int32. |
| __ Goto(&done, ChangeSmiToInt32(value)); |
| |
| // Otherwise, check that it's a heap number or oddball and truncate the value |
| // to int32. |
| __ Bind(&if_not_smi); |
| Node* number = BuildCheckedHeapNumberOrOddballToFloat64( |
| params.mode(), params.feedback(), value, frame_state); |
| number = __ TruncateFloat64ToWord32(number); |
| __ Goto(&done, number); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerAllocate(Node* node) { |
| Node* size = node->InputAt(0); |
| AllocationType allocation = AllocationTypeOf(node->op()); |
| Node* new_node = __ Allocate(allocation, size); |
| return new_node; |
| } |
| |
| Node* EffectControlLinearizer::LowerNumberToString(Node* node) { |
| Node* argument = node->InputAt(0); |
| |
| Callable const callable = |
| Builtins::CallableFor(isolate(), Builtins::kNumberToString); |
| Operator::Properties properties = Operator::kEliminatable; |
| CallDescriptor::Flags flags = CallDescriptor::kNoFlags; |
| auto call_descriptor = Linkage::GetStubCallDescriptor( |
| graph()->zone(), callable.descriptor(), |
| callable.descriptor().GetStackParameterCount(), flags, properties); |
| return __ Call(call_descriptor, __ HeapConstant(callable.code()), argument, |
| __ NoContextConstant()); |
| } |
| |
| Node* EffectControlLinearizer::LowerObjectIsArrayBufferView(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| auto if_smi = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kBit); |
| |
| Node* check = ObjectIsSmi(value); |
| __ GotoIf(check, &if_smi); |
| |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| Node* value_instance_type = |
| __ LoadField(AccessBuilder::ForMapInstanceType(), value_map); |
| Node* vfalse = __ Uint32LessThan( |
| __ Int32Sub(value_instance_type, |
| __ Int32Constant(FIRST_JS_ARRAY_BUFFER_VIEW_TYPE)), |
| __ Int32Constant(LAST_JS_ARRAY_BUFFER_VIEW_TYPE - |
| FIRST_JS_ARRAY_BUFFER_VIEW_TYPE + 1)); |
| __ Goto(&done, vfalse); |
| |
| __ Bind(&if_smi); |
| __ Goto(&done, __ Int32Constant(0)); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerObjectIsBigInt(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| auto if_smi = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kBit); |
| |
| Node* check = ObjectIsSmi(value); |
| __ GotoIf(check, &if_smi); |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| Node* vfalse = __ TaggedEqual(value_map, __ BigIntMapConstant()); |
| __ Goto(&done, vfalse); |
| |
| __ Bind(&if_smi); |
| __ Goto(&done, __ Int32Constant(0)); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerObjectIsCallable(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| auto if_smi = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kBit); |
| |
| Node* check = ObjectIsSmi(value); |
| __ GotoIf(check, &if_smi); |
| |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| Node* value_bit_field = |
| __ LoadField(AccessBuilder::ForMapBitField(), value_map); |
| Node* vfalse = __ Word32Equal( |
| __ Int32Constant(Map::Bits1::IsCallableBit::kMask), |
| __ Word32And(value_bit_field, |
| __ Int32Constant(Map::Bits1::IsCallableBit::kMask))); |
| __ Goto(&done, vfalse); |
| |
| __ Bind(&if_smi); |
| __ Goto(&done, __ Int32Constant(0)); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerObjectIsConstructor(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| auto if_smi = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kBit); |
| |
| Node* check = ObjectIsSmi(value); |
| __ GotoIf(check, &if_smi); |
| |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| Node* value_bit_field = |
| __ LoadField(AccessBuilder::ForMapBitField(), value_map); |
| Node* vfalse = __ Word32Equal( |
| __ Int32Constant(Map::Bits1::IsConstructorBit::kMask), |
| __ Word32And(value_bit_field, |
| __ Int32Constant(Map::Bits1::IsConstructorBit::kMask))); |
| __ Goto(&done, vfalse); |
| |
| __ Bind(&if_smi); |
| __ Goto(&done, __ Int32Constant(0)); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerObjectIsDetectableCallable(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| auto if_smi = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kBit); |
| |
| Node* check = ObjectIsSmi(value); |
| __ GotoIf(check, &if_smi); |
| |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| Node* value_bit_field = |
| __ LoadField(AccessBuilder::ForMapBitField(), value_map); |
| Node* vfalse = __ Word32Equal( |
| __ Int32Constant(Map::Bits1::IsCallableBit::kMask), |
| __ Word32And(value_bit_field, |
| __ Int32Constant((Map::Bits1::IsCallableBit::kMask) | |
| (Map::Bits1::IsUndetectableBit::kMask)))); |
| __ Goto(&done, vfalse); |
| |
| __ Bind(&if_smi); |
| __ Goto(&done, __ Int32Constant(0)); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerNumberIsFloat64Hole(Node* node) { |
| Node* value = node->InputAt(0); |
| Node* check = __ Word32Equal(__ Float64ExtractHighWord32(value), |
| __ Int32Constant(kHoleNanUpper32)); |
| return check; |
| } |
| |
| Node* EffectControlLinearizer::LowerNumberIsFinite(Node* node) { |
| Node* number = node->InputAt(0); |
| Node* diff = __ Float64Sub(number, number); |
| Node* check = __ Float64Equal(diff, diff); |
| return check; |
| } |
| |
| Node* EffectControlLinearizer::LowerObjectIsFiniteNumber(Node* node) { |
| Node* object = node->InputAt(0); |
| Node* zero = __ Int32Constant(0); |
| Node* one = __ Int32Constant(1); |
| |
| auto done = __ MakeLabel(MachineRepresentation::kBit); |
| |
| // Check if {object} is a Smi. |
| __ GotoIf(ObjectIsSmi(object), &done, one); |
| |
| // Check if {object} is a HeapNumber. |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), object); |
| __ GotoIfNot(__ TaggedEqual(value_map, __ HeapNumberMapConstant()), &done, |
| zero); |
| |
| // {object} is a HeapNumber. |
| Node* value = __ LoadField(AccessBuilder::ForHeapNumberValue(), object); |
| Node* diff = __ Float64Sub(value, value); |
| Node* check = __ Float64Equal(diff, diff); |
| __ Goto(&done, check); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerNumberIsInteger(Node* node) { |
| Node* number = node->InputAt(0); |
| Node* trunc = BuildFloat64RoundTruncate(number); |
| Node* diff = __ Float64Sub(number, trunc); |
| Node* check = __ Float64Equal(diff, __ Float64Constant(0)); |
| return check; |
| } |
| |
| Node* EffectControlLinearizer::LowerObjectIsInteger(Node* node) { |
| Node* object = node->InputAt(0); |
| Node* zero = __ Int32Constant(0); |
| Node* one = __ Int32Constant(1); |
| |
| auto done = __ MakeLabel(MachineRepresentation::kBit); |
| |
| // Check if {object} is a Smi. |
| __ GotoIf(ObjectIsSmi(object), &done, one); |
| |
| // Check if {object} is a HeapNumber. |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), object); |
| __ GotoIfNot(__ TaggedEqual(value_map, __ HeapNumberMapConstant()), &done, |
| zero); |
| |
| // {object} is a HeapNumber. |
| Node* value = __ LoadField(AccessBuilder::ForHeapNumberValue(), object); |
| Node* trunc = BuildFloat64RoundTruncate(value); |
| Node* diff = __ Float64Sub(value, trunc); |
| Node* check = __ Float64Equal(diff, __ Float64Constant(0)); |
| __ Goto(&done, check); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerNumberIsSafeInteger(Node* node) { |
| Node* number = node->InputAt(0); |
| Node* zero = __ Int32Constant(0); |
| auto done = __ MakeLabel(MachineRepresentation::kBit); |
| |
| Node* trunc = BuildFloat64RoundTruncate(number); |
| Node* diff = __ Float64Sub(number, trunc); |
| Node* check = __ Float64Equal(diff, __ Float64Constant(0)); |
| __ GotoIfNot(check, &done, zero); |
| Node* in_range = __ Float64LessThanOrEqual( |
| __ Float64Abs(trunc), __ Float64Constant(kMaxSafeInteger)); |
| __ Goto(&done, in_range); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerObjectIsSafeInteger(Node* node) { |
| Node* object = node->InputAt(0); |
| Node* zero = __ Int32Constant(0); |
| Node* one = __ Int32Constant(1); |
| |
| auto done = __ MakeLabel(MachineRepresentation::kBit); |
| |
| // Check if {object} is a Smi. |
| __ GotoIf(ObjectIsSmi(object), &done, one); |
| |
| // Check if {object} is a HeapNumber. |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), object); |
| __ GotoIfNot(__ TaggedEqual(value_map, __ HeapNumberMapConstant()), &done, |
| zero); |
| |
| // {object} is a HeapNumber. |
| Node* value = __ LoadField(AccessBuilder::ForHeapNumberValue(), object); |
| Node* trunc = BuildFloat64RoundTruncate(value); |
| Node* diff = __ Float64Sub(value, trunc); |
| Node* check = __ Float64Equal(diff, __ Float64Constant(0)); |
| __ GotoIfNot(check, &done, zero); |
| Node* in_range = __ Float64LessThanOrEqual( |
| __ Float64Abs(trunc), __ Float64Constant(kMaxSafeInteger)); |
| __ Goto(&done, in_range); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| namespace { |
| |
| // There is no (currently) available constexpr version of bit_cast, so we have |
| // to make do with constructing the -0.0 bits manually (by setting the sign bit |
| // to 1 and everything else to 0). |
| // TODO(leszeks): Revisit when upgrading to C++20. |
| constexpr int32_t kMinusZeroLoBits = static_cast<int32_t>(0); |
| constexpr int32_t kMinusZeroHiBits = static_cast<int32_t>(1) << 31; |
| constexpr int64_t kMinusZeroBits = |
| (static_cast<uint64_t>(kMinusZeroHiBits) << 32) | kMinusZeroLoBits; |
| |
| } // namespace |
| |
| Node* EffectControlLinearizer::LowerObjectIsMinusZero(Node* node) { |
| Node* value = node->InputAt(0); |
| Node* zero = __ Int32Constant(0); |
| |
| auto done = __ MakeLabel(MachineRepresentation::kBit); |
| |
| // Check if {value} is a Smi. |
| __ GotoIf(ObjectIsSmi(value), &done, zero); |
| |
| // Check if {value} is a HeapNumber. |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| __ GotoIfNot(__ TaggedEqual(value_map, __ HeapNumberMapConstant()), &done, |
| zero); |
| |
| // Check if {value} contains -0. |
| Node* value_value = __ LoadField(AccessBuilder::ForHeapNumberValue(), value); |
| if (machine()->Is64()) { |
| Node* value64 = __ BitcastFloat64ToInt64(value_value); |
| __ Goto(&done, __ Word64Equal(value64, __ Int64Constant(kMinusZeroBits))); |
| } else { |
| Node* value_lo = __ Float64ExtractLowWord32(value_value); |
| __ GotoIfNot(__ Word32Equal(value_lo, __ Int32Constant(kMinusZeroLoBits)), |
| &done, zero); |
| Node* value_hi = __ Float64ExtractHighWord32(value_value); |
| __ Goto(&done, |
| __ Word32Equal(value_hi, __ Int32Constant(kMinusZeroHiBits))); |
| } |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerNumberIsMinusZero(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| if (machine()->Is64()) { |
| Node* value64 = __ BitcastFloat64ToInt64(value); |
| return __ Word64Equal(value64, __ Int64Constant(kMinusZeroBits)); |
| } else { |
| auto done = __ MakeLabel(MachineRepresentation::kBit); |
| |
| Node* value_lo = __ Float64ExtractLowWord32(value); |
| __ GotoIfNot(__ Word32Equal(value_lo, __ Int32Constant(kMinusZeroLoBits)), |
| &done, __ Int32Constant(0)); |
| Node* value_hi = __ Float64ExtractHighWord32(value); |
| __ Goto(&done, |
| __ Word32Equal(value_hi, __ Int32Constant(kMinusZeroHiBits))); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| } |
| |
| Node* EffectControlLinearizer::LowerObjectIsNaN(Node* node) { |
| Node* value = node->InputAt(0); |
| Node* zero = __ Int32Constant(0); |
| |
| auto done = __ MakeLabel(MachineRepresentation::kBit); |
| |
| // Check if {value} is a Smi. |
| __ GotoIf(ObjectIsSmi(value), &done, zero); |
| |
| // Check if {value} is a HeapNumber. |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| __ GotoIfNot(__ TaggedEqual(value_map, __ HeapNumberMapConstant()), &done, |
| zero); |
| |
| // Check if {value} contains a NaN. |
| Node* value_value = __ LoadField(AccessBuilder::ForHeapNumberValue(), value); |
| __ Goto(&done, |
| __ Word32Equal(__ Float64Equal(value_value, value_value), zero)); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerNumberIsNaN(Node* node) { |
| Node* number = node->InputAt(0); |
| Node* diff = __ Float64Equal(number, number); |
| Node* check = __ Word32Equal(diff, __ Int32Constant(0)); |
| return check; |
| } |
| |
| Node* EffectControlLinearizer::LowerObjectIsNonCallable(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| auto if_primitive = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kBit); |
| |
| Node* check0 = ObjectIsSmi(value); |
| __ GotoIf(check0, &if_primitive); |
| |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| Node* value_instance_type = |
| __ LoadField(AccessBuilder::ForMapInstanceType(), value_map); |
| STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE); |
| Node* check1 = __ Uint32LessThanOrEqual( |
| __ Uint32Constant(FIRST_JS_RECEIVER_TYPE), value_instance_type); |
| __ GotoIfNot(check1, &if_primitive); |
| |
| Node* value_bit_field = |
| __ LoadField(AccessBuilder::ForMapBitField(), value_map); |
| Node* check2 = __ Word32Equal( |
| __ Int32Constant(0), |
| __ Word32And(value_bit_field, |
| __ Int32Constant(Map::Bits1::IsCallableBit::kMask))); |
| __ Goto(&done, check2); |
| |
| __ Bind(&if_primitive); |
| __ Goto(&done, __ Int32Constant(0)); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerObjectIsNumber(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| auto if_smi = __ MakeLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kBit); |
| |
| __ GotoIf(ObjectIsSmi(value), &if_smi); |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| __ Goto(&done, __ TaggedEqual(value_map, __ HeapNumberMapConstant())); |
| |
| __ Bind(&if_smi); |
| __ Goto(&done, __ Int32Constant(1)); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerObjectIsReceiver(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| auto if_smi = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kBit); |
| |
| __ GotoIf(ObjectIsSmi(value), &if_smi); |
| |
| STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE); |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| Node* value_instance_type = |
| __ LoadField(AccessBuilder::ForMapInstanceType(), value_map); |
| Node* result = __ Uint32LessThanOrEqual( |
| __ Uint32Constant(FIRST_JS_RECEIVER_TYPE), value_instance_type); |
| __ Goto(&done, result); |
| |
| __ Bind(&if_smi); |
| __ Goto(&done, __ Int32Constant(0)); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerObjectIsSmi(Node* node) { |
| Node* value = node->InputAt(0); |
| return ObjectIsSmi(value); |
| } |
| |
| Node* EffectControlLinearizer::LowerObjectIsString(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| auto if_smi = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kBit); |
| |
| Node* check = ObjectIsSmi(value); |
| __ GotoIf(check, &if_smi); |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| Node* value_instance_type = |
| __ LoadField(AccessBuilder::ForMapInstanceType(), value_map); |
| Node* vfalse = __ Uint32LessThan(value_instance_type, |
| __ Uint32Constant(FIRST_NONSTRING_TYPE)); |
| __ Goto(&done, vfalse); |
| |
| __ Bind(&if_smi); |
| __ Goto(&done, __ Int32Constant(0)); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerObjectIsSymbol(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| auto if_smi = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kBit); |
| |
| Node* check = ObjectIsSmi(value); |
| __ GotoIf(check, &if_smi); |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| Node* value_instance_type = |
| __ LoadField(AccessBuilder::ForMapInstanceType(), value_map); |
| Node* vfalse = |
| __ Word32Equal(value_instance_type, __ Uint32Constant(SYMBOL_TYPE)); |
| __ Goto(&done, vfalse); |
| |
| __ Bind(&if_smi); |
| __ Goto(&done, __ Int32Constant(0)); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerObjectIsUndetectable(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| auto if_smi = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kBit); |
| |
| Node* check = ObjectIsSmi(value); |
| __ GotoIf(check, &if_smi); |
| |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| Node* value_bit_field = |
| __ LoadField(AccessBuilder::ForMapBitField(), value_map); |
| Node* vfalse = __ Word32Equal( |
| __ Word32Equal( |
| __ Int32Constant(0), |
| __ Word32And(value_bit_field, |
| __ Int32Constant(Map::Bits1::IsUndetectableBit::kMask))), |
| __ Int32Constant(0)); |
| __ Goto(&done, vfalse); |
| |
| __ Bind(&if_smi); |
| __ Goto(&done, __ Int32Constant(0)); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerTypeOf(Node* node) { |
| Node* obj = node->InputAt(0); |
| Callable const callable = Builtins::CallableFor(isolate(), Builtins::kTypeof); |
| Operator::Properties const properties = Operator::kEliminatable; |
| CallDescriptor::Flags const flags = CallDescriptor::kNoAllocate; |
| auto call_descriptor = Linkage::GetStubCallDescriptor( |
| graph()->zone(), callable.descriptor(), |
| callable.descriptor().GetStackParameterCount(), flags, properties); |
| return __ Call(call_descriptor, __ HeapConstant(callable.code()), obj, |
| __ NoContextConstant()); |
| } |
| |
| void EffectControlLinearizer::LowerTierUpCheck(Node* node) { |
| TierUpCheckNode n(node); |
| TNode<FeedbackVector> vector = n.feedback_vector(); |
| |
| Node* optimization_state = |
| __ LoadField(AccessBuilder::ForFeedbackVectorFlags(), vector); |
| |
| // TODO(jgruber): The branch introduces a sequence of spills before the |
| // branch (and restores at `fallthrough`) that are completely unnecessary |
| // since the IfFalse continuation ends in a tail call. Investigate how to |
| // avoid these and fix it. |
| |
| auto fallthrough = __ MakeLabel(); |
| auto has_optimized_code_or_marker = __ MakeDeferredLabel(); |
| __ BranchWithHint( |
| __ Word32Equal( |
| __ Word32And(optimization_state, |
| __ Uint32Constant( |
| FeedbackVector:: |
| kHasNoTopTierCodeOrCompileOptimizedMarkerMask)), |
| __ Int32Constant(0)), |
| &fallthrough, &has_optimized_code_or_marker, BranchHint::kTrue); |
| |
| __ Bind(&has_optimized_code_or_marker); |
| |
| // The optimization marker field contains a non-trivial value, and some |
| // action has to be taken. For example, perhaps tier-up has been requested |
| // and we need to kick off a compilation job; or optimized code is available |
| // and should be tail-called. |
| // |
| // Currently we delegate these tasks to the InterpreterEntryTrampoline. |
| // TODO(jgruber,v8:8888): Consider a dedicated builtin instead. |
| |
| TNode<HeapObject> code = |
| __ HeapConstant(BUILTIN_CODE(isolate(), InterpreterEntryTrampoline)); |
| |
| JSTrampolineDescriptor descriptor; |
| CallDescriptor::Flags flags = CallDescriptor::kFixedTargetRegister | |
| CallDescriptor::kIsTailCallForTierUp; |
| auto call_descriptor = Linkage::GetStubCallDescriptor( |
| graph()->zone(), descriptor, descriptor.GetStackParameterCount(), flags, |
| Operator::kNoProperties); |
| Node* nodes[] = {code, n.target(), n.new_target(), n.input_count(), |
| n.context(), __ effect(), __ control()}; |
| |
| #ifdef DEBUG |
| static constexpr int kCodeContextEffectControl = 4; |
| DCHECK_EQ(arraysize(nodes), |
| descriptor.GetParameterCount() + kCodeContextEffectControl); |
| #endif // DEBUG |
| |
| __ TailCall(call_descriptor, arraysize(nodes), nodes); |
| |
| __ Bind(&fallthrough); |
| } |
| |
| void EffectControlLinearizer::LowerUpdateInterruptBudget(Node* node) { |
| UpdateInterruptBudgetNode n(node); |
| TNode<FeedbackCell> feedback_cell = n.feedback_cell(); |
| TNode<Int32T> budget = __ LoadField<Int32T>( |
| AccessBuilder::ForFeedbackCellInterruptBudget(), feedback_cell); |
| Node* new_budget = __ Int32Add(budget, __ Int32Constant(n.delta())); |
| __ StoreField(AccessBuilder::ForFeedbackCellInterruptBudget(), feedback_cell, |
| new_budget); |
| if (n.delta() < 0) { |
| auto next = __ MakeLabel(); |
| auto if_budget_exhausted = __ MakeDeferredLabel(); |
| __ Branch(__ Int32LessThan(new_budget, __ Int32Constant(0)), |
| &if_budget_exhausted, &next); |
| |
| __ Bind(&if_budget_exhausted); |
| CallBuiltin(Builtins::kBytecodeBudgetInterruptFromCode, |
| node->op()->properties(), feedback_cell); |
| __ Goto(&next); |
| |
| __ Bind(&next); |
| } |
| } |
| |
| Node* EffectControlLinearizer::LowerToBoolean(Node* node) { |
| Node* obj = node->InputAt(0); |
| Callable const callable = |
| Builtins::CallableFor(isolate(), Builtins::kToBoolean); |
| Operator::Properties const properties = Operator::kEliminatable; |
| CallDescriptor::Flags const flags = CallDescriptor::kNoAllocate; |
| auto call_descriptor = Linkage::GetStubCallDescriptor( |
| graph()->zone(), callable.descriptor(), |
| callable.descriptor().GetStackParameterCount(), flags, properties); |
| return __ Call(call_descriptor, __ HeapConstant(callable.code()), obj, |
| __ NoContextConstant()); |
| } |
| |
| Node* EffectControlLinearizer::LowerArgumentsLength(Node* node) { |
| #ifdef V8_NO_ARGUMENTS_ADAPTOR |
| return ChangeIntPtrToSmi( |
| __ Load(MachineType::Pointer(), __ LoadFramePointer(), |
| __ IntPtrConstant(StandardFrameConstants::kArgCOffset))); |
| #else |
| auto done = __ MakeLabel(MachineRepresentation::kTaggedSigned); |
| Node* frame = __ LoadFramePointer(); |
| |
| Node* arguments_frame = NodeProperties::GetValueInput(node, 0); |
| int formal_parameter_count = FormalParameterCountOf(node->op()); |
| DCHECK_LE(0, formal_parameter_count); |
| |
| // The ArgumentsLength node is computing the actual number of arguments. |
| // We have to distinguish the case when there is an arguments adaptor frame |
| // (i.e., arguments_frame != LoadFramePointer()). |
| auto if_adaptor_frame = __ MakeLabel(); |
| __ GotoIf(__ TaggedEqual(arguments_frame, frame), &done, |
| __ SmiConstant(formal_parameter_count)); |
| __ Goto(&if_adaptor_frame); |
| |
| __ Bind(&if_adaptor_frame); |
| Node* arguments_length = __ BitcastWordToTaggedSigned(__ Load( |
| MachineType::Pointer(), arguments_frame, |
| __ IntPtrConstant(ArgumentsAdaptorFrameConstants::kLengthOffset))); |
| __ Goto(&done, arguments_length); |
| __ Bind(&done); |
| return done.PhiAt(0); |
| #endif |
| } |
| |
| Node* EffectControlLinearizer::LowerRestLength(Node* node) { |
| int formal_parameter_count = FormalParameterCountOf(node->op()); |
| DCHECK_LE(0, formal_parameter_count); |
| |
| auto done = __ MakeLabel(MachineRepresentation::kTaggedSigned); |
| Node* frame = __ LoadFramePointer(); |
| |
| #ifdef V8_NO_ARGUMENTS_ADAPTOR |
| Node* arguments_length = ChangeIntPtrToSmi( |
| __ Load(MachineType::Pointer(), frame, |
| __ IntPtrConstant(StandardFrameConstants::kArgCOffset))); |
| #else |
| Node* arguments_frame = NodeProperties::GetValueInput(node, 0); |
| |
| // The RestLength node is computing the number of rest parameters, |
| // which is max(0, actual_parameter_count - formal_parameter_count). |
| // We have to distinguish the case, when there is an arguments adaptor frame |
| // (i.e., arguments_frame != LoadFramePointer()). |
| auto if_adaptor_frame = __ MakeLabel(); |
| __ GotoIf(__ TaggedEqual(arguments_frame, frame), &done, __ SmiConstant(0)); |
| __ Goto(&if_adaptor_frame); |
| |
| __ Bind(&if_adaptor_frame); |
| Node* arguments_length = __ BitcastWordToTaggedSigned(__ Load( |
| MachineType::Pointer(), arguments_frame, |
| __ IntPtrConstant(ArgumentsAdaptorFrameConstants::kLengthOffset))); |
| #endif |
| |
| Node* rest_length = |
| __ SmiSub(arguments_length, __ SmiConstant(formal_parameter_count)); |
| __ GotoIf(__ SmiLessThan(rest_length, __ SmiConstant(0)), &done, |
| __ SmiConstant(0)); |
| __ Goto(&done, rest_length); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerArgumentsFrame(Node* node) { |
| auto done = __ MakeLabel(MachineType::PointerRepresentation()); |
| |
| Node* frame = __ LoadFramePointer(); |
| Node* parent_frame = |
| __ Load(MachineType::Pointer(), frame, |
| __ IntPtrConstant(StandardFrameConstants::kCallerFPOffset)); |
| Node* parent_frame_type = __ Load( |
| MachineType::IntPtr(), parent_frame, |
| __ IntPtrConstant(CommonFrameConstants::kContextOrFrameTypeOffset)); |
| |
| __ GotoIf(__ IntPtrEqual(parent_frame_type, |
| __ IntPtrConstant(StackFrame::TypeToMarker( |
| StackFrame::ARGUMENTS_ADAPTOR))), |
| &done, parent_frame); |
| __ Goto(&done, frame); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerNewDoubleElements(Node* node) { |
| AllocationType const allocation = AllocationTypeOf(node->op()); |
| Node* length = node->InputAt(0); |
| |
| auto done = __ MakeLabel(MachineRepresentation::kTaggedPointer); |
| Node* zero_length = __ IntPtrEqual(length, __ IntPtrConstant(0)); |
| __ GotoIf(zero_length, &done, |
| __ HeapConstant(factory()->empty_fixed_array())); |
| |
| // Compute the effective size of the backing store. |
| Node* size = __ IntAdd(__ WordShl(length, __ IntPtrConstant(kDoubleSizeLog2)), |
| __ IntPtrConstant(FixedDoubleArray::kHeaderSize)); |
| |
| // Allocate the result and initialize the header. |
| Node* result = __ Allocate(allocation, size); |
| __ StoreField(AccessBuilder::ForMap(), result, |
| __ FixedDoubleArrayMapConstant()); |
| __ StoreField(AccessBuilder::ForFixedArrayLength(), result, |
| ChangeIntPtrToSmi(length)); |
| |
| // Initialize the backing store with holes. |
| STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset, |
| Oddball::kToNumberRawOffset); |
| Node* the_hole = |
| __ LoadField(AccessBuilder::ForHeapNumberValue(), __ TheHoleConstant()); |
| auto loop = __ MakeLoopLabel(MachineType::PointerRepresentation()); |
| __ Goto(&loop, __ IntPtrConstant(0)); |
| __ Bind(&loop); |
| { |
| // Check if we've initialized everything. |
| Node* index = loop.PhiAt(0); |
| Node* check = __ UintLessThan(index, length); |
| __ GotoIfNot(check, &done, result); |
| |
| ElementAccess const access = {kTaggedBase, FixedDoubleArray::kHeaderSize, |
| Type::NumberOrHole(), MachineType::Float64(), |
| kNoWriteBarrier}; |
| __ StoreElement(access, result, index, the_hole); |
| |
| // Advance the {index}. |
| index = __ IntAdd(index, __ IntPtrConstant(1)); |
| __ Goto(&loop, index); |
| } |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerNewSmiOrObjectElements(Node* node) { |
| AllocationType const allocation = AllocationTypeOf(node->op()); |
| Node* length = node->InputAt(0); |
| |
| auto done = __ MakeLabel(MachineRepresentation::kTaggedPointer); |
| Node* zero_length = __ IntPtrEqual(length, __ IntPtrConstant(0)); |
| __ GotoIf(zero_length, &done, |
| __ HeapConstant(factory()->empty_fixed_array())); |
| |
| // Compute the effective size of the backing store. |
| Node* size = __ IntAdd(__ WordShl(length, __ IntPtrConstant(kTaggedSizeLog2)), |
| __ IntPtrConstant(FixedArray::kHeaderSize)); |
| |
| // Allocate the result and initialize the header. |
| Node* result = __ Allocate(allocation, size); |
| __ StoreField(AccessBuilder::ForMap(), result, __ FixedArrayMapConstant()); |
| __ StoreField(AccessBuilder::ForFixedArrayLength(), result, |
| ChangeIntPtrToSmi(length)); |
| |
| // Initialize the backing store with holes. |
| Node* the_hole = __ TheHoleConstant(); |
| auto loop = __ MakeLoopLabel(MachineType::PointerRepresentation()); |
| __ Goto(&loop, __ IntPtrConstant(0)); |
| __ Bind(&loop); |
| { |
| // Check if we've initialized everything. |
| Node* index = loop.PhiAt(0); |
| Node* check = __ UintLessThan(index, length); |
| __ GotoIfNot(check, &done, result); |
| |
| // Storing "the_hole" doesn't need a write barrier. |
| ElementAccess const access = {kTaggedBase, FixedArray::kHeaderSize, |
| Type::Any(), MachineType::AnyTagged(), |
| kNoWriteBarrier}; |
| __ StoreElement(access, result, index, the_hole); |
| |
| // Advance the {index}. |
| index = __ IntAdd(index, __ IntPtrConstant(1)); |
| __ Goto(&loop, index); |
| } |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerNewArgumentsElements(Node* node) { |
| const NewArgumentsElementsParameters& parameters = |
| NewArgumentsElementsParametersOf(node->op()); |
| CreateArgumentsType type = parameters.arguments_type(); |
| Operator::Properties const properties = node->op()->properties(); |
| CallDescriptor::Flags const flags = CallDescriptor::kNoFlags; |
| Node* frame = NodeProperties::GetValueInput(node, 0); |
| Node* arguments_count = NodeProperties::GetValueInput(node, 1); |
| Builtins::Name builtin_name; |
| switch (type) { |
| case CreateArgumentsType::kMappedArguments: |
| builtin_name = Builtins::kNewSloppyArgumentsElements; |
| break; |
| case CreateArgumentsType::kUnmappedArguments: |
| builtin_name = Builtins::kNewStrictArgumentsElements; |
| break; |
| case CreateArgumentsType::kRestParameter: |
| builtin_name = Builtins::kNewRestArgumentsElements; |
| break; |
| } |
| Callable const callable = Builtins::CallableFor(isolate(), builtin_name); |
| auto call_descriptor = Linkage::GetStubCallDescriptor( |
| graph()->zone(), callable.descriptor(), |
| callable.descriptor().GetStackParameterCount(), flags, properties); |
| return __ Call(call_descriptor, __ HeapConstant(callable.code()), frame, |
| __ IntPtrConstant(parameters.formal_parameter_count()), |
| arguments_count); |
| } |
| |
| Node* EffectControlLinearizer::LowerNewConsString(Node* node) { |
| Node* length = node->InputAt(0); |
| Node* first = node->InputAt(1); |
| Node* second = node->InputAt(2); |
| |
| // Determine the instance types of {first} and {second}. |
| Node* first_map = __ LoadField(AccessBuilder::ForMap(), first); |
| Node* first_instance_type = |
| __ LoadField(AccessBuilder::ForMapInstanceType(), first_map); |
| Node* second_map = __ LoadField(AccessBuilder::ForMap(), second); |
| Node* second_instance_type = |
| __ LoadField(AccessBuilder::ForMapInstanceType(), second_map); |
| |
| // Determine the proper map for the resulting ConsString. |
| // If both {first} and {second} are one-byte strings, we |
| // create a new ConsOneByteString, otherwise we create a |
| // new ConsString instead. |
| auto if_onebyte = __ MakeLabel(); |
| auto if_twobyte = __ MakeLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kTaggedPointer); |
| STATIC_ASSERT(kOneByteStringTag != 0); |
| STATIC_ASSERT(kTwoByteStringTag == 0); |
| Node* instance_type = __ Word32And(first_instance_type, second_instance_type); |
| Node* encoding = |
| __ Word32And(instance_type, __ Int32Constant(kStringEncodingMask)); |
| __ Branch(__ Word32Equal(encoding, __ Int32Constant(kTwoByteStringTag)), |
| &if_twobyte, &if_onebyte); |
| __ Bind(&if_onebyte); |
| __ Goto(&done, __ HeapConstant(factory()->cons_one_byte_string_map())); |
| __ Bind(&if_twobyte); |
| __ Goto(&done, __ HeapConstant(factory()->cons_string_map())); |
| __ Bind(&done); |
| Node* result_map = done.PhiAt(0); |
| |
| // Allocate the resulting ConsString. |
| Node* result = |
| __ Allocate(AllocationType::kYoung, __ IntPtrConstant(ConsString::kSize)); |
| __ StoreField(AccessBuilder::ForMap(), result, result_map); |
| __ StoreField(AccessBuilder::ForNameHashField(), result, |
| __ Int32Constant(Name::kEmptyHashField)); |
| __ StoreField(AccessBuilder::ForStringLength(), result, length); |
| __ StoreField(AccessBuilder::ForConsStringFirst(), result, first); |
| __ StoreField(AccessBuilder::ForConsStringSecond(), result, second); |
| return result; |
| } |
| |
| Node* EffectControlLinearizer::LowerSameValue(Node* node) { |
| Node* lhs = node->InputAt(0); |
| Node* rhs = node->InputAt(1); |
| |
| Callable const callable = |
| Builtins::CallableFor(isolate(), Builtins::kSameValue); |
| Operator::Properties properties = Operator::kEliminatable; |
| CallDescriptor::Flags flags = CallDescriptor::kNoFlags; |
| auto call_descriptor = Linkage::GetStubCallDescriptor( |
| graph()->zone(), callable.descriptor(), |
| callable.descriptor().GetStackParameterCount(), flags, properties); |
| return __ Call(call_descriptor, __ HeapConstant(callable.code()), lhs, rhs, |
| __ NoContextConstant()); |
| } |
| |
| Node* EffectControlLinearizer::LowerSameValueNumbersOnly(Node* node) { |
| Node* lhs = node->InputAt(0); |
| Node* rhs = node->InputAt(1); |
| |
| Callable const callable = |
| Builtins::CallableFor(isolate(), Builtins::kSameValueNumbersOnly); |
| Operator::Properties properties = Operator::kEliminatable; |
| CallDescriptor::Flags flags = CallDescriptor::kNoFlags; |
| auto call_descriptor = Linkage::GetStubCallDescriptor( |
| graph()->zone(), callable.descriptor(), |
| callable.descriptor().GetStackParameterCount(), flags, properties); |
| return __ Call(call_descriptor, __ HeapConstant(callable.code()), lhs, rhs, |
| __ NoContextConstant()); |
| } |
| |
| Node* EffectControlLinearizer::LowerNumberSameValue(Node* node) { |
| Node* lhs = node->InputAt(0); |
| Node* rhs = node->InputAt(1); |
| |
| auto is_float64_equal = __ MakeLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kBit); |
| |
| __ GotoIf(__ Float64Equal(lhs, rhs), &is_float64_equal); |
| |
| // Return true iff both {lhs} and {rhs} are NaN. |
| __ GotoIf(__ Float64Equal(lhs, lhs), &done, __ Int32Constant(0)); |
| __ GotoIf(__ Float64Equal(rhs, rhs), &done, __ Int32Constant(0)); |
| __ Goto(&done, __ Int32Constant(1)); |
| |
| __ Bind(&is_float64_equal); |
| // Even if the values are float64-equal, we still need to distinguish |
| // zero and minus zero. |
| Node* lhs_hi = __ Float64ExtractHighWord32(lhs); |
| Node* rhs_hi = __ Float64ExtractHighWord32(rhs); |
| __ Goto(&done, __ Word32Equal(lhs_hi, rhs_hi)); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerDeadValue(Node* node) { |
| Node* input = NodeProperties::GetValueInput(node, 0); |
| if (input->opcode() != IrOpcode::kUnreachable) { |
| // There is no fundamental reason not to connect to end here, except it |
| // integrates into the way the graph is constructed in a simpler way at |
| // this point. |
| // TODO(jgruber): Connect to end here as well. |
| Node* unreachable = __ UnreachableWithoutConnectToEnd(); |
| NodeProperties::ReplaceValueInput(node, unreachable, 0); |
| } |
| return gasm()->AddNode(node); |
| } |
| |
| Node* EffectControlLinearizer::LowerStringToNumber(Node* node) { |
| Node* string = node->InputAt(0); |
| |
| Callable const callable = |
| Builtins::CallableFor(isolate(), Builtins::kStringToNumber); |
| Operator::Properties properties = Operator::kEliminatable; |
| CallDescriptor::Flags flags = CallDescriptor::kNoFlags; |
| auto call_descriptor = Linkage::GetStubCallDescriptor( |
| graph()->zone(), callable.descriptor(), |
| callable.descriptor().GetStackParameterCount(), flags, properties); |
| return __ Call(call_descriptor, __ HeapConstant(callable.code()), string, |
| __ NoContextConstant()); |
| } |
| |
| Node* EffectControlLinearizer::LowerStringCharCodeAt(Node* node) { |
| Node* receiver = node->InputAt(0); |
| Node* position = node->InputAt(1); |
| |
| // We need a loop here to properly deal with indirect strings |
| // (SlicedString, ConsString and ThinString). |
| auto loop = __ MakeLoopLabel(MachineRepresentation::kTagged, |
| MachineType::PointerRepresentation()); |
| auto loop_next = __ MakeLabel(MachineRepresentation::kTagged, |
| MachineType::PointerRepresentation()); |
| auto loop_done = __ MakeLabel(MachineRepresentation::kWord32); |
| __ Goto(&loop, receiver, position); |
| __ Bind(&loop); |
| { |
| Node* receiver = loop.PhiAt(0); |
| Node* position = loop.PhiAt(1); |
| Node* receiver_map = __ LoadField(AccessBuilder::ForMap(), receiver); |
| Node* receiver_instance_type = |
| __ LoadField(AccessBuilder::ForMapInstanceType(), receiver_map); |
| Node* receiver_representation = __ Word32And( |
| receiver_instance_type, __ Int32Constant(kStringRepresentationMask)); |
| |
| // Dispatch on the current {receiver}s string representation. |
| auto if_lessthanoreq_cons = __ MakeLabel(); |
| auto if_greaterthan_cons = __ MakeLabel(); |
| auto if_seqstring = __ MakeLabel(); |
| auto if_consstring = __ MakeLabel(); |
| auto if_thinstring = __ MakeLabel(); |
| auto if_externalstring = __ MakeLabel(); |
| auto if_slicedstring = __ MakeLabel(); |
| auto if_runtime = __ MakeDeferredLabel(); |
| |
| __ Branch(__ Int32LessThanOrEqual(receiver_representation, |
| __ Int32Constant(kConsStringTag)), |
| &if_lessthanoreq_cons, &if_greaterthan_cons); |
| |
| __ Bind(&if_lessthanoreq_cons); |
| { |
| __ Branch(__ Word32Equal(receiver_representation, |
| __ Int32Constant(kConsStringTag)), |
| &if_consstring, &if_seqstring); |
| } |
| |
| __ Bind(&if_greaterthan_cons); |
| { |
| __ GotoIf(__ Word32Equal(receiver_representation, |
| __ Int32Constant(kThinStringTag)), |
| &if_thinstring); |
| __ GotoIf(__ Word32Equal(receiver_representation, |
| __ Int32Constant(kExternalStringTag)), |
| &if_externalstring); |
| __ Branch(__ Word32Equal(receiver_representation, |
| __ Int32Constant(kSlicedStringTag)), |
| &if_slicedstring, &if_runtime); |
| } |
| |
| __ Bind(&if_seqstring); |
| { |
| Node* receiver_is_onebyte = __ Word32Equal( |
| __ Word32Equal(__ Word32And(receiver_instance_type, |
| __ Int32Constant(kStringEncodingMask)), |
| __ Int32Constant(kTwoByteStringTag)), |
| __ Int32Constant(0)); |
| Node* result = LoadFromSeqString(receiver, position, receiver_is_onebyte); |
| __ Goto(&loop_done, result); |
| } |
| |
| __ Bind(&if_consstring); |
| { |
| Node* receiver_second = |
| __ LoadField(AccessBuilder::ForConsStringSecond(), receiver); |
| __ GotoIfNot(__ TaggedEqual(receiver_second, __ EmptyStringConstant()), |
| &if_runtime); |
| Node* receiver_first = |
| __ LoadField(AccessBuilder::ForConsStringFirst(), receiver); |
| __ Goto(&loop_next, receiver_first, position); |
| } |
| |
| __ Bind(&if_thinstring); |
| { |
| Node* receiver_actual = |
| __ LoadField(AccessBuilder::ForThinStringActual(), receiver); |
| __ Goto(&loop_next, receiver_actual, position); |
| } |
| |
| __ Bind(&if_externalstring); |
| { |
| // We need to bailout to the runtime for uncached external strings. |
| __ GotoIf(__ Word32Equal( |
| __ Word32And(receiver_instance_type, |
| __ Int32Constant(kUncachedExternalStringMask)), |
| __ Int32Constant(kUncachedExternalStringTag)), |
| &if_runtime); |
| |
| Node* receiver_data = __ LoadField( |
| AccessBuilder::ForExternalStringResourceData(), receiver); |
| |
| auto if_onebyte = __ MakeLabel(); |
| auto if_twobyte = __ MakeLabel(); |
| __ Branch( |
| __ Word32Equal(__ Word32And(receiver_instance_type, |
| __ Int32Constant(kStringEncodingMask)), |
| __ Int32Constant(kTwoByteStringTag)), |
| &if_twobyte, &if_onebyte); |
| |
| __ Bind(&if_onebyte); |
| { |
| Node* result = __ Load(MachineType::Uint8(), receiver_data, position); |
| __ Goto(&loop_done, result); |
| } |
| |
| __ Bind(&if_twobyte); |
| { |
| Node* result = __ Load(MachineType::Uint16(), receiver_data, |
| __ WordShl(position, __ IntPtrConstant(1))); |
| __ Goto(&loop_done, result); |
| } |
| } |
| |
| __ Bind(&if_slicedstring); |
| { |
| Node* receiver_offset = |
| __ LoadField(AccessBuilder::ForSlicedStringOffset(), receiver); |
| Node* receiver_parent = |
| __ LoadField(AccessBuilder::ForSlicedStringParent(), receiver); |
| __ Goto(&loop_next, receiver_parent, |
| __ IntAdd(position, ChangeSmiToIntPtr(receiver_offset))); |
| } |
| |
| __ Bind(&if_runtime); |
| { |
| Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow; |
| Runtime::FunctionId id = Runtime::kStringCharCodeAt; |
| auto call_descriptor = Linkage::GetRuntimeCallDescriptor( |
| graph()->zone(), id, 2, properties, CallDescriptor::kNoFlags); |
| Node* result = __ Call(call_descriptor, __ CEntryStubConstant(1), |
| receiver, ChangeIntPtrToSmi(position), |
| __ ExternalConstant(ExternalReference::Create(id)), |
| __ Int32Constant(2), __ NoContextConstant()); |
| __ Goto(&loop_done, ChangeSmiToInt32(result)); |
| } |
| |
| __ Bind(&loop_next); |
| __ Goto(&loop, loop_next.PhiAt(0), loop_next.PhiAt(1)); |
| } |
| __ Bind(&loop_done); |
| return loop_done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerStringCodePointAt(Node* node) { |
| Node* receiver = node->InputAt(0); |
| Node* position = node->InputAt(1); |
| |
| Callable const callable = |
| Builtins::CallableFor(isolate(), Builtins::kStringCodePointAt); |
| Operator::Properties properties = Operator::kNoThrow | Operator::kNoWrite; |
| CallDescriptor::Flags flags = CallDescriptor::kNoFlags; |
| auto call_descriptor = Linkage::GetStubCallDescriptor( |
| graph()->zone(), callable.descriptor(), |
| callable.descriptor().GetStackParameterCount(), flags, properties); |
| return __ Call(call_descriptor, __ HeapConstant(callable.code()), receiver, |
| position, __ NoContextConstant()); |
| } |
| |
| Node* EffectControlLinearizer::LoadFromSeqString(Node* receiver, Node* position, |
| Node* is_one_byte) { |
| auto one_byte_load = __ MakeLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kWord32); |
| __ GotoIf(is_one_byte, &one_byte_load); |
| Node* two_byte_result = __ LoadElement( |
| AccessBuilder::ForSeqTwoByteStringCharacter(), receiver, position); |
| __ Goto(&done, two_byte_result); |
| |
| __ Bind(&one_byte_load); |
| Node* one_byte_element = __ LoadElement( |
| AccessBuilder::ForSeqOneByteStringCharacter(), receiver, position); |
| __ Goto(&done, one_byte_element); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerStringFromSingleCharCode(Node* node) { |
| Node* value = node->InputAt(0); |
| Node* code = __ Word32And(value, __ Uint32Constant(0xFFFF)); |
| |
| auto if_not_one_byte = __ MakeDeferredLabel(); |
| auto cache_miss = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kTagged); |
| |
| // Check if the {code} is a one byte character |
| Node* check1 = __ Uint32LessThanOrEqual( |
| code, __ Uint32Constant(String::kMaxOneByteCharCode)); |
| __ GotoIfNot(check1, &if_not_one_byte); |
| { |
| // Load the isolate wide single character string cache. |
| Node* cache = __ HeapConstant(factory()->single_character_string_cache()); |
| |
| // Compute the {cache} index for {code}. |
| Node* index = machine()->Is32() ? code : __ ChangeUint32ToUint64(code); |
| |
| // Check if we have an entry for the {code} in the single character string |
| // cache already. |
| Node* entry = |
| __ LoadElement(AccessBuilder::ForFixedArrayElement(), cache, index); |
| |
| Node* check2 = __ TaggedEqual(entry, __ UndefinedConstant()); |
| __ GotoIf(check2, &cache_miss); |
| |
| // Use the {entry} from the {cache}. |
| __ Goto(&done, entry); |
| |
| __ Bind(&cache_miss); |
| { |
| // Allocate a new SeqOneByteString for {code}. |
| Node* vtrue2 = |
| __ Allocate(AllocationType::kYoung, |
| __ IntPtrConstant(SeqOneByteString::SizeFor(1))); |
| __ StoreField(AccessBuilder::ForMap(), vtrue2, |
| __ HeapConstant(factory()->one_byte_string_map())); |
| __ StoreField(AccessBuilder::ForNameHashField(), vtrue2, |
| __ Int32Constant(Name::kEmptyHashField)); |
| __ StoreField(AccessBuilder::ForStringLength(), vtrue2, |
| __ Int32Constant(1)); |
| __ Store( |
| StoreRepresentation(MachineRepresentation::kWord8, kNoWriteBarrier), |
| vtrue2, |
| __ IntPtrConstant(SeqOneByteString::kHeaderSize - kHeapObjectTag), |
| code); |
| |
| // Remember it in the {cache}. |
| __ StoreElement(AccessBuilder::ForFixedArrayElement(), cache, index, |
| vtrue2); |
| __ Goto(&done, vtrue2); |
| } |
| } |
| |
| __ Bind(&if_not_one_byte); |
| { |
| // Allocate a new SeqTwoByteString for {code}. |
| Node* vfalse1 = |
| __ Allocate(AllocationType::kYoung, |
| __ IntPtrConstant(SeqTwoByteString::SizeFor(1))); |
| __ StoreField(AccessBuilder::ForMap(), vfalse1, |
| __ HeapConstant(factory()->string_map())); |
| __ StoreField(AccessBuilder::ForNameHashField(), vfalse1, |
| __ Int32Constant(Name::kEmptyHashField)); |
| __ StoreField(AccessBuilder::ForStringLength(), vfalse1, |
| __ Int32Constant(1)); |
| __ Store( |
| StoreRepresentation(MachineRepresentation::kWord16, kNoWriteBarrier), |
| vfalse1, |
| __ IntPtrConstant(SeqTwoByteString::kHeaderSize - kHeapObjectTag), |
| code); |
| __ Goto(&done, vfalse1); |
| } |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| #ifdef V8_INTL_SUPPORT |
| |
| Node* EffectControlLinearizer::LowerStringToLowerCaseIntl(Node* node) { |
| Node* receiver = node->InputAt(0); |
| |
| Callable callable = |
| Builtins::CallableFor(isolate(), Builtins::kStringToLowerCaseIntl); |
| Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow; |
| CallDescriptor::Flags flags = CallDescriptor::kNoFlags; |
| auto call_descriptor = Linkage::GetStubCallDescriptor( |
| graph()->zone(), callable.descriptor(), |
| callable.descriptor().GetStackParameterCount(), flags, properties); |
| return __ Call(call_descriptor, __ HeapConstant(callable.code()), receiver, |
| __ NoContextConstant()); |
| } |
| |
| Node* EffectControlLinearizer::LowerStringToUpperCaseIntl(Node* node) { |
| Node* receiver = node->InputAt(0); |
| Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow; |
| Runtime::FunctionId id = Runtime::kStringToUpperCaseIntl; |
| auto call_descriptor = Linkage::GetRuntimeCallDescriptor( |
| graph()->zone(), id, 1, properties, CallDescriptor::kNoFlags); |
| return __ Call(call_descriptor, __ CEntryStubConstant(1), receiver, |
| __ ExternalConstant(ExternalReference::Create(id)), |
| __ Int32Constant(1), __ NoContextConstant()); |
| } |
| |
| #else |
| |
| Node* EffectControlLinearizer::LowerStringToLowerCaseIntl(Node* node) { |
| UNREACHABLE(); |
| return nullptr; |
| } |
| |
| Node* EffectControlLinearizer::LowerStringToUpperCaseIntl(Node* node) { |
| UNREACHABLE(); |
| return nullptr; |
| } |
| |
| #endif // V8_INTL_SUPPORT |
| |
| Node* EffectControlLinearizer::LowerStringFromSingleCodePoint(Node* node) { |
| Node* value = node->InputAt(0); |
| Node* code = value; |
| |
| auto if_not_single_code = __ MakeDeferredLabel(); |
| auto if_not_one_byte = __ MakeDeferredLabel(); |
| auto cache_miss = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kTagged); |
| |
| // Check if the {code} is a single code unit |
| Node* check0 = __ Uint32LessThanOrEqual(code, __ Uint32Constant(0xFFFF)); |
| __ GotoIfNot(check0, &if_not_single_code); |
| |
| { |
| // Check if the {code} is a one byte character |
| Node* check1 = __ Uint32LessThanOrEqual( |
| code, __ Uint32Constant(String::kMaxOneByteCharCode)); |
| __ GotoIfNot(check1, &if_not_one_byte); |
| { |
| // Load the isolate wide single character string cache. |
| Node* cache = __ HeapConstant(factory()->single_character_string_cache()); |
| |
| // Compute the {cache} index for {code}. |
| Node* index = machine()->Is32() ? code : __ ChangeUint32ToUint64(code); |
| |
| // Check if we have an entry for the {code} in the single character string |
| // cache already. |
| Node* entry = |
| __ LoadElement(AccessBuilder::ForFixedArrayElement(), cache, index); |
| |
| Node* check2 = __ TaggedEqual(entry, __ UndefinedConstant()); |
| __ GotoIf(check2, &cache_miss); |
| |
| // Use the {entry} from the {cache}. |
| __ Goto(&done, entry); |
| |
| __ Bind(&cache_miss); |
| { |
| // Allocate a new SeqOneByteString for {code}. |
| Node* vtrue2 = |
| __ Allocate(AllocationType::kYoung, |
| __ IntPtrConstant(SeqOneByteString::SizeFor(1))); |
| __ StoreField(AccessBuilder::ForMap(), vtrue2, |
| __ HeapConstant(factory()->one_byte_string_map())); |
| __ StoreField(AccessBuilder::ForNameHashField(), vtrue2, |
| __ Int32Constant(Name::kEmptyHashField)); |
| __ StoreField(AccessBuilder::ForStringLength(), vtrue2, |
| __ Int32Constant(1)); |
| __ Store( |
| StoreRepresentation(MachineRepresentation::kWord8, kNoWriteBarrier), |
| vtrue2, |
| __ IntPtrConstant(SeqOneByteString::kHeaderSize - kHeapObjectTag), |
| code); |
| |
| // Remember it in the {cache}. |
| __ StoreElement(AccessBuilder::ForFixedArrayElement(), cache, index, |
| vtrue2); |
| __ Goto(&done, vtrue2); |
| } |
| } |
| |
| __ Bind(&if_not_one_byte); |
| { |
| // Allocate a new SeqTwoByteString for {code}. |
| Node* vfalse1 = |
| __ Allocate(AllocationType::kYoung, |
| __ IntPtrConstant(SeqTwoByteString::SizeFor(1))); |
| __ StoreField(AccessBuilder::ForMap(), vfalse1, |
| __ HeapConstant(factory()->string_map())); |
| __ StoreField(AccessBuilder::ForNameHashField(), vfalse1, |
| __ IntPtrConstant(Name::kEmptyHashField)); |
| __ StoreField(AccessBuilder::ForStringLength(), vfalse1, |
| __ Int32Constant(1)); |
| __ Store( |
| StoreRepresentation(MachineRepresentation::kWord16, kNoWriteBarrier), |
| vfalse1, |
| __ IntPtrConstant(SeqTwoByteString::kHeaderSize - kHeapObjectTag), |
| code); |
| __ Goto(&done, vfalse1); |
| } |
| } |
| |
| __ Bind(&if_not_single_code); |
| // Generate surrogate pair string |
| { |
| // Convert UTF32 to UTF16 code units, and store as a 32 bit word. |
| Node* lead_offset = __ Int32Constant(0xD800 - (0x10000 >> 10)); |
| |
| // lead = (codepoint >> 10) + LEAD_OFFSET |
| Node* lead = |
| __ Int32Add(__ Word32Shr(code, __ Int32Constant(10)), lead_offset); |
| |
| // trail = (codepoint & 0x3FF) + 0xDC00; |
| Node* trail = __ Int32Add(__ Word32And(code, __ Int32Constant(0x3FF)), |
| __ Int32Constant(0xDC00)); |
| |
| // codpoint = (trail << 16) | lead; |
| #if V8_TARGET_BIG_ENDIAN |
| code = __ Word32Or(__ Word32Shl(lead, __ Int32Constant(16)), trail); |
| #else |
| code = __ Word32Or(__ Word32Shl(trail, __ Int32Constant(16)), lead); |
| #endif |
| |
| // Allocate a new SeqTwoByteString for {code}. |
| Node* vfalse0 = |
| __ Allocate(AllocationType::kYoung, |
| __ IntPtrConstant(SeqTwoByteString::SizeFor(2))); |
| __ StoreField(AccessBuilder::ForMap(), vfalse0, |
| __ HeapConstant(factory()->string_map())); |
| __ StoreField(AccessBuilder::ForNameHashField(), vfalse0, |
| __ Int32Constant(Name::kEmptyHashField)); |
| __ StoreField(AccessBuilder::ForStringLength(), vfalse0, |
| __ Int32Constant(2)); |
| __ Store( |
| StoreRepresentation(MachineRepresentation::kWord32, kNoWriteBarrier), |
| vfalse0, |
| __ IntPtrConstant(SeqTwoByteString::kHeaderSize - kHeapObjectTag), |
| code); |
| __ Goto(&done, vfalse0); |
| } |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerStringIndexOf(Node* node) { |
| Node* subject = node->InputAt(0); |
| Node* search_string = node->InputAt(1); |
| Node* position = node->InputAt(2); |
| |
| Callable callable = |
| Builtins::CallableFor(isolate(), Builtins::kStringIndexOf); |
| Operator::Properties properties = Operator::kEliminatable; |
| CallDescriptor::Flags flags = CallDescriptor::kNoFlags; |
| auto call_descriptor = Linkage::GetStubCallDescriptor( |
| graph()->zone(), callable.descriptor(), |
| callable.descriptor().GetStackParameterCount(), flags, properties); |
| return __ Call(call_descriptor, __ HeapConstant(callable.code()), subject, |
| search_string, position, __ NoContextConstant()); |
| } |
| |
| Node* EffectControlLinearizer::LowerStringFromCodePointAt(Node* node) { |
| Node* string = node->InputAt(0); |
| Node* index = node->InputAt(1); |
| |
| Callable callable = |
| Builtins::CallableFor(isolate(), Builtins::kStringFromCodePointAt); |
| Operator::Properties properties = Operator::kEliminatable; |
| CallDescriptor::Flags flags = CallDescriptor::kNoFlags; |
| auto call_descriptor = Linkage::GetStubCallDescriptor( |
| graph()->zone(), callable.descriptor(), |
| callable.descriptor().GetStackParameterCount(), flags, properties); |
| return __ Call(call_descriptor, __ HeapConstant(callable.code()), string, |
| index, __ NoContextConstant()); |
| } |
| |
| Node* EffectControlLinearizer::LowerStringLength(Node* node) { |
| Node* subject = node->InputAt(0); |
| |
| return __ LoadField(AccessBuilder::ForStringLength(), subject); |
| } |
| |
| Node* EffectControlLinearizer::LowerStringComparison(Callable const& callable, |
| Node* node) { |
| Node* lhs = node->InputAt(0); |
| Node* rhs = node->InputAt(1); |
| |
| Operator::Properties properties = Operator::kEliminatable; |
| CallDescriptor::Flags flags = CallDescriptor::kNoFlags; |
| auto call_descriptor = Linkage::GetStubCallDescriptor( |
| graph()->zone(), callable.descriptor(), |
| callable.descriptor().GetStackParameterCount(), flags, properties); |
| return __ Call(call_descriptor, __ HeapConstant(callable.code()), lhs, rhs, |
| __ NoContextConstant()); |
| } |
| |
| Node* EffectControlLinearizer::LowerStringSubstring(Node* node) { |
| Node* receiver = node->InputAt(0); |
| Node* start = ChangeInt32ToIntPtr(node->InputAt(1)); |
| Node* end = ChangeInt32ToIntPtr(node->InputAt(2)); |
| |
| Callable callable = |
| Builtins::CallableFor(isolate(), Builtins::kStringSubstring); |
| Operator::Properties properties = Operator::kEliminatable; |
| CallDescriptor::Flags flags = CallDescriptor::kNoFlags; |
| auto call_descriptor = Linkage::GetStubCallDescriptor( |
| graph()->zone(), callable.descriptor(), |
| callable.descriptor().GetStackParameterCount(), flags, properties); |
| return __ Call(call_descriptor, __ HeapConstant(callable.code()), receiver, |
| start, end, __ NoContextConstant()); |
| } |
| |
| Node* EffectControlLinearizer::LowerStringEqual(Node* node) { |
| return LowerStringComparison( |
| Builtins::CallableFor(isolate(), Builtins::kStringEqual), node); |
| } |
| |
| Node* EffectControlLinearizer::LowerStringLessThan(Node* node) { |
| return LowerStringComparison( |
| Builtins::CallableFor(isolate(), Builtins::kStringLessThan), node); |
| } |
| |
| Node* EffectControlLinearizer::LowerStringLessThanOrEqual(Node* node) { |
| return LowerStringComparison( |
| Builtins::CallableFor(isolate(), Builtins::kStringLessThanOrEqual), node); |
| } |
| |
| Node* EffectControlLinearizer::LowerBigIntAdd(Node* node, Node* frame_state) { |
| Node* lhs = node->InputAt(0); |
| Node* rhs = node->InputAt(1); |
| |
| Callable const callable = |
| Builtins::CallableFor(isolate(), Builtins::kBigIntAddNoThrow); |
| auto call_descriptor = Linkage::GetStubCallDescriptor( |
| graph()->zone(), callable.descriptor(), |
| callable.descriptor().GetStackParameterCount(), CallDescriptor::kNoFlags, |
| Operator::kFoldable | Operator::kNoThrow); |
| Node* value = __ Call(call_descriptor, __ HeapConstant(callable.code()), lhs, |
| rhs, __ NoContextConstant()); |
| |
| // Check for exception sentinel: Smi is returned to signal BigIntTooBig. |
| __ DeoptimizeIf(DeoptimizeReason::kBigIntTooBig, FeedbackSource{}, |
| ObjectIsSmi(value), frame_state); |
| |
| return value; |
| } |
| |
| Node* EffectControlLinearizer::LowerBigIntSubtract(Node* node, |
| Node* frame_state) { |
| Node* lhs = node->InputAt(0); |
| Node* rhs = node->InputAt(1); |
| |
| Callable const callable = |
| Builtins::CallableFor(isolate(), Builtins::kBigIntSubtractNoThrow); |
| auto call_descriptor = Linkage::GetStubCallDescriptor( |
| graph()->zone(), callable.descriptor(), |
| callable.descriptor().GetStackParameterCount(), CallDescriptor::kNoFlags, |
| Operator::kFoldable | Operator::kNoThrow); |
| Node* value = __ Call(call_descriptor, __ HeapConstant(callable.code()), lhs, |
| rhs, __ NoContextConstant()); |
| |
| // Check for exception sentinel: Smi is returned to signal BigIntTooBig. |
| __ DeoptimizeIf(DeoptimizeReason::kBigIntTooBig, FeedbackSource{}, |
| ObjectIsSmi(value), frame_state); |
| |
| return value; |
| } |
| |
| Node* EffectControlLinearizer::LowerBigIntNegate(Node* node) { |
| Callable const callable = |
| Builtins::CallableFor(isolate(), Builtins::kBigIntUnaryMinus); |
| auto call_descriptor = Linkage::GetStubCallDescriptor( |
| graph()->zone(), callable.descriptor(), |
| callable.descriptor().GetStackParameterCount(), CallDescriptor::kNoFlags, |
| Operator::kFoldable | Operator::kNoThrow); |
| Node* value = __ Call(call_descriptor, __ HeapConstant(callable.code()), |
| node->InputAt(0), __ NoContextConstant()); |
| |
| return value; |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckFloat64Hole(Node* node, |
| Node* frame_state) { |
| // If we reach this point w/o eliminating the {node} that's marked |
| // with allow-return-hole, we cannot do anything, so just deoptimize |
| // in case of the hole NaN. |
| CheckFloat64HoleParameters const& params = |
| CheckFloat64HoleParametersOf(node->op()); |
| Node* value = node->InputAt(0); |
| |
| auto if_nan = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(); |
| |
| // First check whether {value} is a NaN at all... |
| __ Branch(__ Float64Equal(value, value), &done, &if_nan); |
| |
| __ Bind(&if_nan); |
| { |
| // ...and only if {value} is a NaN, perform the expensive bit |
| // check. See http://crbug.com/v8/8264 for details. |
| Node* check = __ Word32Equal(__ Float64ExtractHighWord32(value), |
| __ Int32Constant(kHoleNanUpper32)); |
| __ DeoptimizeIf(DeoptimizeReason::kHole, params.feedback(), check, |
| frame_state); |
| __ Goto(&done); |
| } |
| |
| __ Bind(&done); |
| return value; |
| } |
| |
| Node* EffectControlLinearizer::LowerCheckNotTaggedHole(Node* node, |
| Node* frame_state) { |
| Node* value = node->InputAt(0); |
| Node* check = __ TaggedEqual(value, __ TheHoleConstant()); |
| __ DeoptimizeIf(DeoptimizeReason::kHole, FeedbackSource(), check, |
| frame_state); |
| return value; |
| } |
| |
| Node* EffectControlLinearizer::LowerConvertTaggedHoleToUndefined(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| auto if_is_hole = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kTagged); |
| |
| Node* check = __ TaggedEqual(value, __ TheHoleConstant()); |
| __ GotoIf(check, &if_is_hole); |
| __ Goto(&done, value); |
| |
| __ Bind(&if_is_hole); |
| __ Goto(&done, __ UndefinedConstant()); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| void EffectControlLinearizer::LowerCheckEqualsInternalizedString( |
| Node* node, Node* frame_state) { |
| Node* exp = node->InputAt(0); |
| Node* val = node->InputAt(1); |
| |
| auto if_same = __ MakeLabel(); |
| auto if_notsame = __ MakeDeferredLabel(); |
| auto if_thinstring = __ MakeLabel(); |
| auto if_notthinstring = __ MakeLabel(); |
| |
| // Check if {exp} and {val} are the same, which is the likely case. |
| __ Branch(__ TaggedEqual(exp, val), &if_same, &if_notsame); |
| |
| __ Bind(&if_notsame); |
| { |
| // Now {val} could still be a non-internalized String that matches {exp}. |
| __ DeoptimizeIf(DeoptimizeReason::kWrongName, FeedbackSource(), |
| ObjectIsSmi(val), frame_state); |
| Node* val_map = __ LoadField(AccessBuilder::ForMap(), val); |
| Node* val_instance_type = |
| __ LoadField(AccessBuilder::ForMapInstanceType(), val_map); |
| |
| // Check for the common case of ThinString first. |
| __ GotoIf(__ Word32Equal(val_instance_type, |
| __ Int32Constant(THIN_ONE_BYTE_STRING_TYPE)), |
| &if_thinstring); |
| __ Branch( |
| __ Word32Equal(val_instance_type, __ Int32Constant(THIN_STRING_TYPE)), |
| &if_thinstring, &if_notthinstring); |
| |
| __ Bind(&if_notthinstring); |
| { |
| // Check that the {val} is a non-internalized String, if it's anything |
| // else it cannot match the recorded feedback {exp} anyways. |
| __ DeoptimizeIfNot( |
| DeoptimizeReason::kWrongName, FeedbackSource(), |
| __ Word32Equal(__ Word32And(val_instance_type, |
| __ Int32Constant(kIsNotStringMask | |
| kIsNotInternalizedMask)), |
| __ Int32Constant(kStringTag | kNotInternalizedTag)), |
| frame_state); |
| |
| // Try to find the {val} in the string table. |
| MachineSignature::Builder builder(graph()->zone(), 1, 2); |
| builder.AddReturn(MachineType::AnyTagged()); |
| builder.AddParam(MachineType::Pointer()); |
| builder.AddParam(MachineType::AnyTagged()); |
| Node* try_string_to_index_or_lookup_existing = __ ExternalConstant( |
| ExternalReference::try_string_to_index_or_lookup_existing()); |
| Node* const isolate_ptr = |
| __ ExternalConstant(ExternalReference::isolate_address(isolate())); |
| auto call_descriptor = |
| Linkage::GetSimplifiedCDescriptor(graph()->zone(), builder.Build()); |
| Node* val_internalized = |
| __ Call(common()->Call(call_descriptor), |
| try_string_to_index_or_lookup_existing, isolate_ptr, val); |
| |
| // Now see if the results match. |
| __ DeoptimizeIfNot(DeoptimizeReason::kWrongName, FeedbackSource(), |
| __ TaggedEqual(exp, val_internalized), frame_state); |
| __ Goto(&if_same); |
| } |
| |
| __ Bind(&if_thinstring); |
| { |
| // The {val} is a ThinString, let's check the actual value. |
| Node* val_actual = |
| __ LoadField(AccessBuilder::ForThinStringActual(), val); |
| __ DeoptimizeIfNot(DeoptimizeReason::kWrongName, FeedbackSource(), |
| __ TaggedEqual(exp, val_actual), frame_state); |
| __ Goto(&if_same); |
| } |
| } |
| |
| __ Bind(&if_same); |
| } |
| |
| void EffectControlLinearizer::LowerCheckEqualsSymbol(Node* node, |
| Node* frame_state) { |
| Node* exp = node->InputAt(0); |
| Node* val = node->InputAt(1); |
| Node* check = __ TaggedEqual(exp, val); |
| __ DeoptimizeIfNot(DeoptimizeReason::kWrongName, FeedbackSource(), check, |
| frame_state); |
| } |
| |
| Node* EffectControlLinearizer::AllocateHeapNumberWithValue(Node* value) { |
| Node* result = |
| __ Allocate(AllocationType::kYoung, __ IntPtrConstant(HeapNumber::kSize)); |
| __ StoreField(AccessBuilder::ForMap(), result, __ HeapNumberMapConstant()); |
| __ StoreField(AccessBuilder::ForHeapNumberValue(), result, value); |
| return result; |
| } |
| |
| Node* EffectControlLinearizer::ChangeIntPtrToSmi(Node* value) { |
| // Do shift on 32bit values if Smis are stored in the lower word. |
| if (machine()->Is64() && SmiValuesAre31Bits()) { |
| return ChangeTaggedInt32ToSmi(__ Word32Shl(value, SmiShiftBitsConstant())); |
| } |
| return __ WordShl(value, SmiShiftBitsConstant()); |
| } |
| |
| Node* EffectControlLinearizer::ChangeTaggedInt32ToSmi(Node* value) { |
| DCHECK(SmiValuesAre31Bits()); |
| // In pointer compression, we smi-corrupt. Then, the upper bits are not |
| // important. |
| return COMPRESS_POINTERS_BOOL ? __ BitcastWord32ToWord64(value) |
| : ChangeInt32ToIntPtr(value); |
| } |
| |
| Node* EffectControlLinearizer::ChangeInt32ToIntPtr(Node* value) { |
| if (machine()->Is64()) { |
| value = __ ChangeInt32ToInt64(value); |
| } |
| return value; |
| } |
| |
| Node* EffectControlLinearizer::ChangeIntPtrToInt32(Node* value) { |
| if (machine()->Is64()) { |
| value = __ TruncateInt64ToInt32(value); |
| } |
| return value; |
| } |
| |
| Node* EffectControlLinearizer::ChangeInt32ToSmi(Node* value) { |
| // Do shift on 32bit values if Smis are stored in the lower word. |
| if (machine()->Is64() && SmiValuesAre31Bits()) { |
| return ChangeIntPtrToSmi(value); |
| } |
| return ChangeIntPtrToSmi(ChangeInt32ToIntPtr(value)); |
| } |
| |
| Node* EffectControlLinearizer::ChangeInt64ToSmi(Node* value) { |
| DCHECK(machine()->Is64()); |
| return ChangeIntPtrToSmi(value); |
| } |
| |
| Node* EffectControlLinearizer::ChangeUint32ToUintPtr(Node* value) { |
| if (machine()->Is64()) { |
| value = __ ChangeUint32ToUint64(value); |
| } |
| return value; |
| } |
| |
| Node* EffectControlLinearizer::ChangeUint32ToSmi(Node* value) { |
| // Do shift on 32bit values if Smis are stored in the lower word. |
| if (machine()->Is64() && SmiValuesAre31Bits()) { |
| Node* smi_value = __ Word32Shl(value, SmiShiftBitsConstant()); |
| // In pointer compression, we smi-corrupt. Then, the upper bits are not |
| // important. |
| return COMPRESS_POINTERS_BOOL ? __ BitcastWord32ToWord64(smi_value) |
| : __ ChangeUint32ToUint64(smi_value); |
| } else { |
| return __ WordShl(ChangeUint32ToUintPtr(value), SmiShiftBitsConstant()); |
| } |
| } |
| |
| Node* EffectControlLinearizer::ChangeSmiToIntPtr(Node* value) { |
| if (machine()->Is64() && SmiValuesAre31Bits()) { |
| // First sign-extend the upper half, then shift away the Smi tag. |
| return __ WordSarShiftOutZeros( |
| __ ChangeInt32ToInt64(__ TruncateInt64ToInt32(value)), |
| SmiShiftBitsConstant()); |
| } |
| return __ WordSarShiftOutZeros(value, SmiShiftBitsConstant()); |
| } |
| |
| Node* EffectControlLinearizer::ChangeSmiToInt32(Node* value) { |
| // Do shift on 32bit values if Smis are stored in the lower word. |
| if (machine()->Is64() && SmiValuesAre31Bits()) { |
| return __ Word32SarShiftOutZeros(__ TruncateInt64ToInt32(value), |
| SmiShiftBitsConstant()); |
| } |
| if (machine()->Is64()) { |
| return __ TruncateInt64ToInt32(ChangeSmiToIntPtr(value)); |
| } |
| return ChangeSmiToIntPtr(value); |
| } |
| |
| Node* EffectControlLinearizer::ChangeSmiToInt64(Node* value) { |
| CHECK(machine()->Is64()); |
| return ChangeSmiToIntPtr(value); |
| } |
| |
| Node* EffectControlLinearizer::ObjectIsSmi(Node* value) { |
| return __ Word32Equal(__ Word32And(value, __ Int32Constant(kSmiTagMask)), |
| __ Int32Constant(kSmiTag)); |
| } |
| |
| Node* EffectControlLinearizer::SmiMaxValueConstant() { |
| return __ Int32Constant(Smi::kMaxValue); |
| } |
| |
| Node* EffectControlLinearizer::SmiShiftBitsConstant() { |
| if (machine()->Is64() && SmiValuesAre31Bits()) { |
| return __ Int32Constant(kSmiShiftSize + kSmiTagSize); |
| } |
| return __ IntPtrConstant(kSmiShiftSize + kSmiTagSize); |
| } |
| |
| Node* EffectControlLinearizer::LowerPlainPrimitiveToNumber(Node* node) { |
| Node* value = node->InputAt(0); |
| return __ PlainPrimitiveToNumber(TNode<Object>::UncheckedCast(value)); |
| } |
| |
| Node* EffectControlLinearizer::LowerPlainPrimitiveToWord32(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| auto if_not_smi = __ MakeDeferredLabel(); |
| auto if_to_number_smi = __ MakeLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kWord32); |
| |
| Node* check0 = ObjectIsSmi(value); |
| __ GotoIfNot(check0, &if_not_smi); |
| __ Goto(&done, ChangeSmiToInt32(value)); |
| |
| __ Bind(&if_not_smi); |
| Node* to_number = |
| __ PlainPrimitiveToNumber(TNode<Object>::UncheckedCast(value)); |
| |
| Node* check1 = ObjectIsSmi(to_number); |
| __ GotoIf(check1, &if_to_number_smi); |
| Node* number = __ LoadField(AccessBuilder::ForHeapNumberValue(), to_number); |
| __ Goto(&done, __ TruncateFloat64ToWord32(number)); |
| |
| __ Bind(&if_to_number_smi); |
| __ Goto(&done, ChangeSmiToInt32(to_number)); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerPlainPrimitiveToFloat64(Node* node) { |
| Node* value = node->InputAt(0); |
| |
| auto if_not_smi = __ MakeDeferredLabel(); |
| auto if_to_number_smi = __ MakeLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kFloat64); |
| |
| Node* check0 = ObjectIsSmi(value); |
| __ GotoIfNot(check0, &if_not_smi); |
| Node* from_smi = ChangeSmiToInt32(value); |
| __ Goto(&done, __ ChangeInt32ToFloat64(from_smi)); |
| |
| __ Bind(&if_not_smi); |
| Node* to_number = |
| __ PlainPrimitiveToNumber(TNode<Object>::UncheckedCast(value)); |
| Node* check1 = ObjectIsSmi(to_number); |
| __ GotoIf(check1, &if_to_number_smi); |
| |
| Node* number = __ LoadField(AccessBuilder::ForHeapNumberValue(), to_number); |
| __ Goto(&done, number); |
| |
| __ Bind(&if_to_number_smi); |
| Node* number_from_smi = ChangeSmiToInt32(to_number); |
| number_from_smi = __ ChangeInt32ToFloat64(number_from_smi); |
| __ Goto(&done, number_from_smi); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerEnsureWritableFastElements(Node* node) { |
| Node* object = node->InputAt(0); |
| Node* elements = node->InputAt(1); |
| |
| auto if_not_fixed_array = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kTagged); |
| |
| // Load the current map of {elements}. |
| Node* elements_map = __ LoadField(AccessBuilder::ForMap(), elements); |
| |
| // Check if {elements} is not a copy-on-write FixedArray. |
| Node* check = __ TaggedEqual(elements_map, __ FixedArrayMapConstant()); |
| __ GotoIfNot(check, &if_not_fixed_array); |
| // Nothing to do if the {elements} are not copy-on-write. |
| __ Goto(&done, elements); |
| |
| __ Bind(&if_not_fixed_array); |
| // We need to take a copy of the {elements} and set them up for {object}. |
| Operator::Properties properties = Operator::kEliminatable; |
| Callable callable = |
| Builtins::CallableFor(isolate(), Builtins::kCopyFastSmiOrObjectElements); |
| CallDescriptor::Flags flags = CallDescriptor::kNoFlags; |
| auto call_descriptor = Linkage::GetStubCallDescriptor( |
| graph()->zone(), callable.descriptor(), |
| callable.descriptor().GetStackParameterCount(), flags, properties); |
| Node* result = __ Call(call_descriptor, __ HeapConstant(callable.code()), |
| object, __ NoContextConstant()); |
| __ Goto(&done, result); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerMaybeGrowFastElements(Node* node, |
| Node* frame_state) { |
| GrowFastElementsParameters params = GrowFastElementsParametersOf(node->op()); |
| Node* object = node->InputAt(0); |
| Node* elements = node->InputAt(1); |
| Node* index = node->InputAt(2); |
| Node* elements_length = node->InputAt(3); |
| |
| auto done = __ MakeLabel(MachineRepresentation::kTagged); |
| auto if_grow = __ MakeDeferredLabel(); |
| auto if_not_grow = __ MakeLabel(); |
| |
| // Check if we need to grow the {elements} backing store. |
| Node* check = __ Uint32LessThan(index, elements_length); |
| __ GotoIfNot(check, &if_grow); |
| __ Goto(&done, elements); |
| |
| __ Bind(&if_grow); |
| // We need to grow the {elements} for {object}. |
| Operator::Properties properties = Operator::kEliminatable; |
| Callable callable = |
| (params.mode() == GrowFastElementsMode::kDoubleElements) |
| ? Builtins::CallableFor(isolate(), Builtins::kGrowFastDoubleElements) |
| : Builtins::CallableFor(isolate(), |
| Builtins::kGrowFastSmiOrObjectElements); |
| CallDescriptor::Flags call_flags = CallDescriptor::kNoFlags; |
| auto call_descriptor = Linkage::GetStubCallDescriptor( |
| graph()->zone(), callable.descriptor(), |
| callable.descriptor().GetStackParameterCount(), call_flags, properties); |
| Node* new_elements = |
| __ Call(call_descriptor, __ HeapConstant(callable.code()), object, |
| ChangeInt32ToSmi(index), __ NoContextConstant()); |
| |
| // Ensure that we were able to grow the {elements}. |
| __ DeoptimizeIf(DeoptimizeReason::kCouldNotGrowElements, params.feedback(), |
| ObjectIsSmi(new_elements), frame_state); |
| __ Goto(&done, new_elements); |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| void EffectControlLinearizer::LowerTransitionElementsKind(Node* node) { |
| ElementsTransition const transition = ElementsTransitionOf(node->op()); |
| Node* object = node->InputAt(0); |
| |
| auto if_map_same = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(); |
| |
| Node* source_map = __ HeapConstant(transition.source()); |
| Node* target_map = __ HeapConstant(transition.target()); |
| |
| // Load the current map of {object}. |
| Node* object_map = __ LoadField(AccessBuilder::ForMap(), object); |
| |
| // Check if {object_map} is the same as {source_map}. |
| Node* check = __ TaggedEqual(object_map, source_map); |
| __ GotoIf(check, &if_map_same); |
| __ Goto(&done); |
| |
| __ Bind(&if_map_same); |
| switch (transition.mode()) { |
| case ElementsTransition::kFastTransition: |
| // In-place migration of {object}, just store the {target_map}. |
| __ StoreField(AccessBuilder::ForMap(), object, target_map); |
| break; |
| case ElementsTransition::kSlowTransition: { |
| // Instance migration, call out to the runtime for {object}. |
| Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow; |
| Runtime::FunctionId id = Runtime::kTransitionElementsKind; |
| auto call_descriptor = Linkage::GetRuntimeCallDescriptor( |
| graph()->zone(), id, 2, properties, CallDescriptor::kNoFlags); |
| __ Call(call_descriptor, __ CEntryStubConstant(1), object, target_map, |
| __ ExternalConstant(ExternalReference::Create(id)), |
| __ Int32Constant(2), __ NoContextConstant()); |
| break; |
| } |
| } |
| __ Goto(&done); |
| |
| __ Bind(&done); |
| } |
| |
| Node* EffectControlLinearizer::LowerLoadMessage(Node* node) { |
| Node* offset = node->InputAt(0); |
| Node* object_pattern = |
| __ LoadField(AccessBuilder::ForExternalIntPtr(), offset); |
| return __ BitcastWordToTagged(object_pattern); |
| } |
| |
| void EffectControlLinearizer::LowerStoreMessage(Node* node) { |
| Node* offset = node->InputAt(0); |
| Node* object = node->InputAt(1); |
| Node* object_pattern = __ BitcastTaggedToWord(object); |
| __ StoreField(AccessBuilder::ForExternalIntPtr(), offset, object_pattern); |
| } |
| |
| // TODO(mslekova): Avoid code duplication with simplified lowering. |
| static MachineType MachineTypeFor(CTypeInfo::Type type) { |
| switch (type) { |
| case CTypeInfo::Type::kVoid: |
| return MachineType::AnyTagged(); |
| case CTypeInfo::Type::kBool: |
| return MachineType::Bool(); |
| case CTypeInfo::Type::kInt32: |
| return MachineType::Int32(); |
| case CTypeInfo::Type::kUint32: |
| return MachineType::Uint32(); |
| case CTypeInfo::Type::kInt64: |
| return MachineType::Int64(); |
| case CTypeInfo::Type::kUint64: |
| return MachineType::Uint64(); |
| case CTypeInfo::Type::kFloat32: |
| return MachineType::Float32(); |
| case CTypeInfo::Type::kFloat64: |
| return MachineType::Float64(); |
| case CTypeInfo::Type::kV8Value: |
| return MachineType::AnyTagged(); |
| } |
| } |
| |
| Node* EffectControlLinearizer::LowerFastApiCall(Node* node) { |
| FastApiCallNode n(node); |
| FastApiCallParameters const& params = n.Parameters(); |
| const CFunctionInfo* c_signature = params.signature(); |
| const int c_arg_count = c_signature->ArgumentCount(); |
| CallDescriptor* js_call_descriptor = params.descriptor(); |
| int js_arg_count = static_cast<int>(js_call_descriptor->ParameterCount()); |
| const int value_input_count = node->op()->ValueInputCount(); |
| CHECK_EQ(FastApiCallNode::ArityForArgc(c_arg_count, js_arg_count), |
| value_input_count); |
| |
| if (fast_api_call_stack_slot_ == nullptr) { |
| // Add the { fallback } output parameter. |
| int kAlign = 4; |
| int kSize = sizeof(v8::FastApiCallbackOptions); |
| // If this check fails, probably you've added new fields to |
| // v8::FastApiCallbackOptions, which means you'll need to write code |
| // that initializes and reads from them too (see the Store and Load to |
| // fast_api_call_stack_slot_ below). |
| CHECK_EQ(kSize, 1); |
| fast_api_call_stack_slot_ = __ StackSlot(kSize, kAlign); |
| } |
| |
| // Generate the store to `fast_api_call_stack_slot_`. |
| __ Store(StoreRepresentation(MachineRepresentation::kWord32, kNoWriteBarrier), |
| fast_api_call_stack_slot_, 0, jsgraph()->ZeroConstant()); |
| |
| MachineSignature::Builder builder( |
| graph()->zone(), 1, c_arg_count + FastApiCallNode::kHasErrorInputCount); |
| MachineType return_type = MachineTypeFor(c_signature->ReturnInfo().GetType()); |
| builder.AddReturn(return_type); |
| for (int i = 0; i < c_arg_count; ++i) { |
| MachineType machine_type = |
| MachineTypeFor(c_signature->ArgumentInfo(i).GetType()); |
| builder.AddParam(machine_type); |
| } |
| builder.AddParam(MachineType::Pointer()); // fast_api_call_stack_slot_ |
| |
| CallDescriptor* call_descriptor = |
| Linkage::GetSimplifiedCDescriptor(graph()->zone(), builder.Build()); |
| |
| call_descriptor->SetCFunctionInfo(c_signature); |
| |
| Node** const inputs = graph()->zone()->NewArray<Node*>( |
| c_arg_count + FastApiCallNode::kFastCallExtraInputCount); |
| inputs[0] = NodeProperties::GetValueInput(node, 0); // the target |
| for (int i = FastApiCallNode::kFastTargetInputCount; |
| i < c_arg_count + FastApiCallNode::kFastTargetInputCount; ++i) { |
| if (c_signature->ArgumentInfo(i - 1).GetType() == |
| CTypeInfo::Type::kFloat32) { |
| inputs[i] = |
| __ TruncateFloat64ToFloat32(NodeProperties::GetValueInput(node, i)); |
| } else { |
| inputs[i] = NodeProperties::GetValueInput(node, i); |
| } |
| } |
| inputs[c_arg_count + 1] = fast_api_call_stack_slot_; |
| inputs[c_arg_count + 2] = __ effect(); |
| inputs[c_arg_count + 3] = __ control(); |
| |
| __ Call(call_descriptor, |
| c_arg_count + FastApiCallNode::kFastCallExtraInputCount, inputs); |
| |
| // Generate the load from `fast_api_call_stack_slot_`. |
| Node* load = __ Load(MachineType::Int32(), fast_api_call_stack_slot_, 0); |
| |
| TNode<Boolean> cond = |
| TNode<Boolean>::UncheckedCast(__ Word32Equal(load, __ Int32Constant(0))); |
| // Hint to true. |
| auto if_success = __ MakeLabel(); |
| auto if_error = __ MakeDeferredLabel(); |
| auto merge = __ MakeLabel(MachineRepresentation::kTagged); |
| __ Branch(cond, &if_success, &if_error); |
| |
| // Generate fast call. |
| __ Bind(&if_success); |
| Node* then_result = [&]() { return __ UndefinedConstant(); }(); |
| __ Goto(&merge, then_result); |
| |
| // Generate direct slow call. |
| __ Bind(&if_error); |
| Node* else_result = [&]() { |
| Node** const slow_inputs = graph()->zone()->NewArray<Node*>( |
| n.SlowCallArgumentCount() + |
| FastApiCallNode::kEffectAndControlInputCount); |
| |
| int fast_call_params = c_arg_count + FastApiCallNode::kFastTargetInputCount; |
| CHECK_EQ(value_input_count - fast_call_params, n.SlowCallArgumentCount()); |
| int index = 0; |
| for (; index < n.SlowCallArgumentCount(); ++index) { |
| slow_inputs[index] = n.SlowCallArgument(index); |
| } |
| |
| slow_inputs[index] = __ effect(); |
| slow_inputs[index + 1] = __ control(); |
| Node* slow_call = __ Call( |
| params.descriptor(), |
| index + FastApiCallNode::kEffectAndControlInputCount, slow_inputs); |
| return slow_call; |
| }(); |
| __ Goto(&merge, else_result); |
| |
| __ Bind(&merge); |
| return merge.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerLoadFieldByIndex(Node* node) { |
| Node* object = node->InputAt(0); |
| Node* index = node->InputAt(1); |
| Node* zero = __ IntPtrConstant(0); |
| Node* one = __ IntPtrConstant(1); |
| |
| // Sign-extend the {index} on 64-bit architectures. |
| if (machine()->Is64()) { |
| index = __ ChangeInt32ToInt64(index); |
| } |
| |
| auto if_double = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kTagged); |
| |
| // Check if field is a mutable double field. |
| __ GotoIfNot(__ IntPtrEqual(__ WordAnd(index, one), zero), &if_double); |
| |
| // The field is a proper Tagged field on {object}. The {index} is shifted |
| // to the left by one in the code below. |
| { |
| // Check if field is in-object or out-of-object. |
| auto if_outofobject = __ MakeLabel(); |
| __ GotoIf(__ IntLessThan(index, zero), &if_outofobject); |
| |
| // The field is located in the {object} itself. |
| { |
| Node* offset = |
| __ IntAdd(__ WordShl(index, __ IntPtrConstant(kTaggedSizeLog2 - 1)), |
| __ IntPtrConstant(JSObject::kHeaderSize - kHeapObjectTag)); |
| Node* result = __ Load(MachineType::AnyTagged(), object, offset); |
| __ Goto(&done, result); |
| } |
| |
| // The field is located in the properties backing store of {object}. |
| // The {index} is equal to the negated out of property index plus 1. |
| __ Bind(&if_outofobject); |
| { |
| Node* properties = __ LoadField( |
| AccessBuilder::ForJSObjectPropertiesOrHashKnownPointer(), object); |
| Node* offset = |
| __ IntAdd(__ WordShl(__ IntSub(zero, index), |
| __ IntPtrConstant(kTaggedSizeLog2 - 1)), |
| __ IntPtrConstant((FixedArray::kHeaderSize - kTaggedSize) - |
| kHeapObjectTag)); |
| Node* result = __ Load(MachineType::AnyTagged(), properties, offset); |
| __ Goto(&done, result); |
| } |
| } |
| |
| // The field is a Double field, either unboxed in the object on 64-bit |
| // architectures, or a mutable HeapNumber. |
| __ Bind(&if_double); |
| { |
| auto loaded_field = __ MakeLabel(MachineRepresentation::kTagged); |
| auto done_double = __ MakeLabel(MachineRepresentation::kFloat64); |
| |
| index = __ WordSar(index, one); |
| |
| // Check if field is in-object or out-of-object. |
| auto if_outofobject = __ MakeLabel(); |
| __ GotoIf(__ IntLessThan(index, zero), &if_outofobject); |
| |
| // The field is located in the {object} itself. |
| { |
| Node* offset = |
| __ IntAdd(__ WordShl(index, __ IntPtrConstant(kTaggedSizeLog2)), |
| __ IntPtrConstant(JSObject::kHeaderSize - kHeapObjectTag)); |
| if (FLAG_unbox_double_fields) { |
| Node* result = __ Load(MachineType::Float64(), object, offset); |
| __ Goto(&done_double, result); |
| } else { |
| Node* field = __ Load(MachineType::AnyTagged(), object, offset); |
| __ Goto(&loaded_field, field); |
| } |
| } |
| |
| __ Bind(&if_outofobject); |
| { |
| Node* properties = __ LoadField( |
| AccessBuilder::ForJSObjectPropertiesOrHashKnownPointer(), object); |
| Node* offset = |
| __ IntAdd(__ WordShl(__ IntSub(zero, index), |
| __ IntPtrConstant(kTaggedSizeLog2)), |
| __ IntPtrConstant((FixedArray::kHeaderSize - kTaggedSize) - |
| kHeapObjectTag)); |
| Node* field = __ Load(MachineType::AnyTagged(), properties, offset); |
| __ Goto(&loaded_field, field); |
| } |
| |
| __ Bind(&loaded_field); |
| { |
| Node* field = loaded_field.PhiAt(0); |
| // We may have transitioned in-place away from double, so check that |
| // this is a HeapNumber -- otherwise the load is fine and we don't need |
| // to copy anything anyway. |
| __ GotoIf(ObjectIsSmi(field), &done, field); |
| Node* field_map = __ LoadField(AccessBuilder::ForMap(), field); |
| __ GotoIfNot(__ TaggedEqual(field_map, __ HeapNumberMapConstant()), &done, |
| field); |
| |
| Node* value = __ LoadField(AccessBuilder::ForHeapNumberValue(), field); |
| __ Goto(&done_double, value); |
| } |
| |
| __ Bind(&done_double); |
| { |
| Node* result = AllocateHeapNumberWithValue(done_double.PhiAt(0)); |
| __ Goto(&done, result); |
| } |
| } |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::BuildReverseBytes(ExternalArrayType type, |
| Node* value) { |
| switch (type) { |
| case kExternalInt8Array: |
| case kExternalUint8Array: |
| case kExternalUint8ClampedArray: |
| return value; |
| |
| case kExternalInt16Array: { |
| Node* result = __ Word32ReverseBytes(value); |
| result = __ Word32Sar(result, __ Int32Constant(16)); |
| return result; |
| } |
| |
| case kExternalUint16Array: { |
| Node* result = __ Word32ReverseBytes(value); |
| result = __ Word32Shr(result, __ Int32Constant(16)); |
| return result; |
| } |
| |
| case kExternalInt32Array: // Fall through. |
| case kExternalUint32Array: |
| return __ Word32ReverseBytes(value); |
| |
| case kExternalFloat32Array: { |
| Node* result = __ BitcastFloat32ToInt32(value); |
| result = __ Word32ReverseBytes(result); |
| result = __ BitcastInt32ToFloat32(result); |
| return result; |
| } |
| |
| case kExternalFloat64Array: { |
| if (machine()->Is64()) { |
| Node* result = __ BitcastFloat64ToInt64(value); |
| result = __ Word64ReverseBytes(result); |
| result = __ BitcastInt64ToFloat64(result); |
| return result; |
| } else { |
| Node* lo = __ Word32ReverseBytes(__ Float64ExtractLowWord32(value)); |
| Node* hi = __ Word32ReverseBytes(__ Float64ExtractHighWord32(value)); |
| Node* result = __ Float64Constant(0.0); |
| result = __ Float64InsertLowWord32(result, hi); |
| result = __ Float64InsertHighWord32(result, lo); |
| return result; |
| } |
| } |
| |
| case kExternalBigInt64Array: |
| case kExternalBigUint64Array: |
| UNREACHABLE(); |
| } |
| } |
| |
| Node* EffectControlLinearizer::LowerLoadDataViewElement(Node* node) { |
| ExternalArrayType element_type = ExternalArrayTypeOf(node->op()); |
| Node* object = node->InputAt(0); |
| Node* storage = node->InputAt(1); |
| Node* index = node->InputAt(2); |
| Node* is_little_endian = node->InputAt(3); |
| |
| // We need to keep the {object} (either the JSArrayBuffer or the JSDataView) |
| // alive so that the GC will not release the JSArrayBuffer (if there's any) |
| // as long as we are still operating on it. |
| __ Retain(object); |
| |
| MachineType const machine_type = |
| AccessBuilder::ForTypedArrayElement(element_type, true).machine_type; |
| |
| Node* value = __ LoadUnaligned(machine_type, storage, index); |
| auto big_endian = __ MakeLabel(); |
| auto done = __ MakeLabel(machine_type.representation()); |
| |
| __ GotoIfNot(is_little_endian, &big_endian); |
| { // Little-endian load. |
| #if V8_TARGET_LITTLE_ENDIAN |
| __ Goto(&done, value); |
| #else |
| __ Goto(&done, BuildReverseBytes(element_type, value)); |
| #endif // V8_TARGET_LITTLE_ENDIAN |
| } |
| |
| __ Bind(&big_endian); |
| { // Big-endian load. |
| #if V8_TARGET_LITTLE_ENDIAN |
| __ Goto(&done, BuildReverseBytes(element_type, value)); |
| #else |
| __ Goto(&done, value); |
| #endif // V8_TARGET_LITTLE_ENDIAN |
| } |
| |
| // We're done, return {result}. |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| void EffectControlLinearizer::LowerStoreDataViewElement(Node* node) { |
| ExternalArrayType element_type = ExternalArrayTypeOf(node->op()); |
| Node* object = node->InputAt(0); |
| Node* storage = node->InputAt(1); |
| Node* index = node->InputAt(2); |
| Node* value = node->InputAt(3); |
| Node* is_little_endian = node->InputAt(4); |
| |
| // We need to keep the {object} (either the JSArrayBuffer or the JSDataView) |
| // alive so that the GC will not release the JSArrayBuffer (if there's any) |
| // as long as we are still operating on it. |
| __ Retain(object); |
| |
| MachineType const machine_type = |
| AccessBuilder::ForTypedArrayElement(element_type, true).machine_type; |
| |
| auto big_endian = __ MakeLabel(); |
| auto done = __ MakeLabel(machine_type.representation()); |
| |
| __ GotoIfNot(is_little_endian, &big_endian); |
| { // Little-endian store. |
| #if V8_TARGET_LITTLE_ENDIAN |
| __ Goto(&done, value); |
| #else |
| __ Goto(&done, BuildReverseBytes(element_type, value)); |
| #endif // V8_TARGET_LITTLE_ENDIAN |
| } |
| |
| __ Bind(&big_endian); |
| { // Big-endian store. |
| #if V8_TARGET_LITTLE_ENDIAN |
| __ Goto(&done, BuildReverseBytes(element_type, value)); |
| #else |
| __ Goto(&done, value); |
| #endif // V8_TARGET_LITTLE_ENDIAN |
| } |
| |
| __ Bind(&done); |
| __ StoreUnaligned(machine_type.representation(), storage, index, |
| done.PhiAt(0)); |
| } |
| |
| // Compute the data pointer, handling the case where the {external} pointer |
| // is the effective data pointer (i.e. the {base} is Smi zero). |
| Node* EffectControlLinearizer::BuildTypedArrayDataPointer(Node* base, |
| Node* external) { |
| if (IntPtrMatcher(base).Is(0)) { |
| return external; |
| } else { |
| if (COMPRESS_POINTERS_BOOL) { |
| base = __ BitcastTaggedToWord(base); |
| // Zero-extend Tagged_t to UintPtr according to current compression |
| // scheme so that the addition with |external_pointer| (which already |
| // contains compensated offset value) will decompress the tagged value. |
| // See JSTypedArray::ExternalPointerCompensationForOnHeapArray() for |
| // details. |
| base = ChangeUint32ToUintPtr(base); |
| } |
| return __ UnsafePointerAdd(base, external); |
| } |
| } |
| |
| Node* EffectControlLinearizer::LowerLoadTypedElement(Node* node) { |
| ExternalArrayType array_type = ExternalArrayTypeOf(node->op()); |
| Node* buffer = node->InputAt(0); |
| Node* base = node->InputAt(1); |
| Node* external = node->InputAt(2); |
| Node* index = node->InputAt(3); |
| |
| // We need to keep the {buffer} alive so that the GC will not release the |
| // ArrayBuffer (if there's any) as long as we are still operating on it. |
| __ Retain(buffer); |
| |
| Node* data_ptr = BuildTypedArrayDataPointer(base, external); |
| |
| // Perform the actual typed element access. |
| return __ LoadElement(AccessBuilder::ForTypedArrayElement( |
| array_type, true, LoadSensitivity::kCritical), |
| data_ptr, index); |
| } |
| |
| Node* EffectControlLinearizer::LowerLoadStackArgument(Node* node) { |
| Node* base = node->InputAt(0); |
| Node* index = node->InputAt(1); |
| |
| Node* argument = |
| __ LoadElement(AccessBuilder::ForStackArgument(), base, index); |
| |
| return __ BitcastWordToTagged(argument); |
| } |
| |
| void EffectControlLinearizer::LowerStoreTypedElement(Node* node) { |
| ExternalArrayType array_type = ExternalArrayTypeOf(node->op()); |
| Node* buffer = node->InputAt(0); |
| Node* base = node->InputAt(1); |
| Node* external = node->InputAt(2); |
| Node* index = node->InputAt(3); |
| Node* value = node->InputAt(4); |
| |
| // We need to keep the {buffer} alive so that the GC will not release the |
| // ArrayBuffer (if there's any) as long as we are still operating on it. |
| __ Retain(buffer); |
| |
| Node* data_ptr = BuildTypedArrayDataPointer(base, external); |
| |
| // Perform the actual typed element access. |
| __ StoreElement(AccessBuilder::ForTypedArrayElement(array_type, true), |
| data_ptr, index, value); |
| } |
| |
| void EffectControlLinearizer::TransitionElementsTo(Node* node, Node* array, |
| ElementsKind from, |
| ElementsKind to) { |
| DCHECK(IsMoreGeneralElementsKindTransition(from, to)); |
| DCHECK(to == HOLEY_ELEMENTS || to == HOLEY_DOUBLE_ELEMENTS); |
| |
| Handle<Map> target(to == HOLEY_ELEMENTS ? FastMapParameterOf(node->op()) |
| : DoubleMapParameterOf(node->op())); |
| Node* target_map = __ HeapConstant(target); |
| |
| if (IsSimpleMapChangeTransition(from, to)) { |
| __ StoreField(AccessBuilder::ForMap(), array, target_map); |
| } else { |
| // Instance migration, call out to the runtime for {array}. |
| Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow; |
| Runtime::FunctionId id = Runtime::kTransitionElementsKind; |
| auto call_descriptor = Linkage::GetRuntimeCallDescriptor( |
| graph()->zone(), id, 2, properties, CallDescriptor::kNoFlags); |
| __ Call(call_descriptor, __ CEntryStubConstant(1), array, target_map, |
| __ ExternalConstant(ExternalReference::Create(id)), |
| __ Int32Constant(2), __ NoContextConstant()); |
| } |
| } |
| |
| Node* EffectControlLinearizer::IsElementsKindGreaterThan( |
| Node* kind, ElementsKind reference_kind) { |
| Node* ref_kind = __ Int32Constant(reference_kind); |
| Node* ret = __ Int32LessThan(ref_kind, kind); |
| return ret; |
| } |
| |
| void EffectControlLinearizer::LowerTransitionAndStoreElement(Node* node) { |
| Node* array = node->InputAt(0); |
| Node* index = node->InputAt(1); |
| Node* value = node->InputAt(2); |
| |
| // Possibly transition array based on input and store. |
| // |
| // -- TRANSITION PHASE ----------------- |
| // kind = ElementsKind(array) |
| // if value is not smi { |
| // if kind == HOLEY_SMI_ELEMENTS { |
| // if value is heap number { |
| // Transition array to HOLEY_DOUBLE_ELEMENTS |
| // kind = HOLEY_DOUBLE_ELEMENTS |
| // } else { |
| // Transition array to HOLEY_ELEMENTS |
| // kind = HOLEY_ELEMENTS |
| // } |
| // } else if kind == HOLEY_DOUBLE_ELEMENTS { |
| // if value is not heap number { |
| // Transition array to HOLEY_ELEMENTS |
| // kind = HOLEY_ELEMENTS |
| // } |
| // } |
| // } |
| // |
| // -- STORE PHASE ---------------------- |
| // [make sure {kind} is up-to-date] |
| // if kind == HOLEY_DOUBLE_ELEMENTS { |
| // if value is smi { |
| // float_value = convert smi to float |
| // Store array[index] = float_value |
| // } else { |
| // float_value = value |
| // Store array[index] = float_value |
| // } |
| // } else { |
| // // kind is HOLEY_SMI_ELEMENTS or HOLEY_ELEMENTS |
| // Store array[index] = value |
| // } |
| // |
| Node* map = __ LoadField(AccessBuilder::ForMap(), array); |
| Node* kind; |
| { |
| Node* bit_field2 = __ LoadField(AccessBuilder::ForMapBitField2(), map); |
| Node* mask = __ Int32Constant(Map::Bits2::ElementsKindBits::kMask); |
| Node* andit = __ Word32And(bit_field2, mask); |
| Node* shift = __ Int32Constant(Map::Bits2::ElementsKindBits::kShift); |
| kind = __ Word32Shr(andit, shift); |
| } |
| |
| auto do_store = __ MakeLabel(MachineRepresentation::kWord32); |
| // We can store a smi anywhere. |
| __ GotoIf(ObjectIsSmi(value), &do_store, kind); |
| |
| // {value} is a HeapObject. |
| auto transition_smi_array = __ MakeDeferredLabel(); |
| auto transition_double_to_fast = __ MakeDeferredLabel(); |
| { |
| __ GotoIfNot(IsElementsKindGreaterThan(kind, HOLEY_SMI_ELEMENTS), |
| &transition_smi_array); |
| __ GotoIfNot(IsElementsKindGreaterThan(kind, HOLEY_ELEMENTS), &do_store, |
| kind); |
| |
| // We have double elements kind. Only a HeapNumber can be stored |
| // without effecting a transition. |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| Node* heap_number_map = __ HeapNumberMapConstant(); |
| Node* check = __ TaggedEqual(value_map, heap_number_map); |
| __ GotoIfNot(check, &transition_double_to_fast); |
| __ Goto(&do_store, kind); |
| } |
| |
| __ Bind(&transition_smi_array); // deferred code. |
| { |
| // Transition {array} from HOLEY_SMI_ELEMENTS to HOLEY_DOUBLE_ELEMENTS or |
| // to HOLEY_ELEMENTS. |
| auto if_value_not_heap_number = __ MakeLabel(); |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| Node* heap_number_map = __ HeapNumberMapConstant(); |
| Node* check = __ TaggedEqual(value_map, heap_number_map); |
| __ GotoIfNot(check, &if_value_not_heap_number); |
| { |
| // {value} is a HeapNumber. |
| TransitionElementsTo(node, array, HOLEY_SMI_ELEMENTS, |
| HOLEY_DOUBLE_ELEMENTS); |
| __ Goto(&do_store, __ Int32Constant(HOLEY_DOUBLE_ELEMENTS)); |
| } |
| __ Bind(&if_value_not_heap_number); |
| { |
| TransitionElementsTo(node, array, HOLEY_SMI_ELEMENTS, HOLEY_ELEMENTS); |
| __ Goto(&do_store, __ Int32Constant(HOLEY_ELEMENTS)); |
| } |
| } |
| |
| __ Bind(&transition_double_to_fast); // deferred code. |
| { |
| TransitionElementsTo(node, array, HOLEY_DOUBLE_ELEMENTS, HOLEY_ELEMENTS); |
| __ Goto(&do_store, __ Int32Constant(HOLEY_ELEMENTS)); |
| } |
| |
| // Make sure kind is up-to-date. |
| __ Bind(&do_store); |
| kind = do_store.PhiAt(0); |
| |
| Node* elements = __ LoadField(AccessBuilder::ForJSObjectElements(), array); |
| auto if_kind_is_double = __ MakeLabel(); |
| auto done = __ MakeLabel(); |
| __ GotoIf(IsElementsKindGreaterThan(kind, HOLEY_ELEMENTS), |
| &if_kind_is_double); |
| { |
| // Our ElementsKind is HOLEY_SMI_ELEMENTS or HOLEY_ELEMENTS. |
| __ StoreElement(AccessBuilder::ForFixedArrayElement(HOLEY_ELEMENTS), |
| elements, index, value); |
| __ Goto(&done); |
| } |
| __ Bind(&if_kind_is_double); |
| { |
| // Our ElementsKind is HOLEY_DOUBLE_ELEMENTS. |
| auto do_double_store = __ MakeLabel(); |
| __ GotoIfNot(ObjectIsSmi(value), &do_double_store); |
| { |
| Node* int_value = ChangeSmiToInt32(value); |
| Node* float_value = __ ChangeInt32ToFloat64(int_value); |
| __ StoreElement(AccessBuilder::ForFixedDoubleArrayElement(), elements, |
| index, float_value); |
| __ Goto(&done); |
| } |
| __ Bind(&do_double_store); |
| { |
| Node* float_value = |
| __ LoadField(AccessBuilder::ForHeapNumberValue(), value); |
| __ StoreElement(AccessBuilder::ForFixedDoubleArrayElement(), elements, |
| index, __ Float64SilenceNaN(float_value)); |
| __ Goto(&done); |
| } |
| } |
| |
| __ Bind(&done); |
| } |
| |
| void EffectControlLinearizer::LowerTransitionAndStoreNumberElement(Node* node) { |
| Node* array = node->InputAt(0); |
| Node* index = node->InputAt(1); |
| Node* value = node->InputAt(2); // This is a Float64, not tagged. |
| |
| // Possibly transition array based on input and store. |
| // |
| // -- TRANSITION PHASE ----------------- |
| // kind = ElementsKind(array) |
| // if kind == HOLEY_SMI_ELEMENTS { |
| // Transition array to HOLEY_DOUBLE_ELEMENTS |
| // } else if kind != HOLEY_DOUBLE_ELEMENTS { |
| // This is UNREACHABLE, execute a debug break. |
| // } |
| // |
| // -- STORE PHASE ---------------------- |
| // Store array[index] = value (it's a float) |
| // |
| Node* map = __ LoadField(AccessBuilder::ForMap(), array); |
| Node* kind; |
| { |
| Node* bit_field2 = __ LoadField(AccessBuilder::ForMapBitField2(), map); |
| Node* mask = __ Int32Constant(Map::Bits2::ElementsKindBits::kMask); |
| Node* andit = __ Word32And(bit_field2, mask); |
| Node* shift = __ Int32Constant(Map::Bits2::ElementsKindBits::kShift); |
| kind = __ Word32Shr(andit, shift); |
| } |
| |
| auto do_store = __ MakeLabel(); |
| |
| // {value} is a float64. |
| auto transition_smi_array = __ MakeDeferredLabel(); |
| { |
| __ GotoIfNot(IsElementsKindGreaterThan(kind, HOLEY_SMI_ELEMENTS), |
| &transition_smi_array); |
| // We expect that our input array started at HOLEY_SMI_ELEMENTS, and |
| // climbs the lattice up to HOLEY_DOUBLE_ELEMENTS. Force a debug break |
| // if this assumption is broken. It also would be the case that |
| // loop peeling can break this assumption. |
| __ GotoIf(__ Word32Equal(kind, __ Int32Constant(HOLEY_DOUBLE_ELEMENTS)), |
| &do_store); |
| __ Unreachable(&do_store); |
| } |
| |
| __ Bind(&transition_smi_array); // deferred code. |
| { |
| // Transition {array} from HOLEY_SMI_ELEMENTS to HOLEY_DOUBLE_ELEMENTS. |
| TransitionElementsTo(node, array, HOLEY_SMI_ELEMENTS, |
| HOLEY_DOUBLE_ELEMENTS); |
| __ Goto(&do_store); |
| } |
| |
| __ Bind(&do_store); |
| |
| Node* elements = __ LoadField(AccessBuilder::ForJSObjectElements(), array); |
| __ StoreElement(AccessBuilder::ForFixedDoubleArrayElement(), elements, index, |
| __ Float64SilenceNaN(value)); |
| } |
| |
| void EffectControlLinearizer::LowerTransitionAndStoreNonNumberElement( |
| Node* node) { |
| Node* array = node->InputAt(0); |
| Node* index = node->InputAt(1); |
| Node* value = node->InputAt(2); |
| |
| // Possibly transition array based on input and store. |
| // |
| // -- TRANSITION PHASE ----------------- |
| // kind = ElementsKind(array) |
| // if kind == HOLEY_SMI_ELEMENTS { |
| // Transition array to HOLEY_ELEMENTS |
| // } else if kind == HOLEY_DOUBLE_ELEMENTS { |
| // Transition array to HOLEY_ELEMENTS |
| // } |
| // |
| // -- STORE PHASE ---------------------- |
| // // kind is HOLEY_ELEMENTS |
| // Store array[index] = value |
| // |
| Node* map = __ LoadField(AccessBuilder::ForMap(), array); |
| Node* kind; |
| { |
| Node* bit_field2 = __ LoadField(AccessBuilder::ForMapBitField2(), map); |
| Node* mask = __ Int32Constant(Map::Bits2::ElementsKindBits::kMask); |
| Node* andit = __ Word32And(bit_field2, mask); |
| Node* shift = __ Int32Constant(Map::Bits2::ElementsKindBits::kShift); |
| kind = __ Word32Shr(andit, shift); |
| } |
| |
| auto do_store = __ MakeLabel(); |
| |
| auto transition_smi_array = __ MakeDeferredLabel(); |
| auto transition_double_to_fast = __ MakeDeferredLabel(); |
| { |
| __ GotoIfNot(IsElementsKindGreaterThan(kind, HOLEY_SMI_ELEMENTS), |
| &transition_smi_array); |
| __ GotoIf(IsElementsKindGreaterThan(kind, HOLEY_ELEMENTS), |
| &transition_double_to_fast); |
| __ Goto(&do_store); |
| } |
| |
| __ Bind(&transition_smi_array); // deferred code. |
| { |
| // Transition {array} from HOLEY_SMI_ELEMENTS to HOLEY_ELEMENTS. |
| TransitionElementsTo(node, array, HOLEY_SMI_ELEMENTS, HOLEY_ELEMENTS); |
| __ Goto(&do_store); |
| } |
| |
| __ Bind(&transition_double_to_fast); // deferred code. |
| { |
| TransitionElementsTo(node, array, HOLEY_DOUBLE_ELEMENTS, HOLEY_ELEMENTS); |
| __ Goto(&do_store); |
| } |
| |
| __ Bind(&do_store); |
| |
| Node* elements = __ LoadField(AccessBuilder::ForJSObjectElements(), array); |
| // Our ElementsKind is HOLEY_ELEMENTS. |
| ElementAccess access = AccessBuilder::ForFixedArrayElement(HOLEY_ELEMENTS); |
| Type value_type = ValueTypeParameterOf(node->op()); |
| if (value_type.Is(Type::BooleanOrNullOrUndefined())) { |
| access.type = value_type; |
| access.write_barrier_kind = kNoWriteBarrier; |
| } |
| __ StoreElement(access, elements, index, value); |
| } |
| |
| void EffectControlLinearizer::LowerStoreSignedSmallElement(Node* node) { |
| Node* array = node->InputAt(0); |
| Node* index = node->InputAt(1); |
| Node* value = node->InputAt(2); // int32 |
| |
| // Store a signed small in an output array. |
| // |
| // kind = ElementsKind(array) |
| // |
| // -- STORE PHASE ---------------------- |
| // if kind == HOLEY_DOUBLE_ELEMENTS { |
| // float_value = convert int32 to float |
| // Store array[index] = float_value |
| // } else { |
| // // kind is HOLEY_SMI_ELEMENTS or HOLEY_ELEMENTS |
| // smi_value = convert int32 to smi |
| // Store array[index] = smi_value |
| // } |
| // |
| Node* map = __ LoadField(AccessBuilder::ForMap(), array); |
| Node* kind; |
| { |
| Node* bit_field2 = __ LoadField(AccessBuilder::ForMapBitField2(), map); |
| Node* mask = __ Int32Constant(Map::Bits2::ElementsKindBits::kMask); |
| Node* andit = __ Word32And(bit_field2, mask); |
| Node* shift = __ Int32Constant(Map::Bits2::ElementsKindBits::kShift); |
| kind = __ Word32Shr(andit, shift); |
| } |
| |
| Node* elements = __ LoadField(AccessBuilder::ForJSObjectElements(), array); |
| auto if_kind_is_double = __ MakeLabel(); |
| auto done = __ MakeLabel(); |
| __ GotoIf(IsElementsKindGreaterThan(kind, HOLEY_ELEMENTS), |
| &if_kind_is_double); |
| { |
| // Our ElementsKind is HOLEY_SMI_ELEMENTS or HOLEY_ELEMENTS. |
| // In this case, we know our value is a signed small, and we can optimize |
| // the ElementAccess information. |
| ElementAccess access = AccessBuilder::ForFixedArrayElement(); |
| access.type = Type::SignedSmall(); |
| access.machine_type = MachineType::TaggedSigned(); |
| access.write_barrier_kind = kNoWriteBarrier; |
| Node* smi_value = ChangeInt32ToSmi(value); |
| __ StoreElement(access, elements, index, smi_value); |
| __ Goto(&done); |
| } |
| __ Bind(&if_kind_is_double); |
| { |
| // Our ElementsKind is HOLEY_DOUBLE_ELEMENTS. |
| Node* float_value = __ ChangeInt32ToFloat64(value); |
| __ StoreElement(AccessBuilder::ForFixedDoubleArrayElement(), elements, |
| index, float_value); |
| __ Goto(&done); |
| } |
| |
| __ Bind(&done); |
| } |
| |
| void EffectControlLinearizer::LowerRuntimeAbort(Node* node) { |
| AbortReason reason = AbortReasonOf(node->op()); |
| Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow; |
| Runtime::FunctionId id = Runtime::kAbort; |
| auto call_descriptor = Linkage::GetRuntimeCallDescriptor( |
| graph()->zone(), id, 1, properties, CallDescriptor::kNoFlags); |
| __ Call(call_descriptor, __ CEntryStubConstant(1), |
| __ SmiConstant(static_cast<int>(reason)), |
| __ ExternalConstant(ExternalReference::Create(id)), |
| __ Int32Constant(1), __ NoContextConstant()); |
| } |
| |
| template <typename... Args> |
| Node* EffectControlLinearizer::CallBuiltin(Builtins::Name builtin, |
| Operator::Properties properties, |
| Args... args) { |
| Callable const callable = Builtins::CallableFor(isolate(), builtin); |
| auto call_descriptor = Linkage::GetStubCallDescriptor( |
| graph()->zone(), callable.descriptor(), |
| callable.descriptor().GetStackParameterCount(), CallDescriptor::kNoFlags, |
| properties); |
| return __ Call(call_descriptor, __ HeapConstant(callable.code()), args..., |
| __ NoContextConstant()); |
| } |
| |
| Node* EffectControlLinearizer::LowerAssertType(Node* node) { |
| DCHECK_EQ(node->opcode(), IrOpcode::kAssertType); |
| Type type = OpParameter<Type>(node->op()); |
| DCHECK(type.IsRange()); |
| auto range = type.AsRange(); |
| Node* const input = node->InputAt(0); |
| Node* const min = __ NumberConstant(range->Min()); |
| Node* const max = __ NumberConstant(range->Max()); |
| CallBuiltin(Builtins::kCheckNumberInRange, node->op()->properties(), input, |
| min, max, __ SmiConstant(node->id())); |
| return input; |
| } |
| |
| Node* EffectControlLinearizer::LowerFoldConstant(Node* node) { |
| DCHECK_EQ(node->opcode(), IrOpcode::kFoldConstant); |
| Node* original = node->InputAt(0); |
| Node* constant = node->InputAt(1); |
| CallBuiltin(Builtins::kCheckSameObject, node->op()->properties(), original, |
| constant); |
| return constant; |
| } |
| |
| Node* EffectControlLinearizer::LowerConvertReceiver(Node* node) { |
| ConvertReceiverMode const mode = ConvertReceiverModeOf(node->op()); |
| Node* value = node->InputAt(0); |
| Node* global_proxy = node->InputAt(1); |
| |
| switch (mode) { |
| case ConvertReceiverMode::kNullOrUndefined: { |
| return global_proxy; |
| } |
| case ConvertReceiverMode::kNotNullOrUndefined: { |
| auto convert_to_object = __ MakeDeferredLabel(); |
| auto done_convert = __ MakeLabel(MachineRepresentation::kTagged); |
| |
| // Check if {value} is already a JSReceiver. |
| __ GotoIf(ObjectIsSmi(value), &convert_to_object); |
| STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE); |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| Node* value_instance_type = |
| __ LoadField(AccessBuilder::ForMapInstanceType(), value_map); |
| Node* check = __ Uint32LessThan( |
| value_instance_type, __ Uint32Constant(FIRST_JS_RECEIVER_TYPE)); |
| __ GotoIf(check, &convert_to_object); |
| __ Goto(&done_convert, value); |
| |
| // Wrap the primitive {value} into a JSPrimitiveWrapper. |
| __ Bind(&convert_to_object); |
| Operator::Properties properties = Operator::kEliminatable; |
| Callable callable = Builtins::CallableFor(isolate(), Builtins::kToObject); |
| CallDescriptor::Flags flags = CallDescriptor::kNoFlags; |
| auto call_descriptor = Linkage::GetStubCallDescriptor( |
| graph()->zone(), callable.descriptor(), |
| callable.descriptor().GetStackParameterCount(), flags, properties); |
| Node* native_context = __ LoadField( |
| AccessBuilder::ForJSGlobalProxyNativeContext(), global_proxy); |
| Node* result = __ Call(call_descriptor, __ HeapConstant(callable.code()), |
| value, native_context); |
| __ Goto(&done_convert, result); |
| |
| __ Bind(&done_convert); |
| return done_convert.PhiAt(0); |
| } |
| case ConvertReceiverMode::kAny: { |
| auto convert_to_object = __ MakeDeferredLabel(); |
| auto convert_global_proxy = __ MakeDeferredLabel(); |
| auto done_convert = __ MakeLabel(MachineRepresentation::kTagged); |
| |
| // Check if {value} is already a JSReceiver, or null/undefined. |
| __ GotoIf(ObjectIsSmi(value), &convert_to_object); |
| STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE); |
| Node* value_map = __ LoadField(AccessBuilder::ForMap(), value); |
| Node* value_instance_type = |
| __ LoadField(AccessBuilder::ForMapInstanceType(), value_map); |
| Node* check = __ Uint32LessThan( |
| value_instance_type, __ Uint32Constant(FIRST_JS_RECEIVER_TYPE)); |
| __ GotoIf(check, &convert_to_object); |
| __ Goto(&done_convert, value); |
| |
| // Wrap the primitive {value} into a JSPrimitiveWrapper. |
| __ Bind(&convert_to_object); |
| __ GotoIf(__ TaggedEqual(value, __ UndefinedConstant()), |
| &convert_global_proxy); |
| __ GotoIf(__ TaggedEqual(value, __ NullConstant()), |
| &convert_global_proxy); |
| Operator::Properties properties = Operator::kEliminatable; |
| Callable callable = Builtins::CallableFor(isolate(), Builtins::kToObject); |
| CallDescriptor::Flags flags = CallDescriptor::kNoFlags; |
| auto call_descriptor = Linkage::GetStubCallDescriptor( |
| graph()->zone(), callable.descriptor(), |
| callable.descriptor().GetStackParameterCount(), flags, properties); |
| Node* native_context = __ LoadField( |
| AccessBuilder::ForJSGlobalProxyNativeContext(), global_proxy); |
| Node* result = __ Call(call_descriptor, __ HeapConstant(callable.code()), |
| value, native_context); |
| __ Goto(&done_convert, result); |
| |
| // Replace the {value} with the {global_proxy}. |
| __ Bind(&convert_global_proxy); |
| __ Goto(&done_convert, global_proxy); |
| |
| __ Bind(&done_convert); |
| return done_convert.PhiAt(0); |
| } |
| } |
| |
| UNREACHABLE(); |
| return nullptr; |
| } |
| |
| Maybe<Node*> EffectControlLinearizer::LowerFloat64RoundUp(Node* node) { |
| // Nothing to be done if a fast hardware instruction is available. |
| if (machine()->Float64RoundUp().IsSupported()) { |
| return Nothing<Node*>(); |
| } |
| |
| Node* const input = node->InputAt(0); |
| |
| // General case for ceil. |
| // |
| // if 0.0 < input then |
| // if 2^52 <= input then |
| // input |
| // else |
| // let temp1 = (2^52 + input) - 2^52 in |
| // if temp1 < input then |
| // temp1 + 1 |
| // else |
| // temp1 |
| // else |
| // if input == 0 then |
| // input |
| // else |
| // if input <= -2^52 then |
| // input |
| // else |
| // let temp1 = -0 - input in |
| // let temp2 = (2^52 + temp1) - 2^52 in |
| // let temp3 = (if temp1 < temp2 then temp2 - 1 else temp2) in |
| // -0 - temp3 |
| |
| auto if_not_positive = __ MakeDeferredLabel(); |
| auto if_greater_than_two_52 = __ MakeDeferredLabel(); |
| auto if_less_than_minus_two_52 = __ MakeDeferredLabel(); |
| auto if_zero = __ MakeDeferredLabel(); |
| auto done_temp3 = __ MakeLabel(MachineRepresentation::kFloat64); |
| auto done = __ MakeLabel(MachineRepresentation::kFloat64); |
| |
| Node* const zero = __ Float64Constant(0.0); |
| Node* const two_52 = __ Float64Constant(4503599627370496.0E0); |
| Node* const one = __ Float64Constant(1.0); |
| |
| Node* check0 = __ Float64LessThan(zero, input); |
| __ GotoIfNot(check0, &if_not_positive); |
| { |
| Node* check1 = __ Float64LessThanOrEqual(two_52, input); |
| __ GotoIf(check1, &if_greater_than_two_52); |
| { |
| Node* temp1 = __ Float64Sub(__ Float64Add(two_52, input), two_52); |
| __ GotoIfNot(__ Float64LessThan(temp1, input), &done, temp1); |
| __ Goto(&done, __ Float64Add(temp1, one)); |
| } |
| |
| __ Bind(&if_greater_than_two_52); |
| __ Goto(&done, input); |
| } |
| |
| __ Bind(&if_not_positive); |
| { |
| Node* check1 = __ Float64Equal(input, zero); |
| __ GotoIf(check1, &if_zero); |
| |
| Node* const minus_two_52 = __ Float64Constant(-4503599627370496.0E0); |
| Node* check2 = __ Float64LessThanOrEqual(input, minus_two_52); |
| __ GotoIf(check2, &if_less_than_minus_two_52); |
| |
| { |
| Node* const minus_zero = __ Float64Constant(-0.0); |
| Node* temp1 = __ Float64Sub(minus_zero, input); |
| Node* temp2 = __ Float64Sub(__ Float64Add(two_52, temp1), two_52); |
| Node* check3 = __ Float64LessThan(temp1, temp2); |
| __ GotoIfNot(check3, &done_temp3, temp2); |
| __ Goto(&done_temp3, __ Float64Sub(temp2, one)); |
| |
| __ Bind(&done_temp3); |
| Node* temp3 = done_temp3.PhiAt(0); |
| __ Goto(&done, __ Float64Sub(minus_zero, temp3)); |
| } |
| __ Bind(&if_less_than_minus_two_52); |
| __ Goto(&done, input); |
| |
| __ Bind(&if_zero); |
| __ Goto(&done, input); |
| } |
| __ Bind(&done); |
| return Just(done.PhiAt(0)); |
| } |
| |
| Node* EffectControlLinearizer::BuildFloat64RoundDown(Node* value) { |
| if (machine()->Float64RoundDown().IsSupported()) { |
| return __ Float64RoundDown(value); |
| } |
| |
| Node* const input = value; |
| |
| // General case for floor. |
| // |
| // if 0.0 < input then |
| // if 2^52 <= input then |
| // input |
| // else |
| // let temp1 = (2^52 + input) - 2^52 in |
| // if input < temp1 then |
| // temp1 - 1 |
| // else |
| // temp1 |
| // else |
| // if input == 0 then |
| // input |
| // else |
| // if input <= -2^52 then |
| // input |
| // else |
| // let temp1 = -0 - input in |
| // let temp2 = (2^52 + temp1) - 2^52 in |
| // if temp2 < temp1 then |
| // -1 - temp2 |
| // else |
| // -0 - temp2 |
| |
| auto if_not_positive = __ MakeDeferredLabel(); |
| auto if_greater_than_two_52 = __ MakeDeferredLabel(); |
| auto if_less_than_minus_two_52 = __ MakeDeferredLabel(); |
| auto if_temp2_lt_temp1 = __ MakeLabel(); |
| auto if_zero = __ MakeDeferredLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kFloat64); |
| |
| Node* const zero = __ Float64Constant(0.0); |
| Node* const two_52 = __ Float64Constant(4503599627370496.0E0); |
| |
| Node* check0 = __ Float64LessThan(zero, input); |
| __ GotoIfNot(check0, &if_not_positive); |
| { |
| Node* check1 = __ Float64LessThanOrEqual(two_52, input); |
| __ GotoIf(check1, &if_greater_than_two_52); |
| { |
| Node* const one = __ Float64Constant(1.0); |
| Node* temp1 = __ Float64Sub(__ Float64Add(two_52, input), two_52); |
| __ GotoIfNot(__ Float64LessThan(input, temp1), &done, temp1); |
| __ Goto(&done, __ Float64Sub(temp1, one)); |
| } |
| |
| __ Bind(&if_greater_than_two_52); |
| __ Goto(&done, input); |
| } |
| |
| __ Bind(&if_not_positive); |
| { |
| Node* check1 = __ Float64Equal(input, zero); |
| __ GotoIf(check1, &if_zero); |
| |
| Node* const minus_two_52 = __ Float64Constant(-4503599627370496.0E0); |
| Node* check2 = __ Float64LessThanOrEqual(input, minus_two_52); |
| __ GotoIf(check2, &if_less_than_minus_two_52); |
| |
| { |
| Node* const minus_zero = __ Float64Constant(-0.0); |
| Node* temp1 = __ Float64Sub(minus_zero, input); |
| Node* temp2 = __ Float64Sub(__ Float64Add(two_52, temp1), two_52); |
| Node* check3 = __ Float64LessThan(temp2, temp1); |
| __ GotoIf(check3, &if_temp2_lt_temp1); |
| __ Goto(&done, __ Float64Sub(minus_zero, temp2)); |
| |
| __ Bind(&if_temp2_lt_temp1); |
| __ Goto(&done, __ Float64Sub(__ Float64Constant(-1.0), temp2)); |
| } |
| __ Bind(&if_less_than_minus_two_52); |
| __ Goto(&done, input); |
| |
| __ Bind(&if_zero); |
| __ Goto(&done, input); |
| } |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Maybe<Node*> EffectControlLinearizer::LowerFloat64RoundDown(Node* node) { |
| // Nothing to be done if a fast hardware instruction is available. |
| if (machine()->Float64RoundDown().IsSupported()) { |
| return Nothing<Node*>(); |
| } |
| |
| Node* const input = node->InputAt(0); |
| return Just(BuildFloat64RoundDown(input)); |
| } |
| |
| Maybe<Node*> EffectControlLinearizer::LowerFloat64RoundTiesEven(Node* node) { |
| // Nothing to be done if a fast hardware instruction is available. |
| if (machine()->Float64RoundTiesEven().IsSupported()) { |
| return Nothing<Node*>(); |
| } |
| |
| Node* const input = node->InputAt(0); |
| |
| // Generate case for round ties to even: |
| // |
| // let value = floor(input) in |
| // let temp1 = input - value in |
| // if temp1 < 0.5 then |
| // value |
| // else if 0.5 < temp1 then |
| // value + 1.0 |
| // else |
| // let temp2 = value % 2.0 in |
| // if temp2 == 0.0 then |
| // value |
| // else |
| // value + 1.0 |
| |
| auto if_is_half = __ MakeLabel(); |
| auto done = __ MakeLabel(MachineRepresentation::kFloat64); |
| |
| Node* value = BuildFloat64RoundDown(input); |
| Node* temp1 = __ Float64Sub(input, value); |
| |
| Node* const half = __ Float64Constant(0.5); |
| Node* check0 = __ Float64LessThan(temp1, half); |
| __ GotoIf(check0, &done, value); |
| |
| Node* const one = __ Float64Constant(1.0); |
| Node* check1 = __ Float64LessThan(half, temp1); |
| __ GotoIfNot(check1, &if_is_half); |
| __ Goto(&done, __ Float64Add(value, one)); |
| |
| __ Bind(&if_is_half); |
| Node* temp2 = __ Float64Mod(value, __ Float64Constant(2.0)); |
| Node* check2 = __ Float64Equal(temp2, __ Float64Constant(0.0)); |
| __ GotoIf(check2, &done, value); |
| __ Goto(&done, __ Float64Add(value, one)); |
| |
| __ Bind(&done); |
| return Just(done.PhiAt(0)); |
| } |
| |
| Node* EffectControlLinearizer::BuildFloat64RoundTruncate(Node* input) { |
| if (machine()->Float64RoundTruncate().IsSupported()) { |
| return __ Float64RoundTruncate(input); |
| } |
| // General case for trunc. |
| // |
| // if 0.0 < input then |
| // if 2^52 <= input then |
| // input |
| // else |
| // let temp1 = (2^52 + input) - 2^52 in |
| // if input < temp1 then |
| // temp1 - 1 |
| // else |
| // temp1 |
| // else |
| // if input == 0 then |
| // input |
| // else |
| // if input <= -2^52 then |
| // input |
| // else |
| // let temp1 = -0 - input in |
| // let temp2 = (2^52 + temp1) - 2^52 in |
| // let temp3 = (if temp1 < temp2 then temp2 - 1 else temp2) in |
| // -0 - temp3 |
| // |
| // Note: We do not use the Diamond helper class here, because it really hurts |
| // readability with nested diamonds. |
| |
| auto if_not_positive = __ MakeDeferredLabel(); |
| auto if_greater_than_two_52 = __ MakeDeferredLabel(); |
| auto if_less_than_minus_two_52 = __ MakeDeferredLabel(); |
| auto if_zero = __ MakeDeferredLabel(); |
| auto done_temp3 = __ MakeLabel(MachineRepresentation::kFloat64); |
| auto done = __ MakeLabel(MachineRepresentation::kFloat64); |
| |
| Node* const zero = __ Float64Constant(0.0); |
| Node* const two_52 = __ Float64Constant(4503599627370496.0E0); |
| Node* const one = __ Float64Constant(1.0); |
| |
| Node* check0 = __ Float64LessThan(zero, input); |
| __ GotoIfNot(check0, &if_not_positive); |
| { |
| Node* check1 = __ Float64LessThanOrEqual(two_52, input); |
| __ GotoIf(check1, &if_greater_than_two_52); |
| { |
| Node* temp1 = __ Float64Sub(__ Float64Add(two_52, input), two_52); |
| __ GotoIfNot(__ Float64LessThan(input, temp1), &done, temp1); |
| __ Goto(&done, __ Float64Sub(temp1, one)); |
| } |
| |
| __ Bind(&if_greater_than_two_52); |
| __ Goto(&done, input); |
| } |
| |
| __ Bind(&if_not_positive); |
| { |
| Node* check1 = __ Float64Equal(input, zero); |
| __ GotoIf(check1, &if_zero); |
| |
| Node* const minus_two_52 = __ Float64Constant(-4503599627370496.0E0); |
| Node* check2 = __ Float64LessThanOrEqual(input, minus_two_52); |
| __ GotoIf(check2, &if_less_than_minus_two_52); |
| |
| { |
| Node* const minus_zero = __ Float64Constant(-0.0); |
| Node* temp1 = __ Float64Sub(minus_zero, input); |
| Node* temp2 = __ Float64Sub(__ Float64Add(two_52, temp1), two_52); |
| Node* check3 = __ Float64LessThan(temp1, temp2); |
| __ GotoIfNot(check3, &done_temp3, temp2); |
| __ Goto(&done_temp3, __ Float64Sub(temp2, one)); |
| |
| __ Bind(&done_temp3); |
| Node* temp3 = done_temp3.PhiAt(0); |
| __ Goto(&done, __ Float64Sub(minus_zero, temp3)); |
| } |
| __ Bind(&if_less_than_minus_two_52); |
| __ Goto(&done, input); |
| |
| __ Bind(&if_zero); |
| __ Goto(&done, input); |
| } |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Maybe<Node*> EffectControlLinearizer::LowerFloat64RoundTruncate(Node* node) { |
| // Nothing to be done if a fast hardware instruction is available. |
| if (machine()->Float64RoundTruncate().IsSupported()) { |
| return Nothing<Node*>(); |
| } |
| |
| Node* const input = node->InputAt(0); |
| return Just(BuildFloat64RoundTruncate(input)); |
| } |
| |
| Node* EffectControlLinearizer::LowerFindOrderedHashMapEntry(Node* node) { |
| Node* table = NodeProperties::GetValueInput(node, 0); |
| Node* key = NodeProperties::GetValueInput(node, 1); |
| |
| { |
| Callable const callable = |
| Builtins::CallableFor(isolate(), Builtins::kFindOrderedHashMapEntry); |
| Operator::Properties const properties = node->op()->properties(); |
| CallDescriptor::Flags const flags = CallDescriptor::kNoFlags; |
| auto call_descriptor = Linkage::GetStubCallDescriptor( |
| graph()->zone(), callable.descriptor(), |
| callable.descriptor().GetStackParameterCount(), flags, properties); |
| return __ Call(call_descriptor, __ HeapConstant(callable.code()), table, |
| key, __ NoContextConstant()); |
| } |
| } |
| |
| Node* EffectControlLinearizer::ComputeUnseededHash(Node* value) { |
| // See v8::internal::ComputeUnseededHash() |
| value = __ Int32Add(__ Word32Xor(value, __ Int32Constant(0xFFFFFFFF)), |
| __ Word32Shl(value, __ Int32Constant(15))); |
| value = __ Word32Xor(value, __ Word32Shr(value, __ Int32Constant(12))); |
| value = __ Int32Add(value, __ Word32Shl(value, __ Int32Constant(2))); |
| value = __ Word32Xor(value, __ Word32Shr(value, __ Int32Constant(4))); |
| value = __ Int32Mul(value, __ Int32Constant(2057)); |
| value = __ Word32Xor(value, __ Word32Shr(value, __ Int32Constant(16))); |
| value = __ Word32And(value, __ Int32Constant(0x3FFFFFFF)); |
| return value; |
| } |
| |
| Node* EffectControlLinearizer::LowerFindOrderedHashMapEntryForInt32Key( |
| Node* node) { |
| Node* table = NodeProperties::GetValueInput(node, 0); |
| Node* key = NodeProperties::GetValueInput(node, 1); |
| |
| // Compute the integer hash code. |
| Node* hash = ChangeUint32ToUintPtr(ComputeUnseededHash(key)); |
| |
| Node* number_of_buckets = ChangeSmiToIntPtr(__ LoadField( |
| AccessBuilder::ForOrderedHashMapOrSetNumberOfBuckets(), table)); |
| hash = __ WordAnd(hash, __ IntSub(number_of_buckets, __ IntPtrConstant(1))); |
| Node* first_entry = ChangeSmiToIntPtr(__ Load( |
| MachineType::TaggedSigned(), table, |
| __ IntAdd(__ WordShl(hash, __ IntPtrConstant(kTaggedSizeLog2)), |
| __ IntPtrConstant(OrderedHashMap::HashTableStartOffset() - |
| kHeapObjectTag)))); |
| |
| auto loop = __ MakeLoopLabel(MachineType::PointerRepresentation()); |
| auto done = __ MakeLabel(MachineType::PointerRepresentation()); |
| __ Goto(&loop, first_entry); |
| __ Bind(&loop); |
| { |
| Node* entry = loop.PhiAt(0); |
| Node* check = |
| __ IntPtrEqual(entry, __ IntPtrConstant(OrderedHashMap::kNotFound)); |
| __ GotoIf(check, &done, entry); |
| entry = __ IntAdd( |
| __ IntMul(entry, __ IntPtrConstant(OrderedHashMap::kEntrySize)), |
| number_of_buckets); |
| |
| Node* candidate_key = __ Load( |
| MachineType::AnyTagged(), table, |
| __ IntAdd(__ WordShl(entry, __ IntPtrConstant(kTaggedSizeLog2)), |
| __ IntPtrConstant(OrderedHashMap::HashTableStartOffset() - |
| kHeapObjectTag))); |
| |
| auto if_match = __ MakeLabel(); |
| auto if_notmatch = __ MakeLabel(); |
| auto if_notsmi = __ MakeDeferredLabel(); |
| __ GotoIfNot(ObjectIsSmi(candidate_key), &if_notsmi); |
| __ Branch(__ Word32Equal(ChangeSmiToInt32(candidate_key), key), &if_match, |
| &if_notmatch); |
| |
| __ Bind(&if_notsmi); |
| __ GotoIfNot( |
| __ TaggedEqual(__ LoadField(AccessBuilder::ForMap(), candidate_key), |
| __ HeapNumberMapConstant()), |
| &if_notmatch); |
| __ Branch(__ Float64Equal(__ LoadField(AccessBuilder::ForHeapNumberValue(), |
| candidate_key), |
| __ ChangeInt32ToFloat64(key)), |
| &if_match, &if_notmatch); |
| |
| __ Bind(&if_match); |
| __ Goto(&done, entry); |
| |
| __ Bind(&if_notmatch); |
| { |
| Node* next_entry = ChangeSmiToIntPtr(__ Load( |
| MachineType::TaggedSigned(), table, |
| __ IntAdd( |
| __ WordShl(entry, __ IntPtrConstant(kTaggedSizeLog2)), |
| __ IntPtrConstant(OrderedHashMap::HashTableStartOffset() + |
| OrderedHashMap::kChainOffset * kTaggedSize - |
| kHeapObjectTag)))); |
| __ Goto(&loop, next_entry); |
| } |
| } |
| |
| __ Bind(&done); |
| return done.PhiAt(0); |
| } |
| |
| Node* EffectControlLinearizer::LowerDateNow(Node* node) { |
| Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow; |
| Runtime::FunctionId id = Runtime::kDateCurrentTime; |
| auto call_descriptor = Linkage::GetRuntimeCallDescriptor( |
| graph()->zone(), id, 0, properties, CallDescriptor::kNoFlags); |
| return __ Call(call_descriptor, __ CEntryStubConstant(1), |
| __ ExternalConstant(ExternalReference::Create(id)), |
| __ Int32Constant(0), __ NoContextConstant()); |
| } |
| |
| Node* EffectControlLinearizer::TruncateWordToInt32(Node* value) { |
| if (machine()->Is64()) { |
| return __ TruncateInt64ToInt32(value); |
| } |
| return value; |
| } |
| |
| Node* EffectControlLinearizer::BuildIsStrongReference(Node* value) { |
| return __ Word32Equal( |
| __ Word32And( |
| TruncateWordToInt32(__ BitcastTaggedToWordForTagAndSmiBits(value)), |
| __ Int32Constant(kHeapObjectTagMask)), |
| __ Int32Constant(kHeapObjectTag)); |
| } |
| |
| Node* EffectControlLinearizer::MakeWeakForComparison(Node* heap_object) { |
| // TODO(gsathya): Specialize this for pointer compression. |
| return __ BitcastWordToTagged( |
| __ WordOr(__ BitcastTaggedToWord(heap_object), |
| __ IntPtrConstant(kWeakHeapObjectTag))); |
| } |
| |
| Node* EffectControlLinearizer::BuildStrongReferenceFromWeakReference( |
| Node* maybe_object) { |
| return __ BitcastWordToTagged( |
| __ WordAnd(__ BitcastMaybeObjectToWord(maybe_object), |
| __ IntPtrConstant(~kWeakHeapObjectMask))); |
| } |
| |
| Node* EffectControlLinearizer::BuildIsWeakReferenceTo(Node* maybe_object, |
| Node* value) { |
| if (COMPRESS_POINTERS_BOOL) { |
| return __ Word32Equal( |
| __ Word32And( |
| TruncateWordToInt32(__ BitcastMaybeObjectToWord(maybe_object)), |
| __ Uint32Constant(~static_cast<uint32_t>(kWeakHeapObjectMask))), |
| TruncateWordToInt32(__ BitcastTaggedToWord(value))); |
| } else { |
| return __ WordEqual(__ WordAnd(__ BitcastMaybeObjectToWord(maybe_object), |
| __ IntPtrConstant(~kWeakHeapObjectMask)), |
| __ BitcastTaggedToWord(value)); |
| } |
| } |
| |
| Node* EffectControlLinearizer::BuildIsClearedWeakReference(Node* maybe_object) { |
| return __ Word32Equal( |
| TruncateWordToInt32(__ BitcastMaybeObjectToWord(maybe_object)), |
| __ Int32Constant(kClearedWeakHeapObjectLower32)); |
| } |
| |
| #undef __ |
| |
| void LinearizeEffectControl(JSGraph* graph, Schedule* schedule, Zone* temp_zone, |
| SourcePositionTable* source_positions, |
| NodeOriginTable* node_origins, |
| MaskArrayIndexEnable mask_array_index, |
| MaintainSchedule maintain_schedule, |
| JSHeapBroker* broker) { |
| EffectControlLinearizer linearizer( |
| graph, schedule, temp_zone, source_positions, node_origins, |
| mask_array_index, maintain_schedule, broker); |
| linearizer.Run(); |
| } |
| |
| } // namespace compiler |
| } // namespace internal |
| } // namespace v8 |
