| // Copyright 2017 the V8 project authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "src/wasm/wasm-code-manager.h" |
| |
| #include <iomanip> |
| |
| #include "src/base/adapters.h" |
| #include "src/base/macros.h" |
| #include "src/base/platform/platform.h" |
| #include "src/base/small-vector.h" |
| #include "src/codegen/assembler-inl.h" |
| #include "src/codegen/macro-assembler-inl.h" |
| #include "src/codegen/macro-assembler.h" |
| #include "src/common/globals.h" |
| #include "src/diagnostics/disassembler.h" |
| #include "src/logging/counters.h" |
| #include "src/logging/log.h" |
| #include "src/objects/objects-inl.h" |
| #include "src/snapshot/embedded/embedded-data.h" |
| #include "src/utils/ostreams.h" |
| #include "src/utils/vector.h" |
| #include "src/wasm/compilation-environment.h" |
| #include "src/wasm/function-compiler.h" |
| #include "src/wasm/jump-table-assembler.h" |
| #include "src/wasm/wasm-import-wrapper-cache.h" |
| #include "src/wasm/wasm-module.h" |
| #include "src/wasm/wasm-objects-inl.h" |
| #include "src/wasm/wasm-objects.h" |
| |
| #if defined(V8_OS_WIN_X64) |
| #include "src/diagnostics/unwinding-info-win64.h" |
| #endif |
| |
| #if defined(V8_OS_STARBOARD) |
| #include "src/poems.h" |
| #endif |
| |
| #define TRACE_HEAP(...) \ |
| do { \ |
| if (FLAG_trace_wasm_native_heap) PrintF(__VA_ARGS__); \ |
| } while (false) |
| |
| namespace v8 { |
| namespace internal { |
| namespace wasm { |
| |
| using trap_handler::ProtectedInstructionData; |
| |
| base::AddressRegion DisjointAllocationPool::Merge(base::AddressRegion region) { |
| auto dest_it = regions_.begin(); |
| auto dest_end = regions_.end(); |
| |
| // Skip over dest regions strictly before {region}. |
| while (dest_it != dest_end && dest_it->end() < region.begin()) ++dest_it; |
| |
| // After last dest region: insert and done. |
| if (dest_it == dest_end) { |
| regions_.push_back(region); |
| return region; |
| } |
| |
| // Adjacent (from below) to dest: merge and done. |
| if (dest_it->begin() == region.end()) { |
| base::AddressRegion merged_region{region.begin(), |
| region.size() + dest_it->size()}; |
| DCHECK_EQ(merged_region.end(), dest_it->end()); |
| *dest_it = merged_region; |
| return merged_region; |
| } |
| |
| // Before dest: insert and done. |
| if (dest_it->begin() > region.end()) { |
| regions_.insert(dest_it, region); |
| return region; |
| } |
| |
| // Src is adjacent from above. Merge and check whether the merged region is |
| // now adjacent to the next region. |
| DCHECK_EQ(dest_it->end(), region.begin()); |
| dest_it->set_size(dest_it->size() + region.size()); |
| DCHECK_EQ(dest_it->end(), region.end()); |
| auto next_dest = dest_it; |
| ++next_dest; |
| if (next_dest != dest_end && dest_it->end() == next_dest->begin()) { |
| dest_it->set_size(dest_it->size() + next_dest->size()); |
| DCHECK_EQ(dest_it->end(), next_dest->end()); |
| regions_.erase(next_dest); |
| } |
| return *dest_it; |
| } |
| |
| base::AddressRegion DisjointAllocationPool::Allocate(size_t size) { |
| for (auto it = regions_.begin(), end = regions_.end(); it != end; ++it) { |
| if (size > it->size()) continue; |
| base::AddressRegion ret{it->begin(), size}; |
| if (size == it->size()) { |
| regions_.erase(it); |
| } else { |
| *it = base::AddressRegion{it->begin() + size, it->size() - size}; |
| } |
| return ret; |
| } |
| return {}; |
| } |
| |
| Address WasmCode::constant_pool() const { |
| if (FLAG_enable_embedded_constant_pool) { |
| if (constant_pool_offset_ < code_comments_offset_) { |
| return instruction_start() + constant_pool_offset_; |
| } |
| } |
| return kNullAddress; |
| } |
| |
| Address WasmCode::handler_table() const { |
| return instruction_start() + handler_table_offset_; |
| } |
| |
| uint32_t WasmCode::handler_table_size() const { |
| DCHECK_GE(constant_pool_offset_, handler_table_offset_); |
| return static_cast<uint32_t>(constant_pool_offset_ - handler_table_offset_); |
| } |
| |
| Address WasmCode::code_comments() const { |
| return instruction_start() + code_comments_offset_; |
| } |
| |
| uint32_t WasmCode::code_comments_size() const { |
| DCHECK_GE(unpadded_binary_size_, code_comments_offset_); |
| return static_cast<uint32_t>(unpadded_binary_size_ - code_comments_offset_); |
| } |
| |
| void WasmCode::RegisterTrapHandlerData() { |
| DCHECK(!has_trap_handler_index()); |
| if (kind() != WasmCode::kFunction) return; |
| if (protected_instructions_.empty()) return; |
| |
| Address base = instruction_start(); |
| |
| size_t size = instructions().size(); |
| const int index = |
| RegisterHandlerData(base, size, protected_instructions().size(), |
| protected_instructions().begin()); |
| |
| // TODO(eholk): if index is negative, fail. |
| CHECK_LE(0, index); |
| set_trap_handler_index(index); |
| DCHECK(has_trap_handler_index()); |
| } |
| |
| bool WasmCode::ShouldBeLogged(Isolate* isolate) { |
| // The return value is cached in {WasmEngine::IsolateData::log_codes}. Ensure |
| // to call {WasmEngine::EnableCodeLogging} if this return value would change |
| // for any isolate. Otherwise we might lose code events. |
| return isolate->logger()->is_listening_to_code_events() || |
| isolate->code_event_dispatcher()->IsListeningToCodeEvents() || |
| isolate->is_profiling(); |
| } |
| |
| void WasmCode::LogCode(Isolate* isolate) const { |
| DCHECK(ShouldBeLogged(isolate)); |
| if (IsAnonymous()) return; |
| |
| ModuleWireBytes wire_bytes(native_module()->wire_bytes()); |
| // TODO(herhut): Allow to log code without on-heap round-trip of the name. |
| WireBytesRef name_ref = |
| native_module()->module()->LookupFunctionName(wire_bytes, index()); |
| WasmName name_vec = wire_bytes.GetNameOrNull(name_ref); |
| if (!name_vec.empty()) { |
| HandleScope scope(isolate); |
| MaybeHandle<String> maybe_name = isolate->factory()->NewStringFromUtf8( |
| Vector<const char>::cast(name_vec)); |
| Handle<String> name; |
| if (!maybe_name.ToHandle(&name)) { |
| name = isolate->factory()->NewStringFromAsciiChecked("<name too long>"); |
| } |
| int name_length; |
| auto cname = |
| name->ToCString(AllowNullsFlag::DISALLOW_NULLS, |
| RobustnessFlag::ROBUST_STRING_TRAVERSAL, &name_length); |
| PROFILE(isolate, |
| CodeCreateEvent(CodeEventListener::FUNCTION_TAG, this, |
| {cname.get(), static_cast<size_t>(name_length)})); |
| } else { |
| EmbeddedVector<char, 32> generated_name; |
| int length = SNPrintF(generated_name, "wasm-function[%d]", index()); |
| generated_name.Truncate(length); |
| PROFILE(isolate, CodeCreateEvent(CodeEventListener::FUNCTION_TAG, this, |
| generated_name)); |
| } |
| |
| if (!source_positions().empty()) { |
| LOG_CODE_EVENT(isolate, CodeLinePosInfoRecordEvent(instruction_start(), |
| source_positions())); |
| } |
| } |
| |
| void WasmCode::Validate() const { |
| #ifdef DEBUG |
| // We expect certain relocation info modes to never appear in {WasmCode} |
| // objects or to be restricted to a small set of valid values. Hence the |
| // iteration below does not use a mask, but visits all relocation data. |
| for (RelocIterator it(instructions(), reloc_info(), constant_pool()); |
| !it.done(); it.next()) { |
| RelocInfo::Mode mode = it.rinfo()->rmode(); |
| switch (mode) { |
| case RelocInfo::WASM_CALL: { |
| Address target = it.rinfo()->wasm_call_address(); |
| WasmCode* code = native_module_->Lookup(target); |
| CHECK_NOT_NULL(code); |
| CHECK_EQ(WasmCode::kJumpTable, code->kind()); |
| CHECK_EQ(native_module()->jump_table_, code); |
| CHECK(code->contains(target)); |
| break; |
| } |
| case RelocInfo::WASM_STUB_CALL: { |
| Address target = it.rinfo()->wasm_stub_call_address(); |
| WasmCode* code = native_module_->Lookup(target); |
| CHECK_NOT_NULL(code); |
| #ifdef V8_EMBEDDED_BUILTINS |
| CHECK_EQ(WasmCode::kJumpTable, code->kind()); |
| CHECK_EQ(native_module()->runtime_stub_table_, code); |
| CHECK(code->contains(target)); |
| #else |
| CHECK_EQ(WasmCode::kRuntimeStub, code->kind()); |
| CHECK_EQ(target, code->instruction_start()); |
| #endif |
| break; |
| } |
| case RelocInfo::INTERNAL_REFERENCE: |
| case RelocInfo::INTERNAL_REFERENCE_ENCODED: { |
| Address target = it.rinfo()->target_internal_reference(); |
| CHECK(contains(target)); |
| break; |
| } |
| case RelocInfo::EXTERNAL_REFERENCE: |
| case RelocInfo::CONST_POOL: |
| case RelocInfo::VENEER_POOL: |
| // These are OK to appear. |
| break; |
| default: |
| FATAL("Unexpected mode: %d", mode); |
| } |
| } |
| #endif |
| } |
| |
| void WasmCode::MaybePrint(const char* name) const { |
| // Determines whether flags want this code to be printed. |
| if ((FLAG_print_wasm_code && kind() == kFunction) || |
| (FLAG_print_wasm_stub_code && kind() != kFunction) || FLAG_print_code) { |
| Print(name); |
| } |
| } |
| |
| void WasmCode::Print(const char* name) const { |
| StdoutStream os; |
| os << "--- WebAssembly code ---\n"; |
| Disassemble(name, os); |
| os << "--- End code ---\n"; |
| } |
| |
| void WasmCode::Disassemble(const char* name, std::ostream& os, |
| Address current_pc) const { |
| if (name) os << "name: " << name << "\n"; |
| if (!IsAnonymous()) os << "index: " << index() << "\n"; |
| os << "kind: " << GetWasmCodeKindAsString(kind_) << "\n"; |
| os << "compiler: " << (is_liftoff() ? "Liftoff" : "TurboFan") << "\n"; |
| size_t padding = instructions().size() - unpadded_binary_size_; |
| os << "Body (size = " << instructions().size() << " = " |
| << unpadded_binary_size_ << " + " << padding << " padding)\n"; |
| |
| #ifdef ENABLE_DISASSEMBLER |
| size_t instruction_size = unpadded_binary_size_; |
| if (constant_pool_offset_ < instruction_size) { |
| instruction_size = constant_pool_offset_; |
| } |
| if (safepoint_table_offset_ && safepoint_table_offset_ < instruction_size) { |
| instruction_size = safepoint_table_offset_; |
| } |
| if (handler_table_offset_ < instruction_size) { |
| instruction_size = handler_table_offset_; |
| } |
| DCHECK_LT(0, instruction_size); |
| os << "Instructions (size = " << instruction_size << ")\n"; |
| Disassembler::Decode(nullptr, &os, instructions().begin(), |
| instructions().begin() + instruction_size, |
| CodeReference(this), current_pc); |
| os << "\n"; |
| |
| if (handler_table_size() > 0) { |
| HandlerTable table(this); |
| os << "Exception Handler Table (size = " << table.NumberOfReturnEntries() |
| << "):\n"; |
| table.HandlerTableReturnPrint(os); |
| os << "\n"; |
| } |
| |
| if (!protected_instructions_.empty()) { |
| os << "Protected instructions:\n pc offset land pad\n"; |
| for (auto& data : protected_instructions()) { |
| os << std::setw(10) << std::hex << data.instr_offset << std::setw(10) |
| << std::hex << data.landing_offset << "\n"; |
| } |
| os << "\n"; |
| } |
| |
| if (!source_positions().empty()) { |
| os << "Source positions:\n pc offset position\n"; |
| for (SourcePositionTableIterator it(source_positions()); !it.done(); |
| it.Advance()) { |
| os << std::setw(10) << std::hex << it.code_offset() << std::dec |
| << std::setw(10) << it.source_position().ScriptOffset() |
| << (it.is_statement() ? " statement" : "") << "\n"; |
| } |
| os << "\n"; |
| } |
| |
| if (safepoint_table_offset_ > 0) { |
| SafepointTable table(this); |
| os << "Safepoints (size = " << table.size() << ")\n"; |
| for (uint32_t i = 0; i < table.length(); i++) { |
| uintptr_t pc_offset = table.GetPcOffset(i); |
| os << reinterpret_cast<const void*>(instruction_start() + pc_offset); |
| os << std::setw(6) << std::hex << pc_offset << " " << std::dec; |
| table.PrintEntry(i, os); |
| os << " (sp -> fp)"; |
| SafepointEntry entry = table.GetEntry(i); |
| if (entry.trampoline_pc() != -1) { |
| os << " trampoline: " << std::hex << entry.trampoline_pc() << std::dec; |
| } |
| if (entry.has_deoptimization_index()) { |
| os << " deopt: " << std::setw(6) << entry.deoptimization_index(); |
| } |
| os << "\n"; |
| } |
| os << "\n"; |
| } |
| |
| os << "RelocInfo (size = " << reloc_info_.size() << ")\n"; |
| for (RelocIterator it(instructions(), reloc_info(), constant_pool()); |
| !it.done(); it.next()) { |
| it.rinfo()->Print(nullptr, os); |
| } |
| os << "\n"; |
| |
| if (code_comments_size() > 0) { |
| PrintCodeCommentsSection(os, code_comments(), code_comments_size()); |
| } |
| #endif // ENABLE_DISASSEMBLER |
| } |
| |
| const char* GetWasmCodeKindAsString(WasmCode::Kind kind) { |
| switch (kind) { |
| case WasmCode::kFunction: |
| return "wasm function"; |
| case WasmCode::kWasmToCapiWrapper: |
| return "wasm-to-capi"; |
| case WasmCode::kWasmToJsWrapper: |
| return "wasm-to-js"; |
| case WasmCode::kRuntimeStub: |
| return "runtime-stub"; |
| case WasmCode::kInterpreterEntry: |
| return "interpreter entry"; |
| case WasmCode::kJumpTable: |
| return "jump table"; |
| } |
| return "unknown kind"; |
| } |
| |
| WasmCode::~WasmCode() { |
| if (has_trap_handler_index()) { |
| trap_handler::ReleaseHandlerData(trap_handler_index()); |
| } |
| } |
| |
| V8_WARN_UNUSED_RESULT bool WasmCode::DecRefOnPotentiallyDeadCode() { |
| if (native_module_->engine()->AddPotentiallyDeadCode(this)) { |
| // The code just became potentially dead. The ref count we wanted to |
| // decrement is now transferred to the set of potentially dead code, and |
| // will be decremented when the next GC is run. |
| return false; |
| } |
| // If we reach here, the code was already potentially dead. Decrement the ref |
| // count, and return true if it drops to zero. |
| return DecRefOnDeadCode(); |
| } |
| |
| // static |
| void WasmCode::DecrementRefCount(Vector<WasmCode* const> code_vec) { |
| // Decrement the ref counter of all given code objects. Keep the ones whose |
| // ref count drops to zero. |
| WasmEngine::DeadCodeMap dead_code; |
| WasmEngine* engine = nullptr; |
| for (WasmCode* code : code_vec) { |
| if (!code->DecRef()) continue; // Remaining references. |
| dead_code[code->native_module()].push_back(code); |
| if (!engine) engine = code->native_module()->engine(); |
| DCHECK_EQ(engine, code->native_module()->engine()); |
| } |
| |
| DCHECK_EQ(dead_code.empty(), engine == nullptr); |
| if (engine) engine->FreeDeadCode(dead_code); |
| } |
| |
| WasmCodeAllocator::WasmCodeAllocator(WasmCodeManager* code_manager, |
| VirtualMemory code_space, |
| bool can_request_more, |
| std::shared_ptr<Counters> async_counters) |
| : code_manager_(code_manager), |
| free_code_space_(code_space.region()), |
| can_request_more_memory_(can_request_more), |
| async_counters_(std::move(async_counters)) { |
| owned_code_space_.reserve(can_request_more ? 4 : 1); |
| owned_code_space_.emplace_back(std::move(code_space)); |
| async_counters_->wasm_module_num_code_spaces()->AddSample(1); |
| } |
| |
| WasmCodeAllocator::~WasmCodeAllocator() { |
| code_manager_->FreeNativeModule(VectorOf(owned_code_space_), |
| committed_code_space()); |
| } |
| |
| namespace { |
| // On Windows, we cannot commit a region that straddles different reservations |
| // of virtual memory. Because we bump-allocate, and because, if we need more |
| // memory, we append that memory at the end of the owned_code_space_ list, we |
| // traverse that list in reverse order to find the reservation(s) that guide how |
| // to chunk the region to commit. |
| #if V8_OS_WIN |
| constexpr bool kNeedsToSplitRangeByReservations = true; |
| #else |
| constexpr bool kNeedsToSplitRangeByReservations = false; |
| #endif |
| |
| base::SmallVector<base::AddressRegion, 1> SplitRangeByReservationsIfNeeded( |
| base::AddressRegion range, |
| const std::vector<VirtualMemory>& owned_code_space) { |
| if (!kNeedsToSplitRangeByReservations) return {range}; |
| |
| base::SmallVector<base::AddressRegion, 1> split_ranges; |
| size_t missing_begin = range.begin(); |
| size_t missing_end = range.end(); |
| for (auto& vmem : base::Reversed(owned_code_space)) { |
| Address overlap_begin = std::max(missing_begin, vmem.address()); |
| Address overlap_end = std::min(missing_end, vmem.end()); |
| if (overlap_begin >= overlap_end) continue; |
| split_ranges.emplace_back(overlap_begin, overlap_end - overlap_begin); |
| // Opportunistically reduce the missing range. This might terminate the loop |
| // early. |
| if (missing_begin == overlap_begin) missing_begin = overlap_end; |
| if (missing_end == overlap_end) missing_end = overlap_begin; |
| if (missing_begin >= missing_end) break; |
| } |
| #ifdef ENABLE_SLOW_DCHECKS |
| // The returned vector should cover the full range. |
| size_t total_split_size = 0; |
| for (auto split : split_ranges) total_split_size += split.size(); |
| DCHECK_EQ(range.size(), total_split_size); |
| #endif |
| return split_ranges; |
| } |
| } // namespace |
| |
| Vector<byte> WasmCodeAllocator::AllocateForCode(NativeModule* native_module, |
| size_t size) { |
| base::MutexGuard lock(&mutex_); |
| DCHECK_EQ(code_manager_, native_module->engine()->code_manager()); |
| DCHECK_LT(0, size); |
| v8::PageAllocator* page_allocator = GetPlatformPageAllocator(); |
| // This happens under a lock assumed by the caller. |
| size = RoundUp<kCodeAlignment>(size); |
| base::AddressRegion code_space = free_code_space_.Allocate(size); |
| if (code_space.is_empty()) { |
| if (!can_request_more_memory_) { |
| V8::FatalProcessOutOfMemory(nullptr, "wasm code reservation"); |
| UNREACHABLE(); |
| } |
| |
| Address hint = owned_code_space_.empty() ? kNullAddress |
| : owned_code_space_.back().end(); |
| |
| VirtualMemory new_mem = |
| code_manager_->TryAllocate(size, reinterpret_cast<void*>(hint)); |
| if (!new_mem.IsReserved()) { |
| V8::FatalProcessOutOfMemory(nullptr, "wasm code reservation"); |
| UNREACHABLE(); |
| } |
| code_manager_->AssignRange(new_mem.region(), native_module); |
| |
| free_code_space_.Merge(new_mem.region()); |
| owned_code_space_.emplace_back(std::move(new_mem)); |
| code_space = free_code_space_.Allocate(size); |
| DCHECK(!code_space.is_empty()); |
| async_counters_->wasm_module_num_code_spaces()->AddSample( |
| static_cast<int>(owned_code_space_.size())); |
| } |
| const Address commit_page_size = page_allocator->CommitPageSize(); |
| Address commit_start = RoundUp(code_space.begin(), commit_page_size); |
| Address commit_end = RoundUp(code_space.end(), commit_page_size); |
| // {commit_start} will be either code_space.start or the start of the next |
| // page. {commit_end} will be the start of the page after the one in which |
| // the allocation ends. |
| // We start from an aligned start, and we know we allocated vmem in |
| // page multiples. |
| // We just need to commit what's not committed. The page in which we |
| // start is already committed (or we start at the beginning of a page). |
| // The end needs to be committed all through the end of the page. |
| if (commit_start < commit_end) { |
| committed_code_space_.fetch_add(commit_end - commit_start); |
| // Committed code cannot grow bigger than maximum code space size. |
| DCHECK_LE(committed_code_space_.load(), kMaxWasmCodeMemory); |
| for (base::AddressRegion split_range : SplitRangeByReservationsIfNeeded( |
| {commit_start, commit_end - commit_start}, owned_code_space_)) { |
| if (!code_manager_->Commit(split_range)) { |
| V8::FatalProcessOutOfMemory(nullptr, "wasm code commit"); |
| UNREACHABLE(); |
| } |
| } |
| } |
| DCHECK(IsAligned(code_space.begin(), kCodeAlignment)); |
| allocated_code_space_.Merge(code_space); |
| generated_code_size_.fetch_add(code_space.size(), std::memory_order_relaxed); |
| |
| TRACE_HEAP("Code alloc for %p: 0x%" PRIxPTR ",+%zu\n", this, |
| code_space.begin(), size); |
| return {reinterpret_cast<byte*>(code_space.begin()), code_space.size()}; |
| } |
| |
| bool WasmCodeAllocator::SetExecutable(bool executable) { |
| base::MutexGuard lock(&mutex_); |
| if (is_executable_ == executable) return true; |
| TRACE_HEAP("Setting module %p as executable: %d.\n", this, executable); |
| |
| v8::PageAllocator* page_allocator = GetPlatformPageAllocator(); |
| |
| if (FLAG_wasm_write_protect_code_memory) { |
| PageAllocator::Permission permission = |
| executable ? PageAllocator::kReadExecute : PageAllocator::kReadWrite; |
| #if V8_OS_WIN |
| // On windows, we need to switch permissions per separate virtual memory |
| // reservation. This is really just a problem when the NativeModule is |
| // growable (meaning can_request_more_memory_). That's 32-bit in production, |
| // or unittests. |
| // For now, in that case, we commit at reserved memory granularity. |
| // Technically, that may be a waste, because we may reserve more than we |
| // use. On 32-bit though, the scarce resource is the address space - |
| // committed or not. |
| if (can_request_more_memory_) { |
| for (auto& vmem : owned_code_space_) { |
| if (!SetPermissions(page_allocator, vmem.address(), vmem.size(), |
| permission)) { |
| return false; |
| } |
| TRACE_HEAP("Set %p:%p to executable:%d\n", vmem.address(), vmem.end(), |
| executable); |
| } |
| is_executable_ = executable; |
| return true; |
| } |
| #endif |
| size_t commit_page_size = page_allocator->CommitPageSize(); |
| for (auto& region : allocated_code_space_.regions()) { |
| // allocated_code_space_ is fine-grained, so we need to |
| // page-align it. |
| size_t region_size = RoundUp(region.size(), commit_page_size); |
| if (!SetPermissions(page_allocator, region.begin(), region_size, |
| permission)) { |
| return false; |
| } |
| TRACE_HEAP("Set 0x%" PRIxPTR ":0x%" PRIxPTR " to executable:%d\n", |
| region.begin(), region.end(), executable); |
| } |
| } |
| is_executable_ = executable; |
| return true; |
| } |
| |
| void WasmCodeAllocator::FreeCode(Vector<WasmCode* const> codes) { |
| // Zap code area and collect freed code regions. |
| DisjointAllocationPool freed_regions; |
| size_t code_size = 0; |
| for (WasmCode* code : codes) { |
| ZapCode(code->instruction_start(), code->instructions().size()); |
| FlushInstructionCache(code->instruction_start(), |
| code->instructions().size()); |
| code_size += code->instructions().size(); |
| freed_regions.Merge(base::AddressRegion{code->instruction_start(), |
| code->instructions().size()}); |
| } |
| freed_code_size_.fetch_add(code_size); |
| |
| // Merge {freed_regions} into {freed_code_space_} and discard full pages. |
| base::MutexGuard guard(&mutex_); |
| PageAllocator* allocator = GetPlatformPageAllocator(); |
| size_t commit_page_size = allocator->CommitPageSize(); |
| for (auto region : freed_regions.regions()) { |
| auto merged_region = freed_code_space_.Merge(region); |
| Address discard_start = |
| std::max(RoundUp(merged_region.begin(), commit_page_size), |
| RoundDown(region.begin(), commit_page_size)); |
| Address discard_end = |
| std::min(RoundDown(merged_region.end(), commit_page_size), |
| RoundUp(region.end(), commit_page_size)); |
| if (discard_start >= discard_end) continue; |
| size_t discard_size = discard_end - discard_start; |
| size_t old_committed = committed_code_space_.fetch_sub(discard_size); |
| DCHECK_GE(old_committed, discard_size); |
| USE(old_committed); |
| for (base::AddressRegion split_range : SplitRangeByReservationsIfNeeded( |
| {discard_start, discard_size}, owned_code_space_)) { |
| code_manager_->Decommit(split_range); |
| } |
| } |
| } |
| |
| NativeModule::NativeModule(WasmEngine* engine, const WasmFeatures& enabled, |
| bool can_request_more, VirtualMemory code_space, |
| std::shared_ptr<const WasmModule> module, |
| std::shared_ptr<Counters> async_counters, |
| std::shared_ptr<NativeModule>* shared_this) |
| : code_allocator_(engine->code_manager(), std::move(code_space), |
| can_request_more, async_counters), |
| enabled_features_(enabled), |
| module_(std::move(module)), |
| import_wrapper_cache_(std::unique_ptr<WasmImportWrapperCache>( |
| new WasmImportWrapperCache())), |
| engine_(engine), |
| use_trap_handler_(trap_handler::IsTrapHandlerEnabled() ? kUseTrapHandler |
| : kNoTrapHandler) { |
| // We receive a pointer to an empty {std::shared_ptr}, and install ourselve |
| // there. |
| DCHECK_NOT_NULL(shared_this); |
| DCHECK_NULL(*shared_this); |
| shared_this->reset(this); |
| compilation_state_ = |
| CompilationState::New(*shared_this, std::move(async_counters)); |
| DCHECK_NOT_NULL(module_); |
| |
| #if defined(V8_OS_WIN_X64) |
| // On some platforms, specifically Win64, we need to reserve some pages at |
| // the beginning of an executable space. |
| // See src/heap/spaces.cc, MemoryAllocator::InitializeCodePageAllocator() and |
| // https://cs.chromium.org/chromium/src/components/crash/content/app/crashpad_win.cc?rcl=fd680447881449fba2edcf0589320e7253719212&l=204 |
| // for details. |
| if (engine_->code_manager() |
| ->CanRegisterUnwindInfoForNonABICompliantCodeRange()) { |
| code_allocator_.AllocateForCode(this, Heap::GetCodeRangeReservedAreaSize()); |
| } |
| #endif |
| |
| uint32_t num_wasm_functions = module_->num_declared_functions; |
| if (num_wasm_functions > 0) { |
| code_table_.reset(new WasmCode* [num_wasm_functions] {}); |
| |
| WasmCodeRefScope code_ref_scope; |
| jump_table_ = CreateEmptyJumpTable( |
| JumpTableAssembler::SizeForNumberOfSlots(num_wasm_functions)); |
| } |
| } |
| |
| void NativeModule::ReserveCodeTableForTesting(uint32_t max_functions) { |
| WasmCodeRefScope code_ref_scope; |
| DCHECK_LE(num_functions(), max_functions); |
| WasmCode** new_table = new WasmCode* [max_functions] {}; |
| if (module_->num_declared_functions > 0) { |
| memcpy(new_table, code_table_.get(), |
| module_->num_declared_functions * sizeof(*new_table)); |
| } |
| code_table_.reset(new_table); |
| |
| // Re-allocate jump table. |
| jump_table_ = CreateEmptyJumpTable( |
| JumpTableAssembler::SizeForNumberOfSlots(max_functions)); |
| } |
| |
| void NativeModule::LogWasmCodes(Isolate* isolate) { |
| if (!WasmCode::ShouldBeLogged(isolate)) return; |
| |
| // TODO(titzer): we skip the logging of the import wrappers |
| // here, but they should be included somehow. |
| int start = module()->num_imported_functions; |
| int end = start + module()->num_declared_functions; |
| WasmCodeRefScope code_ref_scope; |
| for (int func_index = start; func_index < end; ++func_index) { |
| if (WasmCode* code = GetCode(func_index)) code->LogCode(isolate); |
| } |
| } |
| |
| CompilationEnv NativeModule::CreateCompilationEnv() const { |
| return {module(), use_trap_handler_, kRuntimeExceptionSupport, |
| enabled_features_}; |
| } |
| |
| WasmCode* NativeModule::AddCodeForTesting(Handle<Code> code) { |
| return AddAndPublishAnonymousCode(code, WasmCode::kFunction); |
| } |
| |
| void NativeModule::UseLazyStub(uint32_t func_index) { |
| DCHECK_LE(module_->num_imported_functions, func_index); |
| DCHECK_LT(func_index, |
| module_->num_imported_functions + module_->num_declared_functions); |
| |
| if (!lazy_compile_table_) { |
| uint32_t num_slots = module_->num_declared_functions; |
| WasmCodeRefScope code_ref_scope; |
| lazy_compile_table_ = CreateEmptyJumpTable( |
| JumpTableAssembler::SizeForNumberOfLazyFunctions(num_slots)); |
| JumpTableAssembler::GenerateLazyCompileTable( |
| lazy_compile_table_->instruction_start(), num_slots, |
| module_->num_imported_functions, |
| runtime_stub_entry(WasmCode::kWasmCompileLazy)); |
| } |
| |
| // Add jump table entry for jump to the lazy compile stub. |
| uint32_t slot_index = func_index - module_->num_imported_functions; |
| DCHECK_NE(runtime_stub_entry(WasmCode::kWasmCompileLazy), kNullAddress); |
| Address lazy_compile_target = |
| lazy_compile_table_->instruction_start() + |
| JumpTableAssembler::LazyCompileSlotIndexToOffset(slot_index); |
| JumpTableAssembler::PatchJumpTableSlot(jump_table_->instruction_start(), |
| slot_index, lazy_compile_target, |
| WasmCode::kFlushICache); |
| } |
| |
| // TODO(mstarzinger): Remove {Isolate} parameter once {V8_EMBEDDED_BUILTINS} |
| // was removed and embedded builtins are no longer optional. |
| void NativeModule::SetRuntimeStubs(Isolate* isolate) { |
| DCHECK_EQ(kNullAddress, runtime_stub_entries_[0]); // Only called once. |
| #ifdef V8_EMBEDDED_BUILTINS |
| WasmCodeRefScope code_ref_scope; |
| WasmCode* jump_table = |
| CreateEmptyJumpTable(JumpTableAssembler::SizeForNumberOfStubSlots( |
| WasmCode::kRuntimeStubCount)); |
| Address base = jump_table->instruction_start(); |
| EmbeddedData embedded_data = EmbeddedData::FromBlob(); |
| #define RUNTIME_STUB(Name) Builtins::k##Name, |
| #define RUNTIME_STUB_TRAP(Name) RUNTIME_STUB(ThrowWasm##Name) |
| Builtins::Name wasm_runtime_stubs[WasmCode::kRuntimeStubCount] = { |
| WASM_RUNTIME_STUB_LIST(RUNTIME_STUB, RUNTIME_STUB_TRAP)}; |
| #undef RUNTIME_STUB |
| #undef RUNTIME_STUB_TRAP |
| Address builtin_address[WasmCode::kRuntimeStubCount]; |
| for (int i = 0; i < WasmCode::kRuntimeStubCount; ++i) { |
| Builtins::Name builtin = wasm_runtime_stubs[i]; |
| CHECK(embedded_data.ContainsBuiltin(builtin)); |
| builtin_address[i] = embedded_data.InstructionStartOfBuiltin(builtin); |
| runtime_stub_entries_[i] = |
| base + JumpTableAssembler::StubSlotIndexToOffset(i); |
| } |
| JumpTableAssembler::GenerateRuntimeStubTable(base, builtin_address, |
| WasmCode::kRuntimeStubCount); |
| DCHECK_NULL(runtime_stub_table_); |
| runtime_stub_table_ = jump_table; |
| #else // V8_EMBEDDED_BUILTINS |
| HandleScope scope(isolate); |
| WasmCodeRefScope code_ref_scope; |
| USE(runtime_stub_table_); // Actually unused, but avoids ifdef's in header. |
| #define COPY_BUILTIN(Name) \ |
| runtime_stub_entries_[WasmCode::k##Name] = \ |
| AddAndPublishAnonymousCode( \ |
| isolate->builtins()->builtin_handle(Builtins::k##Name), \ |
| WasmCode::kRuntimeStub, #Name) \ |
| ->instruction_start(); |
| #define COPY_BUILTIN_TRAP(Name) COPY_BUILTIN(ThrowWasm##Name) |
| WASM_RUNTIME_STUB_LIST(COPY_BUILTIN, COPY_BUILTIN_TRAP) |
| #undef COPY_BUILTIN_TRAP |
| #undef COPY_BUILTIN |
| #endif // V8_EMBEDDED_BUILTINS |
| DCHECK_NE(kNullAddress, runtime_stub_entries_[0]); |
| } |
| |
| WasmCode* NativeModule::AddAndPublishAnonymousCode(Handle<Code> code, |
| WasmCode::Kind kind, |
| const char* name) { |
| // For off-heap builtins, we create a copy of the off-heap instruction stream |
| // instead of the on-heap code object containing the trampoline. Ensure that |
| // we do not apply the on-heap reloc info to the off-heap instructions. |
| const size_t relocation_size = |
| code->is_off_heap_trampoline() ? 0 : code->relocation_size(); |
| OwnedVector<byte> reloc_info; |
| if (relocation_size > 0) { |
| reloc_info = OwnedVector<byte>::New(relocation_size); |
| memcpy(reloc_info.start(), code->relocation_start(), relocation_size); |
| } |
| Handle<ByteArray> source_pos_table(code->SourcePositionTable(), |
| code->GetIsolate()); |
| OwnedVector<byte> source_pos = |
| OwnedVector<byte>::New(source_pos_table->length()); |
| if (source_pos_table->length() > 0) { |
| source_pos_table->copy_out(0, source_pos.start(), |
| source_pos_table->length()); |
| } |
| Vector<const byte> instructions( |
| reinterpret_cast<byte*>(code->InstructionStart()), |
| static_cast<size_t>(code->InstructionSize())); |
| const uint32_t stack_slots = static_cast<uint32_t>( |
| code->has_safepoint_info() ? code->stack_slots() : 0); |
| |
| // TODO(jgruber,v8:8758): Remove this translation. It exists only because |
| // Code objects contains real offsets but WasmCode expects an offset of 0 to |
| // mean 'empty'. |
| const size_t safepoint_table_offset = static_cast<size_t>( |
| code->has_safepoint_table() ? code->safepoint_table_offset() : 0); |
| const size_t handler_table_offset = |
| static_cast<size_t>(code->handler_table_offset()); |
| const size_t constant_pool_offset = |
| static_cast<size_t>(code->constant_pool_offset()); |
| const size_t code_comments_offset = |
| static_cast<size_t>(code->code_comments_offset()); |
| |
| Vector<uint8_t> dst_code_bytes = |
| code_allocator_.AllocateForCode(this, instructions.size()); |
| memcpy(dst_code_bytes.begin(), instructions.begin(), instructions.size()); |
| |
| // Apply the relocation delta by iterating over the RelocInfo. |
| intptr_t delta = reinterpret_cast<Address>(dst_code_bytes.begin()) - |
| code->InstructionStart(); |
| int mode_mask = RelocInfo::kApplyMask | |
| RelocInfo::ModeMask(RelocInfo::WASM_STUB_CALL); |
| Address constant_pool_start = |
| reinterpret_cast<Address>(dst_code_bytes.begin()) + constant_pool_offset; |
| RelocIterator orig_it(*code, mode_mask); |
| for (RelocIterator it(dst_code_bytes, reloc_info.as_vector(), |
| constant_pool_start, mode_mask); |
| !it.done(); it.next(), orig_it.next()) { |
| RelocInfo::Mode mode = it.rinfo()->rmode(); |
| if (RelocInfo::IsWasmStubCall(mode)) { |
| uint32_t stub_call_tag = orig_it.rinfo()->wasm_call_tag(); |
| DCHECK_LT(stub_call_tag, WasmCode::kRuntimeStubCount); |
| Address entry = runtime_stub_entry( |
| static_cast<WasmCode::RuntimeStubId>(stub_call_tag)); |
| it.rinfo()->set_wasm_stub_call_address(entry, SKIP_ICACHE_FLUSH); |
| } else { |
| it.rinfo()->apply(delta); |
| } |
| } |
| |
| // Flush the i-cache after relocation. |
| FlushInstructionCache(dst_code_bytes.begin(), dst_code_bytes.size()); |
| |
| DCHECK_NE(kind, WasmCode::Kind::kInterpreterEntry); |
| std::unique_ptr<WasmCode> new_code{new WasmCode{ |
| this, // native_module |
| kAnonymousFuncIndex, // index |
| dst_code_bytes, // instructions |
| stack_slots, // stack_slots |
| 0, // tagged_parameter_slots |
| safepoint_table_offset, // safepoint_table_offset |
| handler_table_offset, // handler_table_offset |
| constant_pool_offset, // constant_pool_offset |
| code_comments_offset, // code_comments_offset |
| instructions.size(), // unpadded_binary_size |
| OwnedVector<ProtectedInstructionData>{}, // protected_instructions |
| std::move(reloc_info), // reloc_info |
| std::move(source_pos), // source positions |
| kind, // kind |
| ExecutionTier::kNone}}; // tier |
| new_code->MaybePrint(name); |
| new_code->Validate(); |
| |
| return PublishCode(std::move(new_code)); |
| } |
| |
| std::unique_ptr<WasmCode> NativeModule::AddCode( |
| uint32_t index, const CodeDesc& desc, uint32_t stack_slots, |
| uint32_t tagged_parameter_slots, |
| OwnedVector<trap_handler::ProtectedInstructionData> protected_instructions, |
| OwnedVector<const byte> source_position_table, WasmCode::Kind kind, |
| ExecutionTier tier) { |
| return AddCodeWithCodeSpace( |
| index, desc, stack_slots, tagged_parameter_slots, |
| std::move(protected_instructions), std::move(source_position_table), kind, |
| tier, code_allocator_.AllocateForCode(this, desc.instr_size)); |
| } |
| |
| std::unique_ptr<WasmCode> NativeModule::AddCodeWithCodeSpace( |
| uint32_t index, const CodeDesc& desc, uint32_t stack_slots, |
| uint32_t tagged_parameter_slots, |
| OwnedVector<ProtectedInstructionData> protected_instructions, |
| OwnedVector<const byte> source_position_table, WasmCode::Kind kind, |
| ExecutionTier tier, Vector<uint8_t> dst_code_bytes) { |
| OwnedVector<byte> reloc_info; |
| if (desc.reloc_size > 0) { |
| reloc_info = OwnedVector<byte>::New(desc.reloc_size); |
| memcpy(reloc_info.start(), desc.buffer + desc.buffer_size - desc.reloc_size, |
| desc.reloc_size); |
| } |
| |
| // TODO(jgruber,v8:8758): Remove this translation. It exists only because |
| // CodeDesc contains real offsets but WasmCode expects an offset of 0 to mean |
| // 'empty'. |
| const size_t safepoint_table_offset = static_cast<size_t>( |
| desc.safepoint_table_size == 0 ? 0 : desc.safepoint_table_offset); |
| const size_t handler_table_offset = |
| static_cast<size_t>(desc.handler_table_offset); |
| const size_t constant_pool_offset = |
| static_cast<size_t>(desc.constant_pool_offset); |
| const size_t code_comments_offset = |
| static_cast<size_t>(desc.code_comments_offset); |
| const size_t instr_size = static_cast<size_t>(desc.instr_size); |
| |
| memcpy(dst_code_bytes.begin(), desc.buffer, |
| static_cast<size_t>(desc.instr_size)); |
| |
| // Apply the relocation delta by iterating over the RelocInfo. |
| intptr_t delta = dst_code_bytes.begin() - desc.buffer; |
| int mode_mask = RelocInfo::kApplyMask | |
| RelocInfo::ModeMask(RelocInfo::WASM_CALL) | |
| RelocInfo::ModeMask(RelocInfo::WASM_STUB_CALL); |
| Address constant_pool_start = |
| reinterpret_cast<Address>(dst_code_bytes.begin()) + constant_pool_offset; |
| for (RelocIterator it(dst_code_bytes, reloc_info.as_vector(), |
| constant_pool_start, mode_mask); |
| !it.done(); it.next()) { |
| RelocInfo::Mode mode = it.rinfo()->rmode(); |
| if (RelocInfo::IsWasmCall(mode)) { |
| uint32_t call_tag = it.rinfo()->wasm_call_tag(); |
| Address target = GetCallTargetForFunction(call_tag); |
| it.rinfo()->set_wasm_call_address(target, SKIP_ICACHE_FLUSH); |
| } else if (RelocInfo::IsWasmStubCall(mode)) { |
| uint32_t stub_call_tag = it.rinfo()->wasm_call_tag(); |
| DCHECK_LT(stub_call_tag, WasmCode::kRuntimeStubCount); |
| Address entry = runtime_stub_entry( |
| static_cast<WasmCode::RuntimeStubId>(stub_call_tag)); |
| it.rinfo()->set_wasm_stub_call_address(entry, SKIP_ICACHE_FLUSH); |
| } else { |
| it.rinfo()->apply(delta); |
| } |
| } |
| |
| // Flush the i-cache after relocation. |
| FlushInstructionCache(dst_code_bytes.begin(), dst_code_bytes.size()); |
| |
| std::unique_ptr<WasmCode> code{new WasmCode{ |
| this, index, dst_code_bytes, stack_slots, tagged_parameter_slots, |
| safepoint_table_offset, handler_table_offset, constant_pool_offset, |
| code_comments_offset, instr_size, std::move(protected_instructions), |
| std::move(reloc_info), std::move(source_position_table), kind, tier}}; |
| code->MaybePrint(); |
| code->Validate(); |
| |
| return code; |
| } |
| |
| WasmCode* NativeModule::PublishCode(std::unique_ptr<WasmCode> code) { |
| base::MutexGuard lock(&allocation_mutex_); |
| return PublishCodeLocked(std::move(code)); |
| } |
| |
| WasmCode::Kind GetCodeKind(const WasmCompilationResult& result) { |
| switch (result.kind) { |
| case WasmCompilationResult::kWasmToJsWrapper: |
| return WasmCode::Kind::kWasmToJsWrapper; |
| case WasmCompilationResult::kInterpreterEntry: |
| return WasmCode::Kind::kInterpreterEntry; |
| case WasmCompilationResult::kFunction: |
| return WasmCode::Kind::kFunction; |
| default: |
| UNREACHABLE(); |
| } |
| } |
| |
| WasmCode* NativeModule::PublishCodeLocked(std::unique_ptr<WasmCode> code) { |
| // The caller must hold the {allocation_mutex_}, thus we fail to lock it here. |
| DCHECK(!allocation_mutex_.TryLock()); |
| |
| if (!code->IsAnonymous() && |
| code->index() >= module_->num_imported_functions) { |
| DCHECK_LT(code->index(), num_functions()); |
| |
| code->RegisterTrapHandlerData(); |
| |
| // Assume an order of execution tiers that represents the quality of their |
| // generated code. |
| static_assert(ExecutionTier::kNone < ExecutionTier::kInterpreter && |
| ExecutionTier::kInterpreter < ExecutionTier::kLiftoff && |
| ExecutionTier::kLiftoff < ExecutionTier::kTurbofan, |
| "Assume an order on execution tiers"); |
| |
| // Update code table but avoid to fall back to less optimized code. We use |
| // the new code if it was compiled with a higher tier. |
| uint32_t slot_idx = code->index() - module_->num_imported_functions; |
| WasmCode* prior_code = code_table_[slot_idx]; |
| bool update_code_table = !prior_code || prior_code->tier() < code->tier(); |
| if (update_code_table) { |
| code_table_[slot_idx] = code.get(); |
| if (prior_code) { |
| WasmCodeRefScope::AddRef(prior_code); |
| // The code is added to the current {WasmCodeRefScope}, hence the ref |
| // count cannot drop to zero here. |
| CHECK(!prior_code->DecRef()); |
| } |
| } |
| |
| // Populate optimized code to the jump table unless there is an active |
| // redirection to the interpreter that should be preserved. |
| bool update_jump_table = |
| update_code_table && !has_interpreter_redirection(code->index()); |
| |
| // Ensure that interpreter entries always populate to the jump table. |
| if (code->kind_ == WasmCode::Kind::kInterpreterEntry) { |
| SetInterpreterRedirection(code->index()); |
| update_jump_table = true; |
| } |
| |
| if (update_jump_table) { |
| JumpTableAssembler::PatchJumpTableSlot( |
| jump_table_->instruction_start(), slot_idx, code->instruction_start(), |
| WasmCode::kFlushICache); |
| } |
| } |
| WasmCodeRefScope::AddRef(code.get()); |
| WasmCode* result = code.get(); |
| owned_code_.emplace(result->instruction_start(), std::move(code)); |
| return result; |
| } |
| |
| WasmCode* NativeModule::AddDeserializedCode( |
| uint32_t index, Vector<const byte> instructions, uint32_t stack_slots, |
| uint32_t tagged_parameter_slots, size_t safepoint_table_offset, |
| size_t handler_table_offset, size_t constant_pool_offset, |
| size_t code_comments_offset, size_t unpadded_binary_size, |
| OwnedVector<ProtectedInstructionData> protected_instructions, |
| OwnedVector<const byte> reloc_info, |
| OwnedVector<const byte> source_position_table, WasmCode::Kind kind, |
| ExecutionTier tier) { |
| Vector<uint8_t> dst_code_bytes = |
| code_allocator_.AllocateForCode(this, instructions.size()); |
| memcpy(dst_code_bytes.begin(), instructions.begin(), instructions.size()); |
| |
| std::unique_ptr<WasmCode> code{new WasmCode{ |
| this, index, dst_code_bytes, stack_slots, tagged_parameter_slots, |
| safepoint_table_offset, handler_table_offset, constant_pool_offset, |
| code_comments_offset, unpadded_binary_size, |
| std::move(protected_instructions), std::move(reloc_info), |
| std::move(source_position_table), kind, tier}}; |
| |
| // Note: we do not flush the i-cache here, since the code needs to be |
| // relocated anyway. The caller is responsible for flushing the i-cache later. |
| |
| return PublishCode(std::move(code)); |
| } |
| |
| std::vector<WasmCode*> NativeModule::SnapshotCodeTable() const { |
| base::MutexGuard lock(&allocation_mutex_); |
| WasmCode** start = code_table_.get(); |
| WasmCode** end = start + module_->num_declared_functions; |
| return std::vector<WasmCode*>{start, end}; |
| } |
| |
| WasmCode* NativeModule::GetCode(uint32_t index) const { |
| base::MutexGuard guard(&allocation_mutex_); |
| DCHECK_LT(index, num_functions()); |
| DCHECK_LE(module_->num_imported_functions, index); |
| WasmCode* code = code_table_[index - module_->num_imported_functions]; |
| if (code) WasmCodeRefScope::AddRef(code); |
| return code; |
| } |
| |
| bool NativeModule::HasCode(uint32_t index) const { |
| base::MutexGuard guard(&allocation_mutex_); |
| DCHECK_LT(index, num_functions()); |
| DCHECK_LE(module_->num_imported_functions, index); |
| return code_table_[index - module_->num_imported_functions] != nullptr; |
| } |
| |
| WasmCode* NativeModule::CreateEmptyJumpTable(uint32_t jump_table_size) { |
| // Only call this if we really need a jump table. |
| DCHECK_LT(0, jump_table_size); |
| Vector<uint8_t> code_space = |
| code_allocator_.AllocateForCode(this, jump_table_size); |
| ZapCode(reinterpret_cast<Address>(code_space.begin()), code_space.size()); |
| std::unique_ptr<WasmCode> code{new WasmCode{ |
| this, // native_module |
| kAnonymousFuncIndex, // index |
| code_space, // instructions |
| 0, // stack_slots |
| 0, // tagged_parameter_slots |
| 0, // safepoint_table_offset |
| jump_table_size, // handler_table_offset |
| jump_table_size, // constant_pool_offset |
| jump_table_size, // code_comments_offset |
| jump_table_size, // unpadded_binary_size |
| OwnedVector<ProtectedInstructionData>{}, // protected_instructions |
| OwnedVector<const uint8_t>{}, // reloc_info |
| OwnedVector<const uint8_t>{}, // source_pos |
| WasmCode::kJumpTable, // kind |
| ExecutionTier::kNone}}; // tier |
| return PublishCode(std::move(code)); |
| } |
| |
| namespace { |
| class NativeModuleWireBytesStorage final : public WireBytesStorage { |
| public: |
| explicit NativeModuleWireBytesStorage( |
| std::shared_ptr<OwnedVector<const uint8_t>> wire_bytes) |
| : wire_bytes_(std::move(wire_bytes)) {} |
| |
| Vector<const uint8_t> GetCode(WireBytesRef ref) const final { |
| return wire_bytes_->as_vector().SubVector(ref.offset(), ref.end_offset()); |
| } |
| |
| private: |
| const std::shared_ptr<OwnedVector<const uint8_t>> wire_bytes_; |
| }; |
| } // namespace |
| |
| void NativeModule::SetWireBytes(OwnedVector<const uint8_t> wire_bytes) { |
| auto shared_wire_bytes = |
| std::make_shared<OwnedVector<const uint8_t>>(std::move(wire_bytes)); |
| wire_bytes_ = shared_wire_bytes; |
| if (!shared_wire_bytes->empty()) { |
| compilation_state_->SetWireBytesStorage( |
| std::make_shared<NativeModuleWireBytesStorage>( |
| std::move(shared_wire_bytes))); |
| } |
| } |
| |
| WasmCode* NativeModule::Lookup(Address pc) const { |
| base::MutexGuard lock(&allocation_mutex_); |
| auto iter = owned_code_.upper_bound(pc); |
| if (iter == owned_code_.begin()) return nullptr; |
| --iter; |
| WasmCode* candidate = iter->second.get(); |
| DCHECK_EQ(candidate->instruction_start(), iter->first); |
| if (!candidate->contains(pc)) return nullptr; |
| WasmCodeRefScope::AddRef(candidate); |
| return candidate; |
| } |
| |
| uint32_t NativeModule::GetJumpTableOffset(uint32_t func_index) const { |
| uint32_t slot_idx = func_index - module_->num_imported_functions; |
| DCHECK_GT(module_->num_declared_functions, slot_idx); |
| return JumpTableAssembler::JumpSlotIndexToOffset(slot_idx); |
| } |
| |
| Address NativeModule::GetCallTargetForFunction(uint32_t func_index) const { |
| // Return the jump table slot for that function index. |
| DCHECK_NOT_NULL(jump_table_); |
| uint32_t slot_offset = GetJumpTableOffset(func_index); |
| DCHECK_LT(slot_offset, jump_table_->instructions().size()); |
| return jump_table_->instruction_start() + slot_offset; |
| } |
| |
| uint32_t NativeModule::GetFunctionIndexFromJumpTableSlot( |
| Address slot_address) const { |
| DCHECK(is_jump_table_slot(slot_address)); |
| uint32_t slot_offset = |
| static_cast<uint32_t>(slot_address - jump_table_->instruction_start()); |
| uint32_t slot_idx = JumpTableAssembler::SlotOffsetToIndex(slot_offset); |
| DCHECK_LT(slot_idx, module_->num_declared_functions); |
| return module_->num_imported_functions + slot_idx; |
| } |
| |
| const char* NativeModule::GetRuntimeStubName(Address runtime_stub_entry) const { |
| #define RETURN_NAME(Name) \ |
| if (runtime_stub_entries_[WasmCode::k##Name] == runtime_stub_entry) { \ |
| return #Name; \ |
| } |
| #define RETURN_NAME_TRAP(Name) RETURN_NAME(ThrowWasm##Name) |
| WASM_RUNTIME_STUB_LIST(RETURN_NAME, RETURN_NAME_TRAP) |
| #undef RETURN_NAME_TRAP |
| #undef RETURN_NAME |
| return "<unknown>"; |
| } |
| |
| NativeModule::~NativeModule() { |
| TRACE_HEAP("Deleting native module: %p\n", this); |
| // Cancel all background compilation before resetting any field of the |
| // NativeModule or freeing anything. |
| compilation_state_->AbortCompilation(); |
| engine_->FreeNativeModule(this); |
| // Free the import wrapper cache before releasing the {WasmCode} objects in |
| // {owned_code_}. The destructor of {WasmImportWrapperCache} still needs to |
| // decrease reference counts on the {WasmCode} objects. |
| import_wrapper_cache_.reset(); |
| } |
| |
| WasmCodeManager::WasmCodeManager(WasmMemoryTracker* memory_tracker, |
| size_t max_committed) |
| : memory_tracker_(memory_tracker), |
| max_committed_code_space_(max_committed), |
| #if defined(V8_OS_WIN_X64) |
| is_win64_unwind_info_disabled_for_testing_(false), |
| #endif |
| total_committed_code_space_(0), |
| critical_committed_code_space_(max_committed / 2) { |
| DCHECK_LE(max_committed, kMaxWasmCodeMemory); |
| } |
| |
| #if defined(V8_OS_WIN_X64) |
| bool WasmCodeManager::CanRegisterUnwindInfoForNonABICompliantCodeRange() const { |
| return win64_unwindinfo::CanRegisterUnwindInfoForNonABICompliantCodeRange() && |
| FLAG_win64_unwinding_info && |
| !is_win64_unwind_info_disabled_for_testing_; |
| } |
| #endif |
| |
| bool WasmCodeManager::Commit(base::AddressRegion region) { |
| // TODO(v8:8462): Remove eager commit once perf supports remapping. |
| if (FLAG_perf_prof) return true; |
| DCHECK(IsAligned(region.begin(), CommitPageSize())); |
| DCHECK(IsAligned(region.size(), CommitPageSize())); |
| // Reserve the size. Use CAS loop to avoid overflow on |
| // {total_committed_code_space_}. |
| size_t old_value = total_committed_code_space_.load(); |
| while (true) { |
| DCHECK_GE(max_committed_code_space_, old_value); |
| if (region.size() > max_committed_code_space_ - old_value) return false; |
| if (total_committed_code_space_.compare_exchange_weak( |
| old_value, old_value + region.size())) { |
| break; |
| } |
| } |
| PageAllocator::Permission permission = FLAG_wasm_write_protect_code_memory |
| ? PageAllocator::kReadWrite |
| : PageAllocator::kReadWriteExecute; |
| |
| bool ret = SetPermissions(GetPlatformPageAllocator(), region.begin(), |
| region.size(), permission); |
| TRACE_HEAP("Setting rw permissions for 0x%" PRIxPTR ":0x%" PRIxPTR "\n", |
| region.begin(), region.end()); |
| |
| if (!ret) { |
| // Highly unlikely. |
| total_committed_code_space_.fetch_sub(region.size()); |
| return false; |
| } |
| return true; |
| } |
| |
| void WasmCodeManager::Decommit(base::AddressRegion region) { |
| // TODO(v8:8462): Remove this once perf supports remapping. |
| if (FLAG_perf_prof) return; |
| PageAllocator* allocator = GetPlatformPageAllocator(); |
| DCHECK(IsAligned(region.begin(), allocator->CommitPageSize())); |
| DCHECK(IsAligned(region.size(), allocator->CommitPageSize())); |
| size_t old_committed = total_committed_code_space_.fetch_sub(region.size()); |
| DCHECK_LE(region.size(), old_committed); |
| USE(old_committed); |
| TRACE_HEAP("Discarding system pages 0x%" PRIxPTR ":0x%" PRIxPTR "\n", |
| region.begin(), region.end()); |
| CHECK(allocator->DiscardSystemPages(reinterpret_cast<void*>(region.begin()), |
| region.size())); |
| } |
| |
| void WasmCodeManager::AssignRange(base::AddressRegion region, |
| NativeModule* native_module) { |
| base::MutexGuard lock(&native_modules_mutex_); |
| lookup_map_.insert(std::make_pair( |
| region.begin(), std::make_pair(region.end(), native_module))); |
| } |
| |
| VirtualMemory WasmCodeManager::TryAllocate(size_t size, void* hint) { |
| v8::PageAllocator* page_allocator = GetPlatformPageAllocator(); |
| DCHECK_GT(size, 0); |
| size_t allocate_page_size = page_allocator->AllocatePageSize(); |
| size = RoundUp(size, allocate_page_size); |
| if (!memory_tracker_->ReserveAddressSpace(size)) return {}; |
| if (hint == nullptr) hint = page_allocator->GetRandomMmapAddr(); |
| |
| VirtualMemory mem(page_allocator, size, hint, allocate_page_size); |
| if (!mem.IsReserved()) { |
| memory_tracker_->ReleaseReservation(size); |
| return {}; |
| } |
| TRACE_HEAP("VMem alloc: 0x%" PRIxPTR ":0x%" PRIxPTR " (%zu)\n", mem.address(), |
| mem.end(), mem.size()); |
| |
| // TODO(v8:8462): Remove eager commit once perf supports remapping. |
| if (FLAG_perf_prof) { |
| SetPermissions(GetPlatformPageAllocator(), mem.address(), mem.size(), |
| PageAllocator::kReadWriteExecute); |
| } |
| return mem; |
| } |
| |
| void WasmCodeManager::SetMaxCommittedMemoryForTesting(size_t limit) { |
| // This has to be set before committing any memory. |
| DCHECK_EQ(0, total_committed_code_space_.load()); |
| max_committed_code_space_ = limit; |
| critical_committed_code_space_.store(limit / 2); |
| } |
| |
| // static |
| size_t WasmCodeManager::EstimateNativeModuleCodeSize(const WasmModule* module) { |
| constexpr size_t kCodeSizeMultiplier = 4; |
| constexpr size_t kCodeOverhead = 32; // for prologue, stack check, ... |
| constexpr size_t kStaticCodeSize = 512; // runtime stubs, ... |
| constexpr size_t kImportSize = 64 * kSystemPointerSize; |
| |
| size_t estimate = kStaticCodeSize; |
| for (auto& function : module->functions) { |
| estimate += kCodeOverhead + kCodeSizeMultiplier * function.code.length(); |
| } |
| estimate += |
| JumpTableAssembler::SizeForNumberOfSlots(module->num_declared_functions); |
| estimate += kImportSize * module->num_imported_functions; |
| |
| return estimate; |
| } |
| |
| // static |
| size_t WasmCodeManager::EstimateNativeModuleNonCodeSize( |
| const WasmModule* module) { |
| size_t wasm_module_estimate = EstimateStoredSize(module); |
| |
| uint32_t num_wasm_functions = module->num_declared_functions; |
| |
| // TODO(wasm): Include wire bytes size. |
| size_t native_module_estimate = |
| sizeof(NativeModule) + /* NativeModule struct */ |
| (sizeof(WasmCode*) * num_wasm_functions) + /* code table size */ |
| (sizeof(WasmCode) * num_wasm_functions); /* code object size */ |
| |
| return wasm_module_estimate + native_module_estimate; |
| } |
| |
| std::shared_ptr<NativeModule> WasmCodeManager::NewNativeModule( |
| WasmEngine* engine, Isolate* isolate, const WasmFeatures& enabled, |
| size_t code_size_estimate, bool can_request_more, |
| std::shared_ptr<const WasmModule> module) { |
| DCHECK_EQ(this, isolate->wasm_engine()->code_manager()); |
| if (total_committed_code_space_.load() > |
| critical_committed_code_space_.load()) { |
| (reinterpret_cast<v8::Isolate*>(isolate)) |
| ->MemoryPressureNotification(MemoryPressureLevel::kCritical); |
| size_t committed = total_committed_code_space_.load(); |
| DCHECK_GE(max_committed_code_space_, committed); |
| critical_committed_code_space_.store( |
| committed + (max_committed_code_space_ - committed) / 2); |
| } |
| |
| // If the code must be contiguous, reserve enough address space up front. |
| size_t code_vmem_size = |
| kRequiresCodeRange ? kMaxWasmCodeMemory : code_size_estimate; |
| // Try up to two times; getting rid of dead JSArrayBuffer allocations might |
| // require two GCs because the first GC maybe incremental and may have |
| // floating garbage. |
| static constexpr int kAllocationRetries = 2; |
| VirtualMemory code_space; |
| for (int retries = 0;; ++retries) { |
| code_space = TryAllocate(code_vmem_size); |
| if (code_space.IsReserved()) break; |
| if (retries == kAllocationRetries) { |
| V8::FatalProcessOutOfMemory(isolate, "NewNativeModule"); |
| UNREACHABLE(); |
| } |
| // Run one GC, then try the allocation again. |
| isolate->heap()->MemoryPressureNotification(MemoryPressureLevel::kCritical, |
| true); |
| } |
| |
| Address start = code_space.address(); |
| size_t size = code_space.size(); |
| Address end = code_space.end(); |
| std::shared_ptr<NativeModule> ret; |
| new NativeModule(engine, enabled, can_request_more, std::move(code_space), |
| std::move(module), isolate->async_counters(), &ret); |
| // The constructor initialized the shared_ptr. |
| DCHECK_NOT_NULL(ret); |
| TRACE_HEAP("New NativeModule %p: Mem: %" PRIuPTR ",+%zu\n", ret.get(), start, |
| size); |
| |
| #if defined(V8_OS_WIN_X64) |
| if (CanRegisterUnwindInfoForNonABICompliantCodeRange()) { |
| win64_unwindinfo::RegisterNonABICompliantCodeRange( |
| reinterpret_cast<void*>(start), size); |
| } |
| #endif |
| |
| base::MutexGuard lock(&native_modules_mutex_); |
| lookup_map_.insert(std::make_pair(start, std::make_pair(end, ret.get()))); |
| return ret; |
| } |
| |
| void NativeModule::SampleCodeSize( |
| Counters* counters, NativeModule::CodeSamplingTime sampling_time) const { |
| size_t code_size = sampling_time == kSampling |
| ? code_allocator_.committed_code_space() |
| : code_allocator_.generated_code_size(); |
| int code_size_mb = static_cast<int>(code_size / MB); |
| Histogram* histogram = nullptr; |
| switch (sampling_time) { |
| case kAfterBaseline: |
| histogram = counters->wasm_module_code_size_mb_after_baseline(); |
| break; |
| case kAfterTopTier: |
| histogram = counters->wasm_module_code_size_mb_after_top_tier(); |
| break; |
| case kSampling: { |
| histogram = counters->wasm_module_code_size_mb(); |
| // If this is a wasm module of >= 2MB, also sample the freed code size, |
| // absolute and relative. Code GC does not happen on asm.js modules, and |
| // small modules will never trigger GC anyway. |
| size_t generated_size = code_allocator_.generated_code_size(); |
| if (generated_size >= 2 * MB && module()->origin == kWasmOrigin) { |
| size_t freed_size = code_allocator_.freed_code_size(); |
| DCHECK_LE(freed_size, generated_size); |
| int total_freed_mb = static_cast<int>(freed_size / MB); |
| counters->wasm_module_freed_code_size_mb()->AddSample(total_freed_mb); |
| int freed_percent = static_cast<int>(100 * freed_size / generated_size); |
| counters->wasm_module_freed_code_size_percent()->AddSample( |
| freed_percent); |
| } |
| break; |
| } |
| } |
| histogram->AddSample(code_size_mb); |
| } |
| |
| WasmCode* NativeModule::AddCompiledCode(WasmCompilationResult result) { |
| return AddCompiledCode({&result, 1})[0]; |
| } |
| |
| std::vector<WasmCode*> NativeModule::AddCompiledCode( |
| Vector<WasmCompilationResult> results) { |
| DCHECK(!results.empty()); |
| // First, allocate code space for all the results. |
| size_t total_code_space = 0; |
| for (auto& result : results) { |
| DCHECK(result.succeeded()); |
| total_code_space += RoundUp<kCodeAlignment>(result.code_desc.instr_size); |
| } |
| Vector<byte> code_space = |
| code_allocator_.AllocateForCode(this, total_code_space); |
| |
| std::vector<std::unique_ptr<WasmCode>> generated_code; |
| generated_code.reserve(results.size()); |
| |
| // Now copy the generated code into the code space and relocate it. |
| for (auto& result : results) { |
| DCHECK_EQ(result.code_desc.buffer, result.instr_buffer.get()); |
| size_t code_size = RoundUp<kCodeAlignment>(result.code_desc.instr_size); |
| Vector<byte> this_code_space = code_space.SubVector(0, code_size); |
| code_space += code_size; |
| generated_code.emplace_back(AddCodeWithCodeSpace( |
| result.func_index, result.code_desc, result.frame_slot_count, |
| result.tagged_parameter_slots, std::move(result.protected_instructions), |
| std::move(result.source_positions), GetCodeKind(result), |
| result.result_tier, this_code_space)); |
| } |
| DCHECK_EQ(0, code_space.size()); |
| |
| // Under the {allocation_mutex_}, publish the code. The published code is put |
| // into the top-most surrounding {WasmCodeRefScope} by {PublishCodeLocked}. |
| std::vector<WasmCode*> code_vector; |
| code_vector.reserve(results.size()); |
| { |
| base::MutexGuard lock(&allocation_mutex_); |
| for (auto& result : generated_code) |
| code_vector.push_back(PublishCodeLocked(std::move(result))); |
| } |
| |
| return code_vector; |
| } |
| |
| bool NativeModule::IsRedirectedToInterpreter(uint32_t func_index) { |
| base::MutexGuard lock(&allocation_mutex_); |
| return has_interpreter_redirection(func_index); |
| } |
| |
| void NativeModule::FreeCode(Vector<WasmCode* const> codes) { |
| // Free the code space. |
| code_allocator_.FreeCode(codes); |
| |
| // Free the {WasmCode} objects. This will also unregister trap handler data. |
| base::MutexGuard guard(&allocation_mutex_); |
| for (WasmCode* code : codes) { |
| DCHECK_EQ(1, owned_code_.count(code->instruction_start())); |
| owned_code_.erase(code->instruction_start()); |
| } |
| } |
| |
| void WasmCodeManager::FreeNativeModule(Vector<VirtualMemory> owned_code_space, |
| size_t committed_size) { |
| base::MutexGuard lock(&native_modules_mutex_); |
| for (auto& code_space : owned_code_space) { |
| DCHECK(code_space.IsReserved()); |
| TRACE_HEAP("VMem Release: 0x%" PRIxPTR ":0x%" PRIxPTR " (%zu)\n", |
| code_space.address(), code_space.end(), code_space.size()); |
| |
| #if defined(V8_OS_WIN_X64) |
| if (CanRegisterUnwindInfoForNonABICompliantCodeRange()) { |
| win64_unwindinfo::UnregisterNonABICompliantCodeRange( |
| reinterpret_cast<void*>(code_space.address())); |
| } |
| #endif |
| |
| lookup_map_.erase(code_space.address()); |
| memory_tracker_->ReleaseReservation(code_space.size()); |
| code_space.Free(); |
| DCHECK(!code_space.IsReserved()); |
| } |
| |
| DCHECK(IsAligned(committed_size, CommitPageSize())); |
| size_t old_committed = total_committed_code_space_.fetch_sub(committed_size); |
| DCHECK_LE(committed_size, old_committed); |
| USE(old_committed); |
| } |
| |
| NativeModule* WasmCodeManager::LookupNativeModule(Address pc) const { |
| base::MutexGuard lock(&native_modules_mutex_); |
| if (lookup_map_.empty()) return nullptr; |
| |
| auto iter = lookup_map_.upper_bound(pc); |
| if (iter == lookup_map_.begin()) return nullptr; |
| --iter; |
| Address region_start = iter->first; |
| Address region_end = iter->second.first; |
| NativeModule* candidate = iter->second.second; |
| |
| DCHECK_NOT_NULL(candidate); |
| return region_start <= pc && pc < region_end ? candidate : nullptr; |
| } |
| |
| WasmCode* WasmCodeManager::LookupCode(Address pc) const { |
| NativeModule* candidate = LookupNativeModule(pc); |
| return candidate ? candidate->Lookup(pc) : nullptr; |
| } |
| |
| // TODO(v8:7424): Code protection scopes are not yet supported with shared code |
| // enabled and need to be revisited to work with --wasm-shared-code as well. |
| NativeModuleModificationScope::NativeModuleModificationScope( |
| NativeModule* native_module) |
| : native_module_(native_module) { |
| if (FLAG_wasm_write_protect_code_memory && native_module_ && |
| (native_module_->modification_scope_depth_++) == 0) { |
| bool success = native_module_->SetExecutable(false); |
| CHECK(success); |
| } |
| } |
| |
| NativeModuleModificationScope::~NativeModuleModificationScope() { |
| if (FLAG_wasm_write_protect_code_memory && native_module_ && |
| (native_module_->modification_scope_depth_--) == 1) { |
| bool success = native_module_->SetExecutable(true); |
| CHECK(success); |
| } |
| } |
| |
| // Disabling these code currently to fix API leaks, wasm module will be removed |
| // as a whole later. |
| #if !defined(V8_OS_STARBOARD) |
| namespace { |
| thread_local WasmCodeRefScope* current_code_refs_scope = nullptr; |
| } // namespace |
| #endif |
| |
| WasmCodeRefScope::WasmCodeRefScope() |
| #if !defined(V8_OS_STARBOARD) |
| : previous_scope_(current_code_refs_scope) { |
| current_code_refs_scope = this; |
| } |
| #else |
| : previous_scope_(nullptr) {} |
| #endif |
| |
| WasmCodeRefScope::~WasmCodeRefScope() { |
| #if !defined(V8_OS_STARBOARD) |
| DCHECK_EQ(this, current_code_refs_scope); |
| current_code_refs_scope = previous_scope_; |
| std::vector<WasmCode*> code_ptrs; |
| code_ptrs.reserve(code_ptrs_.size()); |
| code_ptrs.assign(code_ptrs_.begin(), code_ptrs_.end()); |
| WasmCode::DecrementRefCount(VectorOf(code_ptrs)); |
| #endif |
| } |
| |
| // static |
| void WasmCodeRefScope::AddRef(WasmCode* code) { |
| #if !defined(V8_OS_STARBOARD) |
| DCHECK_NOT_NULL(code); |
| WasmCodeRefScope* current_scope = current_code_refs_scope; |
| DCHECK_NOT_NULL(current_scope); |
| auto entry = current_scope->code_ptrs_.insert(code); |
| // If we added a new entry, increment the ref counter. |
| if (entry.second) code->IncRef(); |
| #endif |
| } |
| |
| } // namespace wasm |
| } // namespace internal |
| } // namespace v8 |
| #undef TRACE_HEAP |