/dex2oat/linker/arm/relative_patcher_arm_base.cc
C++ | 551 lines | 443 code | 46 blank | 62 comment | 96 complexity | 6ec42efb1acf8fed0fab190388885d3a MD5 | raw file
- /*
- * Copyright (C) 2015 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
- #include "linker/arm/relative_patcher_arm_base.h"
- #include "base/stl_util.h"
- #include "compiled_method-inl.h"
- #include "debug/method_debug_info.h"
- #include "dex/dex_file_types.h"
- #include "linker/linker_patch.h"
- #include "oat.h"
- #include "oat_quick_method_header.h"
- #include "stream/output_stream.h"
- namespace art {
- namespace linker {
- class ArmBaseRelativePatcher::ThunkData {
- public:
- ThunkData(ArrayRef<const uint8_t> code, const std::string& debug_name, uint32_t max_next_offset)
- : code_(code),
- debug_name_(debug_name),
- offsets_(),
- max_next_offset_(max_next_offset),
- pending_offset_(0u) {
- DCHECK(NeedsNextThunk()); // The data is constructed only when we expect to need the thunk.
- }
- ThunkData(ThunkData&& src) = default;
- size_t CodeSize() const {
- return code_.size();
- }
- ArrayRef<const uint8_t> GetCode() const {
- return code_;
- }
- const std::string& GetDebugName() const {
- return debug_name_;
- }
- bool NeedsNextThunk() const {
- return max_next_offset_ != 0u;
- }
- uint32_t MaxNextOffset() const {
- DCHECK(NeedsNextThunk());
- return max_next_offset_;
- }
- void ClearMaxNextOffset() {
- DCHECK(NeedsNextThunk());
- max_next_offset_ = 0u;
- }
- void SetMaxNextOffset(uint32_t max_next_offset) {
- DCHECK(!NeedsNextThunk());
- max_next_offset_ = max_next_offset;
- }
- // Adjust the MaxNextOffset() down if needed to fit the code before the next thunk.
- // Returns true if it was adjusted, false if the old value was kept.
- bool MakeSpaceBefore(const ThunkData& next_thunk, size_t alignment) {
- DCHECK(NeedsNextThunk());
- DCHECK(next_thunk.NeedsNextThunk());
- DCHECK_ALIGNED_PARAM(MaxNextOffset(), alignment);
- DCHECK_ALIGNED_PARAM(next_thunk.MaxNextOffset(), alignment);
- if (next_thunk.MaxNextOffset() - CodeSize() < MaxNextOffset()) {
- max_next_offset_ = RoundDown(next_thunk.MaxNextOffset() - CodeSize(), alignment);
- return true;
- } else {
- return false;
- }
- }
- uint32_t ReserveOffset(size_t offset) {
- DCHECK(NeedsNextThunk());
- DCHECK_LE(offset, max_next_offset_);
- max_next_offset_ = 0u; // The reserved offset should satisfy all pending references.
- offsets_.push_back(offset);
- return offset + CodeSize();
- }
- bool HasReservedOffset() const {
- return !offsets_.empty();
- }
- uint32_t LastReservedOffset() const {
- DCHECK(HasReservedOffset());
- return offsets_.back();
- }
- bool HasPendingOffset() const {
- return pending_offset_ != offsets_.size();
- }
- uint32_t GetPendingOffset() const {
- DCHECK(HasPendingOffset());
- return offsets_[pending_offset_];
- }
- void MarkPendingOffsetAsWritten() {
- DCHECK(HasPendingOffset());
- ++pending_offset_;
- }
- bool HasWrittenOffset() const {
- return pending_offset_ != 0u;
- }
- uint32_t LastWrittenOffset() const {
- DCHECK(HasWrittenOffset());
- return offsets_[pending_offset_ - 1u];
- }
- size_t IndexOfFirstThunkAtOrAfter(uint32_t offset) const {
- size_t number_of_thunks = NumberOfThunks();
- for (size_t i = 0; i != number_of_thunks; ++i) {
- if (GetThunkOffset(i) >= offset) {
- return i;
- }
- }
- return number_of_thunks;
- }
- size_t NumberOfThunks() const {
- return offsets_.size();
- }
- uint32_t GetThunkOffset(size_t index) const {
- DCHECK_LT(index, NumberOfThunks());
- return offsets_[index];
- }
- private:
- const ArrayRef<const uint8_t> code_; // The code of the thunk.
- const std::string debug_name_; // The debug name of the thunk.
- std::vector<uint32_t> offsets_; // Offsets at which the thunk needs to be written.
- uint32_t max_next_offset_; // The maximum offset at which the next thunk can be placed.
- uint32_t pending_offset_; // The index of the next offset to write.
- };
- class ArmBaseRelativePatcher::PendingThunkComparator {
- public:
- bool operator()(const ThunkData* lhs, const ThunkData* rhs) const {
- DCHECK(lhs->HasPendingOffset());
- DCHECK(rhs->HasPendingOffset());
- // The top of the heap is defined to contain the highest element and we want to pick
- // the thunk with the smallest pending offset, so use the reverse ordering, i.e. ">".
- return lhs->GetPendingOffset() > rhs->GetPendingOffset();
- }
- };
- uint32_t ArmBaseRelativePatcher::ReserveSpace(uint32_t offset,
- const CompiledMethod* compiled_method,
- MethodReference method_ref) {
- return ReserveSpaceInternal(offset, compiled_method, method_ref, 0u);
- }
- uint32_t ArmBaseRelativePatcher::ReserveSpaceEnd(uint32_t offset) {
- // For multi-oat compilations (boot image), ReserveSpaceEnd() is called for each oat file.
- // Since we do not know here whether this is the last file or whether the next opportunity
- // to place thunk will be soon enough, we need to reserve all needed thunks now. Code for
- // subsequent oat files can still call back to them.
- if (!unprocessed_method_call_patches_.empty()) {
- ResolveMethodCalls(offset, MethodReference(nullptr, dex::kDexNoIndex));
- }
- for (ThunkData* data : unreserved_thunks_) {
- uint32_t thunk_offset = CompiledCode::AlignCode(offset, instruction_set_);
- offset = data->ReserveOffset(thunk_offset);
- }
- unreserved_thunks_.clear();
- // We also need to delay initiating the pending_thunks_ until the call to WriteThunks().
- // Check that the `pending_thunks_.capacity()` indicates that no WriteThunks() has taken place.
- DCHECK_EQ(pending_thunks_.capacity(), 0u);
- return offset;
- }
- uint32_t ArmBaseRelativePatcher::WriteThunks(OutputStream* out, uint32_t offset) {
- if (pending_thunks_.capacity() == 0u) {
- if (thunks_.empty()) {
- return offset;
- }
- // First call to WriteThunks(), prepare the thunks for writing.
- pending_thunks_.reserve(thunks_.size());
- for (auto& entry : thunks_) {
- ThunkData* data = &entry.second;
- if (data->HasPendingOffset()) {
- pending_thunks_.push_back(data);
- }
- }
- std::make_heap(pending_thunks_.begin(), pending_thunks_.end(), PendingThunkComparator());
- }
- uint32_t aligned_offset = CompiledMethod::AlignCode(offset, instruction_set_);
- while (!pending_thunks_.empty() &&
- pending_thunks_.front()->GetPendingOffset() == aligned_offset) {
- // Write alignment bytes and code.
- uint32_t aligned_code_delta = aligned_offset - offset;
- if (aligned_code_delta != 0u && UNLIKELY(!WriteCodeAlignment(out, aligned_code_delta))) {
- return 0u;
- }
- if (UNLIKELY(!WriteThunk(out, pending_thunks_.front()->GetCode()))) {
- return 0u;
- }
- offset = aligned_offset + pending_thunks_.front()->CodeSize();
- // Mark the thunk as written at the pending offset and update the `pending_thunks_` heap.
- std::pop_heap(pending_thunks_.begin(), pending_thunks_.end(), PendingThunkComparator());
- pending_thunks_.back()->MarkPendingOffsetAsWritten();
- if (pending_thunks_.back()->HasPendingOffset()) {
- std::push_heap(pending_thunks_.begin(), pending_thunks_.end(), PendingThunkComparator());
- } else {
- pending_thunks_.pop_back();
- }
- aligned_offset = CompiledMethod::AlignCode(offset, instruction_set_);
- }
- DCHECK(pending_thunks_.empty() || pending_thunks_.front()->GetPendingOffset() > aligned_offset);
- return offset;
- }
- std::vector<debug::MethodDebugInfo> ArmBaseRelativePatcher::GenerateThunkDebugInfo(
- uint32_t executable_offset) {
- // For multi-oat compilation (boot image), `thunks_` records thunks for all oat files.
- // To return debug info for the current oat file, we must ignore thunks before the
- // `executable_offset` as they are in the previous oat files and this function must be
- // called before reserving thunk positions for subsequent oat files.
- size_t number_of_thunks = 0u;
- for (auto&& entry : thunks_) {
- const ThunkData& data = entry.second;
- number_of_thunks += data.NumberOfThunks() - data.IndexOfFirstThunkAtOrAfter(executable_offset);
- }
- std::vector<debug::MethodDebugInfo> result;
- result.reserve(number_of_thunks);
- for (auto&& entry : thunks_) {
- const ThunkData& data = entry.second;
- size_t start = data.IndexOfFirstThunkAtOrAfter(executable_offset);
- if (start == data.NumberOfThunks()) {
- continue;
- }
- // Get the base name to use for the first occurrence of the thunk.
- const std::string& base_name = data.GetDebugName();
- for (size_t i = start, num = data.NumberOfThunks(); i != num; ++i) {
- debug::MethodDebugInfo info = {};
- if (i == 0u) {
- info.custom_name = base_name;
- } else {
- // Add a disambiguating tag for subsequent identical thunks. Since the `thunks_`
- // keeps records also for thunks in previous oat files, names based on the thunk
- // index shall be unique across the whole multi-oat output.
- info.custom_name = base_name + "_" + std::to_string(i);
- }
- info.isa = instruction_set_;
- info.is_code_address_text_relative = true;
- info.code_address = data.GetThunkOffset(i) - executable_offset;
- info.code_size = data.CodeSize();
- result.push_back(std::move(info));
- }
- }
- return result;
- }
- ArmBaseRelativePatcher::ArmBaseRelativePatcher(RelativePatcherThunkProvider* thunk_provider,
- RelativePatcherTargetProvider* target_provider,
- InstructionSet instruction_set)
- : thunk_provider_(thunk_provider),
- target_provider_(target_provider),
- instruction_set_(instruction_set),
- thunks_(),
- unprocessed_method_call_patches_(),
- method_call_thunk_(nullptr),
- pending_thunks_() {
- }
- ArmBaseRelativePatcher::~ArmBaseRelativePatcher() {
- // All work done by member destructors.
- }
- uint32_t ArmBaseRelativePatcher::ReserveSpaceInternal(uint32_t offset,
- const CompiledMethod* compiled_method,
- MethodReference method_ref,
- uint32_t max_extra_space) {
- // Adjust code size for extra space required by the subclass.
- uint32_t max_code_size = compiled_method->GetQuickCode().size() + max_extra_space;
- uint32_t code_offset;
- uint32_t next_aligned_offset;
- while (true) {
- code_offset = compiled_method->AlignCode(offset + sizeof(OatQuickMethodHeader));
- next_aligned_offset = compiled_method->AlignCode(code_offset + max_code_size);
- if (unreserved_thunks_.empty() ||
- unreserved_thunks_.front()->MaxNextOffset() >= next_aligned_offset) {
- break;
- }
- ThunkData* thunk = unreserved_thunks_.front();
- if (thunk == method_call_thunk_) {
- ResolveMethodCalls(code_offset, method_ref);
- // This may have changed `method_call_thunk_` data, so re-check if we need to reserve.
- if (unreserved_thunks_.empty() ||
- unreserved_thunks_.front()->MaxNextOffset() >= next_aligned_offset) {
- break;
- }
- // We need to process the new `front()` whether it's still the `method_call_thunk_` or not.
- thunk = unreserved_thunks_.front();
- }
- unreserved_thunks_.pop_front();
- uint32_t thunk_offset = CompiledCode::AlignCode(offset, instruction_set_);
- offset = thunk->ReserveOffset(thunk_offset);
- if (thunk == method_call_thunk_) {
- // All remaining method call patches will be handled by this thunk.
- DCHECK(!unprocessed_method_call_patches_.empty());
- DCHECK_LE(thunk_offset - unprocessed_method_call_patches_.front().GetPatchOffset(),
- MaxPositiveDisplacement(GetMethodCallKey()));
- unprocessed_method_call_patches_.clear();
- }
- }
- // Process patches and check that adding thunks for the current method did not push any
- // thunks (previously existing or newly added) before `next_aligned_offset`. This is
- // essentially a check that we never compile a method that's too big. The calls or branches
- // from the method should be able to reach beyond the end of the method and over any pending
- // thunks. (The number of different thunks should be relatively low and their code short.)
- ProcessPatches(compiled_method, code_offset);
- CHECK(unreserved_thunks_.empty() ||
- unreserved_thunks_.front()->MaxNextOffset() >= next_aligned_offset);
- return offset;
- }
- uint32_t ArmBaseRelativePatcher::CalculateMethodCallDisplacement(uint32_t patch_offset,
- uint32_t target_offset) {
- DCHECK(method_call_thunk_ != nullptr);
- // Unsigned arithmetic with its well-defined overflow behavior is just fine here.
- uint32_t displacement = target_offset - patch_offset;
- uint32_t max_positive_displacement = MaxPositiveDisplacement(GetMethodCallKey());
- uint32_t max_negative_displacement = MaxNegativeDisplacement(GetMethodCallKey());
- // NOTE: With unsigned arithmetic we do mean to use && rather than || below.
- if (displacement > max_positive_displacement && displacement < -max_negative_displacement) {
- // Unwritten thunks have higher offsets, check if it's within range.
- DCHECK(!method_call_thunk_->HasPendingOffset() ||
- method_call_thunk_->GetPendingOffset() > patch_offset);
- if (method_call_thunk_->HasPendingOffset() &&
- method_call_thunk_->GetPendingOffset() - patch_offset <= max_positive_displacement) {
- displacement = method_call_thunk_->GetPendingOffset() - patch_offset;
- } else {
- // We must have a previous thunk then.
- DCHECK(method_call_thunk_->HasWrittenOffset());
- DCHECK_LT(method_call_thunk_->LastWrittenOffset(), patch_offset);
- displacement = method_call_thunk_->LastWrittenOffset() - patch_offset;
- DCHECK_GE(displacement, -max_negative_displacement);
- }
- }
- return displacement;
- }
- uint32_t ArmBaseRelativePatcher::GetThunkTargetOffset(const ThunkKey& key, uint32_t patch_offset) {
- auto it = thunks_.find(key);
- CHECK(it != thunks_.end());
- const ThunkData& data = it->second;
- if (data.HasWrittenOffset()) {
- uint32_t offset = data.LastWrittenOffset();
- DCHECK_LT(offset, patch_offset);
- if (patch_offset - offset <= MaxNegativeDisplacement(key)) {
- return offset;
- }
- }
- DCHECK(data.HasPendingOffset());
- uint32_t offset = data.GetPendingOffset();
- DCHECK_GT(offset, patch_offset);
- DCHECK_LE(offset - patch_offset, MaxPositiveDisplacement(key));
- return offset;
- }
- ArmBaseRelativePatcher::ThunkKey ArmBaseRelativePatcher::GetMethodCallKey() {
- return ThunkKey(ThunkType::kMethodCall);
- }
- ArmBaseRelativePatcher::ThunkKey ArmBaseRelativePatcher::GetEntrypointCallKey(
- const LinkerPatch& patch) {
- DCHECK_EQ(patch.GetType(), LinkerPatch::Type::kCallEntrypoint);
- return ThunkKey(ThunkType::kEntrypointCall, patch.EntrypointOffset());
- }
- ArmBaseRelativePatcher::ThunkKey ArmBaseRelativePatcher::GetBakerThunkKey(
- const LinkerPatch& patch) {
- DCHECK_EQ(patch.GetType(), LinkerPatch::Type::kBakerReadBarrierBranch);
- return ThunkKey(ThunkType::kBakerReadBarrier,
- patch.GetBakerCustomValue1(),
- patch.GetBakerCustomValue2());
- }
- void ArmBaseRelativePatcher::ProcessPatches(const CompiledMethod* compiled_method,
- uint32_t code_offset) {
- for (const LinkerPatch& patch : compiled_method->GetPatches()) {
- uint32_t patch_offset = code_offset + patch.LiteralOffset();
- ThunkKey key(static_cast<ThunkType>(-1));
- bool simple_thunk_patch = false;
- ThunkData* old_data = nullptr;
- if (patch.GetType() == LinkerPatch::Type::kCallRelative) {
- key = GetMethodCallKey();
- unprocessed_method_call_patches_.emplace_back(patch_offset, patch.TargetMethod());
- if (method_call_thunk_ == nullptr) {
- uint32_t max_next_offset = CalculateMaxNextOffset(patch_offset, key);
- auto it = thunks_.Put(key, ThunkDataForPatch(patch, max_next_offset));
- method_call_thunk_ = &it->second;
- AddUnreservedThunk(method_call_thunk_);
- } else {
- old_data = method_call_thunk_;
- }
- } else if (patch.GetType() == LinkerPatch::Type::kCallEntrypoint) {
- key = GetEntrypointCallKey(patch);
- simple_thunk_patch = true;
- } else if (patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch) {
- key = GetBakerThunkKey(patch);
- simple_thunk_patch = true;
- }
- if (simple_thunk_patch) {
- auto lb = thunks_.lower_bound(key);
- if (lb == thunks_.end() || thunks_.key_comp()(key, lb->first)) {
- uint32_t max_next_offset = CalculateMaxNextOffset(patch_offset, key);
- auto it = thunks_.PutBefore(lb, key, ThunkDataForPatch(patch, max_next_offset));
- AddUnreservedThunk(&it->second);
- } else {
- old_data = &lb->second;
- }
- }
- if (old_data != nullptr) {
- // Shared path where an old thunk may need an update.
- DCHECK(key.GetType() != static_cast<ThunkType>(-1));
- DCHECK(!old_data->HasReservedOffset() || old_data->LastReservedOffset() < patch_offset);
- if (old_data->NeedsNextThunk()) {
- // Patches for a method are ordered by literal offset, so if we still need to place
- // this thunk for a previous patch, that thunk shall be in range for this patch.
- DCHECK_LE(old_data->MaxNextOffset(), CalculateMaxNextOffset(patch_offset, key));
- } else {
- if (!old_data->HasReservedOffset() ||
- patch_offset - old_data->LastReservedOffset() > MaxNegativeDisplacement(key)) {
- old_data->SetMaxNextOffset(CalculateMaxNextOffset(patch_offset, key));
- AddUnreservedThunk(old_data);
- }
- }
- }
- }
- }
- void ArmBaseRelativePatcher::AddUnreservedThunk(ThunkData* data) {
- DCHECK(data->NeedsNextThunk());
- size_t index = unreserved_thunks_.size();
- while (index != 0u && data->MaxNextOffset() < unreserved_thunks_[index - 1u]->MaxNextOffset()) {
- --index;
- }
- unreserved_thunks_.insert(unreserved_thunks_.begin() + index, data);
- // We may need to update the max next offset(s) if the thunk code would not fit.
- size_t alignment = GetInstructionSetAlignment(instruction_set_);
- if (index + 1u != unreserved_thunks_.size()) {
- // Note: Ignore the return value as we need to process previous thunks regardless.
- data->MakeSpaceBefore(*unreserved_thunks_[index + 1u], alignment);
- }
- // Make space for previous thunks. Once we find a pending thunk that does
- // not need an adjustment, we can stop.
- while (index != 0u && unreserved_thunks_[index - 1u]->MakeSpaceBefore(*data, alignment)) {
- --index;
- data = unreserved_thunks_[index];
- }
- }
- void ArmBaseRelativePatcher::ResolveMethodCalls(uint32_t quick_code_offset,
- MethodReference method_ref) {
- DCHECK(!unreserved_thunks_.empty());
- DCHECK(!unprocessed_method_call_patches_.empty());
- DCHECK(method_call_thunk_ != nullptr);
- uint32_t max_positive_displacement = MaxPositiveDisplacement(GetMethodCallKey());
- uint32_t max_negative_displacement = MaxNegativeDisplacement(GetMethodCallKey());
- // Process as many patches as possible, stop only on unresolved targets or calls too far back.
- while (!unprocessed_method_call_patches_.empty()) {
- MethodReference target_method = unprocessed_method_call_patches_.front().GetTargetMethod();
- uint32_t patch_offset = unprocessed_method_call_patches_.front().GetPatchOffset();
- DCHECK(!method_call_thunk_->HasReservedOffset() ||
- method_call_thunk_->LastReservedOffset() <= patch_offset);
- if (!method_call_thunk_->HasReservedOffset() ||
- patch_offset - method_call_thunk_->LastReservedOffset() > max_negative_displacement) {
- // No previous thunk in range, check if we can reach the target directly.
- if (target_method == method_ref) {
- DCHECK_GT(quick_code_offset, patch_offset);
- if (quick_code_offset - patch_offset > max_positive_displacement) {
- break;
- }
- } else {
- auto result = target_provider_->FindMethodOffset(target_method);
- if (!result.first) {
- break;
- }
- uint32_t target_offset = result.second - CompiledCode::CodeDelta(instruction_set_);
- if (target_offset >= patch_offset) {
- DCHECK_LE(target_offset - patch_offset, max_positive_displacement);
- } else if (patch_offset - target_offset > max_negative_displacement) {
- break;
- }
- }
- }
- unprocessed_method_call_patches_.pop_front();
- }
- if (!unprocessed_method_call_patches_.empty()) {
- // Try to adjust the max next offset in `method_call_thunk_`. Do this conservatively only if
- // the thunk shall be at the end of the `unreserved_thunks_` to avoid dealing with overlaps.
- uint32_t new_max_next_offset =
- unprocessed_method_call_patches_.front().GetPatchOffset() + max_positive_displacement;
- if (new_max_next_offset >
- unreserved_thunks_.back()->MaxNextOffset() + unreserved_thunks_.back()->CodeSize()) {
- method_call_thunk_->ClearMaxNextOffset();
- method_call_thunk_->SetMaxNextOffset(new_max_next_offset);
- if (method_call_thunk_ != unreserved_thunks_.back()) {
- RemoveElement(unreserved_thunks_, method_call_thunk_);
- unreserved_thunks_.push_back(method_call_thunk_);
- }
- }
- } else {
- // We have resolved all method calls, we do not need a new thunk anymore.
- method_call_thunk_->ClearMaxNextOffset();
- RemoveElement(unreserved_thunks_, method_call_thunk_);
- }
- }
- inline uint32_t ArmBaseRelativePatcher::CalculateMaxNextOffset(uint32_t patch_offset,
- const ThunkKey& key) {
- return RoundDown(patch_offset + MaxPositiveDisplacement(key),
- GetInstructionSetAlignment(instruction_set_));
- }
- inline ArmBaseRelativePatcher::ThunkData ArmBaseRelativePatcher::ThunkDataForPatch(
- const LinkerPatch& patch, uint32_t max_next_offset) {
- ArrayRef<const uint8_t> code;
- std::string debug_name;
- thunk_provider_->GetThunkCode(patch, &code, &debug_name);
- DCHECK(!code.empty());
- return ThunkData(code, debug_name, max_next_offset);
- }
- } // namespace linker
- } // namespace art