// Copyright 2018 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "src/compiler/js-heap-broker.h"

#include <optional>

#include "src/common/globals.h"

#ifdef ENABLE_SLOW_DCHECKS
#include <algorithm>
#endif

#include "src/codegen/optimized-compilation-info.h"
#include "src/compiler/js-heap-broker-inl.h"
#include "src/handles/handles-inl.h"
#include "src/handles/maybe-handles.h"
#include "src/heap/heap-inl.h"
#include "src/objects/allocation-site-inl.h"
#include "src/objects/js-array-inl.h"
#include "src/objects/literal-objects-inl.h"
#include "src/objects/map-updater.h"
#include "src/objects/megadom-handler-inl.h"
#include "src/objects/objects-inl.h"
#include "src/objects/property-cell.h"

namespace v8 {
namespace internal {
namespace compiler {

#define TRACE(broker, x) TRACE_BROKER(broker, x)

void JSHeapBroker::IncrementTracingIndentation() { ++trace_indentation_; }

void JSHeapBroker::DecrementTracingIndentation() { --trace_indentation_; }

JSHeapBroker::JSHeapBroker(Isolate* isolate, Zone* broker_zone,
                           bool tracing_enabled, CodeKind code_kind)
    : isolate_(isolate),
#if V8_COMPRESS_POINTERS
      cage_base_(isolate),
#endif  // V8_COMPRESS_POINTERS
      zone_(broker_zone),
      // Note that this initialization of {refs_} with the minimal initial
      // capacity is redundant in the normal use case (concurrent compilation
      // enabled, standard objects to be serialized), as the map is going to be
      // replaced immediately with a larger-capacity one.  It doesn't seem to
      // affect the performance in a noticeable way though.
      refs_(zone()->New<RefsMap>(kMinimalRefsBucketCount, AddressMatcher(),
                                 zone())),
      root_index_map_(isolate),
      tracing_enabled_(tracing_enabled),
      code_kind_(code_kind),
      feedback_(zone()),
      property_access_infos_(zone()) {
  TRACE(this, "Constructing heap broker");
}

JSHeapBroker::~JSHeapBroker() { DCHECK_NULL(local_isolate_); }

std::string JSHeapBroker::Trace() const {
  std::ostringstream oss;
  oss << "[" << this << "] ";
  for (unsigned i = 0; i < trace_indentation_ * 2; ++i) oss.put(' ');
  return oss.str();
}

#ifdef DEBUG
static thread_local JSHeapBroker* current_broker = nullptr;

CurrentHeapBrokerScope::CurrentHeapBrokerScope(JSHeapBroker* broker)
    : prev_broker_(current_broker) {
  current_broker = broker;
}
CurrentHeapBrokerScope::~CurrentHeapBrokerScope() {
  current_broker = prev_broker_;
}

// static
JSHeapBroker* JSHeapBroker::Current() {
  DCHECK_NOT_NULL(current_broker);
  return current_broker;
}
#endif

void JSHeapBroker::AttachLocalIsolate(OptimizedCompilationInfo* info,
                                      LocalIsolate* local_isolate) {
  DCHECK_NULL(local_isolate_);
  local_isolate_ = local_isolate;
  DCHECK_NOT_NULL(local_isolate_);
  local_isolate_->heap()->AttachPersistentHandles(
      info->DetachPersistentHandles());
}

void JSHeapBroker::DetachLocalIsolate(OptimizedCompilationInfo* info) {
  DCHECK_NULL(ph_);
  DCHECK_NOT_NULL(local_isolate_);
  std::unique_ptr<PersistentHandles> ph =
      local_isolate_->heap()->DetachPersistentHandles();
  local_isolate_ = nullptr;
  info->set_persistent_handles(std::move(ph));
}

void JSHeapBroker::StopSerializing() {
  CHECK_EQ(mode_, kSerializing);
  TRACE(this, "Stopping serialization");
  mode_ = kSerialized;
}

void JSHeapBroker::Retire() {
  CHECK_EQ(mode_, kSerialized);
  TRACE(this, "Retiring");
  mode_ = kRetired;
}

void JSHeapBroker::SetTargetNativeContextRef(
    DirectHandle<NativeContext> native_context) {
  DCHECK(!target_native_context_.has_value());
  target_native_context_ = MakeRef(this, *native_context);
}

StringRef JSHeapBroker::GetTypedArrayStringTag(ElementsKind kind) {
  DCHECK(IsTypedArrayOrRabGsabTypedArrayElementsKind(kind));
  switch (kind) {
#define TYPED_ARRAY_STRING_TAG(Type, type, TYPE, ctype) \
  case ElementsKind::TYPE##_ELEMENTS:                   \
    return Type##Array_string();
    TYPED_ARRAYS(TYPED_ARRAY_STRING_TAG)
    RAB_GSAB_TYPED_ARRAYS_WITH_TYPED_ARRAY_TYPE(TYPED_ARRAY_STRING_TAG)
#undef TYPED_ARRAY_STRING_TAG
    default:
      UNREACHABLE();
  }
}

ObjectData* JSHeapBroker::GetOrCreateData(Handle<Object> object,
                                          GetOrCreateDataFlags flags) {
  ObjectData* return_value = TryGetOrCreateData(object, flags | kCrashOnError);
  DCHECK_NOT_NULL(return_value);
  return return_value;
}

bool JSHeapBroker::StackHasOverflowed() const {
  DCHECK_IMPLIES(local_isolate_ == nullptr,
                 ThreadId::Current() == isolate_->thread_id());
  return (local_isolate_ != nullptr)
             ? StackLimitCheck::HasOverflowed(local_isolate_)
             : StackLimitCheck(isolate_).HasOverflowed();
}

bool JSHeapBroker::ObjectMayBeUninitialized(DirectHandle<Object> object) const {
  return ObjectMayBeUninitialized(*object);
}

bool JSHeapBroker::ObjectMayBeUninitialized(Tagged<Object> object) const {
  if (!IsHeapObject(object)) return false;
  return ObjectMayBeUninitialized(Cast<HeapObject>(object));
}

bool JSHeapBroker::ObjectMayBeUninitialized(Tagged<HeapObject> object) const {
  return !IsMainThread() && isolate()->heap()->IsPendingAllocation(object);
}

#define V(Type, name, Name)                                                 \
  void JSHeapBroker::Init##Name() {                                         \
    DCHECK(!name##_);                                                       \
    name##_ = MakeRefAssumeMemoryFence(this, isolate()->factory()->name()); \
  }
READ_ONLY_ROOT_LIST(V)
#undef V

ProcessedFeedback::ProcessedFeedback(Kind kind, FeedbackSlotKind slot_kind)
    : kind_(kind), slot_kind_(slot_kind) {}

KeyedAccessMode ElementAccessFeedback::keyed_mode() const {
  return keyed_mode_;
}

ZoneVector<ElementAccessFeedback::TransitionGroup> const&
ElementAccessFeedback::transition_groups() const {
  return transition_groups_;
}

ElementAccessFeedback const& ElementAccessFeedback::Refine(
    JSHeapBroker* broker, ZoneVector<MapRef> const& inferred_maps) const {
  if (inferred_maps.empty()) {
    return *broker->zone()->New<ElementAccessFeedback>(
        broker->zone(), keyed_mode(), slot_kind());
  }

  ZoneRefSet<Map> inferred(inferred_maps.begin(), inferred_maps.end(),
                           broker->zone());
  return Refine(broker, inferred, false);
}

NamedAccessFeedback const& ElementAccessFeedback::Refine(JSHeapBroker* broker,
                                                         NameRef name) const {
  // Allow swapping megamorphic elements accesses for named accesses when the
  // key is know to be a known name.
  CHECK(transition_groups_.empty());
  ZoneVector<MapRef> maps(broker->zone());
  ZoneVector<OptionalObjectRef> handlers(broker->zone());
  return *broker->zone()->New<NamedAccessFeedback>(broker, name, maps, handlers,
                                                   slot_kind());
}

ElementAccessFeedback const& ElementAccessFeedback::Refine(
    JSHeapBroker* broker, ZoneRefSet<Map> const& inferred_maps,
    bool always_keep_group_target) const {
  ElementAccessFeedback& refined_feedback =
      *broker->zone()->New<ElementAccessFeedback>(broker->zone(), keyed_mode(),
                                                  slot_kind());
  if (inferred_maps.size() == 0) return refined_feedback;

  for (auto const& group : transition_groups()) {
    DCHECK(!group.empty());
    TransitionGroup new_group(broker->zone());
    for (size_t i = 1; i < group.size(); ++i) {
      MapRef source = group[i];
      if (inferred_maps.contains(source)) {
        new_group.push_back(source);
      }
    }

    MapRef target = group.front();
    bool const keep_target = always_keep_group_target ||
                             inferred_maps.contains(target) ||
                             new_group.size() > 1;
    if (keep_target) {
      new_group.push_back(target);
      // The target must be at the front, the order of sources doesn't matter.
      std::swap(new_group[0], new_group[new_group.size() - 1]);
    }

    if (!new_group.empty()) {
      DCHECK(new_group.size() == 1 || new_group.front().equals(target));
      refined_feedback.transition_groups_.push_back(std::move(new_group));
    }
  }
  return refined_feedback;
}

InsufficientFeedback::InsufficientFeedback(FeedbackSlotKind slot_kind)
    : ProcessedFeedback(kInsufficient, slot_kind) {}

GlobalAccessFeedback::GlobalAccessFeedback(PropertyCellRef cell,
                                           FeedbackSlotKind slot_kind)
    : ProcessedFeedback(kGlobalAccess, slot_kind),
      cell_or_context_(cell),
      index_and_immutable_(0 /* doesn't matter */) {
  DCHECK(IsGlobalICKind(slot_kind));
}

GlobalAccessFeedback::GlobalAccessFeedback(FeedbackSlotKind slot_kind)
    : ProcessedFeedback(kGlobalAccess, slot_kind),
      index_and_immutable_(0 /* doesn't matter */) {
  DCHECK(IsGlobalICKind(slot_kind));
}

GlobalAccessFeedback::GlobalAccessFeedback(ContextRef script_context,
                                           int slot_index, bool immutable,
                                           FeedbackSlotKind slot_kind)
    : ProcessedFeedback(kGlobalAccess, slot_kind),
      cell_or_context_(script_context),
      index_and_immutable_(FeedbackNexus::SlotIndexBits::encode(slot_index) |
                           FeedbackNexus::ImmutabilityBit::encode(immutable)) {
  DCHECK_EQ(this->slot_index(), slot_index);
  DCHECK_EQ(this->immutable(), immutable);
  DCHECK(IsGlobalICKind(slot_kind));
}

bool GlobalAccessFeedback::IsMegamorphic() const {
  return !cell_or_context_.has_value();
}
bool GlobalAccessFeedback::IsPropertyCell() const {
  return cell_or_context_.has_value() && cell_or_context_->IsPropertyCell();
}
bool GlobalAccessFeedback::IsScriptContextSlot() const {
  return cell_or_context_.has_value() && cell_or_context_->IsContext();
}
PropertyCellRef GlobalAccessFeedback::property_cell() const {
  CHECK(IsPropertyCell());
  return cell_or_context_->AsPropertyCell();
}
ContextRef GlobalAccessFeedback::script_context() const {
  CHECK(IsScriptContextSlot());
  return cell_or_context_->AsContext();
}
int GlobalAccessFeedback::slot_index() const {
  DCHECK(IsScriptContextSlot());
  return FeedbackNexus::SlotIndexBits::decode(index_and_immutable_);
}
bool GlobalAccessFeedback::immutable() const {
  DCHECK(IsScriptContextSlot());
  return FeedbackNexus::ImmutabilityBit::decode(index_and_immutable_);
}

OptionalObjectRef GlobalAccessFeedback::GetConstantHint(
    JSHeapBroker* broker) const {
  if (IsPropertyCell()) {
    bool cell_cached = property_cell().Cache(broker);
    CHECK(cell_cached);  // Can't fail on the main thread.
    return property_cell().value(broker);
  } else if (IsScriptContextSlot() && immutable()) {
    return script_context().get(broker, slot_index());
  } else {
    return std::nullopt;
  }
}

KeyedAccessMode KeyedAccessMode::FromNexus(FeedbackNexus const& nexus) {
  FeedbackSlotKind kind = nexus.kind();
  constexpr bool kStringKeysUnsupported = false;
  if (IsKeyedLoadICKind(kind)) {
    bool string_keys = nexus.GetKeyType() == IcCheckType::kProperty;
    return KeyedAccessMode(AccessMode::kLoad, nexus.GetKeyedAccessLoadMode(),
                           string_keys);
  }
  if (IsKeyedHasICKind(kind)) {
    return KeyedAccessMode(AccessMode::kHas, nexus.GetKeyedAccessLoadMode(),
                           kStringKeysUnsupported);
  }
  if (IsDefineKeyedOwnICKind(kind)) {
    return KeyedAccessMode(AccessMode::kDefine, nexus.GetKeyedAccessStoreMode(),
                           kStringKeysUnsupported);
  }
  if (IsKeyedStoreICKind(kind)) {
    bool string_keys = nexus.GetKeyType() == IcCheckType::kProperty;
    return KeyedAccessMode(AccessMode::kStore, nexus.GetKeyedAccessStoreMode(),
                           string_keys);
  }
  if (IsStoreInArrayLiteralICKind(kind) ||
      IsDefineKeyedOwnPropertyInLiteralKind(kind)) {
    return KeyedAccessMode(AccessMode::kStoreInLiteral,
                           nexus.GetKeyedAccessStoreMode(),
                           kStringKeysUnsupported);
  }
  UNREACHABLE();
}

AccessMode KeyedAccessMode::access_mode() const { return access_mode_; }

bool KeyedAccessMode::IsLoad() const {
  return access_mode_ == AccessMode::kLoad || access_mode_ == AccessMode::kHas;
}
bool KeyedAccessMode::IsStore() const {
  return access_mode_ == AccessMode::kStore ||
         access_mode_ == AccessMode::kDefine ||
         access_mode_ == AccessMode::kStoreInLiteral;
}

KeyedAccessLoadMode KeyedAccessMode::load_mode() const {
  CHECK(IsLoad());
  return load_mode_;
}

KeyedAccessStoreMode KeyedAccessMode::store_mode() const {
  CHECK(IsStore());
  return store_mode_;
}

KeyedAccessMode::KeyedAccessMode(AccessMode access_mode,
                                 KeyedAccessLoadMode load_mode,
                                 bool string_keys)
    : access_mode_(access_mode),
      load_mode_(load_mode),
      string_keys_(string_keys) {
  CHECK(!IsStore());
  CHECK(IsLoad());
}
KeyedAccessMode::KeyedAccessMode(AccessMode access_mode,
                                 KeyedAccessStoreMode store_mode,
                                 bool string_keys)
    : access_mode_(access_mode),
      store_mode_(store_mode),
      string_keys_(string_keys) {
  CHECK(!IsLoad());
  CHECK(IsStore());
}

ElementAccessFeedback::ElementAccessFeedback(Zone* zone,
                                             KeyedAccessMode const& keyed_mode,
                                             FeedbackSlotKind slot_kind)
    : ProcessedFeedback(kElementAccess, slot_kind),
      keyed_mode_(keyed_mode),
      transition_groups_(zone) {
  DCHECK(IsKeyedLoadICKind(slot_kind) || IsKeyedHasICKind(slot_kind) ||
         IsDefineKeyedOwnPropertyInLiteralKind(slot_kind) ||
         IsKeyedStoreICKind(slot_kind) ||
         IsStoreInArrayLiteralICKind(slot_kind) ||
         IsDefineKeyedOwnICKind(slot_kind));
}

bool ElementAccessFeedback::HasOnlyStringMaps(JSHeapBroker* broker) const {
  for (auto const& group : transition_groups()) {
    for (MapRef map : group) {
      if (!map.IsStringMap()) return false;
    }
  }
  return true;
}

HomomorphicPropertyAccessFeedback::HomomorphicPropertyAccessFeedback(
    NameRef name, WeakHomomorphicFixedArrayRef homomorphic_array,
    Tagged<Smi> handler, FeedbackSlotKind slot_kind)
    : ProcessedFeedback(kHomomorphicPropertyAccess, slot_kind),
      name_(name),
      homomorphic_array_(homomorphic_array),
      handler_(handler) {
  DCHECK(IsLoadICKind(slot_kind));
}

MegaDOMPropertyAccessFeedback::MegaDOMPropertyAccessFeedback(
    FunctionTemplateInfoRef info_ref, FeedbackSlotKind slot_kind)
    : ProcessedFeedback(kMegaDOMPropertyAccess, slot_kind), info_(info_ref) {
  DCHECK(IsLoadICKind(slot_kind));
}

NamedAccessFeedback::NamedAccessFeedback(
    JSHeapBroker* broker, NameRef name, ZoneVector<MapRef> const& maps,
    ZoneVector<OptionalObjectRef> const& handlers, FeedbackSlotKind slot_kind,
    bool has_deprecated_map_without_migration_target)
    : ProcessedFeedback(kNamedAccess, slot_kind),
      name_(name.UnpackIfThin(broker)),
      original_name_maybe_thin_(name),
      maps_(maps),
      handlers_(handlers),
      has_deprecated_map_without_migration_target_(
          has_deprecated_map_without_migration_target) {
  DCHECK(IsLoadICKind(slot_kind) || IsSetNamedICKind(slot_kind) ||
         IsDefineNamedOwnICKind(slot_kind) || IsKeyedLoadICKind(slot_kind) ||
         IsKeyedHasICKind(slot_kind) || IsKeyedStoreICKind(slot_kind) ||
         IsStoreInArrayLiteralICKind(slot_kind) ||
         IsDefineKeyedOwnPropertyInLiteralKind(slot_kind) ||
         IsDefineKeyedOwnICKind(slot_kind));
}

void JSHeapBroker::SetFeedback(FeedbackSource const& source,
                               ProcessedFeedback const* feedback) {
  CHECK(source.IsValid());
  auto insertion = feedback_.insert({source, feedback});
  CHECK(insertion.second);
}

bool JSHeapBroker::HasFeedback(FeedbackSource const& source) const {
  DCHECK(source.IsValid());
  return feedback_.find(source) != feedback_.end();
}

ProcessedFeedback const& JSHeapBroker::GetFeedback(
    FeedbackSource const& source) const {
  DCHECK(source.IsValid());
  auto it = feedback_.find(source);
  CHECK_NE(it, feedback_.end());
  return *it->second;
}

FeedbackSlotKind JSHeapBroker::GetFeedbackSlotKind(
    FeedbackSource const& source) const {
  if (HasFeedback(source)) return GetFeedback(source).slot_kind();
  FeedbackNexus nexus(source.vector, source.slot, feedback_nexus_config());
  return nexus.kind();
}

bool JSHeapBroker::FeedbackIsInsufficient(FeedbackSource const& source) const {
  if (HasFeedback(source)) return GetFeedback(source).IsInsufficient();
  return FeedbackNexus(source.vector, source.slot, feedback_nexus_config())
      .IsUninitialized();
}

const ProcessedFeedback& JSHeapBroker::NewInsufficientFeedback(
    FeedbackSlotKind kind) const {
  return *zone()->New<InsufficientFeedback>(kind);
}

ProcessedFeedback const& JSHeapBroker::ReadFeedbackForPropertyAccess(
    FeedbackSource const& source, AccessMode mode, OptionalNameRef static_name,
    bool allow_homomorphic) {
  FeedbackNexus nexus(source.vector, source.slot, feedback_nexus_config());
  FeedbackSlotKind kind = nexus.kind();
  if (nexus.IsUninitialized()) return NewInsufficientFeedback(kind);

  ZoneVector<MapRef> maps(zone());
  ZoneVector<OptionalObjectRef> handlers(zone());
  bool has_deprecated_map_without_migration_target = false;
  nexus.IterateMapsWithUnclearedHandler(
      [this, &maps, &handlers, &has_deprecated_map_without_migration_target](
          DirectHandle<Map> map_handle, MaybeObjectDirectHandle handler_obj) {
        OptionalMapRef maybe_map_ref = TryMakeRef(this, *map_handle);
        if (!maybe_map_ref.has_value()) return;
        MapRef map = maybe_map_ref.value();
        // May change concurrently at any time - must be guarded by a
        // dependency if non-deprecation is important.
        if (map.is_deprecated()) {
          // TODO(ishell): support fast map updating if we enable it.
          CHECK(!v8_flags.fast_map_update);
          std::optional<Tagged<Map>> maybe_map = MapUpdater::TryUpdateNoLock(
              isolate(), *map.object(), ConcurrencyMode::kConcurrent);
          if (maybe_map.has_value()) {
            OptionalMapRef maybe_updated_ref =
                TryMakeRef(this, maybe_map.value());
            if (!maybe_updated_ref.has_value()) return;
            map = maybe_updated_ref.value();
          } else {
            return;  // Couldn't update the deprecated map.
          }
          if (!map.is_migration_target()) {
            // Maps are marked as migration targets only when an object
            // migrates, so it's possible to have a deprecated map whose updated
            // counterpart is not a migration target.
            has_deprecated_map_without_migration_target = true;
          }
        }
        if (map.is_abandoned_prototype_map()) return;
        maps.push_back(map);

        OptionalObjectRef handler;
        if (!handler_obj.is_null()) {
          handler = TryMakeRef(this, *handler_obj.object());
        }
        handlers.push_back(handler);
      });

  OptionalNameRef name =
      static_name.has_value() ? static_name : GetNameFeedback(nexus);

  if (nexus.ic_state() == InlineCacheState::MEGADOM) {
    DCHECK(maps.empty());
    MaybeObjectHandle maybe_handler = nexus.ExtractMegaDOMHandler();
    if (maybe_handler.is_null()) return NewInsufficientFeedback(kind);

    DirectHandle<MegaDomHandler> handler =
        Cast<MegaDomHandler>(maybe_handler.object());
    if (handler->accessor(kAcquireLoad).IsCleared()) {
      return NewInsufficientFeedback(kind);
    }

    FunctionTemplateInfoRef info = MakeRefAssumeMemoryFence(
        this, Cast<FunctionTemplateInfo>(
                  handler->accessor(kAcquireLoad).GetHeapObject()));
    return *zone()->New<MegaDOMPropertyAccessFeedback>(info, kind);
  }

  if (allow_homomorphic && nexus.ic_state() == InlineCacheState::HOMOMORPHIC) {
    DCHECK(maps.empty());
    Tagged<Smi> handler;
    if (!TryCast<Smi>(nexus.GetFeedbackExtra(), &handler)) {
      return NewInsufficientFeedback(kind);
    }
    Tagged<HeapObject> feedback_object;
    if (!nexus.GetFeedback().GetHeapObjectIfStrong(&feedback_object)) {
      return NewInsufficientFeedback(kind);
    }
    WeakHomomorphicFixedArrayRef homomorphic_array = MakeRefAssumeMemoryFence(
        this, Cast<WeakHomomorphicFixedArray>(feedback_object));

    return *zone()->New<HomomorphicPropertyAccessFeedback>(
        *static_name, homomorphic_array, handler, kind);
  }

  // If no maps were found for a non-megamorphic access, then our maps died
  // and we should soft-deopt.
  if (maps.empty() && nexus.ic_state() != InlineCacheState::MEGAMORPHIC &&
      nexus.ic_state() != InlineCacheState::HOMOMORPHIC) {
    return NewInsufficientFeedback(kind);
  }

  if (name.has_value()) {
    // We rely on this invariant in JSGenericLowering.
    DCHECK_IMPLIES(maps.empty(),
                   nexus.ic_state() == InlineCacheState::MEGAMORPHIC ||
                       nexus.ic_state() == InlineCacheState::HOMOMORPHIC);
    DCHECK_EQ(maps.size(), handlers.size());
    return *zone()->New<NamedAccessFeedback>(
        this, *name, maps, handlers, kind,
        has_deprecated_map_without_migration_target);
  } else if (nexus.GetKeyType() == IcCheckType::kElement && !maps.empty()) {
    return ProcessFeedbackMapsForElementAccess(
        maps, KeyedAccessMode::FromNexus(nexus), kind);
  } else if (nexus.IsOneMapManyNames() && maps.size() == 1) {
    return ProcessFeedbackMapsForKeyedPropertyAccess(
        maps, KeyedAccessMode::FromNexus(nexus), kind);
  } else {
    // No actionable feedback.
    DCHECK(maps.empty());
    DCHECK_EQ(nexus.ic_state(), InlineCacheState::MEGAMORPHIC);
    // TODO(neis): Using ElementAccessFeedback here is kind of an abuse.
    return *zone()->New<ElementAccessFeedback>(
        zone(), KeyedAccessMode::FromNexus(nexus), kind);
  }
}

ProcessedFeedback const& JSHeapBroker::ReadFeedbackForGlobalAccess(
    JSHeapBroker* broker, FeedbackSource const& source) {
  FeedbackNexus nexus(source.vector, source.slot, feedback_nexus_config());
  DCHECK(nexus.kind() == FeedbackSlotKind::kLoadGlobalInsideTypeof ||
         nexus.kind() == FeedbackSlotKind::kLoadGlobalNotInsideTypeof ||
         nexus.kind() == FeedbackSlotKind::kStoreGlobalSloppy ||
         nexus.kind() == FeedbackSlotKind::kStoreGlobalStrict);
  if (nexus.IsUninitialized()) return NewInsufficientFeedback(nexus.kind());
  if (nexus.ic_state() != InlineCacheState::MONOMORPHIC ||
      nexus.GetFeedback().IsCleared()) {
    return *zone()->New<GlobalAccessFeedback>(nexus.kind());
  }

  Handle<Object> feedback_value =
      CanonicalPersistentHandle(nexus.GetFeedback().GetHeapObjectOrSmi());

  if (IsSmi(*feedback_value)) {
    // The wanted name belongs to a script-scope variable and the feedback
    // tells us where to find its value.
    int const number = Object::NumberValue(*feedback_value);
    uint32_t const script_context_index =
        FeedbackNexus::ContextIndexBits::decode(number);
    int const context_slot_index = FeedbackNexus::SlotIndexBits::decode(number);
    ContextRef context = MakeRefAssumeMemoryFence(
        this,
        target_native_context().script_context_table(broker).object()->get(
            script_context_index, kAcquireLoad));

    OptionalObjectRef contents = context.get(broker, context_slot_index);
    if (contents.has_value()) CHECK(!contents->IsTheHole());

    return *zone()->New<GlobalAccessFeedback>(
        context, context_slot_index,
        FeedbackNexus::ImmutabilityBit::decode(number), nexus.kind());
  }

  CHECK(IsPropertyCell(*feedback_value));
  // The wanted name belongs (or did belong) to a property on the global
  // object and the feedback is the cell holding its value.
  return *zone()->New<GlobalAccessFeedback>(
      MakeRefAssumeMemoryFence(this, Cast<PropertyCell>(feedback_value)),
      nexus.kind());
}

ProcessedFeedback const& JSHeapBroker::ReadFeedbackForBinaryOperation(
    FeedbackSource const& source) const {
  FeedbackNexus nexus(source.vector, source.slot, feedback_nexus_config());
  if (nexus.IsUninitialized()) return NewInsufficientFeedback(nexus.kind());
  BinaryOperationHint hint = nexus.GetBinaryOperationFeedback();
  DCHECK_NE(hint, BinaryOperationHint::kNone);  // Not uninitialized.
  return *zone()->New<BinaryOperationFeedback>(hint, nexus.kind());
}

ProcessedFeedback const& JSHeapBroker::ReadFeedbackForTypeOf(
    FeedbackSource const& source) const {
  FeedbackNexus nexus(source.vector, source.slot, feedback_nexus_config());
  if (nexus.IsUninitialized()) return NewInsufficientFeedback(nexus.kind());
  return *zone()->New<TypeOfOpFeedback>(nexus.GetTypeOfFeedback(),
                                        nexus.kind());
}

ProcessedFeedback const& JSHeapBroker::ReadFeedbackForCompareOperation(
    FeedbackSource const& source) const {
  FeedbackNexus nexus(source.vector, source.slot, feedback_nexus_config());
  if (nexus.IsUninitialized()) return NewInsufficientFeedback(nexus.kind());
  CompareOperationHint hint = nexus.GetCompareOperationFeedback();
  DCHECK_NE(hint, CompareOperationHint::kNone);  // Not uninitialized.
  return *zone()->New<CompareOperationFeedback>(hint, nexus.kind());
}

ProcessedFeedback const& JSHeapBroker::ReadFeedbackForForIn(
    FeedbackSource const& source) const {
  FeedbackNexus nexus(source.vector, source.slot, feedback_nexus_config());
  if (nexus.IsUninitialized()) return NewInsufficientFeedback(nexus.kind());
  ForInHint hint = nexus.GetForInFeedback();
  DCHECK_NE(hint, ForInHint::kNone);  // Not uninitialized.
  return *zone()->New<ForInFeedback>(hint, nexus.kind());
}

ProcessedFeedback const& JSHeapBroker::ReadFeedbackForInstanceOf(
    FeedbackSource const& source) {
  FeedbackNexus nexus(source.vector, source.slot, feedback_nexus_config());
  if (nexus.IsUninitialized()) return NewInsufficientFeedback(nexus.kind());

  OptionalJSObjectRef optional_constructor;
  {
    MaybeDirectHandle<JSObject> maybe_constructor =
        nexus.GetConstructorFeedback();
    DirectHandle<JSObject> constructor;
    if (maybe_constructor.ToHandle(&constructor)) {
      optional_constructor = MakeRefAssumeMemoryFence(this, *constructor);
    }
  }
  return *zone()->New<InstanceOfFeedback>(optional_constructor, nexus.kind());
}

ProcessedFeedback const& JSHeapBroker::ReadFeedbackForArrayOrObjectLiteral(
    FeedbackSource const& source) {
  FeedbackNexus nexus(source.vector, source.slot, feedback_nexus_config());
  if (nexus.IsUninitialized()) return NewInsufficientFeedback(nexus.kind());

  Tagged<HeapObject> object;
  if (!nexus.GetFeedback().GetHeapObject(&object)) {
    return NewInsufficientFeedback(nexus.kind());
  }

  AllocationSiteRef site =
      MakeRefAssumeMemoryFence(this, Cast<AllocationSite>(object));
  return *zone()->New<LiteralFeedback>(site, nexus.kind());
}

ProcessedFeedback const& JSHeapBroker::ReadFeedbackForRegExpLiteral(
    FeedbackSource const& source) {
  FeedbackNexus nexus(source.vector, source.slot, feedback_nexus_config());
  if (nexus.IsUninitialized()) return NewInsufficientFeedback(nexus.kind());

  Tagged<HeapObject> object;
  if (!nexus.GetFeedback().GetHeapObject(&object)) {
    return NewInsufficientFeedback(nexus.kind());
  }

  RegExpBoilerplateDescriptionRef boilerplate = MakeRefAssumeMemoryFence(
      this, Cast<RegExpBoilerplateDescription>(object));
  return *zone()->New<RegExpLiteralFeedback>(boilerplate, nexus.kind());
}

ProcessedFeedback const& JSHeapBroker::ReadFeedbackForTemplateObject(
    FeedbackSource const& source) {
  FeedbackNexus nexus(source.vector, source.slot, feedback_nexus_config());
  if (nexus.IsUninitialized()) return NewInsufficientFeedback(nexus.kind());

  Tagged<HeapObject> object;
  if (!nexus.GetFeedback().GetHeapObject(&object)) {
    return NewInsufficientFeedback(nexus.kind());
  }

  JSArrayRef array = MakeRefAssumeMemoryFence(this, Cast<JSArray>(object));
  return *zone()->New<TemplateObjectFeedback>(array, nexus.kind());
}

ProcessedFeedback const& JSHeapBroker::ReadFeedbackForCall(
    FeedbackSource const& source) {
  FeedbackNexus nexus(source.vector, source.slot, feedback_nexus_config());
  if (nexus.IsUninitialized()) return NewInsufficientFeedback(nexus.kind());

  OptionalHeapObjectRef target_ref;
  {
    Tagged<MaybeObject> maybe_target = nexus.GetFeedback();
    Tagged<HeapObject> target_object;
    if (maybe_target.GetHeapObject(&target_object) &&
        (!IsJSFunction(target_object) ||
         // Eval mutates the context on the function, so block specializing for
         // these functions to avoid accidentally inlining the context as a
         // constant.
         Cast<JSFunction>(target_object)
                 ->shared()
                 ->function_literal_id(kRelaxedLoad) != 0)) {
      target_ref = TryMakeRef(this, target_object);
    }
  }

  float frequency = nexus.ComputeCallFrequency();
  SpeculationMode mode = nexus.GetSpeculationMode();
  CallFeedbackContent content = nexus.GetCallFeedbackContent();
  return *zone()->New<CallFeedback>(target_ref, frequency, mode, content,
                                    nexus.kind());
}

BinaryOperationHint JSHeapBroker::GetFeedbackForBinaryOperation(
    FeedbackSource const& source) {
  ProcessedFeedback const& feedback = ProcessFeedbackForBinaryOperation(source);
  return feedback.IsInsufficient() ? BinaryOperationHint::kNone
                                   : feedback.AsBinaryOperation().value();
}

TypeOfFeedback::Result JSHeapBroker::GetFeedbackForTypeOf(
    FeedbackSource const& source) {
  ProcessedFeedback const& feedback = ProcessFeedbackForTypeOf(source);
  return feedback.IsInsufficient() ? TypeOfFeedback::kNone
                                   : feedback.AsTypeOf().value();
}

CompareOperationHint JSHeapBroker::GetFeedbackForCompareOperation(
    FeedbackSource const& source) {
  ProcessedFeedback const& feedback =
      ProcessFeedbackForCompareOperation(source);
  return feedback.IsInsufficient() ? CompareOperationHint::kNone
                                   : feedback.AsCompareOperation().value();
}

ForInHint JSHeapBroker::GetFeedbackForForIn(FeedbackSource const& source) {
  ProcessedFeedback const& feedback = ProcessFeedbackForForIn(source);
  return feedback.IsInsufficient() ? ForInHint::kNone
                                   : feedback.AsForIn().value();
}

ProcessedFeedback const& JSHeapBroker::GetFeedbackForArrayOrObjectLiteral(
    FeedbackSource const& source) {
  if (HasFeedback(source)) return GetFeedback(source);
  ProcessedFeedback const& feedback =
      ReadFeedbackForArrayOrObjectLiteral(source);
  SetFeedback(source, &feedback);
  return feedback;
}

ProcessedFeedback const& JSHeapBroker::GetFeedbackForRegExpLiteral(
    FeedbackSource const& source) {
  if (HasFeedback(source)) return GetFeedback(source);
  ProcessedFeedback const& feedback = ReadFeedbackForRegExpLiteral(source);
  SetFeedback(source, &feedback);
  return feedback;
}

ProcessedFeedback const& JSHeapBroker::GetFeedbackForTemplateObject(
    FeedbackSource const& source) {
  if (HasFeedback(source)) return GetFeedback(source);
  ProcessedFeedback const& feedback = ReadFeedbackForTemplateObject(source);
  SetFeedback(source, &feedback);
  return feedback;
}

ProcessedFeedback const& JSHeapBroker::ProcessFeedbackForTypeOf(
    FeedbackSource const& source) {
  if (HasFeedback(source)) return GetFeedback(source);
  ProcessedFeedback const& feedback = ReadFeedbackForTypeOf(source);
  SetFeedback(source, &feedback);
  return feedback;
}

ProcessedFeedback const& JSHeapBroker::ProcessFeedbackForBinaryOperation(
    FeedbackSource const& source) {
  if (HasFeedback(source)) return GetFeedback(source);
  ProcessedFeedback const& feedback = ReadFeedbackForBinaryOperation(source);
  SetFeedback(source, &feedback);
  return feedback;
}

ProcessedFeedback const& JSHeapBroker::ProcessFeedbackForCompareOperation(
    FeedbackSource const& source) {
  if (HasFeedback(source)) return GetFeedback(source);
  ProcessedFeedback const& feedback = ReadFeedbackForCompareOperation(source);
  SetFeedback(source, &feedback);
  return feedback;
}

ProcessedFeedback const& JSHeapBroker::ProcessFeedbackForForIn(
    FeedbackSource const& source) {
  if (HasFeedback(source)) return GetFeedback(source);
  ProcessedFeedback const& feedback = ReadFeedbackForForIn(source);
  SetFeedback(source, &feedback);
  return feedback;
}

ProcessedFeedback const& JSHeapBroker::GetFeedbackForPropertyAccess(
    FeedbackSource const& source, AccessMode mode, OptionalNameRef static_name,
    bool allow_homomorphic) {
  if (HasFeedback(source)) return GetFeedback(source);
  ProcessedFeedback const& feedback = ReadFeedbackForPropertyAccess(
      source, mode, static_name, allow_homomorphic);
  SetFeedback(source, &feedback);
  return feedback;
}

ProcessedFeedback const& JSHeapBroker::GetFeedbackForInstanceOf(
    FeedbackSource const& source) {
  if (HasFeedback(source)) return GetFeedback(source);
  ProcessedFeedback const& feedback = ReadFeedbackForInstanceOf(source);
  SetFeedback(source, &feedback);
  return feedback;
}

ProcessedFeedback const& JSHeapBroker::GetFeedbackForCall(
    FeedbackSource const& source) {
  if (HasFeedback(source)) return GetFeedback(source);
  ProcessedFeedback const& feedback = ReadFeedbackForCall(source);
  SetFeedback(source, &feedback);
  return feedback;
}

ProcessedFeedback const& JSHeapBroker::GetFeedbackForGlobalAccess(
    FeedbackSource const& source) {
  if (HasFeedback(source)) return GetFeedback(source);
  ProcessedFeedback const& feedback = ReadFeedbackForGlobalAccess(this, source);
  SetFeedback(source, &feedback);
  return feedback;
}

ElementAccessFeedback const&
JSHeapBroker::ProcessFeedbackMapsForKeyedPropertyAccess(
    ZoneVector<MapRef>& maps, KeyedAccessMode const& keyed_mode,
    FeedbackSlotKind slot_kind) {
  DCHECK_EQ(maps.size(), 1);
  DCHECK(keyed_mode.string_keys());
  ElementAccessFeedback* result =
      zone()->New<ElementAccessFeedback>(zone(), keyed_mode, slot_kind);
  result->AddGroup(ElementAccessFeedback::TransitionGroup(1, maps[0], zone()));
  return *result;
}

ElementAccessFeedback const& JSHeapBroker::ProcessFeedbackMapsForElementAccess(
    ZoneVector<MapRef>& maps, KeyedAccessMode const& keyed_mode,
    FeedbackSlotKind slot_kind) {
  DCHECK(!maps.empty());
  DCHECK(!keyed_mode.string_keys());

  // Collect possible transition targets.
  MapHandles possible_transition_targets(isolate());
  possible_transition_targets.reserve(maps.size());
  for (MapRef& map : maps) {
    if (map.CanInlineElementAccess() &&
        IsFastElementsKind(map.elements_kind()) &&
        GetInitialFastElementsKind() != map.elements_kind()) {
      possible_transition_targets.push_back(map.object());
    }
  }

  using TransitionGroup = ElementAccessFeedback::TransitionGroup;
  ZoneRefMap<MapRef, TransitionGroup> transition_groups(zone());

  // Separate the actual receiver maps and the possible transition sources.
  for (MapRef map : maps) {
    Tagged<Map> transition_target;

    // Don't generate elements kind transitions from stable maps.
    if (!map.is_stable()) {
      // The lock is needed for UnusedPropertyFields (called deep inside
      // FindElementsKindTransitionedMap).
      MapUpdaterGuardIfNeeded mumd_scope(this);

      transition_target = map.object()->FindElementsKindTransitionedMap(
          isolate(),
          MapHandlesSpan(possible_transition_targets.begin(),
                         possible_transition_targets.end()),
          ConcurrencyMode::kConcurrent);
    }

    if (transition_target.is_null()) {
      TransitionGroup group(1, map, zone());
      transition_groups.insert({map, group});
    } else {
      MapRef target = MakeRefAssumeMemoryFence(this, transition_target);
      TransitionGroup new_group(1, target, zone());
      TransitionGroup& actual_group =
          transition_groups.insert({target, new_group}).first->second;
      actual_group.push_back(map);
    }
  }

  ElementAccessFeedback* result =
      zone()->New<ElementAccessFeedback>(zone(), keyed_mode, slot_kind);
  for (auto& entry : transition_groups) {
    result->AddGroup(std::move(entry.second));
  }

  CHECK(!result->transition_groups().empty());
  return *result;
}

void ElementAccessFeedback::AddGroup(TransitionGroup&& group) {
  CHECK(!group.empty());
  transition_groups_.push_back(std::move(group));

#ifdef ENABLE_SLOW_DCHECKS
  // Check that each of the group's maps occurs exactly once in the whole
  // feedback. This implies that "a source is not a target".
  for (MapRef map : group) {
    int count = 0;
    for (TransitionGroup const& some_group : transition_groups()) {
      count +=
          std::count_if(some_group.begin(), some_group.end(),
                        [&](MapRef some_map) { return some_map.equals(map); });
    }
    CHECK_EQ(count, 1);
  }
#endif
}

OptionalNameRef JSHeapBroker::GetNameFeedback(FeedbackNexus const& nexus) {
  Tagged<Name> raw_name = nexus.GetName();
  if (raw_name.is_null()) return std::nullopt;
  return MakeRefAssumeMemoryFence(this, raw_name);
}

PropertyAccessInfo JSHeapBroker::GetPropertyAccessInfo(
    MapRef map, NameRef name, AccessMode access_mode,
    OptionalObjectRef handler) {
  DCHECK_NOT_NULL(dependencies_);

  // Only include handler in the cache key for dictionary map loads.
  // For everything else, the handler doesn't affect the resulting
  // PropertyAccessInfo, and including it would cause a cache explosion.
  OptionalObjectRef cache_handler =
      (map.is_dictionary_map() && access_mode == AccessMode::kLoad)
          ? handler
          : std::nullopt;

  PropertyAccessTarget target({map, name, access_mode, cache_handler});
  auto it = property_access_infos_.find(target);
  if (it != property_access_infos_.end()) return it->second;

  AccessInfoFactory factory(this, zone());
  PropertyAccessInfo access_info =
      factory.ComputePropertyAccessInfo(map, name, access_mode, handler);
  TRACE(this, "Storing PropertyAccessInfo for "
                  << access_mode << " of property " << name << " on map "
                  << map);
  property_access_infos_.insert({target, access_info});
  return access_info;
}

TypeOfOpFeedback const& ProcessedFeedback::AsTypeOf() const {
  CHECK_EQ(kTypeOf, kind());
  return *static_cast<TypeOfOpFeedback const*>(this);
}

BinaryOperationFeedback const& ProcessedFeedback::AsBinaryOperation() const {
  CHECK_EQ(kBinaryOperation, kind());
  return *static_cast<BinaryOperationFeedback const*>(this);
}

CallFeedback const& ProcessedFeedback::AsCall() const {
  CHECK_EQ(kCall, kind());
  return *static_cast<CallFeedback const*>(this);
}

CompareOperationFeedback const& ProcessedFeedback::AsCompareOperation() const {
  CHECK_EQ(kCompareOperation, kind());
  return *static_cast<CompareOperationFeedback const*>(this);
}

ElementAccessFeedback const& ProcessedFeedback::AsElementAccess() const {
  CHECK_EQ(kElementAccess, kind());
  return *static_cast<ElementAccessFeedback const*>(this);
}

ForInFeedback const& ProcessedFeedback::AsForIn() const {
  CHECK_EQ(kForIn, kind());
  return *static_cast<ForInFeedback const*>(this);
}

GlobalAccessFeedback const& ProcessedFeedback::AsGlobalAccess() const {
  CHECK_EQ(kGlobalAccess, kind());
  return *static_cast<GlobalAccessFeedback const*>(this);
}

InstanceOfFeedback const& ProcessedFeedback::AsInstanceOf() const {
  CHECK_EQ(kInstanceOf, kind());
  return *static_cast<InstanceOfFeedback const*>(this);
}

NamedAccessFeedback const& ProcessedFeedback::AsNamedAccess() const {
  CHECK_EQ(kNamedAccess, kind());
  return *static_cast<NamedAccessFeedback const*>(this);
}

HomomorphicPropertyAccessFeedback const&
ProcessedFeedback::AsHomomorphicPropertyAccess() const {
  CHECK_EQ(kHomomorphicPropertyAccess, kind());
  return *static_cast<HomomorphicPropertyAccessFeedback const*>(this);
}

MegaDOMPropertyAccessFeedback const&
ProcessedFeedback::AsMegaDOMPropertyAccess() const {
  CHECK_EQ(kMegaDOMPropertyAccess, kind());
  return *static_cast<MegaDOMPropertyAccessFeedback const*>(this);
}

LiteralFeedback const& ProcessedFeedback::AsLiteral() const {
  CHECK_EQ(kLiteral, kind());
  return *static_cast<LiteralFeedback const*>(this);
}

RegExpLiteralFeedback const& ProcessedFeedback::AsRegExpLiteral() const {
  CHECK_EQ(kRegExpLiteral, kind());
  return *static_cast<RegExpLiteralFeedback const*>(this);
}

TemplateObjectFeedback const& ProcessedFeedback::AsTemplateObject() const {
  CHECK_EQ(kTemplateObject, kind());
  return *static_cast<TemplateObjectFeedback const*>(this);
}

#undef TRACE

}  // namespace compiler
}  // namespace internal
}  // namespace v8
