From 8538bdc878098d1dafce4656d36642fbae8f6e43 Mon Sep 17 00:00:00 2001 From: maynemei Date: Fri, 1 Nov 2024 14:22:13 +0800 Subject: [PATCH 01/36] adding store buffer --- core/lsu/LSU.cpp | 140 +++++++++++++++++++++++++++++++++++++++-------- core/lsu/LSU.hpp | 16 ++++++ 2 files changed, 132 insertions(+), 24 deletions(-) diff --git a/core/lsu/LSU.cpp b/core/lsu/LSU.cpp index 5be73076..daef1e81 100644 --- a/core/lsu/LSU.cpp +++ b/core/lsu/LSU.cpp @@ -21,6 +21,8 @@ namespace olympia replay_buffer_size_(p->replay_buffer_size), replay_issue_delay_(p->replay_issue_delay), ready_queue_(), + store_buffer_("store_buffer", p->ldst_inst_queue_size, getClock()), // Add this line + store_buffer_size_(p->ldst_inst_queue_size), load_store_info_allocator_(sparta::notNull(OlympiaAllocators::getOlympiaAllocators(node)) ->load_store_info_allocator), memory_access_allocator_(sparta::notNull(OlympiaAllocators::getOlympiaAllocators(node)) @@ -48,6 +50,7 @@ namespace olympia ldst_pipeline_.enableCollection(node); ldst_inst_queue_.enableCollection(node); replay_buffer_.enableCollection(node); + store_buffer_.enableCollection(node); // Startup handler for sending initial credits sparta::StartupEvent(node, CREATE_SPARTA_HANDLER(LSU, sendInitialCredits_)); @@ -177,6 +180,12 @@ namespace olympia { ILOG("New instruction added to the ldst queue " << inst_ptr); allocateInstToIssueQueue_(inst_ptr); + // allocate to Store buffer + if (inst_ptr->isStoreInst()) + { + allocateInstToStoreBuffer_(inst_ptr); + } + handleOperandIssueCheck_(inst_ptr); lsu_insts_dispatched_++; } @@ -265,7 +274,21 @@ namespace olympia sparta_assert(inst_ptr->getStatus() == Inst::Status::RETIRED, "Get ROB Ack, but the store inst hasn't retired yet!"); - ++stores_retired_; + if (inst_ptr->isStoreInst()) + { + auto oldest_store = getOldestStore_(); + sparta_assert(oldest_store && oldest_store->getUniqueID() == inst_ptr->getUniqueID(), + "Attempting to retire store out of order! Expected: " + << (oldest_store ? oldest_store->getUniqueID() : 0) + << " Got: " << inst_ptr->getUniqueID()); + + // Remove from store buffer and commit to cache + auto store_info_ptr_ = createLoadStoreInst_(inst_ptr); + store_buffer_.erase(store_buffer_.begin());; + out_cache_lookup_req_.send(store_info_ptr_->getMemoryAccessInfoPtr()); + ++stores_retired_; + } + updateIssuePriorityAfterStoreInstRetire_(inst_ptr); if (isReadyToIssueInsts_()) @@ -592,41 +615,60 @@ namespace olympia } const LoadStoreInstInfoPtr & load_store_info_ptr = ldst_pipeline_[cache_read_stage_]; + const InstPtr & inst_ptr = load_store_info_ptr->getInstPtr(); const MemoryAccessInfoPtr & mem_access_info_ptr = load_store_info_ptr->getMemoryAccessInfoPtr(); ILOG(mem_access_info_ptr); - if (false == mem_access_info_ptr->isCacheHit()) - { - ILOG("Cannot complete inst, cache miss: " << mem_access_info_ptr); - if (allow_speculative_load_exec_) + if (!inst_ptr->isStoreInst()) + { + // for load we check whether we could use store forwarding + // checking whether there are address match in store buffer + uint64_t load_addr = inst_ptr->getTargetVAddr(); + auto forwarding_store = findYoungestMatchingStore_(load_addr); + + if (forwarding_store) { - updateInstReplayReady_(load_store_info_ptr); + mem_access_info_ptr->setDataReady(true); + ILOG("Load using forwarded data from store buffer: " << inst_ptr + << " from store: " << forwarding_store); } - // There might not be a wake up because the cache cannot handle nay more instruction - // Change to nack wakeup when implemented - if (!load_store_info_ptr->isInReadyQueue()) + else { - appendToReadyQueue_(load_store_info_ptr); - load_store_info_ptr->setState(LoadStoreInstInfo::IssueState::READY); - if (isReadyToIssueInsts_()) + if (false == mem_access_info_ptr->isCacheHit()) { - uev_issue_inst_.schedule(sparta::Clock::Cycle(0)); + ILOG("Cannot complete inst, cache miss: " << mem_access_info_ptr); + if (allow_speculative_load_exec_) + { + updateInstReplayReady_(load_store_info_ptr); + } + // There might not be a wake up because the cache cannot handle nay more instruction + // Change to nack wakeup when implemented + if (!load_store_info_ptr->isInReadyQueue()) + { + appendToReadyQueue_(load_store_info_ptr); + load_store_info_ptr->setState(LoadStoreInstInfo::IssueState::READY); + if (isReadyToIssueInsts_()) + { + uev_issue_inst_.schedule(sparta::Clock::Cycle(0)); + } + } + ldst_pipeline_.invalidateStage(cache_read_stage_); + return; } - } - ldst_pipeline_.invalidateStage(cache_read_stage_); - return; - } - if (mem_access_info_ptr->isDataReady()) - { - ILOG("Instruction had previously had its data ready"); - return; - } + if (mem_access_info_ptr->isDataReady()) + { + ILOG("Instruction had previously had its data ready"); + return; + } - ILOG("Data ready set for " << mem_access_info_ptr); - mem_access_info_ptr->setDataReady(true); + ILOG("Data ready set for " << mem_access_info_ptr); + mem_access_info_ptr->setDataReady(true); + } + } + if (isReadyToIssueInsts_()) { ILOG("Cache read issue"); @@ -790,6 +832,7 @@ namespace olympia flushIssueQueue_(criteria); flushReplayBuffer_(criteria); flushReadyQueue_(criteria); + flushStoreBuffer_(criteria); // Cancel replay events auto flush = [&criteria](const LoadStoreInstInfoPtr & ldst_info_ptr) -> bool @@ -894,6 +937,40 @@ namespace olympia ILOG("Append new load/store instruction to issue queue!"); } + void LSU::allocateInstToStoreBuffer_(const InstPtr & inst_ptr) + { + auto store_info_ptr = createLoadStoreInst_(inst_ptr); + + sparta_assert(store_buffer_.size() < ldst_inst_queue_size_, + "Appending store buffer causes overflows!"); + + store_buffer_.push_back(store_info_ptr); + ILOG("Store added to store buffer: " << inst_ptr); + } + + InstPtr LSU::findYoungestMatchingStore_(uint64_t addr) + { + InstPtr matching_store = nullptr; + + for (auto it = store_buffer_.begin(); it != store_buffer_.end(); ++it) + { + auto & store = *it; + if (store->getTargetVAddr() == addr) + { + matching_store = store; + } + } + return matching_store; + } + + InstPtr LSU::getOldestStore_() const + { + if(store_buffer_.empty()) { + return nullptr; + } + return store_buffer_.read(0); + } + bool LSU::allOlderStoresIssued_(const InstPtr & inst_ptr) { for (const auto & ldst_info_ptr : ldst_inst_queue_) @@ -1368,4 +1445,19 @@ namespace olympia } } + void LSU::flushStoreBuffer_(const FlushCriteria & criteria) + { + auto sb_iter = store_buffer_.begin(); + while(sb_iter != store_buffer_.end()) { + auto store_ptr = *sb_iter; + if(criteria.includedInFlush(store_ptr)) { + auto delete_iter = sb_iter++; + store_buffer_.erase(delete_iter); + ILOG("Flushed store from store buffer: " << store_ptr); + } else { + ++sb_iter; + } + } + } + } // namespace olympia diff --git a/core/lsu/LSU.hpp b/core/lsu/LSU.hpp index 0896169c..2f71a569 100644 --- a/core/lsu/LSU.hpp +++ b/core/lsu/LSU.hpp @@ -50,6 +50,7 @@ namespace olympia PARAMETER(uint32_t, ldst_inst_queue_size, 8, "LSU ldst inst queue size") PARAMETER(uint32_t, replay_buffer_size, ldst_inst_queue_size, "Replay buffer size") PARAMETER(uint32_t, replay_issue_delay, 3, "Replay Issue delay") + PARAMETER(uint32_t, store_buffer_size, ldst_inst_queue_size, "Size of the store buffer") // LSU microarchitecture parameters PARAMETER( bool, allow_speculative_load_exec, true, @@ -167,6 +168,10 @@ namespace olympia using LoadStorePipeline = sparta::Pipeline; LoadStorePipeline ldst_pipeline_; + // Store Buffer + sparta::Buffer store_buffer_; + const uint32_t store_buffer_size_; + // LSU Microarchitecture parameters const bool allow_speculative_load_exec_; @@ -258,6 +263,15 @@ namespace olympia void allocateInstToIssueQueue_(const InstPtr & inst_ptr); + // allocate store inst to store buffer + void allocateInstToStoreBuffer_(const InstPtr & inst_ptr); + + // Search store buffer in FIFO order for youngest matching store + InstPtr findYoungestMatchingStore_(uint64_t addr); + + // get oldest store + InstPtr getOldestStore_() const; + bool olderStoresExists_(const InstPtr & inst_ptr); bool allOlderStoresIssued_(const InstPtr & inst_ptr); @@ -315,6 +329,8 @@ namespace olympia // Flush Replay Buffer void flushReplayBuffer_(const FlushCriteria &); + void flushStoreBuffer_(const FlushCriteria &); + // Counters sparta::Counter lsu_insts_dispatched_{getStatisticSet(), "lsu_insts_dispatched", "Number of LSU instructions dispatched", From 22dda9fc402cab37d1706b06b0195bfa0f366efe Mon Sep 17 00:00:00 2001 From: maynemei Date: Sat, 2 Nov 2024 00:32:02 +0800 Subject: [PATCH 02/36] modified return type of some store function, and reorder the constructor --- core/lsu/LSU.cpp | 27 +++++++++++++-------------- core/lsu/LSU.hpp | 14 +++++++------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/core/lsu/LSU.cpp b/core/lsu/LSU.cpp index daef1e81..3dbc888e 100644 --- a/core/lsu/LSU.cpp +++ b/core/lsu/LSU.cpp @@ -20,9 +20,9 @@ namespace olympia replay_buffer_("replay_buffer", p->replay_buffer_size, getClock()), replay_buffer_size_(p->replay_buffer_size), replay_issue_delay_(p->replay_issue_delay), - ready_queue_(), store_buffer_("store_buffer", p->ldst_inst_queue_size, getClock()), // Add this line store_buffer_size_(p->ldst_inst_queue_size), + ready_queue_(), load_store_info_allocator_(sparta::notNull(OlympiaAllocators::getOlympiaAllocators(node)) ->load_store_info_allocator), memory_access_allocator_(sparta::notNull(OlympiaAllocators::getOlympiaAllocators(node)) @@ -33,7 +33,7 @@ namespace olympia cache_read_stage_(cache_lookup_stage_ + 1), // Get data from the cache in the cycle after cache lookup complete_stage_( - cache_read_stage_ + cache_read_stage_ + p->cache_read_stage_length), // Complete stage is after the cache read stage ldst_pipeline_("LoadStorePipeline", (complete_stage_ + 1), getClock()), // complete_stage_ + 1 is number of stages @@ -277,15 +277,14 @@ namespace olympia if (inst_ptr->isStoreInst()) { auto oldest_store = getOldestStore_(); - sparta_assert(oldest_store && oldest_store->getUniqueID() == inst_ptr->getUniqueID(), + sparta_assert(oldest_store && oldest_store->getInstPtr()->getUniqueID() == inst_ptr->getUniqueID(), "Attempting to retire store out of order! Expected: " - << (oldest_store ? oldest_store->getUniqueID() : 0) + << (oldest_store ? oldest_store->getInstPtr()->getUniqueID() : 0) << " Got: " << inst_ptr->getUniqueID()); // Remove from store buffer and commit to cache - auto store_info_ptr_ = createLoadStoreInst_(inst_ptr); + out_cache_lookup_req_.send(oldest_store->getMemoryAccessInfoPtr()); store_buffer_.erase(store_buffer_.begin());; - out_cache_lookup_req_.send(store_info_ptr_->getMemoryAccessInfoPtr()); ++stores_retired_; } @@ -631,7 +630,7 @@ namespace olympia { mem_access_info_ptr->setDataReady(true); ILOG("Load using forwarded data from store buffer: " << inst_ptr - << " from store: " << forwarding_store); + << " from store: " << forwarding_store->getInstPtr()); } else { @@ -948,14 +947,14 @@ namespace olympia ILOG("Store added to store buffer: " << inst_ptr); } - InstPtr LSU::findYoungestMatchingStore_(uint64_t addr) + LoadStoreInstInfoPtr LSU::findYoungestMatchingStore_(uint64_t addr) { - InstPtr matching_store = nullptr; + LoadStoreInstInfoPtr matching_store = nullptr; for (auto it = store_buffer_.begin(); it != store_buffer_.end(); ++it) { auto & store = *it; - if (store->getTargetVAddr() == addr) + if (store->getInstPtr()->getTargetVAddr() == addr) { matching_store = store; } @@ -963,7 +962,7 @@ namespace olympia return matching_store; } - InstPtr LSU::getOldestStore_() const + LoadStoreInstInfoPtr LSU::getOldestStore_() const { if(store_buffer_.empty()) { return nullptr; @@ -1449,11 +1448,11 @@ namespace olympia { auto sb_iter = store_buffer_.begin(); while(sb_iter != store_buffer_.end()) { - auto store_ptr = *sb_iter; - if(criteria.includedInFlush(store_ptr)) { + auto inst_ptr = (*sb_iter)->getInstPtr(); + if(criteria.includedInFlush(inst_ptr)) { auto delete_iter = sb_iter++; store_buffer_.erase(delete_iter); - ILOG("Flushed store from store buffer: " << store_ptr); + ILOG("Flushed store from store buffer: " << inst_ptr); } else { ++sb_iter; } diff --git a/core/lsu/LSU.hpp b/core/lsu/LSU.hpp index 2f71a569..9e6175fd 100644 --- a/core/lsu/LSU.hpp +++ b/core/lsu/LSU.hpp @@ -50,7 +50,7 @@ namespace olympia PARAMETER(uint32_t, ldst_inst_queue_size, 8, "LSU ldst inst queue size") PARAMETER(uint32_t, replay_buffer_size, ldst_inst_queue_size, "Replay buffer size") PARAMETER(uint32_t, replay_issue_delay, 3, "Replay Issue delay") - PARAMETER(uint32_t, store_buffer_size, ldst_inst_queue_size, "Size of the store buffer") + // PARAMETER(uint32_t, store_buffer_size, ldst_inst_queue_size, "Size of the store buffer") // LSU microarchitecture parameters PARAMETER( bool, allow_speculative_load_exec, true, @@ -138,6 +138,10 @@ namespace olympia const uint32_t replay_buffer_size_; const uint32_t replay_issue_delay_; + // Store Buffer + sparta::Buffer store_buffer_; + const uint32_t store_buffer_size_; + sparta::PriorityQueue ready_queue_; // MMU unit bool mmu_busy_ = false; @@ -168,10 +172,6 @@ namespace olympia using LoadStorePipeline = sparta::Pipeline; LoadStorePipeline ldst_pipeline_; - // Store Buffer - sparta::Buffer store_buffer_; - const uint32_t store_buffer_size_; - // LSU Microarchitecture parameters const bool allow_speculative_load_exec_; @@ -267,10 +267,10 @@ namespace olympia void allocateInstToStoreBuffer_(const InstPtr & inst_ptr); // Search store buffer in FIFO order for youngest matching store - InstPtr findYoungestMatchingStore_(uint64_t addr); + LoadStoreInstInfoPtr findYoungestMatchingStore_(uint64_t addr); // get oldest store - InstPtr getOldestStore_() const; + LoadStoreInstInfoPtr getOldestStore_() const; bool olderStoresExists_(const InstPtr & inst_ptr); From 903729595bf7b6b0a19c9a421b1589874020a96f Mon Sep 17 00:00:00 2001 From: maynemei Date: Sat, 2 Nov 2024 01:09:51 +0800 Subject: [PATCH 03/36] change forwarding at handleCacheLookupReq_, roll back handleCacheRead_ --- core/lsu/LSU.cpp | 161 +++++++++++++++++++++++------------------------ 1 file changed, 80 insertions(+), 81 deletions(-) diff --git a/core/lsu/LSU.cpp b/core/lsu/LSU.cpp index 3dbc888e..8e792acf 100644 --- a/core/lsu/LSU.cpp +++ b/core/lsu/LSU.cpp @@ -460,6 +460,31 @@ namespace olympia const MemoryAccessInfoPtr & mem_access_info_ptr = load_store_info_ptr->getMemoryAccessInfoPtr(); const bool phy_addr_is_ready = mem_access_info_ptr->getPhyAddrStatus(); + const InstPtr & inst_ptr = mem_access_info_ptr->getInstPtr(); + + // first check physical address and bypass conditions + const bool is_already_hit = + (mem_access_info_ptr->getCacheState() == MemoryAccessInfo::CacheState::HIT); + const bool is_unretired_store = + inst_ptr->isStoreInst() && (inst_ptr->getStatus() != Inst::Status::RETIRED); + const bool cache_bypass = is_already_hit || !phy_addr_is_ready || is_unretired_store; + + if (cache_bypass) + { + if (is_already_hit) + { + ILOG("Cache Lookup is skipped (Cache already hit)"); + } + else if (is_unretired_store) + { + ILOG("Cache Lookup is skipped (store instruction not oldest)"); + } + else + { + sparta_assert(false, "Cache access is bypassed without a valid reason!"); + } + return; + } // If we did not have an MMU hit from previous stage, invalidate and bail if (false == phy_addr_is_ready) @@ -484,66 +509,59 @@ namespace olympia return; } - const InstPtr & inst_ptr = mem_access_info_ptr->getInstPtr(); - ILOG(load_store_info_ptr << " " << mem_access_info_ptr); // If have passed translation and the instruction is a store, // then it's good to be retired (i.e. mark it completed). // Stores typically do not cause a flush after a successful // translation. We now wait for the Retire block to "retire" // it, meaning it's good to go to the cache - if (inst_ptr->isStoreInst() && (inst_ptr->getStatus() == Inst::Status::SCHEDULED)) + if (inst_ptr->isStoreInst()) { - ILOG("Store marked as completed " << inst_ptr); - inst_ptr->setStatus(Inst::Status::COMPLETED); - load_store_info_ptr->setState(LoadStoreInstInfo::IssueState::READY); - ldst_pipeline_.invalidateStage(cache_lookup_stage_); - if (allow_speculative_load_exec_) + if (inst_ptr->getStatus() == Inst::Status::SCHEDULED) { - updateInstReplayReady_(load_store_info_ptr); + ILOG("Store marked as completed " << inst_ptr); + inst_ptr->setStatus(Inst::Status::COMPLETED); + load_store_info_ptr->setState(LoadStoreInstInfo::IssueState::READY); + ldst_pipeline_.invalidateStage(cache_lookup_stage_); + if (allow_speculative_load_exec_) + { + updateInstReplayReady_(load_store_info_ptr); + } + return; } - return; } - - // Loads dont perform a cache lookup if there are older stores present in the load store - // queue - if (!inst_ptr->isStoreInst() && olderStoresExists_(inst_ptr) - && allow_speculative_load_exec_) + else // Loads handling { - ILOG("Dropping speculative load " << inst_ptr); - load_store_info_ptr->setState(LoadStoreInstInfo::IssueState::READY); - ldst_pipeline_.invalidateStage(cache_lookup_stage_); - if (allow_speculative_load_exec_) + // Check for speculative execution constraints + // since we use data forwarding, we only need to check whether all older store was issued + if (allow_speculative_load_exec_ && !allOlderStoresIssued_(inst_ptr)) { + ILOG("Dropping speculative load " << inst_ptr << " due to unissued older stores"); + load_store_info_ptr->setState(LoadStoreInstInfo::IssueState::READY); + ldst_pipeline_.invalidateStage(cache_lookup_stage_); updateInstReplayReady_(load_store_info_ptr); + return; } - return; - } - const bool is_already_hit = - (mem_access_info_ptr->getCacheState() == MemoryAccessInfo::CacheState::HIT); - const bool is_unretired_store = - inst_ptr->isStoreInst() && (inst_ptr->getStatus() != Inst::Status::RETIRED); - const bool cache_bypass = is_already_hit || !phy_addr_is_ready || is_unretired_store; + //check if we can forward from store buffer first + uint64_t load_addr = inst_ptr->getTargetVAddr(); + auto forwarding_store = findYoungestMatchingStore_(load_addr); - if (cache_bypass) - { - if (is_already_hit) - { - ILOG("Cache Lookup is skipped (Cache already hit)"); - } - else if (is_unretired_store) + if (forwarding_store) { - ILOG("Cache Lookup is skipped (store instruction not oldest)"); + ILOG("Found forwarding store for load " << inst_ptr); + mem_access_info_ptr->setDataReady(true); + mem_access_info_ptr->setCacheState(MemoryAccessInfo::CacheState::HIT); + return; } - else - { - sparta_assert(false, "Cache access is bypassed without a valid reason!"); + + // No forwarding possible - need cache access + if (!mem_access_info_ptr->isCacheHit()) { + out_cache_lookup_req_.send(mem_access_info_ptr); } - return; } - out_cache_lookup_req_.send(mem_access_info_ptr); + } void LSU::getAckFromCache_(const MemoryAccessInfoPtr & mem_access_info_ptr) @@ -614,60 +632,41 @@ namespace olympia } const LoadStoreInstInfoPtr & load_store_info_ptr = ldst_pipeline_[cache_read_stage_]; - const InstPtr & inst_ptr = load_store_info_ptr->getInstPtr(); const MemoryAccessInfoPtr & mem_access_info_ptr = load_store_info_ptr->getMemoryAccessInfoPtr(); ILOG(mem_access_info_ptr); - if (!inst_ptr->isStoreInst()) - { - // for load we check whether we could use store forwarding - // checking whether there are address match in store buffer - uint64_t load_addr = inst_ptr->getTargetVAddr(); - auto forwarding_store = findYoungestMatchingStore_(load_addr); - - if (forwarding_store) + if (false == mem_access_info_ptr->isCacheHit()) + { + ILOG("Cannot complete inst, cache miss: " << mem_access_info_ptr); + if (allow_speculative_load_exec_) { - mem_access_info_ptr->setDataReady(true); - ILOG("Load using forwarded data from store buffer: " << inst_ptr - << " from store: " << forwarding_store->getInstPtr()); + updateInstReplayReady_(load_store_info_ptr); } - else + // There might not be a wake up because the cache cannot handle nay more instruction + // Change to nack wakeup when implemented + if (!load_store_info_ptr->isInReadyQueue()) { - if (false == mem_access_info_ptr->isCacheHit()) + appendToReadyQueue_(load_store_info_ptr); + load_store_info_ptr->setState(LoadStoreInstInfo::IssueState::READY); + if (isReadyToIssueInsts_()) { - ILOG("Cannot complete inst, cache miss: " << mem_access_info_ptr); - if (allow_speculative_load_exec_) - { - updateInstReplayReady_(load_store_info_ptr); - } - // There might not be a wake up because the cache cannot handle nay more instruction - // Change to nack wakeup when implemented - if (!load_store_info_ptr->isInReadyQueue()) - { - appendToReadyQueue_(load_store_info_ptr); - load_store_info_ptr->setState(LoadStoreInstInfo::IssueState::READY); - if (isReadyToIssueInsts_()) - { - uev_issue_inst_.schedule(sparta::Clock::Cycle(0)); - } - } - ldst_pipeline_.invalidateStage(cache_read_stage_); - return; + uev_issue_inst_.schedule(sparta::Clock::Cycle(0)); } + } + ldst_pipeline_.invalidateStage(cache_read_stage_); + return; + } - if (mem_access_info_ptr->isDataReady()) - { - ILOG("Instruction had previously had its data ready"); - return; - } + if (mem_access_info_ptr->isDataReady()) + { + ILOG("Instruction had previously had its data ready"); + return; + } - ILOG("Data ready set for " << mem_access_info_ptr); - mem_access_info_ptr->setDataReady(true); + ILOG("Data ready set for " << mem_access_info_ptr); + mem_access_info_ptr->setDataReady(true); - } - } - if (isReadyToIssueInsts_()) { ILOG("Cache read issue"); From 8819e27e79559b02bbd75cd72c6dba60afb98499 Mon Sep 17 00:00:00 2001 From: maynemei Date: Sun, 17 Nov 2024 22:47:41 -0500 Subject: [PATCH 04/36] passed regression test, mode test case needed --- core/lsu/LSU.cpp | 119 ++++++++++++++++++++++------------------------- core/lsu/LSU.hpp | 2 +- 2 files changed, 57 insertions(+), 64 deletions(-) diff --git a/core/lsu/LSU.cpp b/core/lsu/LSU.cpp index 8e792acf..ace6c6b6 100644 --- a/core/lsu/LSU.cpp +++ b/core/lsu/LSU.cpp @@ -21,7 +21,7 @@ namespace olympia replay_buffer_size_(p->replay_buffer_size), replay_issue_delay_(p->replay_issue_delay), store_buffer_("store_buffer", p->ldst_inst_queue_size, getClock()), // Add this line - store_buffer_size_(p->ldst_inst_queue_size), + store_buffer_size_(p->ldst_inst_queue_size), ready_queue_(), load_store_info_allocator_(sparta::notNull(OlympiaAllocators::getOlympiaAllocators(node)) ->load_store_info_allocator), @@ -460,31 +460,6 @@ namespace olympia const MemoryAccessInfoPtr & mem_access_info_ptr = load_store_info_ptr->getMemoryAccessInfoPtr(); const bool phy_addr_is_ready = mem_access_info_ptr->getPhyAddrStatus(); - const InstPtr & inst_ptr = mem_access_info_ptr->getInstPtr(); - - // first check physical address and bypass conditions - const bool is_already_hit = - (mem_access_info_ptr->getCacheState() == MemoryAccessInfo::CacheState::HIT); - const bool is_unretired_store = - inst_ptr->isStoreInst() && (inst_ptr->getStatus() != Inst::Status::RETIRED); - const bool cache_bypass = is_already_hit || !phy_addr_is_ready || is_unretired_store; - - if (cache_bypass) - { - if (is_already_hit) - { - ILOG("Cache Lookup is skipped (Cache already hit)"); - } - else if (is_unretired_store) - { - ILOG("Cache Lookup is skipped (store instruction not oldest)"); - } - else - { - sparta_assert(false, "Cache access is bypassed without a valid reason!"); - } - return; - } // If we did not have an MMU hit from previous stage, invalidate and bail if (false == phy_addr_is_ready) @@ -494,7 +469,7 @@ namespace olympia { updateInstReplayReady_(load_store_info_ptr); } - // There might not be a wake up because the cache cannot handle nay more instruction + // There might not be a wake up because the cache cannot handle any more instruction // Change to nack wakeup when implemented if (!load_store_info_ptr->isInReadyQueue()) { @@ -509,42 +484,45 @@ namespace olympia return; } + const InstPtr & inst_ptr = mem_access_info_ptr->getInstPtr(); + ILOG(load_store_info_ptr << " " << mem_access_info_ptr); // If have passed translation and the instruction is a store, // then it's good to be retired (i.e. mark it completed). // Stores typically do not cause a flush after a successful - // translation. We now wait for the Retire block to "retire" + // translation. We now wait for the Retire block to "retire" // it, meaning it's good to go to the cache - if (inst_ptr->isStoreInst()) + if (inst_ptr->isStoreInst() && (inst_ptr->getStatus() == Inst::Status::SCHEDULED)) { - if (inst_ptr->getStatus() == Inst::Status::SCHEDULED) + ILOG("Store marked as completed " << inst_ptr); + inst_ptr->setStatus(Inst::Status::COMPLETED); + load_store_info_ptr->setState(LoadStoreInstInfo::IssueState::READY); + ldst_pipeline_.invalidateStage(cache_lookup_stage_); + if (allow_speculative_load_exec_) { - ILOG("Store marked as completed " << inst_ptr); - inst_ptr->setStatus(Inst::Status::COMPLETED); - load_store_info_ptr->setState(LoadStoreInstInfo::IssueState::READY); - ldst_pipeline_.invalidateStage(cache_lookup_stage_); - if (allow_speculative_load_exec_) - { - updateInstReplayReady_(load_store_info_ptr); - } - return; + updateInstReplayReady_(load_store_info_ptr); } + return; } - else // Loads handling + + // Loads dont perform a cache lookup if there are older stores haven't issued in the load store queue + if (!inst_ptr->isStoreInst() && !allOlderStoresIssued_(inst_ptr) + && allow_speculative_load_exec_) { - // Check for speculative execution constraints - // since we use data forwarding, we only need to check whether all older store was issued - if (allow_speculative_load_exec_ && !allOlderStoresIssued_(inst_ptr)) + ILOG("Dropping speculative load " << inst_ptr); + load_store_info_ptr->setState(LoadStoreInstInfo::IssueState::READY); + ldst_pipeline_.invalidateStage(cache_lookup_stage_); + if (allow_speculative_load_exec_) { - ILOG("Dropping speculative load " << inst_ptr << " due to unissued older stores"); - load_store_info_ptr->setState(LoadStoreInstInfo::IssueState::READY); - ldst_pipeline_.invalidateStage(cache_lookup_stage_); updateInstReplayReady_(load_store_info_ptr); - return; } + return; + } - //check if we can forward from store buffer first - uint64_t load_addr = inst_ptr->getTargetVAddr(); + // Add store forwarding check here for loads + if (!inst_ptr->isStoreInst()) + { + const uint64_t load_addr = inst_ptr->getTargetVAddr(); auto forwarding_store = findYoungestMatchingStore_(load_addr); if (forwarding_store) @@ -554,14 +532,32 @@ namespace olympia mem_access_info_ptr->setCacheState(MemoryAccessInfo::CacheState::HIT); return; } + } - // No forwarding possible - need cache access - if (!mem_access_info_ptr->isCacheHit()) { - out_cache_lookup_req_.send(mem_access_info_ptr); + const bool is_already_hit = + (mem_access_info_ptr->getCacheState() == MemoryAccessInfo::CacheState::HIT); + const bool is_unretired_store = + inst_ptr->isStoreInst() && (inst_ptr->getStatus() != Inst::Status::RETIRED); + const bool cache_bypass = is_already_hit || !phy_addr_is_ready || is_unretired_store; + + if (cache_bypass) + { + if (is_already_hit) + { + ILOG("Cache Lookup is skipped (Cache already hit)"); + } + else if (is_unretired_store) + { + ILOG("Cache Lookup is skipped (store instruction not oldest)"); + } + else + { + sparta_assert(false, "Cache access is bypassed without a valid reason!"); } + return; } - + out_cache_lookup_req_.send(mem_access_info_ptr); } void LSU::getAckFromCache_(const MemoryAccessInfoPtr & mem_access_info_ptr) @@ -946,19 +942,15 @@ namespace olympia ILOG("Store added to store buffer: " << inst_ptr); } - LoadStoreInstInfoPtr LSU::findYoungestMatchingStore_(uint64_t addr) + LoadStoreInstInfoPtr LSU::findYoungestMatchingStore_(const uint64_t addr) const { LoadStoreInstInfoPtr matching_store = nullptr; - for (auto it = store_buffer_.begin(); it != store_buffer_.end(); ++it) - { - auto & store = *it; - if (store->getInstPtr()->getTargetVAddr() == addr) - { - matching_store = store; - } - } - return matching_store; + auto it = std::find_if(store_buffer_.rbegin(), store_buffer_.rend(), + [addr](const auto& store) { + return store->getInstPtr()->getTargetVAddr() == addr; + }); + return (it != store_buffer_.rend()) ? *it : nullptr; } LoadStoreInstInfoPtr LSU::getOldestStore_() const @@ -1450,6 +1442,7 @@ namespace olympia auto inst_ptr = (*sb_iter)->getInstPtr(); if(criteria.includedInFlush(inst_ptr)) { auto delete_iter = sb_iter++; + // store buffer didn't return an iterator store_buffer_.erase(delete_iter); ILOG("Flushed store from store buffer: " << inst_ptr); } else { diff --git a/core/lsu/LSU.hpp b/core/lsu/LSU.hpp index 9e6175fd..426ee16b 100644 --- a/core/lsu/LSU.hpp +++ b/core/lsu/LSU.hpp @@ -267,7 +267,7 @@ namespace olympia void allocateInstToStoreBuffer_(const InstPtr & inst_ptr); // Search store buffer in FIFO order for youngest matching store - LoadStoreInstInfoPtr findYoungestMatchingStore_(uint64_t addr); + LoadStoreInstInfoPtr findYoungestMatchingStore_(const uint64_t addr) const ; // get oldest store LoadStoreInstInfoPtr getOldestStore_() const; From 14a3a70d994bffc5e0bfc6c2d9a39c67bb7cddc2 Mon Sep 17 00:00:00 2001 From: maynemei Date: Sun, 24 Nov 2024 22:51:35 -0500 Subject: [PATCH 05/36] modified test case, also disable store sned cache lookup req when retire --- core/lsu/LSU.cpp | 3 +-- test/core/lsu/Lsu_test.cpp | 51 ++++++++++++++++++++++++++++++++++++-- test/core/lsu/raw.json | 6 ++--- 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/core/lsu/LSU.cpp b/core/lsu/LSU.cpp index ace6c6b6..d0cacbd8 100644 --- a/core/lsu/LSU.cpp +++ b/core/lsu/LSU.cpp @@ -282,8 +282,7 @@ namespace olympia << (oldest_store ? oldest_store->getInstPtr()->getUniqueID() : 0) << " Got: " << inst_ptr->getUniqueID()); - // Remove from store buffer and commit to cache - out_cache_lookup_req_.send(oldest_store->getMemoryAccessInfoPtr()); + // Remove from store buffer -> don't actually need to send cache request store_buffer_.erase(store_buffer_.begin());; ++stores_retired_; } diff --git a/test/core/lsu/Lsu_test.cpp b/test/core/lsu/Lsu_test.cpp index 50e86c15..874536ee 100644 --- a/test/core/lsu/Lsu_test.cpp +++ b/test/core/lsu/Lsu_test.cpp @@ -51,6 +51,44 @@ class olympia::LSUTester EXPECT_EQUAL(lsu.cache_read_stage_, 4); EXPECT_EQUAL(lsu.complete_stage_, 6); } + + void test_store_address_match(olympia::LSU &lsu, uint64_t addr, bool should_match) { + auto& store_buffer = lsu.store_buffer_; // Friend class can access private members + if(store_buffer.empty()) { + EXPECT_FALSE(should_match); + return; + } + bool found = false; + for(const auto& store : store_buffer) { + if(store->getInstPtr()->getTargetVAddr() == addr) { + found = true; + break; + } + } + EXPECT_EQUAL(found, should_match); + } + + // Helper to verify store forwarding for a specific load/store pair + void test_store_forwarding(olympia::LSU &lsu) { + // Check store buffer has the expected store + EXPECT_TRUE(lsu.store_buffer_.size() > 0); + + // Get store and load from issue queue that should match + bool found_pair = false; + for(const auto& ldst_inst : lsu.ldst_inst_queue_) { + auto inst = ldst_inst->getInstPtr(); + if(!inst->isStoreInst()) { + // Found a load - check if it got forwarded data + auto mem_info = ldst_inst->getMemoryAccessInfoPtr(); + if(mem_info->isDataReady() && + mem_info->getCacheState() == MemoryAccessInfo::CacheState::HIT) { + found_pair = true; + break; + } + } + } + EXPECT_TRUE(found_pair); + } }; const char USAGE[] = @@ -109,8 +147,17 @@ void runTest(int argc, char **argv) lsupipe_tester.test_pipeline_stages(*my_lsu); cls.runSimulator(&sim, 9); lsupipe_tester.test_inst_issue(*my_lsu, 2); // Loads operand dependency meet - cls.runSimulator(&sim, 52); - lsupipe_tester.test_replay_issue_abort(*my_lsu, 3); // Loads operand dependency meet + lsupipe_tester.test_store_address_match(*my_lsu, 0xdeeebeef, true); + + // Run for second load (no match at 0xdeebbeef) + cls.runSimulator(&sim, 5); + lsupipe_tester.test_store_address_match(*my_lsu, 0xdeebbeef, false); + + cls.runSimulator(&sim, 47); + lsupipe_tester.test_replay_issue_abort(*my_lsu, 3); + // Loads operand dependency meet + lsupipe_tester.test_store_address_match(*my_lsu, 0xdeadbeef, true); + cls.runSimulator(&sim); } diff --git a/test/core/lsu/raw.json b/test/core/lsu/raw.json index 63242dac..5451707b 100644 --- a/test/core/lsu/raw.json +++ b/test/core/lsu/raw.json @@ -1,6 +1,6 @@ [ { - "mnemonic": "lw", + "mnemonic": "sw", "rs1": 5, "rs2": 6, "vaddr": "0xdeeebeef" @@ -15,7 +15,7 @@ "mnemonic": "lw", "rs1": 5, "rs2": 6, - "vaddr": "0xdeeebeef" + "vaddr": "0xdeebbeef" }, { "mnemonic": "sw", @@ -33,6 +33,6 @@ "mnemonic": "lw", "rs1": 3, "rd": 4, - "vaddr" : "0xdeadbeef" + "vaddr" : "0xdeeebeef" } ] \ No newline at end of file From dc379a111ca20c9fe8ec28c5ba3a4ac2c254f6d6 Mon Sep 17 00:00:00 2001 From: maynemei Date: Mon, 25 Nov 2024 11:06:15 -0500 Subject: [PATCH 06/36] modified test to check forwarding cycle --- core/lsu/LSU.cpp | 1 + test/core/lsu/Lsu_test.cpp | 69 ++++++++++++++++++++++++-------------- 2 files changed, 44 insertions(+), 26 deletions(-) diff --git a/core/lsu/LSU.cpp b/core/lsu/LSU.cpp index d0cacbd8..94dc66a9 100644 --- a/core/lsu/LSU.cpp +++ b/core/lsu/LSU.cpp @@ -511,6 +511,7 @@ namespace olympia ILOG("Dropping speculative load " << inst_ptr); load_store_info_ptr->setState(LoadStoreInstInfo::IssueState::READY); ldst_pipeline_.invalidateStage(cache_lookup_stage_); + // TODO: double check whether "allow_speculative_load_exec_" means not allow if (allow_speculative_load_exec_) { updateInstReplayReady_(load_store_info_ptr); diff --git a/test/core/lsu/Lsu_test.cpp b/test/core/lsu/Lsu_test.cpp index 874536ee..428f84ad 100644 --- a/test/core/lsu/Lsu_test.cpp +++ b/test/core/lsu/Lsu_test.cpp @@ -68,26 +68,29 @@ class olympia::LSUTester EXPECT_EQUAL(found, should_match); } - // Helper to verify store forwarding for a specific load/store pair - void test_store_forwarding(olympia::LSU &lsu) { - // Check store buffer has the expected store - EXPECT_TRUE(lsu.store_buffer_.size() > 0); - - // Get store and load from issue queue that should match - bool found_pair = false; + void test_forwarding_occurred(olympia::LSU &lsu, uint64_t addr) { + bool found = false; for(const auto& ldst_inst : lsu.ldst_inst_queue_) { - auto inst = ldst_inst->getInstPtr(); - if(!inst->isStoreInst()) { - // Found a load - check if it got forwarded data + if(!ldst_inst->getInstPtr()->isStoreInst() && + ldst_inst->getInstPtr()->getTargetVAddr() == addr) { auto mem_info = ldst_inst->getMemoryAccessInfoPtr(); - if(mem_info->isDataReady() && - mem_info->getCacheState() == MemoryAccessInfo::CacheState::HIT) { - found_pair = true; - break; - } + found = mem_info->isDataReady() && + mem_info->getCacheState() == MemoryAccessInfo::CacheState::HIT; + break; } } - EXPECT_TRUE(found_pair); + EXPECT_TRUE(found); + } + + void test_completion_time(olympia::LSU &lsu, int expected_cycles, uint64_t addr) { + uint64_t start_cycle = lsu.getClock()->currentCycle(); + + // Run until instruction completes + while(!lsu.lsu_insts_completed_ && + (lsu.getClock()->currentCycle() - start_cycle) < static_cast(expected_cycles)) { + // Continue simulation + } + EXPECT_EQUAL(lsu.getClock()->currentCycle() - start_cycle, expected_cycles); } }; @@ -149,16 +152,30 @@ void runTest(int argc, char **argv) lsupipe_tester.test_inst_issue(*my_lsu, 2); // Loads operand dependency meet lsupipe_tester.test_store_address_match(*my_lsu, 0xdeeebeef, true); - // Run for second load (no match at 0xdeebbeef) - cls.runSimulator(&sim, 5); - lsupipe_tester.test_store_address_match(*my_lsu, 0xdeebbeef, false); - - cls.runSimulator(&sim, 47); - lsupipe_tester.test_replay_issue_abort(*my_lsu, 3); - // Loads operand dependency meet - lsupipe_tester.test_store_address_match(*my_lsu, 0xdeadbeef, true); - - cls.runSimulator(&sim); + // First store and store buffer + cls.runSimulator(&sim, 7); + lsupipe_tester.test_store_address_match(*my_lsu, 0xdeeebeef, true); +// lsupipe_tester.test_inst_issue(*my_lsu, 1); + + // First load - forwarding case + auto start_cycle = my_lsu->getClock()->currentCycle(); + cls.runSimulator(&sim, 3); +// lsupipe_tester.test_inst_issue(*my_lsu, 2); + EXPECT_EQUAL(my_lsu->getClock()->currentCycle() - start_cycle, 3); + + // Second load - no forwarding + start_cycle = my_lsu->getClock()->currentCycle(); + cls.runSimulator(&sim, 7); + EXPECT_EQUAL(my_lsu->getClock()->currentCycle() - start_cycle, 7); + lsupipe_tester.test_store_address_match(*my_lsu, 0xdeebbeef, false); + + // Replay mechanism + cls.runSimulator(&sim, 47); + lsupipe_tester.test_replay_issue_abort(*my_lsu, 2); + lsupipe_tester.test_store_address_match(*my_lsu, 0xdeadbeef, true); + + // Final state + cls.runSimulator(&sim); } int main(int argc, char **argv) From 0eb3a6a7789402940ede2dd571163703cc0896d3 Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Thu, 28 Nov 2024 22:30:45 -0800 Subject: [PATCH 07/36] Update Lsu_test.cpp Signed-off-by: MayneMei <69469280+MayneMei@users.noreply.github.com> --- test/core/lsu/Lsu_test.cpp | 54 +++++++++----------------------------- 1 file changed, 13 insertions(+), 41 deletions(-) diff --git a/test/core/lsu/Lsu_test.cpp b/test/core/lsu/Lsu_test.cpp index 428f84ad..1cc4c8af 100644 --- a/test/core/lsu/Lsu_test.cpp +++ b/test/core/lsu/Lsu_test.cpp @@ -52,45 +52,18 @@ class olympia::LSUTester EXPECT_EQUAL(lsu.complete_stage_, 6); } - void test_store_address_match(olympia::LSU &lsu, uint64_t addr, bool should_match) { - auto& store_buffer = lsu.store_buffer_; // Friend class can access private members - if(store_buffer.empty()) { - EXPECT_FALSE(should_match); - return; + void test_store_size(olympia::LSU &lsu, uint64_t size) { + auto& store_buffer = lsu.store_buffer_; + + std::cout << "Store buffer size: " << store_buffer.size() << std::endl; + + if(!store_buffer.empty()) { + std::cout << "First store addr: 0x" << std::hex + << store_buffer.front()->getInstPtr()->getTargetVAddr() << std::endl; } - bool found = false; - for(const auto& store : store_buffer) { - if(store->getInstPtr()->getTargetVAddr() == addr) { - found = true; - break; - } - } - EXPECT_EQUAL(found, should_match); - } - - void test_forwarding_occurred(olympia::LSU &lsu, uint64_t addr) { - bool found = false; - for(const auto& ldst_inst : lsu.ldst_inst_queue_) { - if(!ldst_inst->getInstPtr()->isStoreInst() && - ldst_inst->getInstPtr()->getTargetVAddr() == addr) { - auto mem_info = ldst_inst->getMemoryAccessInfoPtr(); - found = mem_info->isDataReady() && - mem_info->getCacheState() == MemoryAccessInfo::CacheState::HIT; - break; - } - } - EXPECT_TRUE(found); - } - - void test_completion_time(olympia::LSU &lsu, int expected_cycles, uint64_t addr) { - uint64_t start_cycle = lsu.getClock()->currentCycle(); - - // Run until instruction completes - while(!lsu.lsu_insts_completed_ && - (lsu.getClock()->currentCycle() - start_cycle) < static_cast(expected_cycles)) { - // Continue simulation - } - EXPECT_EQUAL(lsu.getClock()->currentCycle() - start_cycle, expected_cycles); + + // Simply check if store was added to buffer + EXPECT_EQUAL(store_buffer.size(), 1); } }; @@ -154,7 +127,7 @@ void runTest(int argc, char **argv) // First store and store buffer cls.runSimulator(&sim, 7); - lsupipe_tester.test_store_address_match(*my_lsu, 0xdeeebeef, true); + lsupipe_tester.test_store_size(*my_lsu, 1); // lsupipe_tester.test_inst_issue(*my_lsu, 1); // First load - forwarding case @@ -167,12 +140,11 @@ void runTest(int argc, char **argv) start_cycle = my_lsu->getClock()->currentCycle(); cls.runSimulator(&sim, 7); EXPECT_EQUAL(my_lsu->getClock()->currentCycle() - start_cycle, 7); - lsupipe_tester.test_store_address_match(*my_lsu, 0xdeebbeef, false); // Replay mechanism cls.runSimulator(&sim, 47); lsupipe_tester.test_replay_issue_abort(*my_lsu, 2); - lsupipe_tester.test_store_address_match(*my_lsu, 0xdeadbeef, true); + lsupipe_tester.test_store_size(*my_lsu, 2); // Final state cls.runSimulator(&sim); From 5404063c2e5c04f5a79206aa2417367a40c7e769 Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Thu, 28 Nov 2024 22:41:27 -0800 Subject: [PATCH 08/36] Update LSU.hpp, add parameter for data forwarding, set default value for mem speculation false Signed-off-by: MayneMei <69469280+MayneMei@users.noreply.github.com> --- core/lsu/LSU.hpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/lsu/LSU.hpp b/core/lsu/LSU.hpp index 426ee16b..ed6242af 100644 --- a/core/lsu/LSU.hpp +++ b/core/lsu/LSU.hpp @@ -53,8 +53,11 @@ namespace olympia // PARAMETER(uint32_t, store_buffer_size, ldst_inst_queue_size, "Size of the store buffer") // LSU microarchitecture parameters PARAMETER( - bool, allow_speculative_load_exec, true, + bool, allow_speculative_load_exec, false, "Allow loads to proceed speculatively before all older store addresses are known") + PARAMETER( + bool, allow_data_forwarding_, true, + "Allow data forwarding to bypass the cache look up / memory access") // Pipeline length PARAMETER(uint32_t, mmu_lookup_stage_length, 1, "Length of the mmu lookup stage") PARAMETER(uint32_t, cache_lookup_stage_length, 1, "Length of the cache lookup stage") @@ -174,6 +177,7 @@ namespace olympia // LSU Microarchitecture parameters const bool allow_speculative_load_exec_; + const bool allow_data_forwarding_; // ROB stopped simulation early, transactions could still be inflight. bool rob_stopped_simulation_ = false; From 5cf7454ba334c6bede4a6ab64ad59b0494d82366 Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Thu, 28 Nov 2024 22:43:21 -0800 Subject: [PATCH 09/36] Update Lsu_test.cpp Signed-off-by: MayneMei <69469280+MayneMei@users.noreply.github.com> --- test/core/lsu/Lsu_test.cpp | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/test/core/lsu/Lsu_test.cpp b/test/core/lsu/Lsu_test.cpp index 1cc4c8af..800f26a5 100644 --- a/test/core/lsu/Lsu_test.cpp +++ b/test/core/lsu/Lsu_test.cpp @@ -53,17 +53,7 @@ class olympia::LSUTester } void test_store_size(olympia::LSU &lsu, uint64_t size) { - auto& store_buffer = lsu.store_buffer_; - - std::cout << "Store buffer size: " << store_buffer.size() << std::endl; - - if(!store_buffer.empty()) { - std::cout << "First store addr: 0x" << std::hex - << store_buffer.front()->getInstPtr()->getTargetVAddr() << std::endl; - } - - // Simply check if store was added to buffer - EXPECT_EQUAL(store_buffer.size(), 1); + EXPECT_EQUAL(lsu.store_buffer_.size(), 1); } }; From 622a0d2812378fcb11bf31554e9edfdb946193de Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Thu, 28 Nov 2024 23:09:25 -0800 Subject: [PATCH 10/36] data forwarding parameterize, modified testcase --- core/lsu/LSU.cpp | 5 +-- core/lsu/LSU.hpp | 2 +- docs/lsu.md | 2 ++ test/core/lsu/Lsu_test.cpp | 69 +++++++++++++++++++++++++------------- test/sim/CMakeLists.txt | 2 ++ 5 files changed, 53 insertions(+), 27 deletions(-) diff --git a/core/lsu/LSU.cpp b/core/lsu/LSU.cpp index 94dc66a9..35ac3c78 100644 --- a/core/lsu/LSU.cpp +++ b/core/lsu/LSU.cpp @@ -37,7 +37,8 @@ namespace olympia + p->cache_read_stage_length), // Complete stage is after the cache read stage ldst_pipeline_("LoadStorePipeline", (complete_stage_ + 1), getClock()), // complete_stage_ + 1 is number of stages - allow_speculative_load_exec_(p->allow_speculative_load_exec) + allow_speculative_load_exec_(p->allow_speculative_load_exec), + allow_data_forwarding_(p->allow_data_forwarding) { sparta_assert(p->mmu_lookup_stage_length > 0, "MMU lookup stage should atleast be one cycle"); @@ -520,7 +521,7 @@ namespace olympia } // Add store forwarding check here for loads - if (!inst_ptr->isStoreInst()) + if (!inst_ptr->isStoreInst() && enable_store_forwarding_) { const uint64_t load_addr = inst_ptr->getTargetVAddr(); auto forwarding_store = findYoungestMatchingStore_(load_addr); diff --git a/core/lsu/LSU.hpp b/core/lsu/LSU.hpp index ed6242af..b928af72 100644 --- a/core/lsu/LSU.hpp +++ b/core/lsu/LSU.hpp @@ -56,7 +56,7 @@ namespace olympia bool, allow_speculative_load_exec, false, "Allow loads to proceed speculatively before all older store addresses are known") PARAMETER( - bool, allow_data_forwarding_, true, + bool, allow_data_forwarding, true, "Allow data forwarding to bypass the cache look up / memory access") // Pipeline length PARAMETER(uint32_t, mmu_lookup_stage_length, 1, "Length of the mmu lookup stage") diff --git a/docs/lsu.md b/docs/lsu.md index e0b1a95b..d0e67bb1 100644 --- a/docs/lsu.md +++ b/docs/lsu.md @@ -24,6 +24,8 @@ out_mmu_lookup_req --> Output to DCache (Send a VA to PA address translation r `allow_speculative_load_exec` - Allow loads to proceed speculatively before all older store addresses are known. +`allow_data_forwarding` - Allow loads to get data from store instead of cache, by pass mem look up. + `replay_buffer_size` - Size of the replay buffer. Defaults to the same size of the LSU instruction queue. `replay_issue_delay` - Delay in cycles to replay the instruction. diff --git a/test/core/lsu/Lsu_test.cpp b/test/core/lsu/Lsu_test.cpp index 800f26a5..4113c668 100644 --- a/test/core/lsu/Lsu_test.cpp +++ b/test/core/lsu/Lsu_test.cpp @@ -111,30 +111,51 @@ void runTest(int argc, char **argv) olympia::LSU *my_lsu = root_node->getChild("cpu.core0.lsu")->getResourceAs(); olympia::LSUTester lsupipe_tester; lsupipe_tester.test_pipeline_stages(*my_lsu); - cls.runSimulator(&sim, 9); - lsupipe_tester.test_inst_issue(*my_lsu, 2); // Loads operand dependency meet - lsupipe_tester.test_store_address_match(*my_lsu, 0xdeeebeef, true); - - // First store and store buffer - cls.runSimulator(&sim, 7); - lsupipe_tester.test_store_size(*my_lsu, 1); -// lsupipe_tester.test_inst_issue(*my_lsu, 1); - - // First load - forwarding case - auto start_cycle = my_lsu->getClock()->currentCycle(); - cls.runSimulator(&sim, 3); -// lsupipe_tester.test_inst_issue(*my_lsu, 2); - EXPECT_EQUAL(my_lsu->getClock()->currentCycle() - start_cycle, 3); - - // Second load - no forwarding - start_cycle = my_lsu->getClock()->currentCycle(); - cls.runSimulator(&sim, 7); - EXPECT_EQUAL(my_lsu->getClock()->currentCycle() - start_cycle, 7); - - // Replay mechanism - cls.runSimulator(&sim, 47); - lsupipe_tester.test_replay_issue_abort(*my_lsu, 2); - lsupipe_tester.test_store_size(*my_lsu, 2); + + if(my_lsu->enable_store_forwarding_) { + // Data forwarding enabled case + + // First store + cls.runSimulator(&sim, 7); + lsupipe_tester.test_store_size(*my_lsu, 1); + + // First load - should get data from store forwarding + auto start_cycle = my_lsu->getClock()->currentCycle(); + cls.runSimulator(&sim, 3); + EXPECT_EQUAL(my_lsu->getClock()->currentCycle() - start_cycle, 3); // Fast path + + // Second load - no matching store, goes to cache + start_cycle = my_lsu->getClock()->currentCycle(); + cls.runSimulator(&sim, 7); + EXPECT_EQUAL(my_lsu->getClock()->currentCycle() - start_cycle, 7); // Cache access path + + // Second store and load + cls.runSimulator(&sim, 47); + lsupipe_tester.test_store_size(*my_lsu, 2); + lsupipe_tester.test_replay_issue_abort(*my_lsu, 2); + } + else { + // Data forwarding disabled case + + // First store + cls.runSimulator(&sim, 7); + lsupipe_tester.test_store_size(*my_lsu, 1); + + // First load - must go to cache + auto start_cycle = my_lsu->getClock()->currentCycle(); + cls.runSimulator(&sim, 7); // Takes longer, must access cache + EXPECT_EQUAL(my_lsu->getClock()->currentCycle() - start_cycle, 7); + + // Second load - also must go to cache + start_cycle = my_lsu->getClock()->currentCycle(); + cls.runSimulator(&sim, 7); + EXPECT_EQUAL(my_lsu->getClock()->currentCycle() - start_cycle, 7); + + // Second store and load + cls.runSimulator(&sim, 47); + lsupipe_tester.test_store_size(*my_lsu, 2); + lsupipe_tester.test_replay_issue_abort(*my_lsu, 2); + } // Final state cls.runSimulator(&sim); diff --git a/test/sim/CMakeLists.txt b/test/sim/CMakeLists.txt index ee44c64d..db87e070 100644 --- a/test/sim/CMakeLists.txt +++ b/test/sim/CMakeLists.txt @@ -91,6 +91,8 @@ list(APPEND test_params_list "top.cpu.core0.lsu.params.cache_lookup_stage_length list(APPEND test_params_list "top.cpu.core0.lsu.params.cache_read_stage_length 3") list(APPEND test_params_list "top.cpu.core0.lsu.params.allow_speculative_load_exec false") list(APPEND test_params_list "top.cpu.core0.lsu.params.allow_speculative_load_exec true") +list(APPEND test_params_list "top.cpu.core0.lsu.params.allow_data_forwarding false") +list(APPEND test_params_list "top.cpu.core0.lsu.params.allow_data_forwarding true") list(APPEND test_params_list "top.cpu.core0.lsu.params.replay_issue_delay 5") # Used to set a custom name for each test From 5bdeefaa7a1321514cd100ca0db3f213273a065a Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Thu, 28 Nov 2024 23:13:24 -0800 Subject: [PATCH 11/36] typo fixing --- core/lsu/LSU.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lsu/LSU.cpp b/core/lsu/LSU.cpp index 35ac3c78..065e6009 100644 --- a/core/lsu/LSU.cpp +++ b/core/lsu/LSU.cpp @@ -521,7 +521,7 @@ namespace olympia } // Add store forwarding check here for loads - if (!inst_ptr->isStoreInst() && enable_store_forwarding_) + if (!inst_ptr->isStoreInst() && allow_data_forwarding) { const uint64_t load_addr = inst_ptr->getTargetVAddr(); auto forwarding_store = findYoungestMatchingStore_(load_addr); From 60d5fbaa840514b3f0a51cf9c160400c0fab8e70 Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Thu, 28 Nov 2024 23:15:51 -0800 Subject: [PATCH 12/36] typo fixing --- core/lsu/LSU.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lsu/LSU.cpp b/core/lsu/LSU.cpp index 065e6009..a482a124 100644 --- a/core/lsu/LSU.cpp +++ b/core/lsu/LSU.cpp @@ -521,7 +521,7 @@ namespace olympia } // Add store forwarding check here for loads - if (!inst_ptr->isStoreInst() && allow_data_forwarding) + if (!inst_ptr->isStoreInst() && allow_data_forwarding_) { const uint64_t load_addr = inst_ptr->getTargetVAddr(); auto forwarding_store = findYoungestMatchingStore_(load_addr); From 0b94a3520ca4c3607899b0469bd068d62c0e4cfa Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Thu, 28 Nov 2024 23:20:29 -0800 Subject: [PATCH 13/36] test typo fixing --- test/core/lsu/Lsu_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core/lsu/Lsu_test.cpp b/test/core/lsu/Lsu_test.cpp index 4113c668..ab711976 100644 --- a/test/core/lsu/Lsu_test.cpp +++ b/test/core/lsu/Lsu_test.cpp @@ -112,7 +112,7 @@ void runTest(int argc, char **argv) olympia::LSUTester lsupipe_tester; lsupipe_tester.test_pipeline_stages(*my_lsu); - if(my_lsu->enable_store_forwarding_) { + if(my_lsu->allow_data_forwarding_) { // Data forwarding enabled case // First store From 662f00c09104a16742f3ef3d328aa02c4c719986 Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Thu, 28 Nov 2024 23:24:37 -0800 Subject: [PATCH 14/36] add helpfer function for test --- core/lsu/LSU.hpp | 5 +++++ test/core/lsu/Lsu_test.cpp | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/core/lsu/LSU.hpp b/core/lsu/LSU.hpp index b928af72..040ba76c 100644 --- a/core/lsu/LSU.hpp +++ b/core/lsu/LSU.hpp @@ -77,6 +77,11 @@ namespace olympia //! name of this resource. static const char name[]; + // return allow_data_forwarding for test + bool allow_data_forwarding_ex() { + return this.allow_data_forwarding_; + } + //////////////////////////////////////////////////////////////////////////////// // Type Name/Alias Declaration //////////////////////////////////////////////////////////////////////////////// diff --git a/test/core/lsu/Lsu_test.cpp b/test/core/lsu/Lsu_test.cpp index ab711976..8fca4b3b 100644 --- a/test/core/lsu/Lsu_test.cpp +++ b/test/core/lsu/Lsu_test.cpp @@ -112,7 +112,7 @@ void runTest(int argc, char **argv) olympia::LSUTester lsupipe_tester; lsupipe_tester.test_pipeline_stages(*my_lsu); - if(my_lsu->allow_data_forwarding_) { + if(my_lsu->allow_data_forwarding_ex()) { // Data forwarding enabled case // First store From 0d52a0e3dfc9caa502993c0717ebf79d34bb2c62 Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Thu, 28 Nov 2024 23:26:27 -0800 Subject: [PATCH 15/36] syntax error --- core/lsu/LSU.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lsu/LSU.hpp b/core/lsu/LSU.hpp index 040ba76c..ccb0b2ad 100644 --- a/core/lsu/LSU.hpp +++ b/core/lsu/LSU.hpp @@ -79,7 +79,7 @@ namespace olympia // return allow_data_forwarding for test bool allow_data_forwarding_ex() { - return this.allow_data_forwarding_; + return allow_data_forwarding_; } //////////////////////////////////////////////////////////////////////////////// From a27fa71d788077ecf5d7c7357c4768dc9670616b Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Thu, 28 Nov 2024 23:41:53 -0800 Subject: [PATCH 16/36] modified test_case and little syntax in lsu --- core/lsu/LSU.cpp | 2 +- test/core/lsu/Lsu_test.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/lsu/LSU.cpp b/core/lsu/LSU.cpp index a482a124..8b115a43 100644 --- a/core/lsu/LSU.cpp +++ b/core/lsu/LSU.cpp @@ -934,7 +934,7 @@ namespace olympia void LSU::allocateInstToStoreBuffer_(const InstPtr & inst_ptr) { - auto store_info_ptr = createLoadStoreInst_(inst_ptr); + const auto & store_info_ptr = createLoadStoreInst_(inst_ptr); sparta_assert(store_buffer_.size() < ldst_inst_queue_size_, "Appending store buffer causes overflows!"); diff --git a/test/core/lsu/Lsu_test.cpp b/test/core/lsu/Lsu_test.cpp index 8fca4b3b..913a8cfb 100644 --- a/test/core/lsu/Lsu_test.cpp +++ b/test/core/lsu/Lsu_test.cpp @@ -52,8 +52,8 @@ class olympia::LSUTester EXPECT_EQUAL(lsu.complete_stage_, 6); } - void test_store_size(olympia::LSU &lsu, uint64_t size) { - EXPECT_EQUAL(lsu.store_buffer_.size(), 1); + void test_store_size(olympia::LSU &lsu, int size) { + EXPECT_EQUAL(lsu.store_buffer_.size(), size); } }; From d7df29020371635e59e07fdd2c1bf3fedb0270cb Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Sun, 1 Dec 2024 23:09:42 -0800 Subject: [PATCH 17/36] chagne erase to pop front(), don't have local machine right now so only can test by using auto regression test --- core/lsu/LSU.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/lsu/LSU.cpp b/core/lsu/LSU.cpp index 8b115a43..edd829cc 100644 --- a/core/lsu/LSU.cpp +++ b/core/lsu/LSU.cpp @@ -284,7 +284,8 @@ namespace olympia << " Got: " << inst_ptr->getUniqueID()); // Remove from store buffer -> don't actually need to send cache request - store_buffer_.erase(store_buffer_.begin());; + sparta_assert(store_buffer_.size() > 0, "Store buffer empty on retiring store"); + store_buffer_.pop_front(); ++stores_retired_; } From 8084f56481742b8554f5f9927466fa45bfb4310c Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Sun, 1 Dec 2024 23:20:46 -0800 Subject: [PATCH 18/36] change store buffer to deque --- core/lsu/LSU.cpp | 8 ++++---- core/lsu/LSU.hpp | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/core/lsu/LSU.cpp b/core/lsu/LSU.cpp index edd829cc..38000357 100644 --- a/core/lsu/LSU.cpp +++ b/core/lsu/LSU.cpp @@ -20,8 +20,8 @@ namespace olympia replay_buffer_("replay_buffer", p->replay_buffer_size, getClock()), replay_buffer_size_(p->replay_buffer_size), replay_issue_delay_(p->replay_issue_delay), - store_buffer_("store_buffer", p->ldst_inst_queue_size, getClock()), // Add this line - store_buffer_size_(p->ldst_inst_queue_size), + // store_buffer_("store_buffer", p->ldst_inst_queue_size, getClock()), // Add this line + // store_buffer_size_(p->ldst_inst_queue_size), ready_queue_(), load_store_info_allocator_(sparta::notNull(OlympiaAllocators::getOlympiaAllocators(node)) ->load_store_info_allocator), @@ -51,7 +51,7 @@ namespace olympia ldst_pipeline_.enableCollection(node); ldst_inst_queue_.enableCollection(node); replay_buffer_.enableCollection(node); - store_buffer_.enableCollection(node); + // store_buffer_.enableCollection(node); // Startup handler for sending initial credits sparta::StartupEvent(node, CREATE_SPARTA_HANDLER(LSU, sendInitialCredits_)); @@ -960,7 +960,7 @@ namespace olympia if(store_buffer_.empty()) { return nullptr; } - return store_buffer_.read(0); + return store_buffer_.front(); } bool LSU::allOlderStoresIssued_(const InstPtr & inst_ptr) diff --git a/core/lsu/LSU.hpp b/core/lsu/LSU.hpp index ccb0b2ad..c98da9e4 100644 --- a/core/lsu/LSU.hpp +++ b/core/lsu/LSU.hpp @@ -147,8 +147,9 @@ namespace olympia const uint32_t replay_issue_delay_; // Store Buffer - sparta::Buffer store_buffer_; - const uint32_t store_buffer_size_; + std::deque store_buffer_; + // sparta::Buffer store_buffer_; + // const uint32_t store_buffer_size_; sparta::PriorityQueue ready_queue_; // MMU unit From d6c92b56da63d0c982aadd4c2fe726535cadc794 Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Sun, 1 Dec 2024 23:37:25 -0800 Subject: [PATCH 19/36] added debug info --- core/lsu/LSU.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/lsu/LSU.cpp b/core/lsu/LSU.cpp index 38000357..98c32046 100644 --- a/core/lsu/LSU.cpp +++ b/core/lsu/LSU.cpp @@ -935,7 +935,9 @@ namespace olympia void LSU::allocateInstToStoreBuffer_(const InstPtr & inst_ptr) { + std::cout << "Creating store info for UID: " << inst_ptr->getUniqueID() << "\n"; const auto & store_info_ptr = createLoadStoreInst_(inst_ptr); + std::cout << "Store info created: " << (store_info_ptr != nullptr) << "\n"; sparta_assert(store_buffer_.size() < ldst_inst_queue_size_, "Appending store buffer causes overflows!"); From b4b60d315aa63f805a514f0ac7665cdb756b50ca Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Sun, 1 Dec 2024 23:48:51 -0800 Subject: [PATCH 20/36] different debug info --- core/lsu/LSU.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/lsu/LSU.cpp b/core/lsu/LSU.cpp index 98c32046..7cf3633f 100644 --- a/core/lsu/LSU.cpp +++ b/core/lsu/LSU.cpp @@ -277,6 +277,8 @@ namespace olympia if (inst_ptr->isStoreInst()) { + std::cout << "RETIRE: Buffer size before:" << store_buffer_.size() + << " UID:" << inst_ptr->getUniqueID() << "\n"; auto oldest_store = getOldestStore_(); sparta_assert(oldest_store && oldest_store->getInstPtr()->getUniqueID() == inst_ptr->getUniqueID(), "Attempting to retire store out of order! Expected: " @@ -935,9 +937,7 @@ namespace olympia void LSU::allocateInstToStoreBuffer_(const InstPtr & inst_ptr) { - std::cout << "Creating store info for UID: " << inst_ptr->getUniqueID() << "\n"; const auto & store_info_ptr = createLoadStoreInst_(inst_ptr); - std::cout << "Store info created: " << (store_info_ptr != nullptr) << "\n"; sparta_assert(store_buffer_.size() < ldst_inst_queue_size_, "Appending store buffer causes overflows!"); @@ -1441,6 +1441,7 @@ namespace olympia void LSU::flushStoreBuffer_(const FlushCriteria & criteria) { + std::cout << "FLUSH: Store buffer size before:" << store_buffer_.size() << "\n"; auto sb_iter = store_buffer_.begin(); while(sb_iter != store_buffer_.end()) { auto inst_ptr = (*sb_iter)->getInstPtr(); @@ -1453,6 +1454,7 @@ namespace olympia ++sb_iter; } } + std::cout << "FLUSH: Store buffer size after:" << store_buffer_.size() << "\n"; } } // namespace olympia From 575256d1312b7268cb7aee69276175c4216d4d06 Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Sun, 1 Dec 2024 23:57:12 -0800 Subject: [PATCH 21/36] store_buffer_initialization --- core/lsu/LSU.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/lsu/LSU.cpp b/core/lsu/LSU.cpp index 7cf3633f..6b489247 100644 --- a/core/lsu/LSU.cpp +++ b/core/lsu/LSU.cpp @@ -23,6 +23,7 @@ namespace olympia // store_buffer_("store_buffer", p->ldst_inst_queue_size, getClock()), // Add this line // store_buffer_size_(p->ldst_inst_queue_size), ready_queue_(), + store_buffer_(), load_store_info_allocator_(sparta::notNull(OlympiaAllocators::getOlympiaAllocators(node)) ->load_store_info_allocator), memory_access_allocator_(sparta::notNull(OlympiaAllocators::getOlympiaAllocators(node)) @@ -275,6 +276,10 @@ namespace olympia sparta_assert(inst_ptr->getStatus() == Inst::Status::RETIRED, "Get ROB Ack, but the store inst hasn't retired yet!"); + if(inst_ptr->getStatus() != Inst::Status::RETIRED) { + return; + } + if (inst_ptr->isStoreInst()) { std::cout << "RETIRE: Buffer size before:" << store_buffer_.size() From 67bb0ccaf04f8d6c8cee17939f5c956e6a4ccf35 Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Sun, 1 Dec 2024 23:59:15 -0800 Subject: [PATCH 22/36] initalization sequence change --- core/lsu/LSU.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lsu/LSU.cpp b/core/lsu/LSU.cpp index 6b489247..b23a3de0 100644 --- a/core/lsu/LSU.cpp +++ b/core/lsu/LSU.cpp @@ -22,8 +22,8 @@ namespace olympia replay_issue_delay_(p->replay_issue_delay), // store_buffer_("store_buffer", p->ldst_inst_queue_size, getClock()), // Add this line // store_buffer_size_(p->ldst_inst_queue_size), - ready_queue_(), store_buffer_(), + ready_queue_(), load_store_info_allocator_(sparta::notNull(OlympiaAllocators::getOlympiaAllocators(node)) ->load_store_info_allocator), memory_access_allocator_(sparta::notNull(OlympiaAllocators::getOlympiaAllocators(node)) From b3980ecac8e20fc59c880c173d8a53e33f694643 Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Mon, 2 Dec 2024 00:05:22 -0800 Subject: [PATCH 23/36] comment cout --- core/lsu/LSU.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/lsu/LSU.cpp b/core/lsu/LSU.cpp index b23a3de0..dfb4453b 100644 --- a/core/lsu/LSU.cpp +++ b/core/lsu/LSU.cpp @@ -282,8 +282,8 @@ namespace olympia if (inst_ptr->isStoreInst()) { - std::cout << "RETIRE: Buffer size before:" << store_buffer_.size() - << " UID:" << inst_ptr->getUniqueID() << "\n"; + // std::cout << "RETIRE: Buffer size before:" << store_buffer_.size() + // << " UID:" << inst_ptr->getUniqueID() << "\n"; auto oldest_store = getOldestStore_(); sparta_assert(oldest_store && oldest_store->getInstPtr()->getUniqueID() == inst_ptr->getUniqueID(), "Attempting to retire store out of order! Expected: " @@ -1446,7 +1446,7 @@ namespace olympia void LSU::flushStoreBuffer_(const FlushCriteria & criteria) { - std::cout << "FLUSH: Store buffer size before:" << store_buffer_.size() << "\n"; + // std::cout << "FLUSH: Store buffer size before:" << store_buffer_.size() << "\n"; auto sb_iter = store_buffer_.begin(); while(sb_iter != store_buffer_.end()) { auto inst_ptr = (*sb_iter)->getInstPtr(); @@ -1459,7 +1459,7 @@ namespace olympia ++sb_iter; } } - std::cout << "FLUSH: Store buffer size after:" << store_buffer_.size() << "\n"; + // std::cout << "FLUSH: Store buffer size after:" << store_buffer_.size() << "\n"; } } // namespace olympia From 020b493260188ed7c0a8d2e1da7c576cbf0d22b7 Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Mon, 2 Dec 2024 00:17:43 -0800 Subject: [PATCH 24/36] cout debugging --- core/lsu/LSU.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/lsu/LSU.cpp b/core/lsu/LSU.cpp index dfb4453b..0a94879d 100644 --- a/core/lsu/LSU.cpp +++ b/core/lsu/LSU.cpp @@ -185,6 +185,8 @@ namespace olympia // allocate to Store buffer if (inst_ptr->isStoreInst()) { + std::cout << "Dispatch: Inst type: " << (inst_ptr->isStoreInst() ? "Store" : "Load") + << " Buffer size: " << store_buffer_.size() << "\n"; allocateInstToStoreBuffer_(inst_ptr); } From 8034e14d7c08d6a40b038270cfc6775d9ec3846c Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Mon, 2 Dec 2024 00:25:16 -0800 Subject: [PATCH 25/36] roll back ti using buffer instead of deque --- core/lsu/LSU.cpp | 22 +++++----------------- core/lsu/LSU.hpp | 5 ++--- 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/core/lsu/LSU.cpp b/core/lsu/LSU.cpp index 0a94879d..8b115a43 100644 --- a/core/lsu/LSU.cpp +++ b/core/lsu/LSU.cpp @@ -20,9 +20,8 @@ namespace olympia replay_buffer_("replay_buffer", p->replay_buffer_size, getClock()), replay_buffer_size_(p->replay_buffer_size), replay_issue_delay_(p->replay_issue_delay), - // store_buffer_("store_buffer", p->ldst_inst_queue_size, getClock()), // Add this line - // store_buffer_size_(p->ldst_inst_queue_size), - store_buffer_(), + store_buffer_("store_buffer", p->ldst_inst_queue_size, getClock()), // Add this line + store_buffer_size_(p->ldst_inst_queue_size), ready_queue_(), load_store_info_allocator_(sparta::notNull(OlympiaAllocators::getOlympiaAllocators(node)) ->load_store_info_allocator), @@ -52,7 +51,7 @@ namespace olympia ldst_pipeline_.enableCollection(node); ldst_inst_queue_.enableCollection(node); replay_buffer_.enableCollection(node); - // store_buffer_.enableCollection(node); + store_buffer_.enableCollection(node); // Startup handler for sending initial credits sparta::StartupEvent(node, CREATE_SPARTA_HANDLER(LSU, sendInitialCredits_)); @@ -185,8 +184,6 @@ namespace olympia // allocate to Store buffer if (inst_ptr->isStoreInst()) { - std::cout << "Dispatch: Inst type: " << (inst_ptr->isStoreInst() ? "Store" : "Load") - << " Buffer size: " << store_buffer_.size() << "\n"; allocateInstToStoreBuffer_(inst_ptr); } @@ -278,14 +275,8 @@ namespace olympia sparta_assert(inst_ptr->getStatus() == Inst::Status::RETIRED, "Get ROB Ack, but the store inst hasn't retired yet!"); - if(inst_ptr->getStatus() != Inst::Status::RETIRED) { - return; - } - if (inst_ptr->isStoreInst()) { - // std::cout << "RETIRE: Buffer size before:" << store_buffer_.size() - // << " UID:" << inst_ptr->getUniqueID() << "\n"; auto oldest_store = getOldestStore_(); sparta_assert(oldest_store && oldest_store->getInstPtr()->getUniqueID() == inst_ptr->getUniqueID(), "Attempting to retire store out of order! Expected: " @@ -293,8 +284,7 @@ namespace olympia << " Got: " << inst_ptr->getUniqueID()); // Remove from store buffer -> don't actually need to send cache request - sparta_assert(store_buffer_.size() > 0, "Store buffer empty on retiring store"); - store_buffer_.pop_front(); + store_buffer_.erase(store_buffer_.begin());; ++stores_retired_; } @@ -969,7 +959,7 @@ namespace olympia if(store_buffer_.empty()) { return nullptr; } - return store_buffer_.front(); + return store_buffer_.read(0); } bool LSU::allOlderStoresIssued_(const InstPtr & inst_ptr) @@ -1448,7 +1438,6 @@ namespace olympia void LSU::flushStoreBuffer_(const FlushCriteria & criteria) { - // std::cout << "FLUSH: Store buffer size before:" << store_buffer_.size() << "\n"; auto sb_iter = store_buffer_.begin(); while(sb_iter != store_buffer_.end()) { auto inst_ptr = (*sb_iter)->getInstPtr(); @@ -1461,7 +1450,6 @@ namespace olympia ++sb_iter; } } - // std::cout << "FLUSH: Store buffer size after:" << store_buffer_.size() << "\n"; } } // namespace olympia diff --git a/core/lsu/LSU.hpp b/core/lsu/LSU.hpp index c98da9e4..ccb0b2ad 100644 --- a/core/lsu/LSU.hpp +++ b/core/lsu/LSU.hpp @@ -147,9 +147,8 @@ namespace olympia const uint32_t replay_issue_delay_; // Store Buffer - std::deque store_buffer_; - // sparta::Buffer store_buffer_; - // const uint32_t store_buffer_size_; + sparta::Buffer store_buffer_; + const uint32_t store_buffer_size_; sparta::PriorityQueue ready_queue_; // MMU unit From e94905748d733cd66d7625198cf5c1ee44148938 Mon Sep 17 00:00:00 2001 From: maynemei Date: Fri, 6 Dec 2024 02:46:27 -0500 Subject: [PATCH 26/36] modified test, cycle matched expectation --- test/core/lsu/Lsu_test.cpp | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/test/core/lsu/Lsu_test.cpp b/test/core/lsu/Lsu_test.cpp index 913a8cfb..58652a7f 100644 --- a/test/core/lsu/Lsu_test.cpp +++ b/test/core/lsu/Lsu_test.cpp @@ -52,9 +52,6 @@ class olympia::LSUTester EXPECT_EQUAL(lsu.complete_stage_, 6); } - void test_store_size(olympia::LSU &lsu, int size) { - EXPECT_EQUAL(lsu.store_buffer_.size(), size); - } }; const char USAGE[] = @@ -114,14 +111,13 @@ void runTest(int argc, char **argv) if(my_lsu->allow_data_forwarding_ex()) { // Data forwarding enabled case - + std::cout << "allow data forwarding " << "\n";; // First store cls.runSimulator(&sim, 7); - lsupipe_tester.test_store_size(*my_lsu, 1); // First load - should get data from store forwarding auto start_cycle = my_lsu->getClock()->currentCycle(); - cls.runSimulator(&sim, 3); + cls.runSimulator(&sim, 3); EXPECT_EQUAL(my_lsu->getClock()->currentCycle() - start_cycle, 3); // Fast path // Second load - no matching store, goes to cache @@ -131,20 +127,18 @@ void runTest(int argc, char **argv) // Second store and load cls.runSimulator(&sim, 47); - lsupipe_tester.test_store_size(*my_lsu, 2); - lsupipe_tester.test_replay_issue_abort(*my_lsu, 2); + lsupipe_tester.test_replay_issue_abort(*my_lsu, 0); } else { // Data forwarding disabled case - + // First store cls.runSimulator(&sim, 7); - lsupipe_tester.test_store_size(*my_lsu, 1); // First load - must go to cache auto start_cycle = my_lsu->getClock()->currentCycle(); cls.runSimulator(&sim, 7); // Takes longer, must access cache - EXPECT_EQUAL(my_lsu->getClock()->currentCycle() - start_cycle, 7); + EXPECT_EQUAL(my_lsu->getClock()->currentCycle() - start_cycle, 7); // Second load - also must go to cache start_cycle = my_lsu->getClock()->currentCycle(); @@ -153,7 +147,6 @@ void runTest(int argc, char **argv) // Second store and load cls.runSimulator(&sim, 47); - lsupipe_tester.test_store_size(*my_lsu, 2); lsupipe_tester.test_replay_issue_abort(*my_lsu, 2); } From b01d7e20bea874dc6062bc25bdc22aba1abd942f Mon Sep 17 00:00:00 2001 From: maynemei Date: Fri, 13 Dec 2024 13:57:46 -0500 Subject: [PATCH 27/36] added documentation --- docs/design_document_template/LSU.adoc | 213 ++++++++++++++++++++ docs/design_document_template/media/LSU.png | Bin 0 -> 136233 bytes 2 files changed, 213 insertions(+) create mode 100644 docs/design_document_template/LSU.adoc create mode 100644 docs/design_document_template/media/LSU.png diff --git a/docs/design_document_template/LSU.adoc b/docs/design_document_template/LSU.adoc new file mode 100644 index 00000000..1de0db21 --- /dev/null +++ b/docs/design_document_template/LSU.adoc @@ -0,0 +1,213 @@ +:doctitle: Olympia Load Store Unit (LSU) Design Document + +:toc: + +[[Document_Information]] +== Document Information + +=== Revision History + +[width="100%",cols="11%,11%,16%,62%",options="header",] +|=== +|*Revision* |*Date* |*Author* |*Summary of Changes* +|0.1 | 2024.12.13 | Team | Initial LSU design document with multi-pipeline and data forwarding +|=== + +=== Conventions and Terminology + +[width="100%",cols="17%,83%",options="header",] +|=== +|Label |Description +|LSU |Load Store Unit - Handles all memory operations +|MMU |Memory Management Unit - Handles virtual to physical address translation +|ROB |ReOrder Buffer - Ensures in-order commitment of instructions +|TLB |Translation Lookaside Buffer - Cache for virtual to physical address translations +|RAW |Read After Write hazard - Load depends on earlier store +|WAW |Write After Write hazard - Store depends on earlier store +|CSB |Committed Store Buffer - Holds retired stores waiting to write to memory +|=== + +=== Related Documents + +[width="100%",cols="25%,75%",options="header",] +|=== +|*Title* |*Description* +|The RISC-V Instruction Set Manual Volume I |Unprivileged Architecture Version 2024041 +|Olympia Core Architecture |Core architecture specification +|Core Memory Model |Memory subsystem specification +|=== + +=== Notes/Open Issues + +* Optimization of store buffer search for data forwarding +* Handling of cache bank conflicts with multiple pipelines +* Fine-tuning of pipeline stage lengths for optimal performance + +== OVERVIEW + +The Load Store Unit (LSU) implements the memory interface for the Olympia processor, managing all load and store operations. It features multiple parallel pipelines, data forwarding capabilities, and ensures memory consistency while maintaining high performance through careful hazard management and efficient queueing structures. + +=== Overview Block Diagram + +image::./media/LSU.png[LSU Block Diagram] + +Figure 1 - LSU Block Diagram + +== Functional Description + +=== Unit Block Description + +The LSU consists of several key functional blocks: + +1. *Instruction Queues* + - Load/Store Instruction Queue (ldst_inst_queue_) + - Store Buffer (store_buffer_) + - Ready Queue (ready_queue_) + +2. *Pipeline Units* + - Multiple parallel Load/Store pipelines + - Address Generation Units + - Data Forwarding Logic + +3. *Interface Controllers* + - MMU Interface + - Cache Interface + - ROB Interface + +=== Key Components Detail + +==== Load/Store Instruction Queue (ldst_inst_queue_) +* Size: Configurable through ldst_inst_queue_size_ parameter +* Purpose: Holds instructions from dispatch until ready for execution +* Implementation: sparta::Buffer template with LoadStoreInstInfoPtr +* Key Methods: +[source,cpp] +---- +void allocateInstToIssueQueue_(const InstPtr & inst_ptr) +void popIssueQueue_(const LoadStoreInstInfoPtr & inst_ptr) +---- + +==== Store Buffer +* Size: Matches ldst_inst_queue_size_ +* Purpose: + - Maintains program order for stores + - Enables store-to-load forwarding + - Tracks uncommitted stores +* Implementation: +[source,cpp] +---- +sparta::Buffer store_buffer_; +LoadStoreInstInfoPtr findYoungestMatchingStore_(const uint64_t addr) const; +---- + +==== Pipeline Stages + +[width="100%",cols="20%,15%,65%",options="header",] +|=== +|Stage |Cycles |Function +|Address Calculation |1 |Virtual address generation +|MMU Lookup |1-N |Address translation +|Cache Lookup |1-N |Cache access initiation +|Cache Read |1 |Data retrieval +|Complete |1 |Instruction completion +|=== + +=== Operation Flow + +1. *Instruction Receipt* + - Receives instructions from dispatch + - Allocates queue entries + - Begins tracking dependencies + +2. *Issue Stage* + - Checks operand readiness + - Verifies no hazards exist + - Selects ready instructions for execution + +3. *Execution* + - Address calculation + - MMU interaction + - Cache access + - Data forwarding when applicable + +4. *Completion* + - Updates architectural state + - Handles exceptions + - Signals ROB for retirement + +=== Data Forwarding Implementation + +The data forwarding logic is implemented through the store buffer and involves: + +1. *Store Buffer Search* +[source,cpp] +---- +LoadStoreInstInfoPtr findYoungestMatchingStore_(const uint64_t addr) const { + auto it = std::find_if(store_buffer_.rbegin(), store_buffer_.rend(), + [addr](const auto& store) { + return store->getInstPtr()->getTargetVAddr() == addr; + }); + return (it != store_buffer_.rend()) ? *it : nullptr; +} +---- + +2. *Forward Detection* +[source,cpp] +---- +void handleCacheLookupReq_() { + // ... + if (!inst_ptr->isStoreInst() && allow_data_forwarding_) { + const uint64_t load_addr = inst_ptr->getTargetVAddr(); + auto forwarding_store = findYoungestMatchingStore_(load_addr); + if (forwarding_store) { + mem_access_info_ptr->setDataReady(true); + mem_access_info_ptr->setCacheState(MemoryAccessInfo::CacheState::HIT); + return; + } + } + // ... +} +---- + +=== Multi-Pipeline Design + +The LSU implements multiple parallel pipelines through: + +1. *Pipeline Configuration* +[source,cpp] +---- +PARAMETER(uint32_t, num_pipelines, 2, "Number of load/store pipelines") +std::vector ldst_pipelines_; +---- + +2. *Pipeline Management* +- Round-robin allocation +- Independent progress tracking +- Shared resource arbitration + +== Test Bench Description + +=== Basic Functionality Tests +* Load/Store instruction handling +* Address translation +* Data forwarding correctness +* Pipeline utilization + +=== Corner Cases +* Pipeline stalls +* Exception handling +* Flush scenarios +* Resource conflicts + +== Future Work + +1. Enhanced store buffer search algorithms +2. Advanced pipeline scheduling +3. Improved hazard detection +4. Extended performance counters + +== References + +[1] RISC-V Specification +[2] Olympia Core Architecture Document +[3] Memory Consistency Model Specification \ No newline at end of file diff --git a/docs/design_document_template/media/LSU.png b/docs/design_document_template/media/LSU.png new file mode 100644 index 0000000000000000000000000000000000000000..c5e66f2f24db2606e0d27057e770b307d4805891 GIT binary patch literal 136233 zcmeEP2RxPg|3@k%QAryOD(?o4q@toN zhW}2`O@Skx9sz;yKWgWV%5qd`<%`>>sP6f?DC)V`yIWY>5vVwXf;!RD+Rn~|1Fs|^AP7Hk>smXQ6L&bn zM~zMJ2PX)Jh2-E9{2(e!{^%|$%mYVc9Ubimx&%{IYq+nHh>)0okSHAHR$aGAU5x`T z51;L6zLl7A8*C6t_Ue=tgjIwkA4|r-6sT z2?&t~?`&pbM<5@@k&l=Xoy-YN#AtX#5 zleq_aL`Vo`gG|WM340LwT;zbZwI?EGkP)kN*5y0&c)jC(}QM22M2-~ zx{K%?O`M#Nts~74J!C<&L#KxxaDqR#=s z`TN(7t%lPWEBr6F+bEgbl&P z#RKJD6IU1F*Y*i{5S$JJ_zJb5hn$J7PNdC-v%Wq7boE_KoGbyqNw@p@iDJl1|9sWH zy6;W|I};acw@-)q>+GNyx>6KOq+{*m;%b6Gfue(plQqC4i9jj13R!wJ6H@@^Px!~g z&e{?YDQ3X*h{TWsDdS=d;3tdzU~g@1j&P=&6T#Wq)5Mg-r3hm>60IFvNK`E%$04Es zN661aZtxi|qR5P145y(xNXesL-$4?se#Z`?oGU1R!-NgWyZpd*22@=L|>d!|g1WLz03n1d?96lqi9}FM_ zcWW0tAXGsCya>=5@>L(cA~FE^tbnXILYAa2$Z{iJ$b0?oqK^3YN}C@-lCn-tCWt7* z2oB}iZ^T71!C_=)@BSHi3kjo)jFPt)_WfH)8M`x_Paf!>(=x^Bpl)K4^v|#P0rZcm z-4yUEOVYx$U5HMIZVyoU-+^EBLKggD78Vx#;%34)eh~>V5q?1tae}arn3<_C z0q7gM0erGVo{f#n+}aJEVs1^gJF&CuT%0H``O4VsL65U`;I|`Kz};j89npKDGlN@V zI?DKcNaz0HKwLB-rtpaQfL_m$fK7Z>q!oV|s_Zm@<%zyV95irmH_BFu4fA<>EKe-0r|Bsf`#gqt+ zgU=`pA*rtr5@vbH;&DJke>g%HbGRCw=j>=gcBWuc0ap$7@vHYosRboeXQCZ&6asxh zf~0@oGxFRqs&XRx1Z4m6L{p-R3(+2k?=R1EL9DGmISo_Z`B9dUC0!urXku<|?O+L) ziXaOk`1xNUIU-zXVTbr8WUWBqC&0pt9pfik-Lj@m1QT2S9pG9bGyNEYe##uc72g2@ zcGeCA{;>_j34q-u2WEuJ*i7KAV>Swk806I;0|t6%Y$?BlkxxfS+WfB(5eZ4b(m}ug zN8V4;Aqx|GYda8Nva(Lrh@mC#Y~tX|?@VyAwt%4mIS~Q(z{Q3U1CW7PB0@nfxU{j1~Fj!f)Gqi zoC(Mp!y`x@6@C~7Vr9!Zg39uhW9X)T1q3lIXM-yNTsWvy0uCmx=%=d;hra+xIEHZG z7blYB#3usfuh<;fV+xM_^o}Gh|58v+@bJIO!xOBEs$xHbRY843L=;(Z{9jpgf}ws1 zp*X=%Kart8k3y^&k|8$1PyY!ebAq3KB0srdVT%ctA_ZN>texN5Zku4KpU6_?CN7Y> zGQm$1{Dk^ZNy^HEyW@M^9cK23lVL*RLvjKp#1l#h6YTW8?BwifX7*3<6XHok%&`fl z;aJwr&*V_kFZSZAuHFeh(*NK`npoo>AHbSe}|9}3$EtQbot`Yg9>EYImbkuo-c$G;37e^paK4t^~U6FE-K zwf)ow1IJ0pz@Hxd>Mom(`)}}|BaA0(;gN(HEHh2}59|M&U7U!v1bK3Poq{>R!o(Hz z)gre;0OYe;BrN+*0cu0Ow^9SG?!k*e@+a~bG_PGm9Q{6?0gM;IQn|6*M$&V!gIJFG zFHjYT)e?`_VIpNS+CR1`aJ;DJ=hXCkF9H1E(uJ|=F`>^0>~p#ymq4ObIS3^J=Y3Pb zBUZ8TL(7B21VujK4`B&(I)87J*jUF)JN{^WzViX35~1Ix{s_sWCxP2|4YfSE@MyCG z(szIytfB;9MI{b7c@A-L1;`Xe62YPGz*t0!@;uUQDIge%%c01AEI;^c}zN>4+r?6 zRRxX&+sG;7^_Qe?$R3fZF_CXtkR%T!^`wRr|8!L*$_sy83r-@I?3&>|vVu*dw46o(mC|E`QAuiHP< zF#zfbkxlq(+4LXnQzDKM<-faw<2L0_Y83IU+y4KkQG^7hf}{9^gDz@h=J z|M-C4%oG&=QYx=^c6Np4S$IJTnf#yW2Si4cZxNJ8H>|M^A(YK1Bt#|1l!s9Y`P&ax z1%GEjEQ$geh3bd-55Rj%VZ&m@^Z!gU3cR_wn2ETFFu56ph?t2PKf&A_&o4p1i3^ID z5=`*If6`WAq8SC$oWt&nJ4dkHnP^5KB=={{DE?^O!~r+3HH1%}B%S`zHHZ%TcdtR% z?T@YjX$g<|YVVP+25A1Og8n-jWqh&{UjcxLBKv>0$evVNj#Xm<(@!+Y_!SyupbFST zp9~0Yk2S^lZzV1`0<-=7hF4$fbu&>j`4ejHmA1U7{(x!(}W>mn`Q$4LU$D0I>6mV+iwndT@fBCfMo!J3BeM7}=XR+fMM* zgbDQ-I!>5S-)lkvPr2-v5K`z1(|%^_r3oAAzhOhU7+J%MQ~x2g5A?%{_GrJ3-|(yD z&lAe}_Va0@1139M?R?VNkdh@@PWjulK6A1tR2;nCz#|g{v56S&k2wzR$@dO_KA%RX< z=M!&3hva`Oi1UqD@l~q^IrzV~U4^^+&P`Uuu%4G_zXlQP`*>wLe(cyMZU3-ttXS^_ z%3e!9saYG^{fVNJg1Y)wBv34(?><4z@ud^Rwq+W8AEiCZ=M$=oQ86UZ&qL`2{?CMxc zMI3Zg#J>NmhKeYO{&mwTQdf}gXvOkFI<5TBKDIcV7}{nM#UODES^!)BR=Zs||M%%L zEB0k;Ni8c6XXHhI;DVARy$T!YZ$^n+(8Tn&>nKS^f$yMXwDle|LKVh~pnQW;2nsAe zgx1IVAfpfChdwySMqt4DTXy^VIsM1Sp%^bZ*2`8OBrZTP{;J!y_{%nLzdI8Dk4;yF#U%tJBqT(I#6?BKM8$ERwm%gY`{t&rKgSRd`_d4Q zH!-tYzzK?d%8fW2ruhGk#sDfsKT~J4fAAYH#wC!jm;@$q5D5e%EGCW} z_|>feGC}`H5%hByRUBp1f0-!Utz8fUkz@?QXEHfpuSglE1^7t;5rqG6Q+;w{+HYuF z{Ns21TPO;l3|ImAAF6l=BTqrV8Ch@mgu`KesQvuE+$;|#OB=<+%Wd%jCR|{ZdLSC64qG*rXftVo*n?OOL{Ai=w-`On0 z-WWtde_y4wA1tNG&-_Qa!hbWkBK=0e1Xbh^ltu9EPyYRk8o!y?kX!zh|LUKAV~GSx zrT?x-{TVwb@x4aQFTR5q2~#koHo?RR>=&mmnA3^q>R=B2$jM5|FA%>Ndubz6OelfA@!((H`JZd`fBUYJd?G&*dcWn~WPKP>ebLvS z3Zhn!h#1K=4Blb#Kd1$ZiF>?+umJKRE>Up_Q4tXwmI8pm4q5Pg#~W2qMS1L9xvJfc>UM~M;J&lmkT z!Jlf$2?{`h1myRkAzUn+J1+TuXaM(Hkm56-`oSoKS!kG(Yn;wd-4dgm@3z7)C?K%y>H>0v1mq;);NnEICCC%)h)!^p1JMDI zDi+pucAp*kH-K6gGqT1AP!P3oC{P+BQc94<$Oz-JUjdJ4F@jEmkRAmtqzzO0%O)^I9xF4)Cp|r@4y)PX@?%&an!{NU4QLAZhCQD3; zSj)J#6A`?B;TokB!5nxYa}$ENg&ADt{E57VUqsEs38ohC%fBI^ad^yKif-Uo{vd^f z{%KhTB!qGz55qd%Y*z&?ORgq-T| z@`5H66+6{>1=&sR+ulB-zqEPt)zQ)P#oF>$dUQQb(`GU=+9aH=RAfqGo)oNbZN+Ii zeg{@Y&OUe5dsKmDrNz7)LmjjM$7If|U6_@+g8H7YhRwmpvu2i*(66(3JtE!EMoejV zdLrl1Dd%(?(d^2juI~A7^KQ*=cogoOo+EWgqQ>DK{BwO0H7%1275up)Bh%m}IXcwU z8CE!?C~9B3!gu(M5#eqqf0pYTohKLVBd!d*PR{Zkbo1!CC)S*rTYr2b zip1ND8@n1`-ytB$jSe?2nmG5TWX63!j=6fa5@%l8`K}qsLkXA#B zS`Jq}2_0a@vW`;T^gHQ3J-pU7Hph)d2Oh}^#>;N1h+ezf_;t8h(6n9W8NFNObq10j z-jN>duVDA{8fZ)0E#oje&?768=>PPU^r%N$b>fjor&1oL`?6ExrC3&zooj?_< zhkLbm!%}4K>{`?Pxng#1)Jf4>$_UHIBx8O5K`zlqqV4R7v%2Zq_9mM=%XjT;jXP5M z`eE{JD$U*b)f#Kmr_5Z;eS67{tO!Zj`x^?)5HD`Q6oz z^^KuQ48a=G!!B(Ng}!aCz5AZ;Zyp(Z&$w&t)y-Wg+2zNxE2P%Y-E}8z2`ep+RC?tA39Hy$sF8R_QAYrk066tw0YorBO~ z2`BRmd|A)_w8aJ)#JmVx&3Bd`ak*;uS{@-^WPVp>vom+u|}XLVvz zUMpfZi)3bJc6QprGo5_1i_==gghv{uO4Ai2ZLd1JUY72xmwo@p{j1f&E3UL9MC>9y z7%JaUFlQ)iDtUEaiDWz`(-oJl*>Pwwd4RSwi^Lp)-+RCJy4JocY^&43t?tkF*3`PC z#x5dW*^pK^GSKK&b$3_TEP~ z?b3??_9hd8-f*^}ZjkkX`4^K%`GP1&>o}0sF^k=q1*+A*^uYengH z=zR!YY^3|xI=t_Jwr0tp1wu{T)tZ|1bQR;bxTEN&#W&AwHF;&&rKt^@T`CplP0m%Z zIocJHzLTTyns`-Lg->rbyYT~qmBdTWr|8s4;M0Ts(cwGLGX>&F7fKl-g(=&PjG2Y;o|Z?DP&oCSgf8K z!B-&5HHjPU^N$nNCO@r1&NSqOnL630|E^K#(WUZDH3mcmoq!K_bY*k{3>~jc&T?tn z*k2V^5d7Y_b;w}QB@khy-=+&Dc+cd1X%}Cx8@z6 zYc@Gz(LRibM_}UpVy`wGLMGmL&hOpQnnGV+{R>vJcyCUZyq#}q*Vpm9v*QM1_H8}? zR|jLsc;M$ZNVCWET?K9~Y02H$Gq%4}YOBjlSBnr;FJJRsl5wTLj)Oa%xrS}^Y>UtG z?6GO9Nj1m|H0dUD4lv<*U_zDE+h;5yQ%*+FE2)op#%9BTs|x0aAEz7Y*e^Xpw0`s+ z(MS9EEFrbAiSollb6ZZ>kR~e0+;>Z?U@5R&SXtDXM>k&IHSD~!{bds(2H8gYZZ`|; zsa#GX3i!hT(82rsEY6&KuR}#C`q@pP}u`g~_50 zPr3KA`CJw%)r+6^d;ET;;2PR`^a3d z*}pPQd1`FQW?jR*N6lz-jl6TLT0LGx?O3g|pm~gSib0m~)<+61Bj0Ar-1`Qc`xzE- zt}5s|bHVn}OrG1c^}eIt65aRZ2_Ie*UrP<9km#QZ!VCe&8hy+4g+9aP*Y`}TsY|$K zN>>)W)<7q93&+QLFCuQrO>Q!Q!=#mgfEGvTmC7MY=BR1hUYmn=Y1Mji!TN+(@rDib z{%MAnOOJD?Htm_CAwz*!9DrEC#@1wML|sQ?cI(iw-Gl7rQt5 zgK}2WSV%F^w}7@!#Nrjni;Ax&S0r0je zNt?@>|3a2XpORc=$NURXXZD+#-3|&d3zAUcR6k`Nc*JdN^Jz!Ysbc3R(vW95dER_U z_l!gHadH6Jk=GAj$nmr3?`+09x2Wl7nkouf7)IrI_PlN!jo57d=23dsQu%$fzPWV{ zmnX-bO50bxdHb3j?}7yk^Ed6fXcfMh7$4s2TRc~nR@3WsQsP~jF z-l%pv-sH@_^&r0JcVz%!Pm$F-=@2ti*D%7VymBg)|gv?!UquS0Xn9D?J8s@ z)iU;%9(v9wd*G}WVeW@n+h(!MOe@p7;pJ}KK^RHN<@b3`Z$zgPz;Bo@{J5HTVMd!u z7S8O}ozf8A-F5hT%N@8`X1*!&F(c3yUua=n>q&d>0+_b~PH%z0ZG>CKqZzjMvMdQq}ddz8@E>C*F-uzI=VJ&gC4O1Gj}k!-I^p zve?G}EAw`~X}xo7IlaeBLGCfiPXPiZL`mE_Orrd~MZrAU3dEQz-Bqge`b1mmkhIQx zn0qsb*$|T(`zAN^>e%VlQXx3Up)FUlW>9Ax!>#V#ja`BbHc7rga{^8> z^gT*5Tx2(7bdwnKU<<47(1*smGrB*)84w&#vd?mmhmbB}2uR8CIeYLR! z?KaOg4#Z~JMeJxA*Px6w0otND_{rkk*wDDa`(sF$k&k}w6H7Mog%^np0&K&r%FT!l zblYzM2Zl0QvjI_$t1;hDK#;gY+xX+N9bv%S1|!WzBgL9j=R;v#G<%;qYkI&hl@v?Q z4NE`kL;||>{29QJ)9QR{b@+|EBBwS-2q}@Y5)j?R^FYRoEXK)^rgdbp9#6}MrqPBf zeV$tqn0TiA8qtOXh?`mHTa5e&_r)yvG=(~WaSH-$R`DAk{9KdQ-S?IXr) zE!df-7A`1TlcK*oLW~$HxIvbw~v=DS2}c*H|^506WbpxWN(l<=|13&f)gD(lPajEEOVxk7x(CqBG4eX zvn?xFhKaM-5InL?zpBF`u$6nsY8R&AE>IiI+p6M21lP-FKfkfxa`vH_J8)AWTGfdF$|;p8J0G)d}jd1{G3+c1)rCTP~~{=y@|w(unPO9hV4mwOQJg&eHQ0 zV!U|}MUTt0I?G>QG^3x4G*i8ZHpRep>dX_tg?TYBPJr@_*XKXRv zw2Jxi%b+!9eMd*G@yF)pm!&MMW-m)=ty+7{#Wmc+Dk8t??vVE-#%DLm_Skz~K0B@9 zOsrmCXt^V}A)==cIlJ+)k^RoI9AYKG;6kK&$g$ z)fBs*b^3LD_I^EEfZ+^qzQV6;+o0x8OONV2Do~oCfjb3OzDu7 zA;#={m-nivaW|vDro8oXxa`F6eMX%31AG<{Tol83YHq1y*2ra^NoO8JrILc<;0i{R z>wxJm1CqO~wPqks=nYN&!Jz}QmN!K@muJ?yEoT6EAXuKOxlinw*rL`qD(ALc^_=YG zdD4J4F8S{HcAkqMhD)7o+Ieb#6ugel_0Y_`{JvmKpM~Zyo`!W?*Kh-^@U3UnLh>|f zCOhP}ae9E)rx_dzxyW-~`xq~*G2C7}*Hp)QZy`t3h17=WPaZtjB*6(%BHvR@?EWKV z+XnoCS9M%H{b7b~ClMc)6DaAW874P&sSn0^h3p0J^}bVb#?0iV=lyK6qq!)SPW7Qa;Taml_YaPy(M7zqJ)P+P0@0;z_H@vWt)hP|tYYL+^1Xv)8na`DQ?|N3+Gp$<& zpsNsQMmBcttT(Q^XkV3Srd&D7>R3cAad?+Q_H6f_hq}#0lNf_j4RWW9Hd9C)Bak`; zJp5$DJ{sWXv-`y?<;5GG1WSczSY2SE@l<^co+&O+R(Wr_&!@#`v+)d7Y zG^VE*Ue?(Mno@+Qy;GJ{z<@7};bpfF4%N^*)3_q3=&{TEWK27u&#bb#!FVsWo;#A|jfFV)_p?%-=Kyb!YitrSO9H$abZ%fpK?hC1Qag zdc2*Ox|am~R_~sNSzt)#0ZBNsPJ1?HfxbgrHPW{?9Uv(|;ePu#Db2byI#Gqql3=+y zJ=maJ(p4eNwkMjx@~D~%zQ#vkHRK&yr`R~~CVk%35GN2ZNf$KZp3u)&fM0Qm&Gw#HX~R}Z z<|3@r6VK{JC=1d?yT3SSOsG7eDa8ZGlu;wVK;VUynG6oLh7z^M_b_hRMz)!5EzNdZ`O-dnI-iDW;Y z-diMA@PXk8$W`-qPxI~ss~Lm;sS}jMk|VxDFOMINd`>1gK+|SKwHRHzk(zw3&b$h5 z4VSoH#0_y6bwdy^p`F`J$`7kK*VDb4e;B%NoIG02@N@CKh0wcnD_FLlI} zzBOSWY@26N%C@Upz;CGY_8NXQ#>GV&KvXMpg=Uc;VQ7wCw#DJ@_YFoY{99Ilx2Kcw z^oqHwj0(2yV`LLxlpe|eXVc{saTej&3j28>w0MtaE{PW1`8w_zjclAUV?KLfU-7CY zaQJlToe7VZ?kJjs_=)s&(zfy#4OE2%NUT%ZOVSSDk2~;fONoLuCo*k6X_|!x7+59s zd`7xrvQ1NDs?&IHmP>Z2>0h+dXFBv@TxGkX3#yEfuP52iV4q8Eh)Y=XzQOZND&F+w zflULaEBk2Z=LrPU3tVDTeCmd^6&{WXEYrSH`X>0)i3F>?Dv zz~$tc9ceQbiKbMxxcl(Z1kPI(3Z_HJgCyDki-g zn-D|^9q4W6l^&=(Vt!<8@4C1}tLk=w!=>p=pulluRu2UPp1=s~s`5XBk)nSVn0g_E zg!x?L;f2!HgI&sg#iP-Gs`;%$3p&9WT)gq_h-eoG3pMa^EK7`XJN4A@21C~1yUT-t zvd!42WYz;wtKc`xAbWGcH}bK;*O#ejvDe#7r}msjS&#G<`MJK&06kYO+Vgc1yRqbO zcXi$U>6#=iAW_L3(C7FVT?fc8a#{um%lqTy28QcV;6abvtbuD$(yaqmdjtJ-1+#fC ztrwF$t_WFbu>%v+s85k6-;I40wc&@eOOMoV^eEv7LmaW;kF=&YyhqHBM@GKhj>%{D z9oQLR`8@Fu^%ysA2JtVB!>p}k8E)-cDv3!CMm0_euLsh&Q;M5a8Df%caqoD9L;iXV z@#lLQ4LEuz4X9FH@S@e{nKhAD^`~@|- zT?0k0w8(!_l%-y#>0Um)4E9S=-4P8#36Z`d{Z38Clvf*xO&*({6ymNGu6qzb;$xXb zlPO|7S}r{g5Q&5Yi8Or&s_pGE-8Bax9KnUyrF;r)Q>?=Az)t=&iyx!uR$rGN$$Yr3?g`Ch> zy&haMo5hpCV^$J(L~3QSZhENE@M~S!zWCg|HW@pfZofU(;mQm;vfc>n!Vda)X1AR+ zNib1Sr_*lpXbxewvO!~jouH%8Hr#)lZK)~I``BzZ)p>|3cE&P#bGwH50=_}Kav%FS z#I<7%oODghh82*a6FlwP2D;nk(7I1u;-dB1EEbD0g8$ZZI#a-0=WX-nlD9B^9-`3f zW`7N49Nn%Y%V-Zyy7R)}1AVup1@^Ogo?Gm6XDQ;#LISObFVLxodg`5m!FRlfPVqM9 z(Y9;+hcCBZj1U$Ro0@A=&{Jd3*%Co4g<#Xt+7a+6n!l+@%>o}>{~M#xm53L02vB?) zjRZqu5Sa4pN7sz>z1;1mQTJ+Gk@E9{A#P_yp-#qvW}(AcaPZ1H#xLr%JsR&HI>lh2 zqxJn9SaJ5gDk`fBZev6#W;TqS~|{vaJVHdR(u9}Lgy@Y=DW9}wqb}${RmKG zYmBN-|1m5Vzyu9vp8muWi%gsu0zWCI=iJCZ$J;oGnxS|E$f@+*<@YnEQB11_rj@dx ziU~b*7NUot`&QbI@iyYIn>oJ3bj8!xuAMOxBl}N1Z0wEuK}%xmQVYR=My3WPr5{+$ zhTeV!L|s*H7GJ}Dr=r2ZtW4#$&qq&aLfj=s)?6KXej1`>O5J#X1_VpsbQLN88DkbG z_c`#K$a3v7L3h1qD$T-xtHZr@SrGM*8OL!yKM~MxY=ec#t_Ug`YnWMltV+cAP{4+n zgnrYqg58qSWZXSwQ>Y*aT$^OpNIs(viL&07)E@IU9nC3nSXLG3n%F%?66mTS#G!6H zx_joa<(hrRhiogf&kp-cEpcd<+H`UDl?BAE!O`HT^ulGMIU~BAi8+PAkGA!{5y+Vu ztLH3~7cOQwc`tb}wD8nA0U>tR=L?Y6Drh>NevhSo*KK~MZm!iOv24Q!7GD~z&w9mr zdCeC4{=1%oQ9B5Aw#3rFRilQ{{aq|=-kZ1<)V`k+a=m*kXRKcTtCQ2occZRrs7=B&&x@OlmnN%s$xuUaa)Z8n@E^ z-9g8PSHE4b^M*vM-ZZWgOUV{O;|ZVz&205l@`5pILOk4CZZ+FZzesnNuonzLvp8nE z51kA%?`qf<@;=f_GFESy#7q>Z{kH;HI+-MDqwK*n>nfr1{hIm^A1>CQ{mW^RS$(%j zRhjIN7!1Cl&(m@zaw8gEC}&e;*cC03f6 zoiHarLNq;?9IW(n1`01%=b22VKdtH1&s5*;iDL{o!g|Gs=hj{Jm-JVLavu?{tz~2i zDCT;(4xP~*c^Uakv4)-6wYkI?<&WJ5flLsBiJrbya&Ut~l#9!|9+pyrcMi*#on3r* z=*b*s3Cj|BQe`?>89BX)W)9I^fz!L^?8d_}6pH}b+MSU< z{qd;z5D6zZn=a%bhnoGX0Ng?xN@p{KYw^vRHx)G*WmthV?HdXVSG%+x{bb5W%>Tnr z9M&FFk8{z|>XLy6#|4Fwg@uou)1 z$^1l3bfY@}ip$y$axpu5`6Qu?#f5BRxKaRFtvT%i_m7XrQkcX^364K)?=CR~7~oUB-l&+9)@=<_AGhWU5lz9X7*ZrZUVQTC zZcNNz`uX+{`(4wDOGV8N%_EZ!<78oGFHlYhqr)FRW_YS#Jbs0C>&#-c)}DvyzN+Oh zj_YMo{2%#_G=;Lyuiih!0`?@zyE_nN3%Ubz5LrXbVY;TJjZWOXWo*$$Kg3lWm)KFy ztL)p^=O=b$_1kbWuh{|3+p8+E?POE)# z3qQ2`hSkTck+^hq#A=6)`tK-99rtK( zAvt0ILPn-vS{$(X^4g8NvjPSB&Odc+SbnW#aG{_5$Q#Z)`{#J;!W~m9L(E@AZO?k@ z_{R5L)>Qok6|?0s@#PE6$a_;H1hTS%Gt!C7@Z1Odcw-+|ydRO~SE!JDWqWNpZq6Zl zgdAS6jW`afuv5@OZAooQ=$};dNcAuH>`C5s`aScq8g?i$@IaXTcl`1wn8Kj0p}& zV69HAG42Cc8e9y2D0vU4u{lJy34DJpXIN1lr1Ez{3|6z~+SMv@nHNhxNMbmx2BxEs z&1%#p@jnW9S(iBJhYakW0R_K(rUtnVArK3Ae9)5{P5z-Rh1Jsu^00ZsJ*~KQMdi z5x{HS*;InJy3fu#Nw3i7Dr7Y%A+DgP)vo935?H0p?=b1-yI2!;y2dcS^jjcgaR-uE z%twa$`Gb(aG+3giiyK_qsuaOFUH#hq@FR#Iqy*uV7LZJKcLf{_Uw}x zfF!Q{NHC{IOHKaLv^ks)W}Lj9@3F_U)>oJ>S>&bl;mWA&Ln_G9AkM5_zQD(`WA$cP z?!}?XCFG-lSG8nxsA4pu(x;??-tf5#Sfi!BL$`hJ6pjuS-bGAYW06g^yv4%z7fQN2 zYV|PRRk@qY$t1M zzZKWcXK;~EugxNe*idkF=Z8R}U{Gq)UnB7hoS|U??^8&m(MvKZkCX^mopbBM%iDKd z9~sV@F?ud8=s3hk^RlWm24f_g6H@dtjtJs@p!4RJ(VNpjF<)PeMOH58 z%gEyxG*-L4ecU4IZaPAAJy?6k?g#4x(E#MD^=0;+b%(-N~sF&s(R(q>yK;s52WrjeO}0T zMA^17zpOpFMRYyG(=#vMf}xcH@vpa4?5dM8@|<5yNq%}^rn5m}jrB<3h>U{0unYJyk<3d{?IgkwdzNHh?{FCYCI#1rnTB2VWQUa$*rs=m7C2M_U$Fs zrQ{@r*KBF;T@Q&4ld{_Da;L_gp5k4p#rO0`&}gk~dPoGZK)+zPojBF&!pW<3DVp32 zh@#GlC=hE)KIb64{=^M8PGLdZ;&6uMo>gZSwR15Pt8vHY3iLvH?re5N`AkpV#N+!+ zTileEr0YM6Pp<5oUMxI+G0t>Pq+6|ZCt%i{>wD?*T(gG47$H$mCWvNUh0i9XnC)lY z=G}fL5wt6e_trM%i8ef^6z56_qK@2#xky4QYSx~T(khHo{Ys*cN5bColFVqTOs@r0Vw;Mt(! z8Wo(q>ZI#P25P~;!}NDqmRsI&(M&?btO{qx?ii`1T#LQW?d)zGRuOgDXQ0{GTzFN0 z{+$QdrL4@-SVq}SFy}gZh!=J^8^paEo;&k&8UMljr3Gmzr$lUZ952U6q~#oLzmZ&C zHe%rE6}xY1e$vPS%gCijkoSJjKHGdb^}0umJmpG*m2>SA7#Zr$+YhDt$JC!n%C*}D ziKm_5tXQolViA;{TUqeVmBqU?K+JG^!lHI%2$r3?P~-3{fAXSsBS`kkk2Vibh=zo zf}eg)0`I-+$(?q+vIcOLACX2?XIOuv;H6>Pju+1hLf+Y3r8In|6-P!9rpr_Lv(U7- zMJNWcY?k8QL&SMNtY??O=JV}us#sLoiDluqYpJWYOW)Z~oLeR38nZ+wx~3p6h2M-` znrkQTv3{@YeplOqQ0zR4qG!eWbM-($e3GuGV?h;f1A``lEcnL1ldjAI+-etaj-vLZTlpJ}T7yRGf-?@g%!Ki*8Gghlu_36 zeR4QYqRxHhce=!bxCX}r@7Ghx)z#Ij0OJ}L-S0N22<6kCJrpHXwy!cZb>#fqfVObg z__|w%7xCsG$ycq)gEd=RPrQ1Qc3GhHjTzfu_|Qk+(H#WW$bJh6h8}nJ`v<@hTka`| z>cq5m(^^x5ZAUIC28}3iNp#1t9h>yAA;PbW#bJ8871PD`9I@u&uu*}EA&HOXjpY%|h{K5cdL2jKQ-zR#rVH#PkHG`fhU*EwY%|M|bcI8e zlGmSa?i-DmYmLNtc6LQH%m9(2&T_RlGpBA<@-r#x0&8^?te;(AeR;+VKWa4U11Y&7 z#tx9XFWY*qAt=ECoo$+e_h~nxIU$47`Zfj*U#IxdDhSm+Tq|OP@znQ>8%fy}LH$@J@b#%S6tNq?0 z7`&H$UTJT89q+ZnPqgh?@5hEvB}0motX90&VQambS`yYB5%1sYqIc z=!Q9~k({)1;>pVyX%~Wb&Q{^bt1&+n-6oJrFRuXpy_$TT`UFo6t?u)OYW57pWvPc3 zZtFqdJqzIdqTLI&ZOC#XS#kG1DjeA? z1@V@InyCz90av;{2o<$F+NsS?vu(PS6~Vz!m^W1I@!4GNQwBq7+@Vdq>%G&`WQ5w& zx8$66;x*~+E$QIj=Q(&*zmmYIc-uM%m%C^mvFDK2;8|<2{0)Uq*wl2r_v$)4nOS@x z`rsU{DY@Ds;4W^dS-&cM^`PP$iqueuweA4^Y^b1mtz0z8-zMVf8yikT#rVNPhgsLI ze*Cd?k7+)euI-}s{lxXcgAQ%>d-ETK*S~hZwZQTY#5-Bu&PzzX?{dQH>Jsg`6u@E) z#lHE&&V4MZUD=*+(T$58?zZ+9(sOqxvDF?)bErPt8<(qTW52~Y*WuZ&!@b~de!n?h94w5a2A-EUSPgNJ>+4liiH|c*Hg^37x+j zweHxp1mI>)_vXcsiMmJ?+^a|#)c(az2f})^g|YMe{Jddkg9#Tf+bk4r%(uI0@%Rbg zU!4tD@9GaRawCVrVYfH!RqP;vcrzewKegd)DJcy}1}T6Dg<=R4s&s<8N8-aZlr`KwDxLPRKfy|2rd=eG|slv=9o zw0UXh&ho&369nv+Hzt9R)G*SUyPFtGnQsVj2IT#>%AT)xz6o z(UT-0(Awqlf(M8(1vW#UD_)isXz=A?i~woJbI^-c!D1G+MA*!_gH)8vu~>}-HZ>6{ zjdCxACn4r^&3bDlYIvgfRMfbm^6!SPRoU?bAo;P_2ObG5>^Z4{ohtkVceK66EmMH@={w<<{r_7QuzJFTQ>DuIZ z2lkm!q<*{w7wwatvm!7iWW5{m5}jx_0(hz(4&67#fJ>QQu0Yn}El|gk?PurGGSR_a zZ(ohl+V4P+drVIXR%4+S_mx0}Z8x<=Y3TGIw(B`V@G>CiVsZPJ4uB*@Iv+6H+3y__MbPNK zu~m+UBn!uxDcrz)HLtG^*1P5XQ*8;FBm^myRBxbo=I2KScq$q<;F!1KRnuwc_8?5U z65JmzdX0pWiG!uQQsunb8x5-Jy8eqy8*lTJ(>>j9V(pCsr|q(_0AU?^5P zypSW`^y`%+BLGhQYd&rYlCc5&N~)|9#FQL%Py%QXgLVl((VCzOAe)!MCyl9?u2K|T z1V&VI)?I-Uh|3SDG@4fIaOHziUej^aCU9~Ws6|UY*|wjQM?Rh;rVmdfB&i!=szZJe zm;%kGch5YIc_Z~x3Gdz~w_T@@V&{J#eBVyr3f==e zv0k#$k8h~&-Ht+DT|?K9XXRE%qHqU@;j$A46Vzz4ARF=S#)tw;luo5)xB)ISZYc7T zUfJx^cY`eyvP-j6%J$FkTR2MwGSlQTH|be3Qx^gm4@LW)$NV5Jv}clw^*BO85KVB#j&nwjn$*RWgw4BN-yo-7R54XF+J zMJ>xC8|%2AsPY-wZh>chgvURPF|Qkm9R8lhr=WeNTR!)sBGr>Kn3 zVxgW_ur@*~h0k))ONG-4u;U!3`1U~tRUZ8wdEC7Ett~T4sZhFN-%Q=b=pk^0xRhD7 zXH)EUSglJ?Yt-C%cU2hFE*M;o)zj8K;R0UqnQObXd3x}kEj%7JCB*vTe9+6ymz}15EIW0{3SvROMc@c{{5bOr7Axn|!JX7)bOYN9M@OJ@Ln2bC7APut3rQ1TM)Hsx zwXb8%7EZ(B%}+#9nY82^!C(o6kZ}|-vWXAT>`mc@#WF9LXs7dKK*AML^oW#ZO<`CV zT(a(EZ=TPvC(<6`EmSt*)1c=DL9|>=5W#LdD3*64BWI2&(P9XjN9vyMRAzUOA!x7A zh>24=48^CR>wsLk-#!s)t(Q!PSW%W~@$|O)vAge?RI+@QgbRabKY9_iy7u~oBQp6k z+i6zTEkF0@RcG^E0#w6hS=?LL28pDB@9sI@KD>wvngwL86|MzA%?LVlW;-MujYkRh8lUQ2lq46K&!b~O|C0hz0QD*N-yZqHmG?syvVl2^}n4HGcjLaX!8 z!5P{)oGpuymKLw?{s2Le$-Xs3s>I!P?V;+195o=fbmUht+_NZ;T#&-z1u0{MJ4cs7 zC3{tA0Dg&P_xlU?pd^NNI2Yn4Sv09zh98b-3@(O{450?RTpHZEN1ek>*pEVVo-b+s z?N=*}>+N2xrdv%y9fl_h?Rw`=`R;a?Q~~4ERTZS(9I!{nU~+PPqtiNPK+tIB9zS{pJTQ8)wkaiLWSnZ@Hf{+kYcn z@xr2)Np#FIr?=6?C!ZD83UIJvyIPEcXisPY?=?}Vy1mdA7!~d@$GM(N<^mW`kunaxM1}4GslWEY1ZLr7_PU&fEUcQ<{bK+`dD7 zA7t;ajO6tvsB|e(Rc5wkdX_ z5!&s0wfHvIR|#uy>zqn5Jx#Z1F5P~;%1LGIM=OV~pYgJ`wh~4vI;)}ZRRHOha^C0V zS=O_>cZD5KK`#mcaKPO**0F<9Y3H!a{kNH<(jX%fDO2u;f}2&UVFGgvm+06R3O!S` zD-Gt6y}V&J68VC$)tK(ATsYXLuO;p5-+PmWQp#i}b z_Q0Z*fl`stE0hCXy4D^$coyxeAPG*OF7q>nJSW8J(#p*1-aH-;Pa9gFMKF^r6y%%V zvwAemYJ+Obz(QNE^baw4s8B-6okO_Pp|1+0@6Rrr6{Jv?;uPdFL~Lu;K~*PG8<6GHTxz&0>jLjX zA2_lSZ2~a)*-0iM6tskVh+jLiR+)X}QAOhrMYb}#53o4`V`aFsOntr}|05aVXE)w+ zOG>yqV2orWfKL0EV=La|$=fH{y9HMs(Xs>B#*?!Rl0Xz@E4Fjae*=xDK;AUu(=#m7 z(j+h4>xU&3UGwM)HC{o9>q%qXr(-;e$>$@!VDmP2?~$e zXG@;Eb*cq6E_x5Y&sf?^!7s&U8DtVVfD;-WGeCcSA(668RN+GaP6kzqfO>2DY z6YwwWnrk<3-+)j3K%#HZ6t96D`7{pKc1^za;rW)bitfVaN7auojz;x9fvzA=@|-O@ zJ}y@dS`lp9$H!+miNtCfG`tIy+Uwr9oU9ncAGvxgx%6Sa;rYAqTV6ya7d%{a`(`h0 zzV5u@=1KE(0-hASuv)d5x&byMK>zx!xWR)tES(Cv0c|nqc`c_@WGt?q;Z47Szr>JT zYky>52ZaAqQ&eo*Yt#wCFBb|L%Ord4-z&fB*_#v{?wWL?LgR)D9i>@T?DH9l&dBO_ zyXz!}bnM9$Z{n-(Hm}^TIQx{!`l8%NZTOgv1Xi!*iRsC%mp6r}=G>If^-dNGI@oR| z$jO}Ssxj-+alN<|$9w5QaBJx*kOm`alM<|q=gusLdYtNIR6^8M(Di}WYq#~9)Mo|O zYnb04O&d0!e0R)(^*MXWHtq!J_s}dP@Mu~Me>8{GlKGx9{Q2r+90SPzccO`q3G%R)BxFrdM*&K9E z8_*#xo>O<*SdCkW+hXwinX6A}cY0+HR1b-Hugn>icnBHEvxrhmNs>e|ff zOE-u!S8sgrR7xLu8XVV3s!cukeET6b>1{~-gI6czs7>;t>#~tSdG+(AE68VN9D#E9 zTtk))g>vh4`d1c|ojDTO*DC5iK5yE@CV~Cm^8DINFkr9vhR+mPxK8ooNI-U_4-7># zuA;KAn!S|P01DL~E0!FpczVk^I*zyUdM^JdDplXKr6(^E3amHA*oxLNj!NcN`nv2) zAUb;yaxxVPB~PEOF3Z@+L|zs$)Ke@=?Y9i;>GquAf-{3V_a4DfZ+3AWsS#~q+;7-o zz{8;2P~gqGRL52!r*}F1RI|#L4q)FvG2SWA=hs{!2o0!5)EWuT5SBePqIN^DD_q*w zr^7c+M7NGHwMhHkeZJ}DA1nj|rTC8D-=soQa8)C$|H`Va$BQJq|Bt=5j_RuI)`bN@ zK~zvm8c`6X1Vl>0Kt#Gbq&ozpOH`B;lrCxM?i2(Bet;m|sDLy|_cs^u>}QYt?sMKV zzJ1O(`;7OmXN<=mYu)R<=RJE~mwM)4QkhyBZCmM#YNeH!DU~N{+@-j%`E>8Kr}#_r ztbb@XwP3#rftjzj4`&93JBH`!Oi{-4XfArB;tQ!w)E*~5gW zB*eWF>UCFnh6nhAy3Nh9i>4Bo_HMOpJO0RAUoI)+U|ZwT`AXQ1RG4ABl6&E;F{R=> zj{ZQWi3m^J2aJ!t7nS%-YnDqi3 zOj^I($f{PN6DLodeS2Lt`7?z~>yIFmL*}c_jKRloW~QH$%1AS5vvGUOqIir7Mt*6O znNbLhy=TnInv7+?)qCwBtE_|73eMVRR6{!Ne z4HQhUIQ8Q!yAS_&Ur(JsgkNTCd78W!RWHkM^#8Sc27;tI*9d&lE%r-pO?}%lG%4;g zWO8uHF;ACGSS8|f7x}YYmicNmruU;&r*+yd>wEM)e|#?!q}yTsUcQH|yQ*IgOX_j2TQJMJj%rR#F4_82(qeN_#J>aCH>@ z)`XzQOq!xk5#bch`!$#eEM2VHzA+WI*|1nJNiE_xt}`fOu`<{o?lf*8Xkm7TZVY?d z(22(Y=~{y!*z5MjC7BltmD+`Qc$-D7P2nd?p!SFcIecgV5e zYIs7op=@a-$JREgN8s62zSBQSjH5Br$`i73uNObVVlq`~wl z{_ExBl#Wk&9l{AZKR2gwKD$KsaMmK0ckA*e=bP+LL^e+5Y`z7!0R(zr?EWy|iamF) zb$&ZqWTradS1A6{amvNU_U?+fOXY9lF}ZZgr9_{CK!I-E$aU3!?st@B<=0~Vk- zSzk5co#RhH3-f2)vg^PVot!GW7VdvJT}n*g>v<}E=TKbj1{(6`;kd6R4Z4SF=MiL! z4Qi4-DaCn}*SuciKfa?3HlFq57*tB3q73wz#d30uz9o;uEFmvIkR11E-e(v_xh914Z$I<7yF+RK#eYh$5|!3(A=ShSP#a zo5q)A_5*9x7AL8QR;KjN7ql(4o2U;OXFFMkyVLWm71YK(^dnQgp3J|++lf*vwB$3| zwrbCGZ0xp9HASJr9-iqU6*^g=w|blEo;hJ_(&j6SCfYF-1S8&O-1_3V|A9x@Wq) zb8C(8G3nB@(u}%dazpt-nJAz*p20p{C6X0+%iqtM>6PceuS)|LyKFiuL)!_mUg z^endzzp3xL+nt1u{_s%jzQx=9N>N!aBCG*uRarwFUdqFX`vRH4rw5)5E%Q6_`fY|T zIsT{w1We-|9YE7cD=T4FvzRM<{w2q#koDF@Zy}3r*5W667rfi=R^T!)BNH3yMYgB3N-&ydEQCu|^{bPME_o@ncWVv7v=U^#r2;^L0w)(=y_2kBeoB6-$ z8n}$EsC!(^{x*)Q4h^W88Ex3r*Z~U@y0x)A=CK*XZ2fyR(u)U6?GO{vwocI zQey3pJf+gAr5fnOsz;rk>RG$cLPL;V9DjopV|qk%9QXIC`AacUjjN?9v(Y}?4dxl? z!<57^KiFPqKN)n43oA`xC%-*G;$I)li|-yT$WoE56qLhZL{BvGqa$iDb64m72GxmL zkD@HRZ+uv`FO5tXi0VFN&Rb?k|9E(x^+(dwWPs~Rg4WBhvY8lV(vZj+)QguQS-TJY z#lK&Srn26)S2d3nyXYO>`^xacz@y=gsf*tIU5S}pm_Zf&5-vAwkl5+{1L~&8n4*yP zLR4lHWbG{y)J?Mm*wj^Ph1rFDSk)?)c*6-Fq0{(Lv1HL&_OJ25`!jx=3-j-cG6>kI zYF6u>*)Kx}r6|(abm@Br0ZzM7HXs?yl_t3K4xUcSG_Z|QU7i_MF<28SgWyyTI?aPC z3ucDXRA0#ZbXbZ_F;xghVf)t=me_3T7P1T(ws?mY#kh6H^oI1dFD*8~urIBOM;Ij2 zsIN_%O?SAhd@ZxZw6JH%>)8vWF*-qLz)Z@Kr7x8}`fE0fyK$xE+)6Isc6RX5w)3qI zjkT}3PJK?zWoj&_uSc~X0Wy^jp(~q^i&(_ox;)#;Im!sb*(MxPW}@@sQ+VDSH@+;h z!`;#6{pQg@g4`=DkD`ic5fup{m#Y-`jRq`lS8NA%TC6uT-4j0Dv!76^9K1GbrWQN# zIM+BbOL#VubEJf9K84WxrDEp`Me{Dz>&XThTqPN6nNKD)_iZJ_>4`M%YP-fv%)g_< z>cYF}tf~u*j;HX(ci+`sPT?0Q3v{(#RkqP_M% zqdMy%eJ?Y&fELat`*#oj%Ff546eW{ww1;lQ4>%}u^3gRQX=a^EG)ZIja@ zhMV}=!Vt-hTHVcIi$>>5y{e|G7e-jJ7BJ+wKa508%f>s^ytH7SgUo!G1$UQD!f24P znP#5lhfBMqnS;>YO}Of}Mz&YFCuT9V1$i~9p+Ct_p03k zzBqTLnNB^}8nDjf+fsgUrbbvx<32JuxyUtT80DByHSYGvkg38vb1^Y4%+=_Dz2;3m zft5%hTf$Uq6%#)e(x?}lk|1a447sQ)ew7IzuxkPAg_A`CQF7Y~J&wg{jofae+;sk; z_6x0x`qoTBZtY&xT8RTC@l*FU>@xQxSkr@AZ`?AwU;UVz+N(73 z{|Yh6R*u5!k}`T`Di}38%0fUAqS>U-O4F_BIOZDix$P5?|Lcl*C3$EpWXU%Bug&-O zn)DpMNiFm^p@Hw+&69#vZ>nik5SoT0za|O?O0rq_k6*eT-x@y&L4YbtMI3rZb{BIedD+Kv2@*NuH=oS5Ke*9LARaF}$#~Y2q{!eZ+zBiIpN;72P z^vD=7o$V!=)q#;BHYma?zZuMp%D1SG;%Bj|j^$el*BY$n^@$ajXLTee^SihXlP`Lmd8IR;W& zH_rvo2B?r08m?>}9EBgx(#{HIo@2B`b&x&Z`-9N_N2|Xk6}ozaV7unCj4oBmuFQ-t#Vd} zkh8{Br7U8RvE>GFmq2Tzm{*(M2veWq96Xi0DZ6dn7w{qdl8aCMV z&!VkVm+#WAc?hbuwQ%)58My4)$!h7mhd$mry(3(QbyS2s0f_9(j9J=AN)@HdtI}5Y za(fgy83BHXPArTXfog~9(*b1cR9iG*0HBqeDi)ndK*Ct97YAW%({oImr{yj(v&a;d zk3%%Si=!T)bmi}e;3q7VD>7eSkut%2Rf&s?g?JaCNEzESfP%gQv{ejOWE9PbvU>|p z!@?iX68O1)__E(9ww!UE18lcXB{or82JDJ34unj&l_F^Ftl`xGZ3)xX(BvJQ>ouE8 zSo;~9YNl1h#G1!-XG4ow>>R{u=geoJ@5I0)rrq`$Lj#)CjCYJ^gJNW_>eFiQewxm4 z4&kZU>5ou4{<<8_V{3d9UA7(Ks2gqu{`n3jPoq>;oNxZ<%ndh58+hDo?AAN5T$uT5 z=at*~=P>Wv6cbFx@a3btUb^<;#}^WT6?R61#ojEKEpgw}#{wTYINiA1E{MwP>5wkEccg*|i?BP{(_trM4X zb(?~S8anXO2WG04MuptWCneS=yz}kUhVaWZhLcBo+AhyuD-%|?muQU&AH0^1$=0#V zz0hYuL0ruIBh@T*6ADlW?El-n-R;w@ky7$*U{7zEvN0FkYc}TSPNyaRU~+=Istv_K zIAYDhb9#KUa}=8nL%qOI`X>3g+}6d2Uz=JwSpzcpey+2n1HU|Pd$ZuJ+m>wvebX?;!lKp@#Xrs^8!BQD(8E!pam( zxRpJpEt{@#b*F@KFEd1dO;~R3z4Baz4e1P%s2T43EruPmkr&!@A0M%8u4*_Nw+2r4 zV&KxLVY~I;?Uc$G=zHfJcW-E5wp*QPF4gY*%c)zW-tuIMmCJnQOyfVmi#Qil+n!0Z z^B^D~ixS07kWSPW;W(2?0O(Sht$d?FLlMIv4#xA`tCQ^yHI-MM=tLC+W(f}UPTPPu z%>EC=;osR%ByJhUjqT>oQ7n(n8i64`7jdd_)s>*R)4>GS7Fd>knVA%tU(Xo5CY)0vh43MUE z{V_4)Q002dBtuVBh&$@9#saDT)}O1i*gPq#Z7qy#7sy<`KKu=_g9LGP#MTwm_klD4 z?;;B6$=u@1@3=7&#|EiQesso9M|=B{FLpiR0mv7GMjou5T5SE)43hj;rL?uUf&com z`gMf@DaH!U;LS10^Ga#+`SOWU86)H`6}^S(zJM=Ps?JgophU?~55<0!l>YpzfvBQq z+q#*{{SBC^i>)t2xt1N=|NV-3JpO|T|9SN}b#-D)ah#78J|cOEkGB>Vy1&fB^i zr`av^-3lz3#z(1!`1?(N*d+K}#JTDa&1@AO9M|ZO$zU9fEHU{XEiq$cihP%Ql;3E( zIiXSxBdDtXcKRJu@qq2Gua`F_i7SSRcj++(FMDCG9W z_ZGSz{0qQ^^BvUtkr;o(P=7+H2XKQ24K+PcOQ8=QOv!PY2aG&ZS#&P4V(oEamo^#A z`BnU8#vV4l@Ltiw*;GZ*m9jJ0OKhVj(sXV*4TW1b?d*MAbym+M9rnEGM#H39~Cp+uCx|m6Chx#V?83vuNN2~jcM0pRy?e* zNz|ZH+%2?DRyLQ{qLT3-{|){Waml7&*6xgk_SQMOGDX>y!E>u?JT4C!$%cI%$EH&) zaA1i#NwQK%9(;Nsk^W4Ypo?Lj9%b1@M(6a@&C*i@bncx~Le=j-Fj(p)sg>SO2hH3RExO5M^k%cn=kLoaMc-19Z!tKHIn$+ZS6-viK zSK&X_3#@FV)%Y!?PrKyEpw@zgol>#D+av6eBVAn${mT!lw9o$t8J_PwEmoV@OFYrB zE}bzhmHK}@&35{)>Vs0IwKr7D2;oUcs~r_z z7G8!6wJNV%DAw?Ak%Z=kow;{wbF&m{c8D^z&$PlpPB)%76a82W{8H#h4OaO})g-u$PJ{I7x& zgd657Z+O@K9&UvFcMmtlVT;c__lGi)$Hg-Mik8A-JYt#LtQye(U9If$-~V>du2(kx zW3~4Gs;K8*(tB7g{hPx2?%|$k#{}6wL*P z^(vs3iQUh*yRn7Vf8_dWtNYRf{Zk4nY(KV=*P}QsZChY(+a&cewZt1PX>QDrlaue= zxoFR9Xuan`dqaaA{Pb@^)^M?D1m$_0^fAaekA`(e2_k?w0KBC?CWY?N$3&-4q~wdKI!E*_3Gh zjZb)Mk~0iMQmeigXR6AoM;Kq<@*a!VC$=WY44vJRof%{_(3%qXp1LQg-`$d92ggl{0Nk(nMvMgknNdyH!7bUgNlQkiu|PV2$BKJ3lz` z%Or-Yo0IDJXR%?-#?M%VI~+^fVoxl7+nY0q=I)-#=qcmqPD%}-X>aM8Vba##COveL z1=26IN1pI9vsmGfjGBoIIQm(U=l)mGu7pLJYI_K8qONf&5jEnlnIE)dvOF_2j0 zPA@bUw>3&j`m)Sc=;#yHS9R21(te|1nXcx^Ael=me#+Kl!qLxu;eQiqWuBy0Vd>q( zA9ERqb;)E;*^25`DNZiC)h+YWAzY*|+vkbNTb%#pScjzr^KV;&T1(3g*99TyEeFju3tNjC8}kKsV`y zjA1u9m*|9_zU94KLlxS;D<`2xe`&e@;zj&R#r=0zadK6X+p3uCe*s1mVgm<3QPF7( z`$BH7U4aYA>2YBvTs)Nk)C$jxzTmaoKBn@K+K#)H@VeyR?AhE&w>8x9$ond`AverY zb@%Rnq!gkjk}NN%$mzO~b*!rQ{-z%LPVdcyAJ25n;D3GU3cX3$`_g>gQj9Urj2xb% z%Bvk4irv2{=N#Ofes=bw=Gv}nA0~G0(JN@~CpZ$sDczax{8KBFUUF=vw*O`tw5T@n z4fKz^p^uV+Et@Z;GOG53SLyHN+7Bd6yi8KCIyRE)DStCe{7VMysH1jnsaW@rWt=!A zm)(FF=@;6+3q)IRL^Ho1o9G3}znMjsvG$noqXC-_{Af&5Fp9RdMb(`Eb7MxKA;cjn z^~Zk1aO~fV@^(!u-|@i+x}dAHvA~Sdsz93X1@HQA%uFndh2ADd?n#-z-&}$<4*@6r zqk(OYEsTOFO|1HvSvR==8UmjbbrlUIW5V7eYiHy3H;3e32l%f8M8o*6xPfwd^hDjY z$_P8jm@>j=n3N^h@+4j^5#y8r)zz=Mj}NiM1j7$5QCOazawa)bB!`EBGz~h|Rl1s4 zYwdHIlBIJe*N)LUKhquDZ=%7rbe&3-+a?rA%Xn~wsSVLIVm-|R^6PVPqomm{9BBik ziw9aHzu@vc%7-ob=la765j4J4tfeirfQYKb9Nd-F_uhwPqp!>dPfm78@qQiB8X=fY ze*q2SBL3JackZj~6ty+9ZRi|0_^I}xZt^jk>dw_*iY)Z+bbtXqYP^CI+3k0W8~eZa zoo@RWevFF9c&GuD2k3o>|L^i#Yc7ePH#Cr7CKZXhQtT!{Z=Kr zxuJ_rk^BH?@8n^}XcYFPdRqChlJGN*XJx(594J=8dHc`3M*SHVirteC-7Sx4@1n1B z3GM&%a+MzbYq|gXgyCPi`@i$5{Xe|B;dILfP~&*XH2{`br*6a&76htHF$`{Zn zVfOdVclJU}G-9t8>Dg5(Mg96@2J%+d0o#t)!D$PDzcr9RBvuLU0NE5Os9giz$fK)L+{=65xj zeit9vm+}WraUxC4w$_d4BH3%(_5M1jgbcM(nW91S@{AY2-=YMRP}rM8Q~bo%A{i8D z=`cm6z)iwP=aGktO{{Jch|IX4?oW^k- zji^kzMwwpJhfxMfNT-Ujv*#uF;K_#Y*BY)S>_yO9@>( z`N4Xel51!fb|Xp>M$SrMeGh+H8ej)KIWTtd1Y<5Yn~NM&Z*A9;x!~bPE?$hh`Q%;^ z5&kQH4{#b{B6?t;^xzSE@MT-iLM}r3*|%SJm~$)W7l`t|jpS|V8`%0D;%Nz%f<%PO zrkmNU`4ssRWnZfn%@B{gjWC970)GHKj5~)4>`qH+@43|P1TPXXEHni>!hx@>g*2;N zfahv0Tm)YGD2PjLZ}YP*=YKjyrt22XrgvtPsb{Q>G2P-Evyazd05kT$ef{n=lri0t z>W@ZxtuGl7#zdH>1LE@#M67NP+sFdPilQzI$gw$ZScY?1X|L^S?@+%57%Qb2Uh46! z7Oty5?cUhV^+^MHqf9tpimqwuhrl=>FvUvd1xQEi8`nwhqU)_>?sYF6A@dKxmjCME zUF+{-$+i1Bvb8LQ8scsMoZKA%{PG=sn<1#|^#V{z9xaYkviBm$DLD0jSP-)7hW~n^ z`XaaT#wi?@|NJ}ZUP3dY*q0NJVOQSY4#`s?wzPU=Na8N1@p4!)DvHoVW^Q`glNE6+z0n*2j_OWKDZhPy22dR z62D<1Yp`@5OR7!TDi3zO3g>t4TkyipIbL}mUCrkfRcEVFQ~*p@#=NEP*6n+UP6zNT zBd6!NlG)B|(*{cvYg~&{ntiAim<+oa;gVtusfhIxUgd_WjkN~>5F_#$G}gGweJt1< z<-s!8^?f*4a)3dgV{nyW?EBlAXzjJ)P@k%PO^z#vFo^(}wtkn_Kh?L^RV3+KX0Hf+ zPuT^<15`Hs(bNdu`uO3K%mgKmF|KF zcHOgwdFagNv4^hfyE&EqirkfF2WJ-P;(i87so+OgGyPns9M7<0^b74wGAp47xH?8_ z(P^c!j01c%35}E@5#BGUgXJ@wm z;VR|sv96(`cZ0n2ku;qt;ks&>61F=(5pS`k*!ucV-}PM>bNTA8b0Z-~Q)b#`*7l-z zq4{S*Bf#TH>bFaql=wGMA;k72uKFN&aMnAVB~Gt`28cjja#qzy=}vwRU?JoNQFb*e9@pnv;lJ8ZGL^t zc-3qIVA@Q8nw?s<7D0?UiAOKuzWymlEBJhitMou0Pc!t#pY-#(3cDQgAo6-%#Ievg z9SMy)vj;#bioi`qdc{GD6=!ha-&zIMJK&2QSTSf#za1C8nM(ovrODWR5BtG@J$J>28LNrbL^m69QD8wuQWfs{!U(ViG<_ z8wEAj^*)595TO`+Rvf$uoNlITy+GC_L+QFPKDU@69vL#s8Zv6O2B1Z^UYY_clj^hL z9_R+kGjUp*3m>Mq4GnOsBiDBan)DF@ zRE!XbT!~ra767u~Al>nf@q07}ut&`N37|kAtEU%RV{rl0e6lS%a+puSAw_9I_$I^s zT)<&6P%59x^N||~EP`?FTB=x-XMxB&x+Zx=_2A(P`joMwP5OKM)v~70h$(J@I$3(> zYl}dZva%+Od`Iv*8tqkVQdn>ChzrF*t>==8aIFUJHtpPAi5f**n5fyLHy~6oCY$Ff zq#v$OK{e#u4|CnIZdq^M{Ynhc^2{JD5tA5vCKC828Ottc#U^lrv8% zGf4{dOJV!bT3T4uz%3twr| z<;*~j77T3w4_@)K1S3_Pl4vynH7;grBU`IVogAmJ4MS>n}C85{|yI#0d4_;P)1HLpgnS~s!45?6i?+Va9<5YFZDFI zVrO5gQg}c=DH`K&AGA46<^7E8Ti~o=fW4TvJ~A{mXMUoErM5YfRB>At6L?Q(s>P0C zhPaJaZl2G9UKRFjvyjPzu+3O1VCn0R<=ppYVVB&K-Aq~!Zf0@`uP)63`ZMh+-O-)w zfyQgC(VDZ{@w=~*x@7Mb@rIkCZ6M=b+^#Jm8No`p zUBQs4eCE}NBwf}p)G9^>Rhh0_{+f4UJDdWX!d0+C-3bjkGKg;**V3gVUsE3!B<0(B z069x+`P*@^`mw@W?-SeH!4k69&=0AQ*&--GSMpUR29aW_k>9|sH#R*Jinx8WSS9wA zEKmB+zHu+!|9F}Y9qSiBgMRCky^(haQYlSCVELOx{1dCcM;ea-LX}Vd8Cb^}cQH0JS=2$Hi&WsCE@zVjD`19gG5d& zjo0bLx~v5PkKZOs7(nx%K!0-v-X!@xB##d(CzkI6J1zPd42a;aLH}`(Y1wr0nY1_z zj5&b8>d)6r`~uz_>;ugp0V6AIMyuotq^w`^&&4v`uYrU8lM+c(i0snQ36lhKfD$qjFidhYJiyL~aF^(~hDc7Wqo zfJFs5T~-&qBpP1W_h;lJwD-`6_k$kL6OA`$HDOVIkU*BibL-3*1@~QYQxm02=yw5H z7ju}BXbKR-P*?b6x4~Qbc$HvzP)8qeFu_dsIwfAPq^)s%M%#J^>ldh)Z{3nto+nHE79&Ft69Rp3h@qJmWD2oAcb_0?rBoF^nB!vJ;Kj5DsEP8>gfj`FsK}{~jY7 zbYq$hf4q_Hqqz^!ohQTlKV{y=g`Y*zDNgHUd<%19;^Rzs7>VOpvRzrVf?6Ldq+~WubysrS)oSYsjrvvz)s2JKSLUae> zK-8bCRSH`#gKYg{F7Goku=TpCN(AVikgXr3A^L?NN#fu6wgq7hvh|||3UD(Ht{};O z@fD?&zf$|`IN(>vgG1D{Gy8A$rFozmk<+9yeVDzoiGftZjow<$W#s`Exs0S{n&i%g! zULduHCZr~zCQUv)5&pLS%%9$^=H4Bc>TZUN%Bl|a^_#T%R6YJDj$)F( znKMvHA#-c-5Ns_mN;woOO~{mpqO5hos-B<`PldiBZaM)u{U|}miPTq|hT!XVNntKE zhBJax+GSv#eYlqzswm*qeOn^9TwY%y%J}Sk;hwF@S|L(F%@aqjKlWed;Hmsb54&PBX!N)7T8ZqXmEC9$DAfeE4gIVlQ7f7*O$m`U8qY`24qY!C^;w-?j3x$SiM3V)`6MAcn+&_#ZFct0uSPlft2kn8>l(Gb4gL( z4^O3j`_nOv=+`n>vgmC3*cqs$h=cVl^3T;!gayG3xyE&9uEz%2o*!gC@LsDR0%J@~ z1Vka5r}ayBODz%Y>!wqLpN-6zj^^Qpt; zMCb(k<{jh`Rl!%hvRJ)epenqtw@d_dC`Y@gN7TZ-~5JUU9EMGJSpINMhnNo|5pvnfr<(eqaW}5(Hyk z0A_2VLzpTUB40J2m4E+I1pp1wowvTlhCotN13}J3vbUy?5mr8-|NJKY>u@a;hAtgF zyGQ?Qb~Q5stE)TF9Q_A3fZdL@8SIbcrUvCV9c_}ZX_K)+E+v>_(? z4dlG!1f?ODngPyZ!z)2Z9q}P#0#1mcU7}tZvTkoebReWALJ=mBI4{Z@ND6azeN+>M zwWy2+i!!_AnbgbV5N2^k%SZvYs-52fZto?XzvK5u0W?hzuwAKv!sAW1X1fGh`{2m_ zZjG0IiyRyKq*7T^7!(Kf2T4k}Ua8na_CNfPl*bpq$+?cq;<@f_HHB8K(6XFuzq9O% zisn^>oWPTCpz)fkSZ;f|(jjbMG5{W5WyTwk3m;*NH2r#^3bXj0kLo5Rjt|_U^2Q> zPBw>7gd<~e((m^HwOPZweExPR;HTyX*qkR}yZVKi<%}4S^1*GM7P<*u3>Vl!Yl*fO zfgm(Oo%A6_M?Q}0=q1pQwfHyf)S`iLykA`b_~tckJyGIlLMYO+KyK!Iwp)cqg?%qw zjx@~^vHz7puZ&zGU90QlX!WoO-jSk4=iugXd~WpAAv*Nr9K3QM1fZu`%CvA{*uM0g zP*Lr&=H8!BpF3B*O#m;NwR9#eJ#V0mXOnFPl1E-2OKQfeVlaP^-urHrW^-kn`QBqqfK_`Se6-eR#w zHpu)S&a)aBp{}NXE`N z6{9Rn#O4hEe@&TQ|9P4yOQqZz+@gCJ0N5KOu)o><2cfPZV$onksy_%iEm|0M1yTiJ zJi&f94Er-G7D$dT^eh0}nKEa@gCPng}UHCkTASx)O9<)?x{Ms zWo;~hfk>c055UU`C*Ft#0XVp5-aLWXVPw~50*f}|Mx=Y__xK3?1a((#y=43pcRH{M9rIO7gLrHvk$W;#oDTkIj+ zUiC7YH`#a!QQ(wU3j(pl7U)owVTcA`vatmn78;~P%l)SCi_O^L^)S=V#?!stY4sJzK!bK$bj%AgP)T! z2|@%&Qiy$e2od;2;+u9vW`*4f&ZaUQ{7U)yeri_rgvyewcAwRO84j_C^l4Id*WrN6 zFjLP1!O&qLr5J23Qoo=-^n_%~9O!X89$>^c{}JJa6ign=q#1yH(8#CACYRzY)#9+c z2XO6MI8FY1YkfA}0VOh^tzJ7Q@ql$gDodFjf2%w|&ZWtm1y~LoJqk^Fn~~(g_Ixpd zSMg927c)asYoNgBDX0U3xV{(I_#e)I!q}v>L=ORDYYVEh+oR!r5R#CIZzWI`KU5}_ z@Nbu~ky&@-cCu6~J!b^FhT3;ST!C&y&0x+E`}-pfQxvRJnV>k+&Wj-a*0HE_5tb~s z@#Y6q@NJBFvT4ggPtS)biS9{OB#oDDpCpiyFs*Uyzhc%&ps!x^t0KJ+b5n+o1*)v@nBENu(t%Yj1n zCyKq@xBQX#})LrfYNk(wVBW+8_u81JcCk*CG3mG~{Az0oe!N-GgG`am<=S zmjl4I+)!(pfP!_28dQ+;>dx~jkwtpo(cf5l$>MysYJYdJg|(4fcIC2~Vi2O{L_`gg z;Hx3Fn!FI3>Ioq%P8lC#me}C}*?7P7fr{Bc3gFjb?Bfwa9_EUrd%KvjPjb*FLwTU` za^H2md=#_1L5^th`%aKItu4*lAdu0=!^u4=a9YM4-&N!fPbRJCY(4duoEA*dcLJ*F z98R0pS(5KI=5+Tzl-7W273ng*2e7O(V8>5rZzUZzgLXVYCIh`<68^iwl<4kwDBm;l=L7Cjt9t-;Lo--Y zYDG;K+cGRU%0%Eez4LbWRF@Z}BmI}vq&#|%5IM$Fe2@3AQ8F{P3wuEN6nuz!)4+p* zfK|3okN%WLCnjqBb+k>V$REV1r;9X|*>V;GaA|Zz(~*M}CUj?f$uz zwawqxz%%)t&7=SDghw7Mt2sz-s1J05_9FFcw(5aa$XHlg62u6?-+*}Rh8LAOg5~Xb53)N#00KUp|(_i?EloW@p!xp<7 zwx&}iUXhu9H1Z9hA$IvnK!#HI*@7T|S^y-x!1O48(kQfz6o?z#xjb zZrs0&R}DwH!V>uzrpcC;*3kQrqrZ9$0ebyU53zSRucrjp zI0ma%Fj%t3!0iX^3qt-IR>I^~FQTAd2MKV5)i6bS8E-oL(PbTVQk-?e`)c|<;Wo&C zaonZFbpEo|D6Hc|-f_;;81Fz3(&H|NFlQ2ZhktD%pLE+>jx~sT$4zht)Gs>{NS8b}l8m#(C<={D0ZyIp@)9COC2D}|`_jNE0948j z&kP2(?y<6uA|5>AIqj)zvP5leI4zVK-0p!&T2!m_d^?!7(Tp~*VJ$$=dt9wn3DOJ8 z!-T*#Y}GF&n5lJR^-L6~a9|a+Be`v=RC#}i3kUHW0Xd=~TgC9|ed;l#nkcu{l#8fF zdZ_38t;`RSZe3mJ4J`=@dg$3mDHXW0BlbpiO0l!W#<~ErW^OzB)p?2tbsbl+G2Mmt zVcgdGYrjSCwZnP8ift4?6*k#~7)(Fi2R16tjI(_x=WhOIn|>C~0C__~1E?p>3zy0# zLR9m#u2$DEQH^4lRd2Rh@c5n8@*YxSe>-W=E@{x2NK2iF#WLniICma8I~7ZTdSko&PfX~H54PMDZ_!+FTl zfb~S}5`CN~m%MS30be(~JSVGaYi5qu8bzUp)V;s0znido6Yl+ow@S>isA`cYy+Snu zkV(ZqsV>j9x|vb2>x&V64aaN6cuHs5vUrMpIh?ogMl~;LNL*pQeOYP_(hh1^s&$j%sYg3ptefYXQ`fYq~`MsZLQ6ygQ`ILV+u299U(6(0y zVemr))dEpzVCBO`mLMHdxFZg zV(dB2Pq4lyepkbVH8#6|aCg*B*r=LOl^r8)eT18Hp4O_)D1I&b_(}P$%kMkAB&3!Cip2(3!4J@ab&2;1MC3^Bd3`WFP%4A%ZL`}+ z-HB+m?kdCTKe1DqA1xLH5!@1moz1D(*j=B*O%2$jonWfiTXCv;(R#s}Sut)Y%9Ls0 zjfA`LF4n?ia0V_0It{=v)A$GQ+gdn!)pT^Y8U$;+YrF{=bm0nyf~Hlf0yC~e^6oAv z6mVQ~3B7+;4Jc(pcJww#ps6ui2p#xE;=_p{5lCd#N*ogz5OtP{zaqKNDwhh{8)PH) zYg=*AnKAbOr>%v#e2xY|3|b?T+1&gnNXez>+8A4UWB+yj=Jt+G3tgD`JdZV}zN?o3 z<%>wx8=li$+gi?R112n+BkNV6JVG~DAqwfc5wb3?8szIK_yc6hkBsfP1gs(2B`FR4 z(ClE6Uid=0NufE(Teq>HLM<+0NixZi0IduCEY@Qwj`X*+KV$VnGA^&2Wto?;wc1`0 zm?D^~X?ibs7gn^mE_qUYfgZx16*Bw&cTWyH@{jM=<;lE#J=~A#O^HW!3g{m@sBe;z z)+RyLdziKdB8)lC9e{VG2-HX4RKGTuL?WYGaeY(#>)foP zOh^+QwbX-}9c?dB=F+R>_q_t7vlv3Ww88x6gat&7Sk|5E7rrLRHU45QRLhrw_jY;u zb=-QDa(}6eE&qzC7^j~Mnw#FJC~n#FJik8B42GDHLmji4i_t3;{QE#Cr(7{iegj-P zjAbFYTB?UTTU1uhg_TIVG`#4lh;(?ej(>*y=6i+TOmdN4+Th8oPh{k`8K0>^xK}qP zIjj3ud>H;nWfab0)$@D1?C-9(AQACo!p}x>>$6TQf{4d@qD5Z>$Ulx)Sk*G&)dM{; zN+!rFVVzlFqx6mNwTZM^wP6pd!*-&q+X9iSC#tcJe>GJy39}@|7_?J}s(I5w4Ts$; z@hC3oGMi}{klqbp=+?@*La_!jUmYkW*x(Zg9b6BfEoo= zv*6v3S;&l>y#n+rUDRM+5M0p8+4w)*}%y_n7<1&(3xUr3og`dnSKBHzaEi5GHA>*-ass4VQpsq9EMQ zH;Ug_vD*1U{K{JHlmVE{fs64Qq3xTrdz3P0=vkn9)O^@A#o+$4+WHi#5ROS11;6`* z2yP9F>u;RR?rc}u?5Zu}H8S)7ND(lSe?RTPhKW+wp?l9kCCoRS_^vt1J6H9>*iJ1G zcm^u)Kx4umTmNPXYT|;CS4m{kigmjo(*lHtPd;^J(z%!J|VB>&)vx1QpDL+W2(F^5fYe4#u-+ z7{&xjdAcHNfz^=p6YPVub=~N8ERFt*<&e&;z8f-Kcbnuj#76^b*DKB(WHtPMdsJD8G zJ_ae&l}%6q!m|(;ejCT{9jQ>=?-p=;q~}7)I>gC&3QJj!abK$rwd9h6bv0>a#DT|1_qK}31`!LyrwF(d0o+c zS+umu*Wdji+!{aiBsx{twxX*f*Lcv*2X_?uVqSS$Py=}9%w(oH9y{NOs~-(`e6J(q z`u_UL&y61e?ffGxk`9Ps$k)@wD1d5}(LAGVKWM7_xG$HaLAreN%os^Mu41Ar)*Zq2g%nQ zCQC?;Hs?I4(;chvDDu+3gOPED+O#`K)hV7lsew&hqtk*M|>k9Zo8G2p5Tz z@aOLiWHCrf7vKgYeO zB$D;d_$iJ-{1uGU2u9rYckO+!oF}3*(Jk;=?o$iGcbtf<4SEiZ3^>VzjGqhP20))K zBt%!S!rsQgCj1#N$<}*vQUL1(BT%#XGJcNcb9(lr#n;g+N;4k$^co*9@x;&Q{16K6 zpoh6)+psC~k!;Ww{{D*P_czy?5&lA8<6$&Fe<)NFRP*-oeO)N_y@$4j!Oqubzp24> zbb@Y*|NJ`AXf2dMP#AV+doDkhG2E7pehE0=n~AG8gp{kCOA9PU`R}4}(ptM({Xj8d z86wR}UE;>MVAb?h-~W!kFeXJJo~}fXj`jLWjmOC(fDvGOHm7t44@+k3S3*DtB)P;O z1wi<7tANpa*(XFo<}~^O=$Ij8c3Ee*)4x}J2~{tGQ976!sd-a_G!{dk6gmX?x~E?* zdvS%o6M+=jO@OlBT-@JVG2^(wWVYDP?V#px4Iu)@lTp)z%swfcIn<|%1cN$4}MK%K=hM4J72q-Sy=jL?^T zmm3cYB9Sjwg6Y$p*aK&#c<8B5*yx|$|Bt=zj_3Mc|1Ke;L1rqH$cn6xLWJyj|yqHB<RwHvorwS%jCP?lp{o72PK z)N!N0c)8XilTO^*57^utl+QUvNqbk6P8~h?hd7cBrYc|RJ@{D5`+Dj?(h6x=eXejC0dha+IWtbQH~?+URaT>AkS1sRM|Ku+sk;lqM= z3sJ#Yot|0GMIf%71ZIF_z!=G^-7d>FoL1*rQAM;GD-*)q+Mcy}UMFyr@9(OGE&lL_ zYvwzShIPU`&|2`EVM$tT#&(EvoOZyS2%Lh*IDt6WgQrT0}~amDZXV&HT2=Q;XtvxNR@fJa7peY!oWmnH*u=bobgLF{jzTor+y2h6h z2RSBfXqMZmj9xxH8vudxCFo-MGH&RrX_4(|0X3)`uwWk`whaB@`w2>-P%AM0;VVKo z6k85Hlt9@AzJKKfQk15<&s+12{bB zvmYG7hHL3o31qz68-^2Q=`_AO!qOtM#P@S3#SC}De)K~5~4eX8T zQ-V|Y{G&-Pzr#NSgRM%t&fl{-;ZU)XP`*5g!L-oZI`6?6r1KkG#?-NEQ}Wvy?V+3c z{26?ONB8lThr*6*~Dc1YqVRcUSsp*sWsUOSnJ{D#uq! z2S8YZ^2;u>gw`$%P^7$62J=?KZYWlMfpj_{IkHJ95LP>)u#ggQ#x#00dzMsh@qR^d z64L9f_Z`}Ono!HWS!FtE@~#A$nmy8k5z=~$XYC>x*AVpBO$@I8ldyg9kQmEoOEV=U z!Q*0;v=A|;iT#XaC*NZGCzlZC-LB9FGe+}B;w=8_^|fVAh{`$E1MGg}x7G{^tvlU3 zbf|X)+N7Asc1y!rhtM7tkB9fHd8KrP7>t!zP$4e7pQ%bVx&kri#uFTYBQW#m zQ`0tGRy_pvCdcS7P{4^884#JI#@BVlaDzir68+#H&GQwjWUb)&3(-JpM&(`r+(J?BtqbH5 zH7FRpkGnqr(vcv3V{5}6r=ciY$mXhn2E`7scM6p1y#?Og8&jer1U`m1jXDk%2SmFS zO{ZH%uR_R~&o_fZR~Ew=rkX_rOgUPTUoX}dX^!AY>i307 z;ovFeJr?^z_2s*=G`V|&zwLz=<~9xnD|mULM1E@sW+)n)TylRLic~>Q&wfq%aOPvC zM!s2aT_mr(3vixl(FDS&Zx!te9?m1ux;0pDpg`NO7!h{H1Mo`bx_%I7mn@Nl^R7s3bV*yMUQ5IY%`ef{&hsX+YErIeI zr7pna>Es0$<6ugKdY0bBTMs*S?>$PZ3#nK<)VIxqa7Rfasy${1BZ2ewFN057B1{=( zZI0auBH0a-jH-oo+Qtmez5*yW!1ZELX1;^fp-q1;OnVN5E671u`>M@=5KWEFyV0Qv zbObs;2D3MYe(gWoM!RPbIC`f|I$_4R+6EU1-4ht^o$wlnx;i(`mH{J7Ya@8fzeCS+?Y*pnLMvxV2&=Ho z2I;~F2`3SNf4wzBn>_YL(Ll&FaOF4&TTyTLc)u^hO|HHpx<0xlyoXDaAdX*#R{DRv zlwiE@-ZPXf(j%5C_6m6EML1B~MTMLIh?PS2)&1P$iPGw6yUjeYSHis{4E52lC30oG zJ#ZxV_fCB-gb!AB38?~&6;U|7AOKL>=;%Nhr=iRFs|D}}6KhJ^gb~l5wz&WhRcUaZdAHeW?dE0=r?W-!FwyQg`1Vmv z8Tb7t{Bo-j?MLC4!@4@K2$5N4A^@+FJ=WwcYzQBGx;SOp1^->VENy=p;1i=e=ribT zE#Pep#wf?c%AiIVwoaXO&l(JJ`H%f9EE-@wtIPIn{w$#Xf(utaWh{){1S^$n#es7c zfSs0I)CZqx!UsE74d*t0SsWuQX+?%I7uHj-Jv@U|FYVzEMkd&s^JyUh{{a#B#s{Uh0UWYeZyHiF`4hEdCfHy782y7tz9xjL^H(pbnqIbn0Drh&c0%A43sUU4b z-Sm4L&vRSC2W=L7n>Sa-FBX=Rv_!5RkDCErEsVyu1+H*H3Fb=cJ-E8B&>Z*we>eUA z-9|&B5vn8-Qn1ZpC{^8r^-D0Cxvu_$%2`l3?8Ua_c(RyfwV{AgMG92 zf@z7Kl*NiY`EIVeOhxm;J6=ESOER23WlIJ^NjOWFyYx0Ic`4Ix;bybA2ct-bPcJIM8a&XDNl^CXRo7yED^XdM{nFZjKSr#j&EovJM za&Vc{nJ%d(;8)aXYt;*VRR3YU-rmE(VX)>E;sU=4dog*GTYoPVunjwG9?K8NPGToZ@kaunDF7M1iaXI^`(I zzw^)V{d4mhOphX9P;jblUeDdd#jY!h+8PXQDfFaV`r$=S(iSBcW}K#l(-(Py9TLI? zx<^6$V<~>u;xU^Gcjie+^73zVwAc-W0cm&RdqfZwM73o+D2|uz5A2Q{dp4v<+3DI2GQdU)pM(SA&+TP=g2)fT2H|==rnjgB z(0Yj6V=ELvpW09p!G%nl8Te+Yl&y)7)Q2b)B-A zs1jwsLkmmwjp+%klUNUj^=-8Zl@){c9E;oc$n7>buDR0*Ww@|&RAv1!Ht<335?4gX zhP`zs1y2*#W^X1Q%>lo!a>X&v68@lTHNKb}o9{}_g{_-NdlsB$w!QbzjWN9Lx&U10%f$ThOq3*@B58igfkDhR_m0%aWYUA_-1PlMN-Sj)b> zd8m{CCk!xXO+M^iiC%4{zo8iZ;O0W;=vAU#h3ff@0}gN423gqq7LJxD&>aj7iWl^7 z`oN@nb{tVv&4DyH!2R&{F--$#BwMb=4KE1+Q9T9(7SPI*w);FaRRMgkaJg!9b9F8& z!ZQB%HMxzSZGdtL%_?KViz+4XcU!)6ujOKG;7&le)P&v|_M~Ot+#Uh)reS)i{Kx)1f0r#=$i?0|#yN z!L8^0XQThGZ}gd%w&cX;%&yQzMs?(o1YNWOEWExlQ*s!b4FfqX&#l|o2N)54B1Iau zyE+_8gAvlPSrCR96piA)56ANGr6cNFqFZ=Pv}KF&xU6hqyWC~;0H6f58HEW_Lm^rP zK!d$+_@}{xjWp#^XB|BI5(^|I8;U=#;r=_ly!H(bo`1shkTk0VM4hPCxb+4DkeV=k z3h^iY48|ZKKex$P6t3KZ6NZs^2L*%uvY`krN90mOcZK?%reOu1U6;p$d^QE7zvi1 z{=ywuo*b||LBJjOBi-=3p&r&*U(wF1 z`a>Mnp; ztq|4Z2#vv{zFVi7(lm07zJKuNwCX;OGW%N(Tay%D3fis$Ss`Kqm&-wEZmd-vAr}bv zMoovFlccrplKAZgjeK7SI9T3lRhua(oi2IJoXKnM0sbA62$JX1&E)g9zI-G zSDx1QW_cnIIIXC_g<__Xm7foV>N0o)p`tApXgqK`Rf9^%1=*e#SkT*idA|7L1{n&1 zjkF}v)_7o+hk(W8C&@bI)lThu_V{P5=xI3)l3(9xuGa|>I3f_%ZTFrW;p;%tHz&Z< z@ijxlF|;UwSKl+F-wxj%>3Inzr^kn7wE`hD(97j+<6WbMm9Vk;uhT_{^=V=;V^dM% z%Pt2sDkz0FkoN9B%9q7-xpsfc0+2*;2gfeV)N$qdN)FCM+RXSm`cQHhM4g*nv?`rb z<%eF4NRjduR@t8)&}*YpPfG;t>*_LH2Ia>xMGy6^A-O>gytDHNJEa zzKjv<5xTmJE$7EM?NqzFjJZg;jt4G2%GO9+WK1fOKJB5viYdwX%Fb?Td3`37AnV3l?t(~*E!3lGp=>}8MNDaO zp8<^16ad*)OKhAoly(o>tz2dz>rSmF8LZhz+now7(oFM)zb( zElM!X2vpU*7|_kLTI;@B5K?ptN1OXt4K#3~_?n z52D(6NBxv+<}Fvc5k@>h#3G~v)e75?Q4!8gxx3?d$d9Re?gpSq^80bLL_pC~&r3J# zI{+5-C@q&**yd^VOs-7XJ~b$Aq6Kbfc35%6Rc!#5ipa0M(ZjZQsQ#*o5ZB9d$K%|D z^Wgg$Uio+mKn!H(R!6E&&~_l(!j~d}Dplln7zxHO4EdT-a&!AF)b}3o#_?dMWJJC1 ze_pWxOrKcCIr0gJZ0dG%Dtj>L3B1p_pc7`6Mz9sE|7QUaZ2q4G+&tC)If4K0uCNT_ zEyTZok+KG$S1wle7>pN*2?M_3rfnsvS0~wVmGRy~ZxOi~Mm=2?zc=2AbkpHw^QAkQKYgCF{6l#ZFaj zDWGpY9FjVbY*Yl$2CGukWRelRWyJ`dRQzSdg20L?zIrMq0A|?^yPmrbf76ORr)2rd zinSpt25`PT@N$qE+r2}ZR*aFAd_$Xf+t4Z^E7pFN{09&P>WI0&c%0g_VvYuB+gj?I zRt$JPPXMHEMI~e)%MnOzu4HtU`o5hjc?rIc+_*zMj}BH+;)&TEPv(Asn>WsB<6f4LPXF_T{mRFCBaw_-IbGp@Pux$`}Ob=zg> zjpt9ni&pfAGBf@kKJO>yM;BqCoblTF)Fe&y^^S9XMr34;n#%>}=lk04^nA*l z$Zxr$trq&V#V&Jlbt$81ZTwCb;a0Lxj=1q_)SUNGGgkb^5dUKo8!r2kjbM!^Bq$9j zEFRljfbEdPod{qi&uGaY$_-NctQMc?mK&Y#Tl#CkT_C>rx!`v=4mDfZzw#b0+}uH* z8O6UA+5@r~R&3eW_hk@3V7(kFzeu4M>5e!-ratzr5aQx zPC#Le)3pmkl5(ITWi98OLYxJ{3nqty`pW6H0*jN$q&w3itwV*tT^oS*G~!ii)DGe-UyJ~1$jngW!Web)$jfagGvUma%_7^wzpTaw7V@kqy0$0vtjpDw+ z`p9^T_8g_yOZbaZRW$XN)G+A3h1g2OtEu zOM4%|cE^VMWBIsk%N}%8p0WAsMPMBp;i`q;8nN|lCGeIhiex)~8v2n9kV8KarwJD$ z-umwHpkIHZ0m6Fk9_){Gb)KrA$gv+&1xYEZ??4L;1@`7ec$rfV4;%{s^8Mw{c{qaqJ)Qi*0Ph)b-AO*2x+22YkLu3 zHt|3(w|$F4aT&&B#EEzks9wu9t|iwfu#Cd*MDnlDz9Lkguzb57{y`F6A`aRKRWqyg z@H{TUaoVRHTN#A6aWz8&B)Oo8;LlfLGa&Jr4^Zfp5k?~BBW>9)5nn@exFjByXfqptd>##{6fe1*km49p=J-`qj;B8;DuA7jCF7}kDXdC zHs4>j_8Ix-GvlyNXH(xU&5fM`Lf!K(bGIwfLHb6=X@D%8!|;JqTrLPu%i%i3l*L_6 zgc{H@%Yzm1N}Jx|%Ezee-|!xsXIgu#;%{bcS^6fy&7ia_O7dWUVS}>4SjA+En4RI=PmriMbuoBymPY#=8VRN{GkkNX@GP(@{bBf&#n zcEB^d0KG*y;T?eU8Ug8`1;ob)@VvZ(U;*R4RvQs)Sv>v0a*$m8Z5n8a(#l}`M4jOS zkmh;NE+o@4L0MtAx-@6vn}@h>NNpdA*N?w(hbzmVEUwWs!6|LiQq4kF-?r@swYB3v zLG>o=LIL;5;(i~2a>bh*sZG45f*{jB4fIl^TYd_$_r3VPxe7fG4yFf+2wPT^(kXl#`$bY7!qoHsyFwgpqu5t?KstMyyd$%rCQD z+(YKy4-c9DuLV|C^IeSvJe5Dx3$0`DE4Y)WfNTNHv;308NS$yRI&nd;Z%(1d{HkF8u7+6()`)M7 zzPLwOR9swK$=+J&HSGJhu66ZK-3T0htEGP)-wu7b+ajXB9p04Z&)Sj z)txccZU>xO6P-*z+z5s?Olj0#=0GTGXiOzHg1ge=x?bv(r*xmWE6f!dRZVb0aUa!3 z^787dn3$MIv3x00(mjv`>W>|0s_hW7QSabj-aFA*iNye~?!O=dF&(5E99Q$z4 zJw+00s5AwYnu9%;k_y}4H7gj5+_JVjj;h+I_@4*S)?jHCSn{yE2bPaE^*chRmzcFB zbEnbd#P9Gveu%?wBA*8yz~ioJkS-;>Qn{&(W!#q2B|RIsNj%$lAnggP;+$b`VgdP z7RaHN)J3q2@eq0Vs7h*xk$qNiLo6RxXa+!ST0pB+R($x)?l>Z3NWNtJ@@Dgj&R?tB z8o9A~#2Hk04RRon zLmjDLX0n5fCQd?4GBTccby5BN#?Ea&B0Fd8TXQMkKrkDb2W{kmvz3^ld` z8B8+YtT}+{Vc@sZXP~1Rb($6VeiT|?aWuSSF%Kg2=8RveHHAY3$K>zMo<0F0EFr+N zxol#RaP-{G%Lix#EkZ6{Gcr0UYkKE(9DCj49hZvMFS86pO)aUN?tru#{3Tl>^t3^t z@>pcQT@R^-@Z_m3X-hCMCdS7lxZzjEHb|KG$GKUCBr#!dudeljO%4TSI(xO^+9Y>J zKUwiE!9QLB{ue*iQdkGH^60z#a_)c1TSK$m zR1l~+VAEhLoru;?o#5fsFZT%Y{uDg|^cv|*UNt*mS{r*cMYYFH%Gnp5@3|DT<0XX| zu?2ti?m>;}^eim3D=EI0ZE5|a+-Y1_6k%EhI})wUv+Oz-9BMGm4Y^7D?r`Zs`d-~G zm1imj-DwuS|KMAD6Hupj{b&{{#RBtf(8T`=nWQ_!UELlAu)`7hQCTnAGGzLsBH^%-&!P)O>#;u2+H6hp2A;opitNINYMUf z1({mh>Q}ALThiYI{%#o)qa6jAi@v6c^M|CLrTmx@xg?qVXXWtXWP72LI1wq;7CTI` zfeEEYtcP_21pDX!cj@Ju3|=Ub&)BcITH#cFSFZ+K3di_*4DcWo`M zw>Uk3&P(oFCy-`%mi$EI8&fp9)tCJXnB@t9 zU5eGyZ;sRb-blQYr%(eWiXZC(so~bkPKZA96!ya5jw=N{*zYN+8B$Zhl!7W?OOm0# z`Z)LFd0AoCmtDFhsC#JRIfW;F&U_N8g^m(-IQmAS&T0`N$Hawor+ z?pNzSlng0?v?sG6)ozhM(;mE8-mk01x3<^SjS#EYw+)0mSzxwU5T|iXp~4{^T0gp< zldkKUX!2KsE`K>fl0Uxjj5QMpRLD#IAlBG(;>*;<-|-K0po3#L3G+<@!SbD2#FrDL zb;d%*5Ofe*O>&)mtO48h%(aS1iuinz^aiu7|7^jBn|Fz}_e7W$-yQleE+ImJuJfGA zp=)*oSR)i|uWm%PPs+It;3dK;@GX;jrQpkoG$fp$kaT?_Q?(c7aMWXg*CcZm2y)PZ zMg`6rU%6*|iw}N)*wt>WC@G#^N3?26_f)2h8VoyvHif{t7ecyn7v`Gc*n2=ns{OT7 zE;3!rSCaf@hxm}NI9^~GFdga$oO4$j{d@HzqBNJQ^tfN z7c;er8a*^MG&CCDINwQzv-8IAlnYz<*w|6fC7^;67yz0&T41A})m8EBMQ;^VMjxtm z0(mJdh#6v1qwe(|M^?4LdD3fnop|cAtzC$Rf#p2?Ar<0?OR-O#fFNzsN zNr_jRmuYO+AI-F8x`(?yUu!mBGrs^jxC|#}MMTpj4_aju$69{yJ3eTBRe+@_cG?PW zM!4*TJ5Xn8z)TS22ekY+tf<|Dy#`E>%Rs7)hwW30ylOEh%19omn&=wq!7#!rKSVZS zbuPb%91D+BqKszPvCo(Hxodpb`edn}v~G5H?lpP4V%&?+?9Xo|7RD=Q854#pQ_P>Y zMp}ftV-pn0oSNWxDSh*NI=ZmEXzGbXVfcPm%J@QD!;cUkHlu zWAuxPI!!Mf5(H_s+kLk{tlPd2R7R{@wj?ROtEOT~LBjC&p7OnXgyq2?=0R9pHEea6 zN-P}yl&)77XCU2e`d$_?x4~LEZT&c6sAb9N635gS711J%CMzT97dt!6>Vbn5uP;=r zuoNCri@&a2Y#=?>T>5@iYk%6!frvF{$ZG1n*e5@jf8t!{d2PpYrfGk=W}c}{ofqHa zUDd}^?4)plK^}xvyHP7K36eHVSE9M4u}lhkJi#I8?Fic=`^XI6gF*4fxa*MErWq3s z!dZzNOXMV-*{f8%JJWJL^LQ)tzlDxTaqR!V?T2LIAh{b#^7}aU(6u>ofoIbHL-_-P z$GamiL)~(G2mlHRH6-ti*aw^*Pl`U*shR6&1F^Jn$5zBG7& z+;n_beGi7dNqrZtXiUQdd#)%gGZxjvO~`G>4-Te}H|k0HzJEGDbLVbj#~Go_r&EMq zO?&Q;Hr&Z;NUg>w$*gzZA&IB(D_xnc<@c4p9TT+PQ1M=ZX4O}Iv~nHPfsjypIK_TP z_~)0Aw?WCYeOx2q;Q}X?Gxq!3DUnhTfsn)9N9i{cW@6c=d1ZoP5Rh3*e3Z`*63@JM zKH8NMVD^nMdR<#POV3?{sgf92quJIApGHRUWuIBfXFqv~fdG6LHqqrk;cuhN>n1bZ zlVo`fYXGYmf$>cE_WJDC)?mH0cdp0QUq(_X{pc?3I3JL_d~am*j5BuQ9S>#Rn${}P zxgP#PP$4gaoi!R(welbJ+cvfgFtV4WxtyyoaXo+`hh`5Hkt56F%OTR_O{U*dL?d{bf+X0`i zzJXQ4VkpJRJvR<5k^Z3a)L}|*$Oll}pfE}7Rje4vD+0?sP7OV#3!&}Dw*xj$V#k%K zDq|orDWzOtS_G>JYJry7!Uo zDmZe-%4h=Q-aa1jE2U_#=ziU9adI7$caP0DP2zBS7D<%foq!&~|02$#2{=Mz~RW(S_ z(Sh7E%h@In-xz?-Szh1lRglAB1kTd2=;&aBbL{K=K1-jv`~8-}kaIGIK?VVoUW@N? z-YZyuACa(e^3`6d!b`n&(SB^*07=JuJ6w>b?3iXJr=hzfefRwA=3UC+@%&kBKX?L&BYpY&4c#+=DB<5lyxXutgo#y;lEoEM$$AO$)E|YkO5%ZKnM?oW?ej^ zgmtW5G*KKIPFO0gena6kQnWOEA9NXx{M z-%QOPksh3?FaKEIJY)Y6<;%u^ZB(WLewPsfi70k!N5@-HfiR87N`Yd64zbYUMms_~9(-SBDgJvp_( zk-qf{`wyt4sUL@qXik=@n_?QEY>)OU;MaV0Csd1QvH`b?$nKvVHnO~-`^qA`=o6GF z%v61&fFp9C`&dvA7<%;_rDtlU5UYv=dXDay<-PJF76gPhH;gtOYspaqnnuztFldAs ztj{J2RReZJ(VlJi&`DGe1U>4^7!P18F-50VatAe+BE9RcS9H9xl@Uu+q_gng4+GP0 z>&cJi6QO5kkmp^p&U|WX{g2b^`fnt(#;w9GBySEZL6K>+BNpmTLXZqd-8nSuAJEJ< zqpy$P;o2RZ!O$jptb{zm(H-^hScL2! z^UoOS!>rncG4+~?1X61@q?!y)|+yjgWN9qSYqRXCT&oA7m6SRcw4TV{`ao-Mk zhl{{NNxxM(o3@H%cj{=T=?5n)Es@Z#j`*)PSFEWRbsy1rWzlZ%HN|y!#5}(A(^%_G zP(w4kFq^1&+wvRN5<;?$=-ke{)M7sB-(f#D27UIS2&sUP0Z3GwU;jDbLWudQ{aX*J z-{t|kljt)ogoD&1+80_9>xUyRVhJDD;s+8QiG2>mh<_w+ub(O(uGqbtHlJNyg ztdPAv8@pkUd_-$>?8)-n*tL&GAaXeT4kJ>(em1l-BbCQ=*U;C#tFGEJG1S{9VVC81Fn*mA z@=2+LkSVyvO@q-F8V#=Wn!K=1#U5|5!g<$oh*y-8!@J zWHFiv0q0;a?VQ~Y6FgB@ZLXJpe*5latcM3_&aU^Q$?FZTYrcSr!908UBRo=(-q zobp6n5Qc#VPsBwo(c`vE#>iFESeCc5pQ|WJ-(g0jL)FJ#`aJe!zU`|$s-stY(e;}Y zhbv=9k5u;ipLG}|)`BqxL9daoH~Rl^(VGZ|*Gdpsx+z^#4awj&>H4wI`gxR9K0PAK zw+xA=Bv|smU)YJ*h{zRmcVEWXRnvf9crRZG-_NNIJQcZrdM>|y4FVTu%vLgS_Q-d0 z#)KCQ18v3Z;nZtcEF8i!)YV7%ZR0v22nQHTS^h$^McSmN!Pl8G z<-S-o#mV-7fy#U2!2-;<;yOJ=?b{SBb#Psib`>+!^Uwm|upEO5f*Ex~z_EM?DsJvu z=Q-3P7_P~fupB&atIONl`{mgm4nL@EvoRTmcb<(=Y|A%`CQOR49xEgAu2NlMl!x67 zHUy(hUYI(+4t)pHggJPZ4?Rx7lReQ;dH&SJe}Dy8qpcQ>0A2>Nlc(51C2w$0s6(VS z3zU0+zx%_a3k94-I2`VfEY;x2iu9Dc0;@Gt`=r56#(|CudIH|%T$B2`C(Sqe zoewNPP|!IhX~cYj`>_m*T@>Ex(_60L$F@PL-jp(E3aATVEIxH+FIEQO`Ni*nsB~8Y zFsn8@PskjxS4GZ`k#O!9$=lEF7Za@RmwG<7)yumJYl_?vzIp&ARrrPK+-B+~suLu| zS~AE-)0+XuC-%DCuX-{@z4P4j%Edzwal`^{(hWVkJ{O(b{(3hzZ|VzIb8>R(GLJ%m zEgJA~baQF}g{FL9Yp6m8(_Ftsbyve5w+eKry~K==58eI~Q!raYZ$ULv;4F**CNP)56TefiOfx%4`b4GVi-3km;Oy2Dg@C-ZnCpNw*V47f-_p);aj1Spd#rK-gPzitt zQW>U8LrfV~I)ep*-hNO?l0foEL*`En-8~F!#&1Uq%9sW)47;(3y6(9*l=Ej3Oqylk z>>0tbnbP*Al@;5MeFv?^bc?QH6=v16VT&jmHg5&ik6rRpo%pg!`;PwnIm_FNYwfiC z184^~)b()pQ+L$uPyR}Py!JMooZe1Cy5xGLpE5ykFqco zcBRiT7BpsMSaU4}po9l3w)oh#6dyi*3_-(~&RTV|Qni6v&tc{&@D@j{6vApDJ+fOM z$cS1aw|KvUsxv}N31gM;D@{H=yFP!oUzoNFtC-mWgc=)jjHQvXVln%tIWGuvR4?%f zs%Gtqk$fwzFL;0LLD8M2-^7JW@_I;X2jq#6m{*+W!vhVGv|P4I!VCMqDBOamdayM$ zqqwH}922txtj>gY5I#{voOjD(_mNDJ8`0Y>mk^A-C_^#PNPBK#Ckz{z22B^e`@%KY zXs9^V`;QSkwXd~pQMQv~R2`)4>#htv%OO;1;$*}KgT(zCgA`WB0p)TQH1q>yreL>^ z&+r}Di**(o1vT9ai*S3xmZjgdLSOgg3NNx7`#w?b21pynjXG5NP{gH@FaPkoHk?@w zDQCi7kjgUaR)9i&SC;(ar7x@-peCy5Bc(G4`DM7ROJOEP9AJp&V zu@!0{Ryf`d|INuE+U*YORSi{?jX`XXv`c~L(*Wp(Jt?aT6H&yT{o!|8e!K^x3&!g^ zs1f_Yz_%JGp*z1de}MW(H5&5);Uvrziy$4yxmyV{1vwr@ zlFT>a$(DD&T^URK+d9Rb%)0i8I(PjGlC6OQ8DgGMB#xAiPet8Sl1f;7yO-tzl+{^K zciM0d$@_Q@{^W}wNObIaL#(Gpo+2ckOxneHC=0F$Ww)ZfbAIYQ$pDn;LXg@p6eKJs ze9<@y6J!093Ewz^p}0I$H}K?tP66Qct)3C9yye>Q?eNe&iQqM;6q(2;VW1$=_}a;C z#?ct!Wv^pe>7Z}l?byFRcWMbShCoV(!4b_tGCgAlwdPB~P3=zUx)>d{qVI=LSuhUF z&Bqo={f!pjYcz{2B(S9w|FES^I1p@{YA8@)(<=NtP7&{0zY4$whi@&9>F2UE^2T7% z;;Tsr%dEc#bvIIN`+0!SBWHwGXO$s1O&|PL<;{@kIyG^bRzW2W)gaUezpxi81pD4qezGR})7c#Re%q#9SlPtG zQ0miodKPB-^+NvHC#E_y0o)aK25iiWHHMjDQi01JcW zJKWMBG=Tz&0`>iFFwW#=T8&;U|J`3%=sI$Oq}=uPoOt|0N4QOiEEKga%)+9@SRE&^ z7NbqFW=WdK9Nt>O;hR3;e;!EN8-n7$@6Z<6#ZguF@)YSr(w-G!wfTvaNsiMAo# zJb*Ui9Cka23yki*LThavVzAi(P(}NIWZO|G3g+u^=N+kcg$nUBc%)n$N z`ecQedqFH}ay~+9j!Y^kK|zd)##4qlWPM1CFxjD^%WNbz0TF@akjpwv~*oROBs+FoGw;^=wD&)E7BK1YdHT}1cr zIGzwONN60i6;dNkg5#{uPocJx6l zX-3AhO(WOZEO?#vtkDM|vX)0%Xc?*ZIuK>CEEBf}!C_C>6?FePzL<3nL-j zSt@2{PG8MCV>L9T8h4}TO;?)2mwqTSv0LY>FHCd|z=rZ>CbjfOm}hWYd`?j}!jfK^ zO$|Dux%Xr-3089&MYi#a7beZCb-pyefoWw>B1>~yxEA+3)0O--%e7LW&=O#k^}fJV z>qvi^QKeZHf|Bo4+QeyZ&LL2NFUFyE?A=`i;a{V(M#e)oJ20*v^F8jBGjDn%?TNqz zyqCC=RNSSrjne!vUr@&}Dz_~%48fc~o*usIeECi@CM3^63H3d;CUme@&rC^9+EhW3 z8E&^?NWc6m?l~_EV4B$<_24>H)>BFj5g=jN$vCzG=L=C~!tXEqilg}M7>;(AgBa$Vrl&%-5wY$UF(4pYz$F4zpLsUAyP?(EN1VtwZlCHRz1&leO$H@2HQ5X zzvqyUT9~;l%wVVWGjk81m#%?%zN|~7*n6>TKuZ6K-9mVY%I1+YC;K$5r*kI0hjoPD z%>GP-EF*9U)!VsQzcd>aM?`});~z3D3#N1&P`&Chbn=MTbs{y8aC#*TXBe#CHm4;}TKy%e@5ys$h*`v~CRt+Y$Ra1X# zRX7HsFC$bzXCVEOz2eJvyT{_XLa+>DNJr(;ngZ*T;<3ponbHi` zm#eQo(vjxg73BbP%tznrX@CSB&`eaMLUT+D2Xu)CWzQ7=jCcPLJfsHP#WgIp3l5!yHvBBUrBkz?oG$>U z-EeKWFSDLW%L`=G+VEhu-`5pA;ddMdu)hIIpL}S7%OQ2^hA}@Mc)Rj8XpMUv7-L*& zkbgX6g~8{uN5YWbyGs}Uupol9J=oy!z!C<((JCvzY1v4JrQ&rnPIm|U%vGiYo?q|I zzp2i-OS19fSbA5T5`{W{x|GGZzPj^p?Qkbd%6sNaQ<^z;@XXB$YltiDy64B%XEg8U zcXt2ORDeklk*+Ec1yGTy`t&1PTKlwWPywAl2N8kjE_VfjqpmAcgX@oZjzUWM5w4TX zgrHr))^$q%ra@gt;{~<);>@;*t`cJyjb=jcCMY-Zqar@EWoB+6pi7yR!;QJ-xxt!M z&dMDx`UeJABMZDPn$Nw_m)=*gGL)@UH@&~Y8ft?ui{D;aaUWJTDI?T(7S|2`<@u4m z>!GM-JQ8!K?)$6Qnq`LSJIh8u%#`^R$5J(YZpD{v3C2vld0r>}sk`IU+{{wF(o-s) zx^BKTUeg26<^xV&|K9mkaVfQjBQZU9Mrh)qL{>yFtrG?sv&mI40GnG`RYOGEZ8=9+E(21v5m^l!jaA@;Iz<&r| z!Up8W^c7j(=|bY`j9(a`@k8Bzw&_%wD}g865jG6dZ^C06)&(r!))_%0 zz*H)J1=p%E?lEF)L3_wcPWr*VF!&X4!%5JuxC4v$U;PTLL{!y&?ZkE5n@&51#)tbv zxv`P4aW&*KQjFw%)Lc9}Ucw7{d{nhA3ephl7NIQodP%5rRDv}1IO!7uweX#7>9^pA zjNj&rc!}k4#**iOOJTDwqw32Tb$A&u*QZNw`vVmx&&>QG8D0*ColG%0+{y&cP5i1q zuiI~Gj#j4l4@7ep^gG7#Ke^snYR-$^7K-~@lp#&AL5N3@^dns7Aup^4xnc2_1;1TL%JVG-;Q=PgOJC z3g!PzFTtbLgMQCCAG()(Y{Gc`s`rN}Jz`{rG=tYq&{K-{G)fkfM+}6ZzZ$E}`G&$^8my z5tRzDi8@5m3;_dK&%*QKLH>#2(C>H8kP*8R0vQSJ^=PL0D7r@2h_lcz%1%`&INxRn1L=t1FTM9so+%q(B27L!Ps(!W2+fXzCk#v{{1YsG z;o;Y*qnI^v;^3wzFso&-0^J}0n#O8d(lwE{@{cNcLh_P^?H2u29w69hrd&xjP<<4_ zrhOIW0qjMckEs0s;s;|$y_MeFT{M-wdI|a@T9p|7Tu#5u60%+R?8bzdX&9>g0m;2K zRoD--0X?))SD`NRvXFtxp1A8vKn@B-+`;*Yt~*U+_Qu1Hi0K-yh>fzpWYGs~gbmKh znqo{FaHZ9%Yia_HSQZ|i6SIb?7{8%I5j1Fd8du3>4GRs));KmOq)t!XsVypWFW0v= zcwxMqF7cVIm&UQtV?b-+>2rh|apGMxi#9Zi+$s7VAYE6(w>q?~;SNKdg8>d_=d?3E zvBToUNde}flH)z*0p!G!s*iCx$d`!IT(vT~F|MvK>tHP)_bRJd>iaxjU`BAWyq}%2 zc9vSZW;nzCTpbKgcelaR)17y6fl6bfEbz|25S6{3BUNc5E>%89Xsr!B{2{8@hMc zmQ;%&0?7>x+Y=Q-bAbCeF8#1311RrVbc;Jdf1Q|ZWw#lJv(c4UZ2iT2FWm z(|co62W;peltRM&j{R|Tr?;YoVew^9#0v&#G;!PDu-0?yW*qQ@Uk^;o{;U=glm?{n zS?;H}b!#0hB7`1f&6jW6sf^wCw2y1LQ}knF2M~R3HUQ!ar>_?rtcS734{vLoZ+NVe z0HlOGBe_O)QP_%2#KW-9qNI&F6EEGTLc-cdEK-!ex+a)K8vfo03SKd3FJgBfH+X2L z7#AtELWzzqFm&9?zM!+uvi^iD@z9BEvL(@%622h>WXz;Xb~bBekRzIQ=fRi-m9c$5 zIJbq8hdNE@?j29Fh9=fJPqs^WxFXo#*}J}Wei|7u0T5P}u?0ZHSuMZR?@Xw@@zDc= zBWEpycLXEum%*8Wzp#|?B60`SL4gK2vd&MpjuKLdEmRi6xr>8>tu4$QqPTeu^Wwzr ze3&BR{~D-jh(}JU?Ww=xHSkp7N2#x0){TNku}bmsE!Zs;FTzCR2)+N+-j&8vy>{)= zR)*BB%sOm|2$@N#%u~orWojErks=jB3JE1?*t^UkMTQh*PKL~LDWb>@$vhnMvu?%S z^_+9QykFk;{hdDeWw*Qk_x~Q(y4JO>l?)Li5S}g^GbaeL`Jh7rCDh`K<29KK$(sZl zfXHJuhdE}8X*ST5Tq>adZd5quP%BOhm3I7Kb15BChEY+>uqm3ardWcwZD26FR0ReP z*k`ui1~#ZKw%pb^FeTc)YxfB9C__rwoOb{H70{cUmyjf4lVTu{GuNj?xEs7#aIOs z_Z}HD2mprx?QDNn>9w%@3D||a5DALW3g20uY{%aZzU1+jy$Sbiz>9Z=#6ZsR(@&Bh zoAD`Jbk<(AyTt?r_d^=02(bTlk*3Qpw0kSGe%2K4avKLCgjPzyynDvgvq!FrS(mcr z07!_gZMspt`gRCq>LlnyxNFr0wZ6-x8=g!8sM&2IUvyNn^rfz7xEVjnnW_YWoc*PA zR7P7js{2bWHOd6t6tD+sLpejGEQKYhQg}bm0syzCd0fag<=TEZn9+G==9%K z!ca0IxN|KOi|EzlFsHrAO2^k2T;!{EwTt?Xz;2dUW;+XCe0iI|L@kPF#d|Nz+gW`) z>;O;_A8V3P(w(`W5{F^{Gzs!ro=#9?n@}CuyG(td!4-CewXG6h9J)fozIQE~c(K`c zsN|6cOd{^1-8c3T>N+NkZGcQvF`Jii@;F3!5J!PJg>f`Bpy3C|DXwWk?3KAVeeGEa zO>4eq&a-3niV)8_>SZ}8yuAu&_3tX@~Np&pim<)Ya)=TbV7F&is{^A@`r;(+I!ux0+gb&DcA_& z$(hn65ijOvk{2w_^)|$-?@Qz#7@u0I5rG!|4w^Lm#lywF%{UN(zU^|^4`ckjBoba# zp^ET8qm@V^j13t}+6QDJT(wvTa8)_?M;`P{&qm5}{JmXo`C9?q%bkIte2n8HgZHyL zMZS25z*s~^O}-8TM(! z`S0`#n|?ma8P$}gQC9CVHdNZ6Te0xx19z6J)b{fqOcvr&cV%)~u-;__)%#;dNZoVa zh`}z(3r6eC90Jez1`u8~;3%%P4r9Nb2TVx5;9#ueyly=G-dmCUVcHgiYeIFgAub?161Ny7@K10oC0^GJ2OG3NNli6(P$J_d-#8aEQ}wO#1BsX}S8^8LrQx#&@PJ3o@wA zCI!)P1olde%v?}-&bJBAM6%O__YsX3%sM004vB9nlP9u2`*S$ENBcZj9eCBlk(YyW zip#bs&A=p%*P?Zy$oEpR+?oZ}19HR4H9V15Ln`~G=F6lvnFRLs?OC8-bu;4b^`x44 zOY?b;)VGMaw0F8EuBc5%C(>i+lY%T@UAzK7AK(=|@nlY8 z*WXa4omcipP@~D5T|VGGu}+nZ9?FwM&9#HGg4n#gUcg4*n+n{Ch=`Cd8XR<)xff~E z^{s?W$>Sc8c#zOq{?f$~^cfcUsB@3hpagA|=P59P%28%tvb$!!?et*Y{QKg*>@U?j z$wD2t4`2z^4>(0b8m9W6`CuBl#fg6yb&$enI|0H|oRL2An zwV`Gr#aMabKzi($(Vkp}5!Vb?Cnt#oQ#h%lJhqwxN6T$MAffBLU^C3#yC9oz`bL== zZJlUn=qW@&t&2nl8~a0Uhj)RC&~imsc6$xkHVlB`l%~P=A_g8YGX{1~l9Zk_FB}VB z%{`qU>%?nd;WAucuU9+F2^emQ<^BsZ5BAdaI*vuXcwCvI(`oHBpLM`JD(Yvcv9C|= zO&|1Dzx((UFENczjInXr)$dvg>#(tzn+X3<-Xml-jkA`c?Lts>R(I6-B2;O;r}k|Gu;r(tnnoQB zbA#5bsSoEG)e<4-8j&F`l;t+WS2CMZ$+7$O`AI0(vvgU@?}8hxA!c*qiLIO77KfH* z27MpyYzea=1-poAPSn<0X>oN6_taS#b|kidwLWTh@cIH>PY|7C+B+H7w~vkJnJLT% z>=5{@r*}Wl!ZC4Q^7b>vf1WSv3ltYvs7NfX{YLtpDV>n{9q*Bw+wa`k)V@0lST%M` zgp^%^6`>XR%<++Jd&71dJv?4|db4?T;;R;(5IH<&$F&(Tppm?UTAJFpvrwF{G~)6U zg(AeMV4$MNsDK~7!TI5H)cUQDSZ*%XucsuDPK?y676{jGh4}4SwR(X>eXc*=WmO%= zy#+gu(aeXN;=rLPfQkOoqo;o@GV;RnWP@~BCtWtwQBA)A9jmIthEmuD7LnrbrVtOH zUF|x40`ic%q&-Q?Ney5gG{B=bB&&ZG9lg}sv*l^=3688ZTNz_4oqRIkqV(w}rrrH^|G%H|2DTDC8(Da^l4*tG0P_gIZ1%Vf$lP+|+ z26A7(pGJZQe5|{Hgkf;u`KK!#&+Nkvawj+ieTQ8(M6I`Sd45+=T?aT_wmn+(t{PZ% z%%mBa^s@tBPn~svg7G9?Huhj62)KubsmJyqO&km!(CNG2Rr3wg>X6yb;4cDa&j&br zTL6^kC(&714SZcuCv1*1NbafxF%zwl!7aa|>Uxv_x#tA=WH8TTUAmY3OgQy?zQ9Vm z>C&0Jg$A|4^f~kN1W#LSO(x%?qBG-7;1E23ya9~}u7ogDk`%(tEapsQzetpo!>jN* zl@R?HabP|Rei7~(>usym$=dd)Q#FKBm}$GAdS6>-WD_h@WMu{I*3yrv1;-b7HrpP# z$Wr(hD}k{9s_{A8wp3_%1#(_eu@YuvSBzmTZ_e-*SI3cA8W2PS@H(NnH!AV3Ju+Wi z6o$|CWF*HKu$urTCrIuZXiSEbTga+92tad}&gbIU-YdX%3_vxL;<0yQD3)Iul%TMK zD!*;a0QV&A8+-l4OZS>54Esz%FamR58l!u}mx}VjS!x_=g3I98E zFu(tUp<|#pJV54F4VLoGvwQqq_+W5*7YFz6zHt&bkchk$6Yd5wERfO06O+6rHTJ%w z{T8SY03`D&_3EH34!040VHv$=dW2f2uclERh~Fp+=#sQ^}J8!gSw9nmXq`l~_c?o_URg%WNvm4YuG zfi9qVbnjGUj@Kf|p;S_9)O*L~Y{RQ?(k{8tNGP$WQE>aUNAmDQl#k?#7yG`iF;nD* zCI1AFG|K&0EFj$L5DH#oH|F5gq&$@H4$d^zyt+A7g&y&Odiqi!CF*OozK~ekNGcV$ zS{^m5yIO&?+Y;Q}9OJ#uzF})dHbg;fS3dwmt<0<}pOJDyVqt%$qXbVCB+^HvZ9}I+ z`-Lfx*UPhtAV#WJ@!aeT>?uA>qf$XWAY`$P9RUHh6gEF1+ThWu*7;LSPb+{x;!I!sMkUor~`KgZL-VMhfd zt6gQuM-k+^ErGY7v;lAB*ks1fN9D&*Vgc?RB6`=?8(B~D$E7d|>hP1+x1oA<@q~z0A3gG$;1PqOizG4rlSFF>wQxjkjYvOvSCLn?0lD$tRA;h7 z2q`@cqbQ*PRq;M3H33vl-skoCDI+6X(|>k@Senf9Vl4Fpi!H25*f@Saq z^c^0+B8j=rLCu3@f&cm9%sn^qh2(n>)<@OHxQRD(lE>=g5e+e;;QMq*?o@BU!D(W16J$3vbKB3>3K-3;rw*xWXKaJc5n7Py$1W9-D4mTyC!Hwc`}e6A(b zPX#+l&Q8hG_^MxCQWMudy%2r4=fwM`pqnFc06^dTk6NONFyb4I^BIuste-$*WpOMn z&AyQj>cU+o(j^tg*6ES{X|PObno`UUgNb1ar&~HfBup8HYB%fkY;hr5U;#+i6~z@5 zMe0DKrW_JQfClLAwL@!#&kJPY#5Yz^_-|Fvz>xvY=V}(8!0E0g#;vvn-z#7?oQR{+ zL(-6nEW}mh5p04HjD_$U%b}Gb*!w`jC0?GFcx8I5RpYZ5k`+Jrni8=@CHNHIO9x)h zmdE2r)pxCg`XY{c>7j|y+(snB*`Naf&9<47m^FQJsl+a&>t=?fUz*6_pe zq2Aa_Hl9(#8TqijBJAFiun{)Ri#6o8z@9>8;T)&l5nqh{Vp~0SN4Eb%TWc+F=Fpbu zP+kJn-?oLpNMb8tiJ?4WSLxn$oejHWclcj?4gFasbESde@qdB}o!CigjJ^=o5J%Du zvJxs`fJ$G^uo${B|LT>wnQ3;R(H^c=>ivgqY8A+P`5@}2wgOz|yC0`PlxqLav~gKW z5tEc~A-0n0t|pu;ai-zmC|o2Oim6 zP6ssY4bY zjg5`uKD~1rY+6Rp9Zo)F5l-Go7v-_gbA?2gYIVAvuApq^6Q#jy)HSJTz#`Oj0P6{% zWcwkGdY!gpEM0SM8j(RicncL+IY0)~NRZ{M;yjSKgH~J5$AQwQGcfM=gJoG|Ieo=% za*}$^#p^4xU3l>~%=p14msj}|!7;|98stK@46rm%v`UEeTnf7au%*tpHiK*oeYp7% zC9#FLP-){pdmji#NeC7Jg^Zs)^S1C|W67GBwRto4FnH*`!4=gZ66z%ry>sgq<#BN z4~m)-0WJ}A1DM?9;|U=d0T9Ks^b#p-ed+Qr81PX@dA3Kn8dK^3v3&ulin|##yDGK& zB~7~5Pdn4kogHj%JgRCigTED4z4GFp>BihaFkeH5bnSz(P*1)M^3w3X0DtMb+O`D~;F^9X3Z+55XG z5^;ZY^Z^oF?k+Y1YG?|^T7|Z_vrUF{ec7&E+&x%0CDVH*32du{uxCV7am+&uh20d3 z1N8!&Mn;#X!i-E(KGB!NsL~TZYE&U%EQ%g3cP@H`&zUvWz4@Fnzdi>Zb_7UgpaPO8 zHAdA-_qf3|qE5z__P4PuMuEu8Cr3LKXdmoK(;WX(;pJ)bU!Da0aRA%meUpt4<|d=M z(W%>yPltswKct}J{CF`?TuZNe7N{7)7FjS0bOg8V? zx5?A(pxBs;-;qBAcV9zAbqUBwqtx8kg)KT!d-K%gFnASwmlB2XiQYsE4yK?fhQhIr zSpMTMnk3bQM9SDEyb10CZ1>OK*qGD|qtjlo`j{#^VG=0fXXvRnn4#~_L|ryuZf;VS zX63=|uL;Yb9B#BA-asOAFjQJC!_d){(iCq#!P@=XVEXFAVBT_!>q(^l!W6tAKqYUK zf_b>OsBq@co8FtGU2tsT#NbJSTX7F5EZ4=MyaDGqYt|a7b@b<75llMaeE1{f4(0u{ zxDb|4eKDpUP+BHVh0D6&w--M9Z80s2SO5MwYGgBvpG&U7EiyxCu>*@gto|FCS^mer zG1wPhvGBjamBr7|k5jNis)kqps{Q_R6fgcavb4OUKfQpx*u@Yl_xsXBlk^wZ(9+im z5%Z{wL)q=Ob+h>EU!XfnU%S`^({x=XVC45jfClX^psnSv{YY%acK^OR(8&D-IJLZh zejuW_MaBK)#(uuVvKO^D?Z5u`-!XxzAJivTtf2m+p{(%JM)-F!+yAdl=8sMO?*#rk zfxjO_^8X&EzlJLQzukw<5ZY0aSQI^6zIvD5ae!{to2n>UqY)RtwF3TVsOTuCC|X_m E4+FGCJ^%m! literal 0 HcmV?d00001 From 7db7fccaecbddf49ff96e9396799cd2d16394112 Mon Sep 17 00:00:00 2001 From: maynemei Date: Sun, 23 Mar 2025 12:40:57 -0700 Subject: [PATCH 28/36] rebased again and resolved comments --- core/lsu/LSU.cpp | 4 ++-- docs/design_document_template/LSU.adoc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/lsu/LSU.cpp b/core/lsu/LSU.cpp index 8b115a43..d7ac8cf2 100644 --- a/core/lsu/LSU.cpp +++ b/core/lsu/LSU.cpp @@ -33,7 +33,7 @@ namespace olympia cache_read_stage_(cache_lookup_stage_ + 1), // Get data from the cache in the cycle after cache lookup complete_stage_( - cache_read_stage_ + cache_read_stage_ + p->cache_read_stage_length), // Complete stage is after the cache read stage ldst_pipeline_("LoadStorePipeline", (complete_stage_ + 1), getClock()), // complete_stage_ + 1 is number of stages @@ -505,7 +505,7 @@ namespace olympia return; } - // Loads dont perform a cache lookup if there are older stores haven't issued in the load store queue + // Loads don't perform a cache lookup if there are older stores haven't issued in the load store queue if (!inst_ptr->isStoreInst() && !allOlderStoresIssued_(inst_ptr) && allow_speculative_load_exec_) { diff --git a/docs/design_document_template/LSU.adoc b/docs/design_document_template/LSU.adoc index 1de0db21..c9896fb7 100644 --- a/docs/design_document_template/LSU.adoc +++ b/docs/design_document_template/LSU.adoc @@ -45,7 +45,7 @@ == OVERVIEW -The Load Store Unit (LSU) implements the memory interface for the Olympia processor, managing all load and store operations. It features multiple parallel pipelines, data forwarding capabilities, and ensures memory consistency while maintaining high performance through careful hazard management and efficient queueing structures. +The Load Store Unit (LSU) implements the memory interface for the Olympia processor. Managing all load and store operations. The LSU features multiple parallel pipelines, data forwarding capabilities, and ensures memory consistency while maintaining high performance through careful hazard management and efficient queueing structures. === Overview Block Diagram From 423ba58d6abe57a70d98e92b58b7fddf778458e8 Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Mon, 7 Apr 2025 14:10:58 -0700 Subject: [PATCH 29/36] Update core/lsu/LSU.hpp Co-authored-by: Knute Lingaard (MIPS) <155678575+knute-mips@users.noreply.github.com> Signed-off-by: MayneMei <69469280+MayneMei@users.noreply.github.com> --- core/lsu/LSU.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lsu/LSU.hpp b/core/lsu/LSU.hpp index ccb0b2ad..696e6747 100644 --- a/core/lsu/LSU.hpp +++ b/core/lsu/LSU.hpp @@ -78,7 +78,7 @@ namespace olympia static const char name[]; // return allow_data_forwarding for test - bool allow_data_forwarding_ex() { + bool allowDataForwardingEX() const { return allow_data_forwarding_; } From 69a3e4105221450a5d974b5ff96b9a559920f74c Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Mon, 7 Apr 2025 14:12:50 -0700 Subject: [PATCH 30/36] Update Lsu_test.cpp Signed-off-by: MayneMei <69469280+MayneMei@users.noreply.github.com> --- test/core/lsu/Lsu_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core/lsu/Lsu_test.cpp b/test/core/lsu/Lsu_test.cpp index 58652a7f..8ad1bebd 100644 --- a/test/core/lsu/Lsu_test.cpp +++ b/test/core/lsu/Lsu_test.cpp @@ -109,7 +109,7 @@ void runTest(int argc, char **argv) olympia::LSUTester lsupipe_tester; lsupipe_tester.test_pipeline_stages(*my_lsu); - if(my_lsu->allow_data_forwarding_ex()) { + if(my_lsu->allowDataForwardingEX()) { // Data forwarding enabled case std::cout << "allow data forwarding " << "\n";; // First store From 82ce2e76290be9bcf75d0a625bf701103bf1a1cf Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Sun, 18 May 2025 06:20:12 +0000 Subject: [PATCH 31/36] modifying forward logic --- core/Inst.hpp | 3 + core/lsu/LSU.cpp | 72 ++++++++++++++++--- core/lsu/LSU.hpp | 4 +- docs/design_document_template/LSU.adoc | 95 +++++++++++++++++++------- 4 files changed, 138 insertions(+), 36 deletions(-) diff --git a/core/Inst.hpp b/core/Inst.hpp index a5987f4d..c1556940 100644 --- a/core/Inst.hpp +++ b/core/Inst.hpp @@ -259,6 +259,9 @@ namespace olympia uint32_t getOpCode() const { return static_cast(opcode_info_->getOpcode()); } + // Get the data size in bytes + uint32_t getMemAccessSize() const { opcode_info->getDataSize() / 8; } // opcode_info's data size is in bits + mavis::InstructionUniqueID getMavisUid() const { return opcode_info_->getInstructionUniqueID(); diff --git a/core/lsu/LSU.cpp b/core/lsu/LSU.cpp index d7ac8cf2..2424e2cd 100644 --- a/core/lsu/LSU.cpp +++ b/core/lsu/LSU.cpp @@ -524,7 +524,10 @@ namespace olympia if (!inst_ptr->isStoreInst() && allow_data_forwarding_) { const uint64_t load_addr = inst_ptr->getTargetVAddr(); - auto forwarding_store = findYoungestMatchingStore_(load_addr); + const uint32_t load_size = inst_ptr->getMemAccessSize(); + + // passing both load address and load size + auto forwarding_store = tryStoreToLoadForwarding(inst_ptr) if (forwarding_store) { @@ -943,17 +946,70 @@ namespace olympia ILOG("Store added to store buffer: " << inst_ptr); } - LoadStoreInstInfoPtr LSU::findYoungestMatchingStore_(const uint64_t addr) const + bool LSU::tryStoreToLoadForwarding(const InstPtr& load_inst_ptr) const { - LoadStoreInstInfoPtr matching_store = nullptr; + const uint64_t load_addr = load_inst_ptr->getTargetVAddr(); + const uint32_t load_size = load_inst_ptr->getMemAccessSize(); + + // A load must have a non-zero size to access memory. + if (load_size == 0) { + return false; + } + + std::vector coverage_mask(load_size, false); + uint32_t bytes_covered_count = 0; + + // Iterate through the store_buffer_ from youngest to oldest. + // This ensures that if multiple stores write to the same byte, + // the data from the youngest store is effectively used. + for (auto it = store_buffer_.rbegin(); it != store_buffer_.rend(); ++it) + { + const auto& store_info_ptr = *it; // LoadStoreInstInfoPtr + const InstPtr& store_inst_ptr = store_info_ptr->getInstPtr(); + + const uint64_t store_addr = store_inst_ptr->getTargetVAddr(); + const uint32_t store_size = store_inst_ptr->getMemAccessSize(); + + if (store_size == 0) { + continue; // Skip stores that don't actually write data. + } + + // Determine the overlapping region [overlap_start_addr, overlap_end_addr) + // The overlap is in terms of global memory addresses. + uint64_t overlap_start_addr = std::max(load_addr, store_addr); + uint64_t overlap_end_addr = std::min(load_addr + load_size, store_addr + store_size); - auto it = std::find_if(store_buffer_.rbegin(), store_buffer_.rend(), - [addr](const auto& store) { - return store->getInstPtr()->getTargetVAddr() == addr; - }); - return (it != store_buffer_.rend()) ? *it : nullptr; + // If there's an actual overlap (i.e., the range is not empty) + if (overlap_start_addr < overlap_end_addr) + { + // Iterate over the bytes *within the load's address range* that this store covers. + for (uint64_t current_byte_global_addr = overlap_start_addr; current_byte_global_addr < overlap_end_addr; ++current_byte_global_addr) + { + // Calculate the index of this byte relative to the load's start address. + // This index is used for the coverage_mask. + uint32_t load_byte_idx = static_cast(current_byte_global_addr - load_addr); + + // If this byte within the load's coverage_mask hasn't been marked true yet + // (meaning it hasn't been covered by an even younger store), mark it. + if (!coverage_mask[load_byte_idx]) + { + coverage_mask[load_byte_idx] = true; + bytes_covered_count++; + } + } + } + + // If all bytes of the load are now covered, no need to check even older stores + if (bytes_covered_count == load_size) { + break; + } + } + + // Check if all bytes required by the load were covered by stores in the buffer. + return bytes_covered_count == load_size; } + LoadStoreInstInfoPtr LSU::getOldestStore_() const { if(store_buffer_.empty()) { diff --git a/core/lsu/LSU.hpp b/core/lsu/LSU.hpp index 696e6747..ea20adef 100644 --- a/core/lsu/LSU.hpp +++ b/core/lsu/LSU.hpp @@ -275,8 +275,8 @@ namespace olympia // allocate store inst to store buffer void allocateInstToStoreBuffer_(const InstPtr & inst_ptr); - // Search store buffer in FIFO order for youngest matching store - LoadStoreInstInfoPtr findYoungestMatchingStore_(const uint64_t addr) const ; + // check whether load inst could be forwarded by store + bool tryStoreToLoadForwarding(const InstPtr& load_inst_ptr) const ; // get oldest store LoadStoreInstInfoPtr getOldestStore_() const; diff --git a/docs/design_document_template/LSU.adoc b/docs/design_document_template/LSU.adoc index c9896fb7..c277686b 100644 --- a/docs/design_document_template/LSU.adoc +++ b/docs/design_document_template/LSU.adoc @@ -140,34 +140,77 @@ LoadStoreInstInfoPtr findYoungestMatchingStore_(const uint64_t addr) const; The data forwarding logic is implemented through the store buffer and involves: 1. *Store Buffer Search* -[source,cpp] ----- -LoadStoreInstInfoPtr findYoungestMatchingStore_(const uint64_t addr) const { - auto it = std::find_if(store_buffer_.rbegin(), store_buffer_.rend(), - [addr](const auto& store) { - return store->getInstPtr()->getTargetVAddr() == addr; - }); - return (it != store_buffer_.rend()) ? *it : nullptr; -} ----- + [source,cpp] + ---- + LoadStoreInstInfoPtr findYoungestMatchingStore_(const uint64_t addr) const { +     auto it = std::find_if(store_buffer_.rbegin(), store_buffer_.rend(), +                           [addr](const auto& store) { +                               return store->getInstPtr()->getTargetVAddr() == addr; +                           }); +     return (it != store_buffer_.rend()) ? *it : nullptr; + } + ---- + This initial search method identifies the youngest store that matches a specific address. However, for comprehensive forwarding, especially when a load might be covered by multiple stores or a store might partially cover a load, a more detailed check is required. 2. *Forward Detection* -[source,cpp] ----- -void handleCacheLookupReq_() { - // ... - if (!inst_ptr->isStoreInst() && allow_data_forwarding_) { - const uint64_t load_addr = inst_ptr->getTargetVAddr(); - auto forwarding_store = findYoungestMatchingStore_(load_addr); - if (forwarding_store) { - mem_access_info_ptr->setDataReady(true); - mem_access_info_ptr->setCacheState(MemoryAccessInfo::CacheState::HIT); - return; - } - } - // ... -} ----- + [source,cpp] + ---- + void handleCacheLookupReq_() { +     // ... +     if (!inst_ptr->isStoreInst() && allow_data_forwarding_) { +         const uint64_t load_addr = inst_ptr->getTargetVAddr(); + // The original findYoungestMatchingStore_ might be a quick initial check +         // auto forwarding_store = findYoungestMatchingStore_(load_addr); + // A more comprehensive check like tryStoreToLoadForwarding is needed for full/partial coverage: +         if (tryStoreToLoadForwarding(inst_ptr, mem_access_info_ptr)) { // <1> +             mem_access_info_ptr->setDataReady(true); +             mem_access_info_ptr->setCacheState(MemoryAccessInfo::CacheState::HIT); +             return; +         } +     } +     // ... + } + ---- + <1> `tryStoreToLoadForwarding` is a more detailed function to check full data coverage. + +[[Comprehensive_Forwarding_Check]] +==== Comprehensive Forwarding Check (tryStoreToLoadForwarding) + +To determine if a load instruction can receive all its data from the `store_buffer_`, a more thorough check is performed. This mechanism is crucial for performance, as it can prevent stalls by bypassing cache access if data is available more locally. + +The process involves the following steps: + +1. *Initialization*: + * The load's target virtual address (`load_addr`) and its size (`load_size`) are obtained. + * A `coverage_mask` (typically a `std::vector` or a bitmask like `uint64_t` for smaller loads) is created with a size equal to `load_size`. Each element/bit corresponds to a byte of the load, initialized to indicate "not covered." + * A counter, `bytes_covered_count`, is initialized to zero. + +2. *Store Buffer Iteration*: + * The `store_buffer_` is iterated from the youngest (most recently added) store to the oldest. This order is critical to ensure that if multiple stores write to the same byte, the data from the youngest store is considered for forwarding. + * For each store in the buffer: + * The store's virtual address (`store_addr`) and size (`store_size`) are retrieved. + * The overlapping memory region between the current load and the store is calculated: + * `overlap_start_addr = std::max(load_addr, store_addr)` + * `overlap_end_addr = std::min(load_addr + load_size, store_addr + store_size)` + * If `overlap_start_addr < overlap_end_addr`, an actual overlap exists. + +3. *Updating Coverage*: + * For each byte within the calculated overlapping region: + * The byte's 0-based index relative to the `load_addr` (`load_byte_idx`) is determined. + * If the `coverage_mask` indicates that this `load_byte_idx` has *not* yet been covered by an even younger store (i.e., `!coverage_mask[load_byte_idx]`): + * The `coverage_mask` for `load_byte_idx` is marked as "covered" (e.g., set to `true`). + * `bytes_covered_count` is incremented. + * This ensures each byte of the load is counted only once towards being covered, and by the youngest store that provides it. + +4. *Early Exit Optimization*: + * If, after processing any store, `bytes_covered_count` becomes equal to `load_size`, it means all bytes of the load are covered. The iteration through the `store_buffer_` can stop early, as no further stores need to be checked. + +5. *Final Determination*: + * After iterating through the necessary stores, if `bytes_covered_count == load_size`, the load can be fully forwarded. + * In this case, the load's `MemoryAccessInfoPtr` is updated: `setDataReady(true)` and `setCacheState(MemoryAccessInfo::CacheState::HIT)`. The LSU can then proceed as if the data was instantaneously available, bypassing a full cache lookup for the data itself. + * If `bytes_covered_count < load_size`, the load cannot be fully forwarded from the store buffer and must proceed to the cache for the remaining (or all) data. + +This comprehensive check allows the LSU to forward data even if it's sourced from multiple (potentially partial) stores, significantly improving performance for common RAW (Read-After-Write) hazard scenarios. === Multi-Pipeline Design From 411ef1c14010cfd895ec87f9be2bd05b584fc035 Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Sun, 18 May 2025 06:26:56 +0000 Subject: [PATCH 32/36] grammar issue fixed --- core/Inst.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/Inst.hpp b/core/Inst.hpp index c1556940..b015e97b 100644 --- a/core/Inst.hpp +++ b/core/Inst.hpp @@ -260,7 +260,7 @@ namespace olympia uint32_t getOpCode() const { return static_cast(opcode_info_->getOpcode()); } // Get the data size in bytes - uint32_t getMemAccessSize() const { opcode_info->getDataSize() / 8; } // opcode_info's data size is in bits + uint32_t getMemAccessSize() const { opcode_info_->getDataSize() / 8; } // opcode_info's data size is in bits mavis::InstructionUniqueID getMavisUid() const { From 4994499fa41a2ec23c96e8b6c104f001030d5bda Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Sun, 18 May 2025 06:30:22 +0000 Subject: [PATCH 33/36] grammar issue fixed --- core/Inst.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/Inst.hpp b/core/Inst.hpp index b015e97b..94cf26ea 100644 --- a/core/Inst.hpp +++ b/core/Inst.hpp @@ -260,7 +260,7 @@ namespace olympia uint32_t getOpCode() const { return static_cast(opcode_info_->getOpcode()); } // Get the data size in bytes - uint32_t getMemAccessSize() const { opcode_info_->getDataSize() / 8; } // opcode_info's data size is in bits + uint32_t getMemAccessSize() const { return opcode_info_->getDataSize() / 8; } // opcode_info's data size is in bits mavis::InstructionUniqueID getMavisUid() const { From 711f66ab843eddd5728de39a676675ea0fd502a9 Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Sun, 18 May 2025 06:39:47 +0000 Subject: [PATCH 34/36] grammar issue fixed --- core/Inst.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/Inst.hpp b/core/Inst.hpp index 94cf26ea..8ae06469 100644 --- a/core/Inst.hpp +++ b/core/Inst.hpp @@ -260,7 +260,7 @@ namespace olympia uint32_t getOpCode() const { return static_cast(opcode_info_->getOpcode()); } // Get the data size in bytes - uint32_t getMemAccessSize() const { return opcode_info_->getDataSize() / 8; } // opcode_info's data size is in bits + uint32_t getMemAccessSize() const { return static_cast(opcode_info_->getDataSize() / 8); } // opcode_info's data size is in bits mavis::InstructionUniqueID getMavisUid() const { From 287b102e7f1899869511a7ba67bb3ba49a9362f0 Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Tue, 20 May 2025 05:55:06 +0000 Subject: [PATCH 35/36] fixed syntax error --- core/lsu/LSU.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lsu/LSU.cpp b/core/lsu/LSU.cpp index 2424e2cd..168391b8 100644 --- a/core/lsu/LSU.cpp +++ b/core/lsu/LSU.cpp @@ -527,7 +527,7 @@ namespace olympia const uint32_t load_size = inst_ptr->getMemAccessSize(); // passing both load address and load size - auto forwarding_store = tryStoreToLoadForwarding(inst_ptr) + auto forwarding_store = tryStoreToLoadForwarding(inst_ptr); if (forwarding_store) { From 4d1270a18398ad315cc81e6e8ceb653919fff069 Mon Sep 17 00:00:00 2001 From: MayneMei <69469280+MayneMei@users.noreply.github.com> Date: Tue, 20 May 2025 06:01:50 +0000 Subject: [PATCH 36/36] deleted unused variable --- core/lsu/LSU.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/lsu/LSU.cpp b/core/lsu/LSU.cpp index 168391b8..642ac6c9 100644 --- a/core/lsu/LSU.cpp +++ b/core/lsu/LSU.cpp @@ -523,9 +523,6 @@ namespace olympia // Add store forwarding check here for loads if (!inst_ptr->isStoreInst() && allow_data_forwarding_) { - const uint64_t load_addr = inst_ptr->getTargetVAddr(); - const uint32_t load_size = inst_ptr->getMemAccessSize(); - // passing both load address and load size auto forwarding_store = tryStoreToLoadForwarding(inst_ptr);