Delete restored keys from hot archive

src/bucket/test/BucketTestUtils.cpp

@@ -235,15 +235,19 @@ LedgerManagerForBucketTests::transferLedgerEntriesToBucketList(
                         ltxEvictions, lh.ledgerSeq, keys, initialLedgerVers,
                         mApp.getLedgerManager()
                             .getSorobanNetworkConfigForApply());
-
+#ifdef ENABLE_NEXT_PROTOCOL_VERSION_UNSAFE_FOR_PRODUCTION
                 if (protocolVersionStartsFrom(
                         initialLedgerVers,
                         BucketBase::
                             FIRST_PROTOCOL_SUPPORTING_PERSISTENT_EVICTION))
                 {
+                    std::vector<LedgerKey> restoredKeys;
+                    ltx.getRestoredHotArchiveKeys(restoredKeys);
                     mApp.getBucketManager().addHotArchiveBatch(
-                        mApp, lh, evictedState.archivedEntries, {}, {});
+                        mApp, lh, evictedState.archivedEntries, restoredKeys,
+                        {});
                 }
+#endif
                 if (ledgerCloseMeta)
                 {
                     ledgerCloseMeta->populateEvictedEntries(evictedState);

src/ledger/LedgerManagerImpl.cpp

@@ -45,6 +45,7 @@
 
 #include <fmt/format.h>
 
+#include "xdr/Stellar-ledger-entries.h"
 #include "xdr/Stellar-ledger.h"
 #include "xdr/Stellar-transaction.h"
 #include "xdrpp/types.h"
@@ -1787,13 +1788,17 @@ LedgerManagerImpl::transferLedgerEntriesToBucketList(
                     ltxEvictions, lh.ledgerSeq, keys, initialLedgerVers,
                     *mSorobanNetworkConfigForApply);
 
+#ifdef ENABLE_NEXT_PROTOCOL_VERSION_UNSAFE_FOR_PRODUCTION
             if (protocolVersionStartsFrom(
                     initialLedgerVers,
                     BucketBase::FIRST_PROTOCOL_SUPPORTING_PERSISTENT_EVICTION))
             {
+                std::vector<LedgerKey> restoredKeys;
+                ltx.getRestoredHotArchiveKeys(restoredKeys);
                 mApp.getBucketManager().addHotArchiveBatch(
-                    mApp, lh, evictedState.archivedEntries, {}, {});
+                    mApp, lh, evictedState.archivedEntries, restoredKeys, {});
             }
+#endif
 
             if (ledgerCloseMeta)
             {

src/ledger/LedgerTxn.cpp

@@ -16,12 +16,14 @@
 #include "main/Application.h"
 #include "transactions/TransactionUtils.h"
 #include "util/GlobalChecks.h"
+#include "util/UnorderedSet.h"
 #include "util/types.h"
 #include "xdr/Stellar-ledger-entries.h"
 #include <Tracy.hpp>
 #include <soci.h>
 
 #include <algorithm>
+#include <stdexcept>
 
 namespace stellar
 {
@@ -501,14 +503,17 @@ LedgerTxn::Impl::commit() noexcept
     maybeUpdateLastModifiedThenInvokeThenSeal([&](EntryMap const& entries) {
         // getEntryIterator has the strong exception safety guarantee
         // commitChild has the strong exception safety guarantee
-        mParent.commitChild(getEntryIterator(entries), mConsistency);
+        mParent.commitChild(getEntryIterator(entries), mRestoredHotArchiveKeys,
+                            mConsistency);
     });
 }
 
 void
-LedgerTxn::commitChild(EntryIterator iter, LedgerTxnConsistency cons) noexcept
+LedgerTxn::commitChild(EntryIterator iter,
+                       UnorderedSet<LedgerKey> const& restoredHotArchiveKeys,
+                       LedgerTxnConsistency cons) noexcept
 {
-    getImpl()->commitChild(std::move(iter), cons);
+    getImpl()->commitChild(std::move(iter), restoredHotArchiveKeys, cons);
 }
 
 static LedgerTxnConsistency
@@ -526,8 +531,9 @@ joinConsistencyLevels(LedgerTxnConsistency c1, LedgerTxnConsistency c2)
 }
 
 void
-LedgerTxn::Impl::commitChild(EntryIterator iter,
-                             LedgerTxnConsistency cons) noexcept
+LedgerTxn::Impl::commitChild(
+    EntryIterator iter, UnorderedSet<LedgerKey> const& restoredHotArchiveKeys,
+    LedgerTxnConsistency cons) noexcept
 {
     // Assignment of xdrpp objects does not have the strong exception safety
     // guarantee, so use std::unique_ptr<...>::swap to achieve it
@@ -632,6 +638,15 @@ LedgerTxn::Impl::commitChild(EntryIterator iter,
         printErrorAndAbort("unknown fatal error during commit to LedgerTxn");
     }
 
+    for (auto const& key : restoredHotArchiveKeys)
+    {
+        auto [_, inserted] = mRestoredHotArchiveKeys.emplace(key);
+        if (!inserted)
+        {
+            printErrorAndAbort("restored hot archive entry already exists");
+        }
+    }
+
     // std::unique_ptr<...>::swap does not throw
     mHeader.swap(childHeader);
     mChild = nullptr;
@@ -802,6 +817,37 @@ LedgerTxn::Impl::erase(InternalLedgerKey const& key)
     }
 }
 
+void
+LedgerTxn::removeFromHotArchive(LedgerKey const& key)
+{
+    getImpl()->removeFromHotArchive(key);
+}
+
+void
+LedgerTxn::Impl::removeFromHotArchive(LedgerKey const& key)
+{
+    throwIfSealed();
+    throwIfChild();
+
+    if (!isPersistentEntry(key))
+    {
+        throw std::runtime_error("Key type not supported in Hot Archive");
+    }
+
+    auto restoredEntry = mEntry.find(key);
+    if (restoredEntry == mEntry.end() || !restoredEntry->second.isInit())
+    {
+        throw std::runtime_error(
+            "Restored entry must be created before Hot Archive removal");
+    }
+
+    auto [_, inserted] = mRestoredHotArchiveKeys.insert(key);
+    if (!inserted)
+    {
+        throw std::runtime_error("Key already removed from hot archive");
+    }
+}
+
 void
 LedgerTxn::eraseWithoutLoading(InternalLedgerKey const& key)
 {
@@ -1470,6 +1516,24 @@ LedgerTxn::Impl::getAllEntries(std::vector<LedgerEntry>& initEntries,
     deadEntries.swap(resDead);
 }
 
+void
+LedgerTxn::getRestoredHotArchiveKeys(
+    std::vector<LedgerKey>& restoredHotArchiveKeys)
+{
+    getImpl()->getRestoredHotArchiveKeys(restoredHotArchiveKeys);
+}
+
+void
+LedgerTxn::Impl::getRestoredHotArchiveKeys(
+    std::vector<LedgerKey>& restoredHotArchiveKeys)
+{
+    restoredHotArchiveKeys.reserve(mRestoredHotArchiveKeys.size());
+    for (auto const& key : mRestoredHotArchiveKeys)
+    {
+        restoredHotArchiveKeys.emplace_back(key);
+    }
+}
+
 LedgerKeySet
 LedgerTxn::getAllTTLKeysWithoutSealing() const
 {
@@ -1957,6 +2021,7 @@ LedgerTxn::Impl::rollback() noexcept
     }
 
     mEntry.clear();
+    mRestoredHotArchiveKeys.clear();
     mMultiOrderBook.clear();
     mActive.clear();
     mActiveHeader.reset();
@@ -2559,10 +2624,11 @@ LedgerTxnRoot::Impl::throwIfChild() const
 }
 
 void
-LedgerTxnRoot::commitChild(EntryIterator iter,
-                           LedgerTxnConsistency cons) noexcept
+LedgerTxnRoot::commitChild(
+    EntryIterator iter, UnorderedSet<LedgerKey> const& restoredHotArchiveKeys,
+    LedgerTxnConsistency cons) noexcept
 {
-    mImpl->commitChild(std::move(iter), cons);
+    mImpl->commitChild(std::move(iter), restoredHotArchiveKeys, cons);
 }
 
 static void
@@ -2618,8 +2684,9 @@ LedgerTxnRoot::Impl::bulkApply(BulkLedgerEntryChangeAccumulator& bleca,
 }
 
 void
-LedgerTxnRoot::Impl::commitChild(EntryIterator iter,
-                                 LedgerTxnConsistency cons) noexcept
+LedgerTxnRoot::Impl::commitChild(
+    EntryIterator iter, UnorderedSet<LedgerKey> const& restoredHotArchiveKeys,
+    LedgerTxnConsistency cons) noexcept
 {
     ZoneScoped;
 

src/ledger/LedgerTxn.h

@@ -420,8 +420,10 @@ class AbstractLedgerTxnParent
     // commitChild and rollbackChild are called by a child AbstractLedgerTxn
     // to trigger an atomic commit or an atomic rollback of the data stored in
     // the child.
-    virtual void commitChild(EntryIterator iter,
-                             LedgerTxnConsistency cons) noexcept = 0;
+    virtual void
+    commitChild(EntryIterator iter,
+                UnorderedSet<LedgerKey> const& restoredHotArchiveKeys,
+                LedgerTxnConsistency cons) noexcept = 0;
     virtual void rollbackChild() noexcept = 0;
 
     // getAllOffers, getBestOffer, and getOffersByAccountAndAsset are used to
@@ -539,10 +541,10 @@ class AbstractLedgerTxn : public AbstractLedgerTxnParent
     virtual void commit() noexcept = 0;
     virtual void rollback() noexcept = 0;
 
-    // loadHeader, create, erase, load, and loadWithoutRecord provide the main
-    // interface to interact with data stored in the AbstractLedgerTxn. These
-    // functions only allow one instance of a particular data to be active at a
-    // time.
+    // loadHeader, create, erase, load, loadWithoutRecord, and
+    // removeFromHotArchive provide the main interface to interact with data
+    // stored in the AbstractLedgerTxn. These functions only allow one instance
+    // of a particular data to be active at a time.
     // - loadHeader
     //     Loads the current LedgerHeader. Throws if there is already an active
     //     LedgerTxnHeader.
@@ -566,11 +568,17 @@ class AbstractLedgerTxn : public AbstractLedgerTxnParent
     //     then it will still be recorded after calling loadWithoutRecord.
     //     Throws if there is an active LedgerTxnEntry associated with this
     //     key.
+    // - removeFromHotArchive:
+    //     Indicates that an entry in the Hot Archive has been restored and must
+    //     be removed from the Hot Archive. Prior to this call, the key must
+    //     be created via a call to create. Throws if key has not been
+    //     previously created in this LedgerTxn.
     // All of these functions throw if the AbstractLedgerTxn is sealed or if
     // the AbstractLedgerTxn has a child.
     virtual LedgerTxnHeader loadHeader() = 0;
     virtual LedgerTxnEntry create(InternalLedgerEntry const& entry) = 0;
     virtual void erase(InternalLedgerKey const& key) = 0;
+    virtual void removeFromHotArchive(LedgerKey const& key) = 0;
     virtual LedgerTxnEntry load(InternalLedgerKey const& key) = 0;
     virtual ConstLedgerTxnEntry
     loadWithoutRecord(InternalLedgerKey const& key) = 0;
@@ -613,6 +621,8 @@ class AbstractLedgerTxn : public AbstractLedgerTxnParent
     virtual void getAllEntries(std::vector<LedgerEntry>& initEntries,
                                std::vector<LedgerEntry>& liveEntries,
                                std::vector<LedgerKey>& deadEntries) = 0;
+    virtual void
+    getRestoredHotArchiveKeys(std::vector<LedgerKey>& restoredKeys) = 0;
 
     // Returns all TTL keys that have been modified (create, update, and
     // delete), but does not cause the AbstractLedgerTxn or update last
@@ -705,11 +715,13 @@ class LedgerTxn : public AbstractLedgerTxn
     void commit() noexcept override;
 
     void commitChild(EntryIterator iter,
+                     UnorderedSet<LedgerKey> const& restoredHotArchiveKeys,
                      LedgerTxnConsistency cons) noexcept override;
 
     LedgerTxnEntry create(InternalLedgerEntry const& entry) override;
 
     void erase(InternalLedgerKey const& key) override;
+    void removeFromHotArchive(LedgerKey const& key) override;
 
     UnorderedMap<LedgerKey, LedgerEntry> getAllOffers() override;
 
@@ -744,6 +756,8 @@ class LedgerTxn : public AbstractLedgerTxn
     void getAllEntries(std::vector<LedgerEntry>& initEntries,
                        std::vector<LedgerEntry>& liveEntries,
                        std::vector<LedgerKey>& deadEntries) override;
+    void
+    getRestoredHotArchiveKeys(std::vector<LedgerKey>& restoredKeys) override;
     LedgerKeySet getAllTTLKeysWithoutSealing() const override;
 
     std::shared_ptr<InternalLedgerEntry const>
@@ -831,6 +845,7 @@ class LedgerTxnRoot : public AbstractLedgerTxnParent
     void addChild(AbstractLedgerTxn& child, TransactionMode mode) override;
 
     void commitChild(EntryIterator iter,
+                     UnorderedSet<LedgerKey> const& restoredHotArchiveKeys,
                      LedgerTxnConsistency cons) noexcept override;
 
     uint64_t countOffers(LedgerRange const& ledgers) const override;

src/ledger/LedgerTxnImpl.h

@@ -8,6 +8,7 @@
 #include "database/Database.h"
 #include "ledger/LedgerTxn.h"
 #include "util/RandomEvictionCache.h"
+#include "util/UnorderedSet.h"
 #include <list>
 #include <optional>
 #ifdef USE_POSTGRES
@@ -94,6 +95,10 @@ class LedgerTxn::Impl
     std::unique_ptr<LedgerHeader> mHeader;
     std::shared_ptr<LedgerTxnHeader::Impl> mActiveHeader;
     EntryMap mEntry;
+
+    // Contains the set of keys for entries that currently exist in the hot
+    // archive and were restored.
+    UnorderedSet<LedgerKey> mRestoredHotArchiveKeys;
     UnorderedMap<InternalLedgerKey, std::shared_ptr<EntryImplBase>> mActive;
     bool const mShouldUpdateLastModified;
     bool mIsSealed;
@@ -335,7 +340,9 @@ class LedgerTxn::Impl
 
     void commit() noexcept;
 
-    void commitChild(EntryIterator iter, LedgerTxnConsistency cons) noexcept;
+    void commitChild(EntryIterator iter,
+                     UnorderedSet<LedgerKey> const& restoredHotArchiveKeys,
+                     LedgerTxnConsistency cons) noexcept;
 
     // create has the basic exception safety guarantee. If it throws an
     // exception, then
@@ -357,6 +364,12 @@ class LedgerTxn::Impl
     // - the entry cache may be, but is not guaranteed to be, cleared.
     void erase(InternalLedgerKey const& key);
 
+    // removeFromHotArchive has the basic exception safety guarantee. If it
+    // throws an exception, then
+    // - the prepared statement cache may be, but is not guaranteed to be,
+    //   modified
+    void removeFromHotArchive(LedgerKey const& key);
+
     // getAllOffers has the basic exception safety guarantee. If it throws an
     // exception, then
     // - the prepared statement cache may be, but is not guaranteed to be,
@@ -430,6 +443,8 @@ class LedgerTxn::Impl
     void getAllEntries(std::vector<LedgerEntry>& initEntries,
                        std::vector<LedgerEntry>& liveEntries,
                        std::vector<LedgerKey>& deadEntries);
+    // getRestoredHotArchiveKeys has the strong exception safety guarantee
+    void getRestoredHotArchiveKeys(std::vector<LedgerKey>& restoredKeys);
 
     LedgerKeySet getAllTTLKeysWithoutSealing() const;
 
@@ -706,7 +721,9 @@ class LedgerTxnRoot::Impl
     // addChild has the strong exception safety guarantee.
     void addChild(AbstractLedgerTxn& child, TransactionMode mode);
 
-    void commitChild(EntryIterator iter, LedgerTxnConsistency cons) noexcept;
+    void commitChild(EntryIterator iter,
+                     UnorderedSet<LedgerKey> const& restoredHotArchiveKeys,
+                     LedgerTxnConsistency cons) noexcept;
 
     // countOffers has the strong exception safety guarantee.
     uint64_t countOffers(LedgerRange const& ledgers) const;