From c42168af4279261f211ee73a27699df060a5b42e Mon Sep 17 00:00:00 2001 From: Garand Tyson Date: Thu, 2 Jan 2025 11:46:06 -0800 Subject: [PATCH] Add restore meta for p23 --- src/main/CommandLine.cpp | 2 +- ...ger-close-meta-v1-protocol-23-soroban.json | 31 +++++- .../InvokeHostFunctionOpFrame.cpp | 3 +- src/transactions/OperationFrame.cpp | 8 ++ src/transactions/OperationFrame.h | 2 + src/transactions/TransactionFrame.cpp | 102 +++++++++++++++++- .../test/InvokeHostFunctionTests.cpp | 102 ++++++++++++++++-- src/util/MetaUtils.cpp | 17 ++- .../InvokeHostFunctionTests.json | 11 +- 9 files changed, 255 insertions(+), 23 deletions(-) diff --git a/src/main/CommandLine.cpp b/src/main/CommandLine.cpp index 6f76de0e04..a2c1c110cb 100644 --- a/src/main/CommandLine.cpp +++ b/src/main/CommandLine.cpp @@ -4,12 +4,12 @@ // clang-format off // This needs to be included first -#include "bucket/LiveBucketList.h" #include "rust/RustVecXdrMarshal.h" // clang-format on #include "main/CommandLine.h" #include "bucket/BucketManager.h" +#include "bucket/LiveBucketList.h" #include "catchup/CatchupConfiguration.h" #include "catchup/CatchupRange.h" #include "catchup/ReplayDebugMetaWork.h" diff --git a/src/testdata/ledger-close-meta-v1-protocol-23-soroban.json b/src/testdata/ledger-close-meta-v1-protocol-23-soroban.json index 117751c6b8..4dee599015 100644 --- a/src/testdata/ledger-close-meta-v1-protocol-23-soroban.json +++ b/src/testdata/ledger-close-meta-v1-protocol-23-soroban.json @@ -1898,6 +1898,33 @@ "operations": [ { "changes": [ + { + "type": "LEDGER_ENTRY_RESTORED", + "restored": { + "lastModifiedLedgerSeq": 7, + "data": { + "type": "CONTRACT_DATA", + "contractData": { + "ext": { + "v": 0 + }, + "contract": "CAA3QKIP2SNVXUJTB4HKOGF55JTSSMQGED3FZYNHMNSXYV3DRRMAWA3Y", + "key": { + "type": "SCV_SYMBOL", + "sym": "archived" + }, + "durability": "PERSISTENT", + "val": { + "type": "SCV_U64", + "u64": 42 + } + } + }, + "ext": { + "v": 0 + } + } + }, { "type": "LEDGER_ENTRY_STATE", "state": { @@ -1915,8 +1942,8 @@ } }, { - "type": "LEDGER_ENTRY_UPDATED", - "updated": { + "type": "LEDGER_ENTRY_RESTORED", + "restored": { "lastModifiedLedgerSeq": 28, "data": { "type": "TTL", diff --git a/src/transactions/InvokeHostFunctionOpFrame.cpp b/src/transactions/InvokeHostFunctionOpFrame.cpp index 0416733dde..5135a64a84 100644 --- a/src/transactions/InvokeHostFunctionOpFrame.cpp +++ b/src/transactions/InvokeHostFunctionOpFrame.cpp @@ -4,8 +4,8 @@ // clang-format off // This needs to be included first +#include "rust/RustVecXdrMarshal.h" #include "TransactionUtils.h" -#include "main/AppConnector.h" #include "util/GlobalChecks.h" #include "util/ProtocolVersion.h" #include "xdr/Stellar-ledger-entries.h" @@ -14,7 +14,6 @@ #include #include #include "xdr/Stellar-contract.h" -#include "rust/RustVecXdrMarshal.h" // clang-format on #include "ledger/LedgerTxnImpl.h" diff --git a/src/transactions/OperationFrame.cpp b/src/transactions/OperationFrame.cpp index 5108b15106..00d2d46041 100644 --- a/src/transactions/OperationFrame.cpp +++ b/src/transactions/OperationFrame.cpp @@ -33,6 +33,7 @@ #include "transactions/SetTrustLineFlagsOpFrame.h" #include "transactions/TransactionFrame.h" #include "transactions/TransactionUtils.h" +#include "util/GlobalChecks.h" #include "util/Logging.h" #include "util/ProtocolVersion.h" #include "util/XDRCereal.h" @@ -329,4 +330,11 @@ OperationFrame::insertLedgerKeysToPrefetch(UnorderedSet& keys) const // Do nothing by default return; } + +SorobanResources const& +OperationFrame::getSorobanResources() const +{ + releaseAssertOrThrow(isSoroban()); + return mParentTx.sorobanResources(); +} } diff --git a/src/transactions/OperationFrame.h b/src/transactions/OperationFrame.h index c260d8f11f..a18778d54c 100644 --- a/src/transactions/OperationFrame.h +++ b/src/transactions/OperationFrame.h @@ -96,5 +96,7 @@ class OperationFrame virtual bool isDexOperation() const; virtual bool isSoroban() const; + + SorobanResources const& getSorobanResources() const; }; } diff --git a/src/transactions/TransactionFrame.cpp b/src/transactions/TransactionFrame.cpp index 4d9b2d6ae1..cc41b6ccdb 100644 --- a/src/transactions/TransactionFrame.cpp +++ b/src/transactions/TransactionFrame.cpp @@ -48,6 +48,8 @@ #include #include +#include +#include #include namespace stellar @@ -57,6 +59,83 @@ namespace // Limit to the maximum resource fee allowed for transaction, // roughly 112 million lumens. int64_t const MAX_RESOURCE_FEE = 1LL << 50; + +#ifdef ENABLE_NEXT_PROTOCOL_VERSION_UNSAFE_FOR_PRODUCTION +// Starting in protocol 23, some operation meta needs to be modified +// to be consumed by downstream systems. In particular, restoration is +// logically a new entry creation from the perspective of ltx and stellar-core +// as a whole, but this change type is reclassified to LEDGER_ENTRY_RESTORED +// for easier consumption downstream. +LedgerEntryChanges +processOpLedgerEntryChanges(std::shared_ptr op, + AbstractLedgerTxn& ltx) +{ + if (op->getOperation().body.type() != RESTORE_FOOTPRINT) + { + return ltx.getChanges(); + } + + auto const& restoreKeys = op->getSorobanResources().footprint.readWrite; + std::unordered_map ttlToDataKey(restoreKeys.size()); + for (auto const& key : restoreKeys) + { + ttlToDataKey[getTTLKey(key)] = key; + } + + LedgerEntryChanges changes = ltx.getChanges(); + + // If an entry being restored does not exist in the live BucketList (i.e. + // the entry was previously evicted), the restoreOp will have already + // created the entry in the ltx. In this case, we need to update the + // creation change type to LEDGER_ENTRY_RESTORED. If the entry being + // restored still exists in the BucketList, RestoreOp will only update the + // TTL, so we need to insert the data LEDGER_ENTRY_RESTORED change directly. + std::unordered_set restoreChangesToInsert; + for (auto& change : changes) + { + if (change.type() == LEDGER_ENTRY_CREATED) + { + auto le = change.created(); + change.type(LEDGER_ENTRY_RESTORED); + change.restored() = le; + } + else if (change.type() == LEDGER_ENTRY_UPDATED) + { + auto ttlLe = change.updated(); + releaseAssertOrThrow(ttlLe.data.type() == TTL); + + // Update the TTL change from LEDGER_ENTRY_UPDATED to + // LEDGER_ENTRY_RESTORED. + change.type(LEDGER_ENTRY_RESTORED); + change.restored() = ttlLe; + auto dataKey = ttlToDataKey.at(LedgerEntryKey(ttlLe)); + restoreChangesToInsert.insert(dataKey); + } + } + + // Now insert all the data entries that were not created but already existed + // on the live BucketList + for (auto const& key : restoreChangesToInsert) + { + LedgerEntryChange change; + change.type(LEDGER_ENTRY_RESTORED); + + // Note: this is already in the cache since the RestoreOp loaded + // all data keys for size calculation during apply already + auto entry = ltx.getNewestVersion(key); + + // If TTL already exists and is just being updated, the + // data entry must also already exist + releaseAssertOrThrow(entry); + + change.restored() = entry->ledgerEntry(); + changes.push_back(change); + } + + return changes; +} +#endif + } // namespace using namespace std; @@ -1648,15 +1727,30 @@ TransactionFrame::applyOperations(SignatureChecker& signatureChecker, { success = false; } + + // The operation meta will be empty if the transaction + // doesn't succeed so we may as well not do any work in that + // case if (success) { app.checkOnOperationApply(op->getOperation(), opResult, ltxOp.getDelta()); - // The operation meta will be empty if the transaction - // doesn't succeed so we may as well not do any work in that - // case - operationMetas.emplace_back(ltxOp.getChanges()); + LedgerEntryChanges changes; +#ifdef ENABLE_NEXT_PROTOCOL_VERSION_UNSAFE_FOR_PRODUCTION + if (protocolVersionStartsFrom( + ledgerVersion, + LiveBucket:: + FIRST_PROTOCOL_SUPPORTING_PERSISTENT_EVICTION)) + { + changes = processOpLedgerEntryChanges(op, ltxOp); + } + else +#endif + { + changes = ltxOp.getChanges(); + } + operationMetas.emplace_back(changes); } if (txRes || diff --git a/src/transactions/test/InvokeHostFunctionTests.cpp b/src/transactions/test/InvokeHostFunctionTests.cpp index bb30a76451..58ed5998b8 100644 --- a/src/transactions/test/InvokeHostFunctionTests.cpp +++ b/src/transactions/test/InvokeHostFunctionTests.cpp @@ -2752,6 +2752,92 @@ TEST_CASE_VERSIONS("entry eviction", "[tx][soroban][archival]") REQUIRE(!evicted); } } + + SECTION("Restoration Meta") + { + test.invokeRestoreOp({persistentKey}, 20'048); + auto targetRestorationLedger = test.getLCLSeq(); + + XDRInputFileStream in; + in.open(metaPath); + LedgerCloseMeta lcm; + bool restoreMeta = false; + + LedgerKeySet keysToRestore = {persistentKey, + getTTLKey(persistentKey)}; + while (in.readOne(lcm)) + { + REQUIRE(lcm.v() == 1); + if (lcm.v1().ledgerHeader.header.ledgerSeq == + targetRestorationLedger) + { + REQUIRE(lcm.v1().evictedTemporaryLedgerKeys.empty()); + REQUIRE( + lcm.v1().evictedPersistentLedgerEntries.empty()); + + REQUIRE(lcm.v1().txProcessing.size() == 1); + auto txMeta = lcm.v1().txProcessing.front(); + REQUIRE( + txMeta.txApplyProcessing.v3().operations.size() == + 1); + + REQUIRE(txMeta.txApplyProcessing.v3() + .operations[0] + .changes.size() == 2); + for (auto const& change : txMeta.txApplyProcessing.v3() + .operations[0] + .changes) + { + + // Only support persistent eviction meta >= p23 + LedgerKey lk; + if (protocolVersionStartsFrom( + cfg.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION, + BucketBase:: + FIRST_PROTOCOL_SUPPORTING_PERSISTENT_EVICTION)) + { + REQUIRE(change.type() == + LedgerEntryChangeType:: + LEDGER_ENTRY_RESTORED); + lk = LedgerEntryKey(change.restored()); + REQUIRE(keysToRestore.find(lk) != + keysToRestore.end()); + keysToRestore.erase(lk); + } + else + { + if (change.type() == + LedgerEntryChangeType::LEDGER_ENTRY_STATE) + { + lk = LedgerEntryKey(change.state()); + REQUIRE(lk == getTTLKey(persistentKey)); + keysToRestore.erase(lk); + } + else + { + REQUIRE(change.type() == + LedgerEntryChangeType:: + LEDGER_ENTRY_UPDATED); + lk = LedgerEntryKey(change.updated()); + REQUIRE(lk == getTTLKey(persistentKey)); + + // While we will see the TTL key twice, + // remove the TTL key in the path above and + // the persistent key here to make the check + // easier + keysToRestore.erase(persistentKey); + } + } + } + + restoreMeta = true; + break; + } + } + + REQUIRE(restoreMeta); + REQUIRE(keysToRestore.empty()); + } } #endif @@ -2942,11 +3028,6 @@ TEST_CASE("persistent entry archival", "[tx][soroban][archival]") auto lk = client.getContract().getDataKey( makeSymbolSCVal("key"), ContractDataDurability::PERSISTENT); - auto hotArchive = test.getApp() - .getBucketManager() - .getBucketSnapshotManager() - .copySearchableHotArchiveBucketListSnapshot(); - auto evictionLedger = 14; // Close ledgers until entry is evicted @@ -2955,6 +3036,11 @@ TEST_CASE("persistent entry archival", "[tx][soroban][archival]") closeLedgerOn(test.getApp(), i, 2, 1, 2016); } + auto hotArchive = test.getApp() + .getBucketManager() + .getBucketSnapshotManager() + .copySearchableHotArchiveBucketListSnapshot(); + if (evict) { REQUIRE(hotArchive->load(lk)); @@ -3022,7 +3108,6 @@ TEST_CASE("persistent entry archival", "[tx][soroban][archival]") SECTION("key accessible after restore") { test.invokeRestoreOp({lk}, 20'048); - auto const& stateArchivalSettings = test.getNetworkCfg().stateArchivalSettings(); auto newExpectedLiveUntilLedger = @@ -3031,6 +3116,11 @@ TEST_CASE("persistent entry archival", "[tx][soroban][archival]") client.get("key", ContractDataDurability::PERSISTENT, 123); + test.getApp() + .getBucketManager() + .getBucketSnapshotManager() + .maybeCopySearchableHotArchiveBucketListSnapshot(hotArchive); + // Restored entries are deleted from Hot Archive REQUIRE(!hotArchive->load(lk)); } diff --git a/src/util/MetaUtils.cpp b/src/util/MetaUtils.cpp index ecd87e5644..a88cd996be 100644 --- a/src/util/MetaUtils.cpp +++ b/src/util/MetaUtils.cpp @@ -8,6 +8,7 @@ #include "util/GlobalChecks.h" #include "util/XDROperators.h" #include "util/types.h" +#include "xdr/Stellar-ledger.h" #include namespace @@ -20,9 +21,14 @@ struct CmpLedgerEntryChanges { // order that we want is: // LEDGER_ENTRY_STATE, LEDGER_ENTRY_CREATED, - // LEDGER_ENTRY_UPDATED, LEDGER_ENTRY_REMOVED - static constexpr std::array reindex = {1, 2, 3, 0}; - releaseAssert(let >= 0 && let < 4); + // LEDGER_ENTRY_UPDATED, LEDGER_ENTRY_REMOVED, LEDGER_ENTRY_RESTORED + static constexpr std::array reindex = {1, 2, 3, 0 +#ifdef ENABLE_NEXT_PROTOCOL_VERSION_UNSAFE_FOR_PRODUCTION + , + 4 +#endif + }; + releaseAssert(let >= 0 && let < 5); return reindex[let]; } @@ -44,6 +50,11 @@ struct CmpLedgerEntryChanges case LEDGER_ENTRY_REMOVED: res = change.removed(); break; +#ifdef ENABLE_NEXT_PROTOCOL_VERSION_UNSAFE_FOR_PRODUCTION + case LEDGER_ENTRY_RESTORED: + res = LedgerEntryKey(change.restored()); + break; +#endif } return res; } diff --git a/test-tx-meta-baseline-next/InvokeHostFunctionTests.json b/test-tx-meta-baseline-next/InvokeHostFunctionTests.json index f43a35a8f5..2548c2d5c3 100644 --- a/test-tx-meta-baseline-next/InvokeHostFunctionTests.json +++ b/test-tx-meta-baseline-next/InvokeHostFunctionTests.json @@ -1325,11 +1325,10 @@ "bKDF6V5IzTo=" ], "contract storage|footprint|unused readWrite key" : [ "VzQb0aWq2bE=" ], - "entry eviction|protocol version 20" : [ "bKDF6V5IzTo=", "bKDF6V5IzTo=", "bKDF6V5IzTo=" ], - "entry eviction|protocol version 21" : [ "bKDF6V5IzTo=", "bKDF6V5IzTo=", "bKDF6V5IzTo=" ], - "entry eviction|protocol version 22" : [ "bKDF6V5IzTo=", "bKDF6V5IzTo=", "bKDF6V5IzTo=" ], - "entry eviction|protocol version 23" : [ "bKDF6V5IzTo=", "bKDF6V5IzTo=", "bKDF6V5IzTo=" ], - "evicted persistent entries" : [ "bKDF6V5IzTo=" ], + "entry eviction|protocol version 20" : [ "bKDF6V5IzTo=", "bKDF6V5IzTo=", "bKDF6V5IzTo=", "bKDF6V5IzTo=" ], + "entry eviction|protocol version 21" : [ "bKDF6V5IzTo=", "bKDF6V5IzTo=", "bKDF6V5IzTo=", "bKDF6V5IzTo=" ], + "entry eviction|protocol version 22" : [ "bKDF6V5IzTo=", "bKDF6V5IzTo=", "bKDF6V5IzTo=", "bKDF6V5IzTo=" ], + "entry eviction|protocol version 23" : [ "bKDF6V5IzTo=", "bKDF6V5IzTo=", "bKDF6V5IzTo=", "bKDF6V5IzTo=" ], "failure diagnostics" : [ "bKDF6V5IzTo=" ], "ledger entry size limit enforced" : [ "bKDF6V5IzTo=", "bKDF6V5IzTo=" ], "loadgen Wasm executes properly" : [ "bKDF6V5IzTo=" ], @@ -1345,6 +1344,8 @@ "bKDF6V5IzTo=", "bKDF6V5IzTo=" ], + "persistent entry archival|eviction" : [ "bKDF6V5IzTo=", "bKDF6V5IzTo=" ], + "persistent entry archival|expiration without eviction" : [ "bKDF6V5IzTo=", "bKDF6V5IzTo=" ], "refund account merged" : [ "bKDF6V5IzTo=", "398Rd+u+jdE=", "b4KxXJCxvM0=", "7/h1q7KjqKA=" ], "refund is sent to fee-bump source|protocol version 20" : [ "bKDF6V5IzTo=", "JRiUwIP1h2Q=", "LyDINL0VaLk=" ], "refund is sent to fee-bump source|protocol version 21" : [ "bKDF6V5IzTo=", "JRiUwIP1h2Q=", "LyDINL0VaLk=" ],