diff --git a/src/Nethermind/Nethermind.Core/Caching/LruCacheLowObject.cs b/src/Nethermind/Nethermind.Core/Caching/LruCacheLowObject.cs index c5a0079878d..6e27ae087ca 100644 --- a/src/Nethermind/Nethermind.Core/Caching/LruCacheLowObject.cs +++ b/src/Nethermind/Nethermind.Core/Caching/LruCacheLowObject.cs @@ -27,7 +27,7 @@ public LruCacheLowObject(int maxCapacity, string name) _name = name; _maxCapacity = maxCapacity; - _cacheMap = new Dictionary(maxCapacity); + _cacheMap = new Dictionary(maxCapacity / 2); _items = new LruCacheItem[maxCapacity]; } diff --git a/src/Nethermind/Nethermind.Core/Caching/LruKeyCacheLowObject.cs b/src/Nethermind/Nethermind.Core/Caching/LruKeyCacheLowObject.cs index 0fbc2d2fb4e..92318716948 100644 --- a/src/Nethermind/Nethermind.Core/Caching/LruKeyCacheLowObject.cs +++ b/src/Nethermind/Nethermind.Core/Caching/LruKeyCacheLowObject.cs @@ -27,7 +27,7 @@ public LruKeyCacheLowObject(int maxCapacity, string name) _name = name; _maxCapacity = maxCapacity; - _cacheMap = new Dictionary(maxCapacity); // do not initialize it at the full capacity + _cacheMap = new Dictionary(maxCapacity / 2); // do not initialize it at the full capacity _items = new LruCacheItem[maxCapacity]; } diff --git a/src/Nethermind/Nethermind.Core/IKeyValueStore.cs b/src/Nethermind/Nethermind.Core/IKeyValueStore.cs index d218c3dcd7c..717ae542c71 100644 --- a/src/Nethermind/Nethermind.Core/IKeyValueStore.cs +++ b/src/Nethermind/Nethermind.Core/IKeyValueStore.cs @@ -55,7 +55,7 @@ public interface IWriteOnlyKeyValueStore /// is preferable. Unless you plan to reuse the array somehow (pool), then you'd just use span. /// public bool PreferWriteByArray => false; - void PutSpan(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) => Set(key, value.ToArray(), flags); + void PutSpan(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) => Set(key, value.IsNull() ? null : value.ToArray(), flags); void Remove(ReadOnlySpan key) => Set(key, null); } diff --git a/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/SyncPeerProtocolHandlerBase.cs b/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/SyncPeerProtocolHandlerBase.cs index 723691b9e40..bc5ebe0ef08 100644 --- a/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/SyncPeerProtocolHandlerBase.cs +++ b/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/SyncPeerProtocolHandlerBase.cs @@ -74,7 +74,8 @@ public abstract class SyncPeerProtocolHandlerBase : ZeroProtocolHandlerBase, ISy initialRequestSize: 4 ); - protected LruKeyCacheLowObject NotifiedTransactions { get; } = new(2 * MemoryAllowance.MemPoolSize, "notifiedTransactions"); + protected LruKeyCacheLowObject? _notifiedTransactions; + protected LruKeyCacheLowObject NotifiedTransactions => _notifiedTransactions ??= new(2 * MemoryAllowance.MemPoolSize, "notifiedTransactions"); protected SyncPeerProtocolHandlerBase(ISession session, IMessageSerializationService serializer, diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Eth62ProtocolHandler.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Eth62ProtocolHandler.cs index 783c85e47d9..8c531355cc1 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Eth62ProtocolHandler.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Eth62ProtocolHandler.cs @@ -33,7 +33,8 @@ public class Eth62ProtocolHandler : SyncPeerProtocolHandlerBase, IZeroProtocolHa protected readonly ITxPool _txPool; private readonly IGossipPolicy _gossipPolicy; private readonly ITxGossipPolicy _txGossipPolicy; - private readonly LruKeyCache _lastBlockNotificationCache = new(10, "LastBlockNotificationCache"); + private LruKeyCache? _lastBlockNotificationCache; + private LruKeyCache LastBlockNotificationCache => _lastBlockNotificationCache ??= new(10, "LastBlockNotificationCache"); public Eth62ProtocolHandler(ISession session, IMessageSerializationService serializer, @@ -338,7 +339,7 @@ public override void NotifyOfNewBlock(Block block, SendBlockMode mode) return; } - if (_lastBlockNotificationCache.Set(block.Hash)) + if (LastBlockNotificationCache.Set(block.Hash)) { switch (mode) { diff --git a/src/Nethermind/Nethermind.Synchronization/FastSync/StateSyncItem.cs b/src/Nethermind/Nethermind.Synchronization/FastSync/StateSyncItem.cs index 547e12937b3..5795bdd7458 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastSync/StateSyncItem.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastSync/StateSyncItem.cs @@ -63,7 +63,7 @@ public StateSyncItem(Hash256 hash, byte[]? accountPathNibbles, byte[]? pathNibbl public readonly struct NodeKey(Hash256? address, TreePath? path, Hash256 hash) : IEquatable { private readonly ValueHash256 Address = address ?? default; - private readonly TreePath? Path = path ?? default; + private readonly TreePath? Path = path; private readonly ValueHash256 Hash = hash; public readonly bool Equals(NodeKey other) diff --git a/src/Nethermind/Nethermind.Trie/INodeStorage.cs b/src/Nethermind/Nethermind.Trie/INodeStorage.cs index 0deb378ec00..91f8abc08d6 100644 --- a/src/Nethermind/Nethermind.Trie/INodeStorage.cs +++ b/src/Nethermind/Nethermind.Trie/INodeStorage.cs @@ -31,6 +31,7 @@ public interface INodeStorage /// Used by StateSync to make sure values are flushed. /// void Flush(); + void Compact(); public enum KeyScheme { diff --git a/src/Nethermind/Nethermind.Trie/NodeStorage.cs b/src/Nethermind/Nethermind.Trie/NodeStorage.cs index cfbe47de704..066b5c19fb8 100644 --- a/src/Nethermind/Nethermind.Trie/NodeStorage.cs +++ b/src/Nethermind/Nethermind.Trie/NodeStorage.cs @@ -192,6 +192,14 @@ public void Flush() } } + public void Compact() + { + if (_keyValueStore is IDb db) + { + db.Compact(); + } + } + private class WriteBatch : INodeStorage.WriteBatch { private readonly IWriteBatch _writeBatch; diff --git a/src/Nethermind/Nethermind.Trie/PreCachedTrieStore.cs b/src/Nethermind/Nethermind.Trie/PreCachedTrieStore.cs index e7a09bcf679..564d9167547 100644 --- a/src/Nethermind/Nethermind.Trie/PreCachedTrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/PreCachedTrieStore.cs @@ -10,65 +10,81 @@ namespace Nethermind.Trie; -public class PreCachedTrieStore(ITrieStore inner, - ConcurrentDictionary preBlockCache) - : ITrieStore +public class PreCachedTrieStore : ITrieStore { + private readonly ITrieStore _inner; + private readonly ConcurrentDictionary _preBlockCache; + private readonly Func _loadRlp; + private readonly Func _tryLoadRlp; + + public PreCachedTrieStore(ITrieStore inner, + ConcurrentDictionary preBlockCache) + { + _inner = inner; + _preBlockCache = preBlockCache; + + // Capture the delegate once for default path to avoid the allocation of the lambda per call + _loadRlp = (NodeKey key) => _inner.LoadRlp(key.Address, in key.Path, key.Hash, flags: ReadFlags.None); + _tryLoadRlp = (NodeKey key) => _inner.TryLoadRlp(key.Address, in key.Path, key.Hash, flags: ReadFlags.None); + } + public void Dispose() { - inner.Dispose(); + _inner.Dispose(); } public void CommitNode(long blockNumber, Hash256? address, in NodeCommitInfo nodeCommitInfo, WriteFlags writeFlags = WriteFlags.None) { - inner.CommitNode(blockNumber, address, in nodeCommitInfo, writeFlags); + _inner.CommitNode(blockNumber, address, in nodeCommitInfo, writeFlags); } public void FinishBlockCommit(TrieType trieType, long blockNumber, Hash256? address, TrieNode? root, WriteFlags writeFlags = WriteFlags.None) { - inner.FinishBlockCommit(trieType, blockNumber, address, root, writeFlags); - preBlockCache.Clear(); + _inner.FinishBlockCommit(trieType, blockNumber, address, root, writeFlags); + _preBlockCache.Clear(); } public bool IsPersisted(Hash256? address, in TreePath path, in ValueHash256 keccak) { - byte[]? rlp = preBlockCache.GetOrAdd(new(address, in path, in keccak), - key => inner.TryLoadRlp(key.Address, in key.Path, key.Hash)); + byte[]? rlp = _preBlockCache.GetOrAdd(new(address, in path, in keccak), + key => _inner.TryLoadRlp(key.Address, in key.Path, key.Hash)); return rlp is not null; } - public IReadOnlyTrieStore AsReadOnly(INodeStorage? keyValueStore = null) => inner.AsReadOnly(keyValueStore); + public IReadOnlyTrieStore AsReadOnly(INodeStorage? keyValueStore = null) => _inner.AsReadOnly(keyValueStore); public event EventHandler? ReorgBoundaryReached { - add => inner.ReorgBoundaryReached += value; - remove => inner.ReorgBoundaryReached -= value; + add => _inner.ReorgBoundaryReached += value; + remove => _inner.ReorgBoundaryReached -= value; } - public IReadOnlyKeyValueStore TrieNodeRlpStore => inner.TrieNodeRlpStore; + public IReadOnlyKeyValueStore TrieNodeRlpStore => _inner.TrieNodeRlpStore; public void Set(Hash256? address, in TreePath path, in ValueHash256 keccak, byte[] rlp) { - preBlockCache[new(address, in path, in keccak)] = rlp; - inner.Set(address, in path, in keccak, rlp); + _preBlockCache[new(address, in path, in keccak)] = rlp; + _inner.Set(address, in path, in keccak, rlp); } - public bool HasRoot(Hash256 stateRoot) => inner.HasRoot(stateRoot); + public bool HasRoot(Hash256 stateRoot) => _inner.HasRoot(stateRoot); public IScopedTrieStore GetTrieStore(Hash256? address) => new ScopedTrieStore(this, address); - public TrieNode FindCachedOrUnknown(Hash256? address, in TreePath path, Hash256 hash) => inner.FindCachedOrUnknown(address, in path, hash); + public TrieNode FindCachedOrUnknown(Hash256? address, in TreePath path, Hash256 hash) => _inner.FindCachedOrUnknown(address, in path, hash); public byte[]? LoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => - preBlockCache.GetOrAdd(new(address, in path, hash), - key => inner.LoadRlp(key.Address, in key.Path, key.Hash, flags)); + _preBlockCache.GetOrAdd(new(address, in path, hash), + flags == ReadFlags.None ? _loadRlp : + key => _inner.LoadRlp(key.Address, in key.Path, key.Hash, flags)); public byte[]? TryLoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => - preBlockCache.GetOrAdd(new(address, in path, hash), - key => inner.TryLoadRlp(key.Address, in key.Path, key.Hash, flags)); + _preBlockCache.GetOrAdd(new(address, in path, hash), + flags == ReadFlags.None ? _tryLoadRlp : + key => _inner.TryLoadRlp(key.Address, in key.Path, key.Hash, flags)); - public INodeStorage.KeyScheme Scheme => inner.Scheme; + public INodeStorage.KeyScheme Scheme => _inner.Scheme; } public class NodeKey : IEquatable diff --git a/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs index a65b2cf4774..84105f69092 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs @@ -88,7 +88,7 @@ public TrieNode FromCachedRlpOrUnknown(in Key key) // we returning a copy to avoid multithreaded access trieNode = new TrieNode(NodeType.Unknown, key.Keccak, trieNode.FullRlp); - trieNode.ResolveNode(_trieStore.GetTrieStore(key.Address), key.Path); + trieNode.ResolveNode(_trieStore.GetTrieStore(key.AddressAsHash256), key.Path); trieNode.Keccak = key.Keccak; Metrics.LoadedFromCacheNodesCount++; @@ -108,8 +108,11 @@ void Trace(TrieNode trieNode) } } - private readonly ConcurrentDictionary _byKeyObjectCache = new(); - private readonly ConcurrentDictionary _byHashObjectCache = new(); + private static readonly int _concurrencyLevel = HashHelpers.GetPrime(Environment.ProcessorCount * 4); + private static readonly int _initialBuckets = HashHelpers.GetPrime(Math.Max(31, Environment.ProcessorCount * 16)); + + private readonly ConcurrentDictionary _byKeyObjectCache = new(_concurrencyLevel, _initialBuckets); + private readonly ConcurrentDictionary _byHashObjectCache = new(_concurrencyLevel, _initialBuckets); public bool IsNodeCached(in Key key) { @@ -211,13 +214,20 @@ public void Clear() internal readonly struct Key : IEquatable { internal const long MemoryUsage = 8 + 36 + 8; // (address (probably shared), path, keccak pointer (shared with TrieNode)) - public Hash256? Address { get; } + public readonly ValueHash256 Address; + public Hash256? AddressAsHash256 => Address == default ? null : Address.ToCommitment(); // Direct member rather than property for large struct, so members are called directly, // rather than struct copy through the property. Could also return a ref through property. public readonly TreePath Path; public Hash256 Keccak { get; } public Key(Hash256? address, in TreePath path, Hash256 keccak) + { + Address = address ?? default; + Path = path; + Keccak = keccak; + } + public Key(in ValueHash256 address, in TreePath path, Hash256 keccak) { Address = address; Path = path; @@ -227,13 +237,7 @@ public Key(Hash256? address, in TreePath path, Hash256 keccak) [SkipLocalsInit] public override int GetHashCode() { - Hash256? address = Address; - var addressHash = 0; - if (address is not null) - { - addressHash = address.ValueHash256.GetHashCode(); - } - + var addressHash = Address != default ? Address.GetHashCode() : 1; return Keccak.ValueHash256.GetChainedHashCode((uint)Path.GetHashCode()) ^ addressHash; } @@ -276,7 +280,7 @@ public readonly void Dispose() // Track some of the persisted path hash. Used to be able to remove keys when it is replaced. // If null, disable removing key. - private LruCache? _pastPathHash; + private LruCacheLowObject? _pastPathHash; // Track ALL of the recently re-committed persisted nodes. This is so that we don't accidentally remove // recommitted persisted nodes (which will not get re-persisted). @@ -768,7 +772,7 @@ private void RemovePastKeys(Dictionary? persistedHash { if (persistedHashes is null) return; - bool CanRemove(Hash256? address, TinyTreePath path, in TreePath fullPath, in ValueHash256 keccak, Hash256? currentlyPersistingKeccak) + bool CanRemove(in ValueHash256 address, TinyTreePath path, in TreePath fullPath, in ValueHash256 keccak, Hash256? currentlyPersistingKeccak) { // Multiple current hash that we don't keep track for simplicity. Just ignore this case. if (currentlyPersistingKeccak is null) return false; @@ -787,33 +791,48 @@ bool CanRemove(Hash256? address, TinyTreePath path, in TreePath fullPath, in Val return true; } - using INodeStorage.WriteBatch writeBatch = _nodeStorage.StartWriteBatch(); + ActionBlock actionBlock = + new ActionBlock(static (batch) => batch.Dispose()); - void DoAct(KeyValuePair keyValuePair) + INodeStorage.WriteBatch writeBatch = _nodeStorage.StartWriteBatch(); + try { - HashAndTinyPath key = keyValuePair.Key; - if (_pastPathHash.TryGet(key, out ValueHash256 prevHash)) + int round = 0; + foreach (KeyValuePair keyValuePair in persistedHashes) { - TreePath fullPath = key.path.ToTreePath(); // Micro op to reduce double convert - Hash256? hash = key.addr == default ? null : key.addr.ToCommitment(); - if (CanRemove(hash, key.path, fullPath, prevHash, keyValuePair.Value)) + HashAndTinyPath key = keyValuePair.Key; + if (_pastPathHash.TryGet(key, out ValueHash256 prevHash)) + { + TreePath fullPath = key.path.ToTreePath(); // Micro op to reduce double convert + if (CanRemove(key.addr, key.path, fullPath, prevHash, keyValuePair.Value)) + { + Metrics.RemovedNodeCount++; + Hash256? address = key.addr == default ? null : key.addr.ToCommitment(); + writeBatch.Set(address, fullPath, prevHash, default, WriteFlags.DisableWAL); + round++; + } + } + + // Batches of 256 + if (round > 256) { - Metrics.RemovedNodeCount++; - writeBatch.Remove(hash, fullPath, prevHash); + actionBlock.Post(writeBatch); + writeBatch = _nodeStorage.StartWriteBatch(); + round = 0; } } } - - ActionBlock> actionBlock = - new ActionBlock>(DoAct); - - foreach (KeyValuePair keyValuePair in persistedHashes) + catch (Exception ex) { - actionBlock.Post(keyValuePair); + if (_logger.IsError) _logger.Error($"Failed to remove past keys. {ex}"); + } + finally + { + writeBatch.Dispose(); + actionBlock.Complete(); + actionBlock.Completion.Wait(); + _nodeStorage.Compact(); } - - actionBlock.Complete(); - actionBlock.Completion.Wait(); } /// @@ -871,7 +890,7 @@ private void PruneCache(bool skipRecalculateMemory = false) if (keccak is null) { TreePath path2 = key.Path; - keccak = node.GenerateKey(this.GetTrieStore(key.Address), ref path2, isRoot: true); + keccak = node.GenerateKey(this.GetTrieStore(key.AddressAsHash256), ref path2, isRoot: true); if (keccak != key.Keccak) { throw new InvalidOperationException($"Persisted {node} {key} != {keccak}"); @@ -1261,7 +1280,8 @@ void PersistNode(TrieNode n, Hash256? address, TreePath path) if (cancellationToken.IsCancellationRequested) return; DirtyNodesCache.Key key = nodesCopy[i].Key; TreePath path = key.Path; - nodesCopy[i].Value.CallRecursively(PersistNode, key.Address, ref path, GetTrieStore(key.Address), false, _logger, false); + Hash256? address = key.AddressAsHash256; + nodesCopy[i].Value.CallRecursively(PersistNode, address, ref path, GetTrieStore(address), false, _logger, false); }); PruneCache(); @@ -1322,34 +1342,184 @@ public bool HasRoot(Hash256 stateRoot) } [StructLayout(LayoutKind.Auto)] - private readonly struct HashAndTinyPath(Hash256? hash, in TinyTreePath path) : IEquatable + private readonly struct HashAndTinyPath : IEquatable { - public readonly ValueHash256 addr = hash ?? default; - public readonly TinyTreePath path = path; + public readonly ValueHash256 addr; + public readonly TinyTreePath path; + + public HashAndTinyPath(Hash256? hash, in TinyTreePath path) + { + addr = hash ?? default; + this.path = path; + } + public HashAndTinyPath(in ValueHash256 hash, in TinyTreePath path) + { + addr = hash; + this.path = path; + } - public bool Equals(HashAndTinyPath other) => addr == other.addr && path.Equals(other.path); + public bool Equals(HashAndTinyPath other) => addr == other.addr && path.Equals(in other.path); public override bool Equals(object? obj) => obj is HashAndTinyPath other && Equals(other); public override int GetHashCode() { - var addressHash = addr.GetHashCode(); + var addressHash = addr != default ? addr.GetHashCode() : 1; return path.GetHashCode() ^ addressHash; } } [StructLayout(LayoutKind.Auto)] - private readonly struct HashAndTinyPathAndHash(Hash256? hash, in TinyTreePath path, in ValueHash256 valueHash) : IEquatable + private readonly struct HashAndTinyPathAndHash : IEquatable { - public readonly ValueHash256 hash = hash ?? default; - public readonly TinyTreePath path = path; - public readonly ValueHash256 valueHash = valueHash; + public readonly ValueHash256 hash; + public readonly TinyTreePath path; + public readonly ValueHash256 valueHash; - public bool Equals(HashAndTinyPathAndHash other) => hash.Equals(in other.hash) && path.Equals(in other.path) && valueHash.Equals(in other.valueHash); + public HashAndTinyPathAndHash(Hash256? hash, in TinyTreePath path, in ValueHash256 valueHash) + { + this.hash = hash ?? default; + this.path = path; + this.valueHash = valueHash; + } + public HashAndTinyPathAndHash(in ValueHash256 hash, in TinyTreePath path, in ValueHash256 valueHash) + { + this.hash = hash; + this.path = path; + this.valueHash = valueHash; + } + + public bool Equals(HashAndTinyPathAndHash other) => hash == other.hash && path.Equals(in other.path) && valueHash.Equals(in other.valueHash); public override bool Equals(object? obj) => obj is HashAndTinyPath other && Equals(other); public override int GetHashCode() { - var hashHash = hash.GetHashCode(); + var hashHash = hash != default ? hash.GetHashCode() : 1; return valueHash.GetChainedHashCode((uint)path.GetHashCode()) ^ hashHash; } } + + internal static class HashHelpers + { + private const int HashPrime = 101; + + private static bool IsPrime(int candidate) + { + if ((candidate & 1) != 0) + { + int limit = (int)Math.Sqrt(candidate); + for (int divisor = 3; divisor <= limit; divisor += 2) + { + if ((candidate % divisor) == 0) + return false; + } + return true; + } + return candidate == 2; + } + + public static int GetPrime(int min) + { + foreach (int prime in Primes) + { + if (prime >= min) + return prime; + } + + // Outside of our predefined table. Compute the hard way. + for (int i = (min | 1); i < int.MaxValue; i += 2) + { + if (IsPrime(i) && ((i - 1) % HashPrime != 0)) + return i; + } + return min; + } + + // Table of prime numbers to use as hash table sizes. + // A typical resize algorithm would pick the smallest prime number in this array + // that is larger than twice the previous capacity. + // Suppose our Hashtable currently has capacity x and enough elements are added + // such that a resize needs to occur. Resizing first computes 2x then finds the + // first prime in the table greater than 2x, i.e. if primes are ordered + // p_1, p_2, ..., p_i, ..., it finds p_n such that p_n-1 < 2x < p_n. + // Doubling is important for preserving the asymptotic complexity of the + // hashtable operations such as add. Having a prime guarantees that double + // hashing does not lead to infinite loops. IE, your hash function will be + // h1(key) + i*h2(key), 0 <= i < size. h2 and the size must be relatively prime. + // We prefer the low computation costs of higher prime numbers over the increased + // memory allocation of a fixed prime number i.e. when right sizing a HashSet. + private static ReadOnlySpan Primes => + [ + 3, + 7, + 11, + 17, + 23, + 29, + 37, + 47, + 59, + 71, + 89, + 107, + 131, + 163, + 197, + 239, + 293, + 353, + 431, + 521, + 631, + 761, + 919, + 1103, + 1327, + 1597, + 1931, + 2333, + 2801, + 3371, + 4049, + 4861, + 5839, + 7013, + 8419, + 10103, + 12143, + 14591, + 17519, + 21023, + 25229, + 30293, + 36353, + 43627, + 52361, + 62851, + 75431, + 90523, + 108631, + 130363, + 156437, + 187751, + 225307, + 270371, + 324449, + 389357, + 467237, + 560689, + 672827, + 807403, + 968897, + 1162687, + 1395263, + 1674319, + 2009191, + 2411033, + 2893249, + 3471899, + 4166287, + 4999559, + 5999471, + 7199369 + ]; + } } }