From 4aff14832b3676b910585852359f692f4d7e68a7 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Mon, 13 Jan 2020 13:56:06 +0300 Subject: [PATCH 01/10] initial commit --- .../view/history/BlockDownloadProcessor.scala | 16 +- .../view/history/FastSyncProcessor.scala | 8 +- .../scala/encry/view/history/History.scala | 88 +++--- .../scala/encry/view/history/HistoryApi.scala | 255 ++++++++++-------- .../encry/view/history/HistoryDBApi.scala | 95 ++++--- .../history/HistoryModifiersValidator.scala | 185 ++++++++----- .../encry/view/history/ValidationError.scala | 2 +- 7 files changed, 386 insertions(+), 263 deletions(-) diff --git a/src/main/scala/encry/view/history/BlockDownloadProcessor.scala b/src/main/scala/encry/view/history/BlockDownloadProcessor.scala index 14495dfc84..5a6a520b72 100644 --- a/src/main/scala/encry/view/history/BlockDownloadProcessor.scala +++ b/src/main/scala/encry/view/history/BlockDownloadProcessor.scala @@ -5,7 +5,7 @@ import org.encryfoundation.common.modifiers.history.Header import org.encryfoundation.common.utils.constants.Constants /** Class that keeps and calculates minimal height for full blocks starting from which we need to download these full - * blocks from the network and keep them in our history. */ + * blocks from the network and keep them in our history. */ case class BlockDownloadProcessor(nodeSettings: NodeSettings, constants: Constants) { private[history] var minimalBlockHeightVar: Int = Int.MaxValue @@ -13,26 +13,26 @@ case class BlockDownloadProcessor(nodeSettings: NodeSettings, constants: Constan def updateMinimalBlockHeightVar(height: Int): Unit = minimalBlockHeightVar = height /** Start height to download full blocks. - * Int.MaxValue when headers chain is not synchronized with the network and no full blocks download needed */ + * Int.MaxValue when headers chain is not synchronized with the network and no full blocks download needed */ def minimalBlockHeight: Int = minimalBlockHeightVar /** Update minimal full block height - * - * @param header - header of new best full block - * @return minimal height to process best full block */ + * + * @param header - header of new best full block + * @return minimal height to process best full block */ def updateBestBlock(header: Header): Int = { minimalBlockHeightVar = minimalBlockHeightAfter(header) minimalBlockHeightVar } - private def minimalBlockHeightAfter(header: Header): Int = { + private def minimalBlockHeightAfter(header: Header): Int = if (minimalBlockHeightVar == Int.MaxValue) { // just synced with the headers chain - determine first full block to apply if (nodeSettings.blocksToKeep < 0) constants.GenesisHeight // keep all blocks in history // TODO: start with the height of UTXO snapshot applied. Start from genesis until this is implemented // Start from config.blocksToKeep blocks back else Math.max(constants.GenesisHeight, header.height - nodeSettings.blocksToKeep + 1) - } else if (nodeSettings.blocksToKeep >= 0) Math.max(header.height - nodeSettings.blocksToKeep + 1, minimalBlockHeightVar) + } else if (nodeSettings.blocksToKeep >= 0) + Math.max(header.height - nodeSettings.blocksToKeep + 1, minimalBlockHeightVar) else constants.GenesisHeight - } } diff --git a/src/main/scala/encry/view/history/FastSyncProcessor.scala b/src/main/scala/encry/view/history/FastSyncProcessor.scala index 6e71c10856..d5d6ff2a5d 100644 --- a/src/main/scala/encry/view/history/FastSyncProcessor.scala +++ b/src/main/scala/encry/view/history/FastSyncProcessor.scala @@ -2,7 +2,7 @@ package encry.view.history import cats.syntax.option.none import encry.consensus.HistoryConsensus.ProgressInfo -import encry.storage.VersionalStorage.{StorageKey, StorageValue, StorageVersion} +import encry.storage.VersionalStorage.{ StorageKey, StorageValue, StorageVersion } import org.encryfoundation.common.modifiers.history.Payload trait FastSyncProcessor extends HistoryApi { @@ -18,8 +18,10 @@ trait FastSyncProcessor extends HistoryApi { StorageVersion @@ validityKey(block.payload.id).untag(StorageKey), List(block.header.id, block.payload.id).map(id => validityKey(id) -> StorageValue @@ Array(1.toByte)) ) - logger.info(s"Finished processing block ${block.encodedId}. " + - s"Processing time is ${(System.currentTimeMillis() - startTime) / 1000} s") + logger.info( + s"Finished processing block ${block.encodedId}. " + + s"Processing time is ${(System.currentTimeMillis() - startTime) / 1000} s" + ) } ProgressInfo(none, Seq.empty, Seq.empty, none) } diff --git a/src/main/scala/encry/view/history/History.scala b/src/main/scala/encry/view/history/History.scala index d2a6154eb3..dee83c6bf1 100644 --- a/src/main/scala/encry/view/history/History.scala +++ b/src/main/scala/encry/view/history/History.scala @@ -1,26 +1,25 @@ package encry.view.history import java.io.File +import cats.syntax.either._ import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus.ProgressInfo import encry.settings._ import encry.storage.VersionalStorage -import encry.storage.VersionalStorage.{StorageKey, StorageValue, StorageVersion} +import encry.storage.VersionalStorage.{ StorageKey, StorageValue, StorageVersion } import encry.storage.iodb.versionalIODB.IODBHistoryWrapper -import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} +import encry.storage.levelDb.versionalLevelDB.{ LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion } import encry.utils.NetworkTimeProvider import encry.view.history.storage.HistoryStorage -import io.iohk.iodb.{ByteArrayWrapper, LSMStore} +import io.iohk.iodb.LSMStore import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId import org.iq80.leveldb.Options -import cats.syntax.either._ -import supertagged.@@ /** - * History implementation. It is processing persistent modifiers generated locally or received from the network. + * History implementation. It is processing persistent modifiers generated locally or received from the network. **/ trait History extends HistoryModifiersValidator with AutoCloseable { @@ -30,7 +29,7 @@ trait History extends HistoryModifiersValidator with AutoCloseable { def append(modifier: PersistentModifier): Either[Throwable, (History, ProgressInfo)] = { logger.info(s"Trying to append modifier ${Algos.encode(modifier.id)} of type ${modifier.modifierTypeId} to history") Either.catchNonFatal(modifier match { - case header: Header => + case header: Header => logger.info(s"Append header ${header.encodedId} at height ${header.height} to history") (this, processHeader(header)) case payload: Payload => (this, processPayload(payload)) @@ -49,22 +48,28 @@ trait History extends HistoryModifiersValidator with AutoCloseable { } /** - * Marks modifier and all modifiers in child chains as invalid - * - * @param modifier that is invalid against the State - * @return ProgressInfo with next modifier to try to apply - */ + * Marks modifier and all modifiers in child chains as invalid + * + * @param modifier that is invalid against the State + * @return ProgressInfo with next modifier to try to apply + */ def reportModifierIsInvalid(modifier: PersistentModifier): (History, ProgressInfo) = { logger.info(s"Modifier ${modifier.encodedId} of type ${modifier.modifierTypeId} is marked as invalid") correspondingHeader(modifier) match { case Some(invalidatedHeader) => val invalidatedHeaders: Seq[Header] = continuationHeaderChains(invalidatedHeader, _ => true).flatten.distinct val validityRow: List[(StorageKey, StorageValue)] = invalidatedHeaders - .flatMap(h => Seq(h.id, h.payloadId) - .map(id => validityKey(id) -> StorageValue @@ Array(0.toByte))).toList + .flatMap( + h => + Seq(h.id, h.payloadId) + .map(id => validityKey(id) -> StorageValue @@ Array(0.toByte)) + ) + .toList logger.info(s"Going to invalidate ${invalidatedHeader.encodedId} and ${invalidatedHeaders.map(_.encodedId)}") - val bestHeaderIsInvalidated: Boolean = getBestHeaderId.exists(id => invalidatedHeaders.exists(_.id sameElements id)) - val bestFullIsInvalidated: Boolean = getBestBlockId.exists(id => invalidatedHeaders.exists(_.id sameElements id)) + val bestHeaderIsInvalidated: Boolean = + getBestHeaderId.exists(id => invalidatedHeaders.exists(_.id sameElements id)) + val bestFullIsInvalidated: Boolean = + getBestBlockId.exists(id => invalidatedHeaders.exists(_.id sameElements id)) (bestHeaderIsInvalidated, bestFullIsInvalidated) match { case (false, false) => // Modifiers from best header and best full chain are not involved, no rollback and links change required. @@ -86,17 +91,20 @@ trait History extends HistoryModifiersValidator with AutoCloseable { this -> ProgressInfo(None, Seq.empty, Seq.empty, None) } else { val invalidatedChain: Seq[Block] = getBestBlock.toSeq - .flatMap(f => headerChainBack(getBestBlockHeight + 1, f.header, h => !invalidatedHeaders.contains(h)).headers) + .flatMap( + f => headerChainBack(getBestBlockHeight + 1, f.header, h => !invalidatedHeaders.contains(h)).headers + ) .flatMap(h => getBlockByHeader(h)) .ensuring(_.lengthCompare(1) > 0, "invalidatedChain should contain at least bestFullBlock and parent") val branchPoint: Block = invalidatedChain.head val validChain: Seq[Block] = - continuationHeaderChains(branchPoint.header, h => getBlockByHeader(h).isDefined && !invalidatedHeaders.contains(h)) + continuationHeaderChains(branchPoint.header, + h => getBlockByHeader(h).isDefined && !invalidatedHeaders.contains(h)) .maxBy(chain => scoreOf(chain.last.id).getOrElse(BigInt(0))) .flatMap(h => getBlockByHeader(h)) val changedLinks: Seq[(StorageKey, StorageValue)] = List( - BestBlockKey -> StorageValue @@ validChain.last.id.untag(ModifierId), + BestBlockKey -> StorageValue @@ validChain.last.id.untag(ModifierId), BestHeaderKey -> StorageValue @@ newBestHeader.id.untag(ModifierId) ) val toInsert: List[(StorageKey, StorageValue)] = validityRow ++ changedLinks @@ -115,21 +123,22 @@ trait History extends HistoryModifiersValidator with AutoCloseable { } /** - * Marks modifier as valid - * - * @param modifier that is valid against the State - * @return ProgressInfo with next modifier to try to apply - */ + * Marks modifier as valid + * + * @param modifier that is valid against the State + * @return ProgressInfo with next modifier to try to apply + */ def reportModifierIsValid(modifier: PersistentModifier): History = { logger.info(s"Modifier ${modifier.encodedId} of type ${modifier.modifierTypeId} is marked as valid ") modifier match { case block: Block => val nonMarkedIds: Seq[ModifierId] = Seq(block.header.id, block.payload.id) .filter(id => historyStorage.get(validityKey(id)).isEmpty) - if (nonMarkedIds.nonEmpty) historyStorage.insert( - StorageVersion @@ validityKey(nonMarkedIds.head).untag(StorageKey), - nonMarkedIds.map(id => validityKey(id) -> StorageValue @@ Array(1.toByte)).toList - ) + if (nonMarkedIds.nonEmpty) + historyStorage.insert( + StorageVersion @@ validityKey(nonMarkedIds.head).untag(StorageKey), + nonMarkedIds.map(id => validityKey(id) -> StorageValue @@ Array(1.toByte)).toList + ) this case _ => historyStorage.insert( @@ -166,8 +175,8 @@ object History extends StrictLogging { case VersionalStorage.IODB => logger.info("Init history with iodb storage") val historyObjectsDir: File = getHistoryObjectsDir(settingsEncry) - val indexStore: LSMStore = new LSMStore(historyIndexDir, keepVersions = 0) - val objectsStore: LSMStore = new LSMStore(historyObjectsDir, keepVersions = 0) + val indexStore: LSMStore = new LSMStore(historyIndexDir, keepVersions = 0) + val objectsStore: LSMStore = new LSMStore(historyObjectsDir, keepVersions = 0) IODBHistoryWrapper(indexStore, objectsStore) case VersionalStorage.LevelDB => logger.info("Init history with levelDB storage") @@ -176,18 +185,17 @@ object History extends StrictLogging { } if (settingsEncry.snapshotSettings.enableFastSynchronization && !settingsEncry.node.offlineGeneration) new History with HistoryHeadersProcessor with FastSyncProcessor { - override val settings: EncryAppSettings = settingsEncry - override var isFullChainSynced: Boolean = settings.node.offlineGeneration - override val historyStorage: HistoryStorage = HistoryStorage(vldbInit) + override val settings: EncryAppSettings = settingsEncry + override var isFullChainSynced: Boolean = settings.node.offlineGeneration + override val historyStorage: HistoryStorage = HistoryStorage(vldbInit) override val timeProvider: NetworkTimeProvider = new NetworkTimeProvider(settingsEncry.ntp) - } - else + } else new History with HistoryHeadersProcessor with HistoryPayloadsProcessor { - override val settings: EncryAppSettings = settingsEncry - override var isFullChainSynced: Boolean = settings.node.offlineGeneration - override val historyStorage: HistoryStorage = HistoryStorage(vldbInit) + override val settings: EncryAppSettings = settingsEncry + override var isFullChainSynced: Boolean = settings.node.offlineGeneration + override val historyStorage: HistoryStorage = HistoryStorage(vldbInit) override val timeProvider: NetworkTimeProvider = new NetworkTimeProvider(settingsEncry.ntp) } } -} \ No newline at end of file +} diff --git a/src/main/scala/encry/view/history/HistoryApi.scala b/src/main/scala/encry/view/history/HistoryApi.scala index a367c509ad..d0160bc314 100644 --- a/src/main/scala/encry/view/history/HistoryApi.scala +++ b/src/main/scala/encry/view/history/HistoryApi.scala @@ -11,7 +11,7 @@ import io.iohk.iodb.ByteArrayWrapper import org.encryfoundation.common.modifiers.history._ import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, Height, ModifierId, ModifierTypeId} +import org.encryfoundation.common.utils.TaggedTypes.{ Difficulty, Height, ModifierId, ModifierTypeId } import scala.annotation.tailrec import scala.collection.immutable.HashSet @@ -41,61 +41,85 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore lazy val fastSyncInProgress: FastSyncProcessor = FastSyncProcessor(settings) - def getHeaderById(id: ModifierId): Option[Header] = headersCache - .get(ByteArrayWrapper(id)) - .orElse(blocksCache.get(ByteArrayWrapper(id)).map(_.header)) - .orElse(getHeaderByIdDB(id)) - - def getBlockByHeaderId(id: ModifierId): Option[Block] = - blocksCache - .get(ByteArrayWrapper(id)) - .orElse(headersCache.get(ByteArrayWrapper(id)) - .flatMap(h => getPayloadByIdDB(h.payloadId).map(p => Block(h, p)))) - .orElse(getBlockByHeaderIdDB(id)) - - def getBlockByHeader(header: Header): Option[Block] = blocksCache - .get(ByteArrayWrapper(header.id)) - .orElse(getPayloadByIdDB(header.payloadId).map(p => Block(header, p))) - - def getBestHeader: Option[Header] = getBestHeaderId.flatMap(id => + def getHeaderById(id: ModifierId): Option[Header] = headersCache .get(ByteArrayWrapper(id)) .orElse(blocksCache.get(ByteArrayWrapper(id)).map(_.header)) .orElse(getHeaderByIdDB(id)) - ) - - def getBestHeaderHeight: Int = getBestHeaderId.flatMap(id => - headersCache.get(ByteArrayWrapper(id)).map(_.height) - .orElse(blocksCache.get(ByteArrayWrapper(id)).map(_.header.height)) - .orElse(getHeightByHeaderId(id)) - ).getOrElse(settings.constants.PreGenesisHeight) - def getBestBlock: Option[Block] = getBestBlockId.flatMap(id => - blocksCache.get(ByteArrayWrapper(id)) + def getBlockByHeaderId(id: ModifierId): Option[Block] = + blocksCache + .get(ByteArrayWrapper(id)) + .orElse( + headersCache + .get(ByteArrayWrapper(id)) + .flatMap(h => getPayloadByIdDB(h.payloadId).map(p => Block(h, p))) + ) .orElse(getBlockByHeaderIdDB(id)) - ) - def getBestBlockHeight: Int = getBestBlockId - .flatMap(id => blocksCache.get(ByteArrayWrapper(id)).map(_.header.height).orElse(getHeightByHeaderId(id))) - .getOrElse(settings.constants.PreGenesisHeight) - - def getHeaderOfBestBlock: Option[Header] = getBestBlockId.flatMap(id => - headersCache.get(ByteArrayWrapper(id)) - .orElse(blocksCache.get(ByteArrayWrapper(id)).map(_.header)) - .orElse(getHeaderByIdDB(id)) - ) + def getBlockByHeader(header: Header): Option[Block] = + blocksCache + .get(ByteArrayWrapper(header.id)) + .orElse(getPayloadByIdDB(header.payloadId).map(p => Block(header, p))) + + def getBestHeader: Option[Header] = + getBestHeaderId.flatMap( + id => + headersCache + .get(ByteArrayWrapper(id)) + .orElse(blocksCache.get(ByteArrayWrapper(id)).map(_.header)) + .orElse(getHeaderByIdDB(id)) + ) + + def getBestHeaderHeight: Int = + getBestHeaderId + .flatMap( + id => + headersCache + .get(ByteArrayWrapper(id)) + .map(_.height) + .orElse(blocksCache.get(ByteArrayWrapper(id)).map(_.header.height)) + .orElse(getHeightByHeaderId(id)) + ) + .getOrElse(settings.constants.PreGenesisHeight) + + def getBestBlock: Option[Block] = + getBestBlockId.flatMap( + id => + blocksCache + .get(ByteArrayWrapper(id)) + .orElse(getBlockByHeaderIdDB(id)) + ) + + def getBestBlockHeight: Int = + getBestBlockId + .flatMap(id => blocksCache.get(ByteArrayWrapper(id)).map(_.header.height).orElse(getHeightByHeaderId(id))) + .getOrElse(settings.constants.PreGenesisHeight) + + def getHeaderOfBestBlock: Option[Header] = + getBestBlockId.flatMap( + id => + headersCache + .get(ByteArrayWrapper(id)) + .orElse(blocksCache.get(ByteArrayWrapper(id)).map(_.header)) + .orElse(getHeaderByIdDB(id)) + ) def getBestHeaderAtHeight(h: Int): Option[Header] = getBestHeaderAtHeightDB(h) - def getBlockByPayload(payload: Payload): Option[Block] = headersCache - .get(ByteArrayWrapper(payload.headerId)).map(h => Block(h, payload)) - .orElse(blocksCache.get(ByteArrayWrapper(payload.headerId))) - .orElse(getHeaderById(payload.headerId).flatMap(h => Some(Block(h, payload)))) + def getBlockByPayload(payload: Payload): Option[Block] = + headersCache + .get(ByteArrayWrapper(payload.headerId)) + .map(h => Block(h, payload)) + .orElse(blocksCache.get(ByteArrayWrapper(payload.headerId))) + .orElse(getHeaderById(payload.headerId).flatMap(h => Some(Block(h, payload)))) - def getHeightByHeaderId(id: ModifierId): Option[Int] = headersCache - .get(ByteArrayWrapper(id)).map(_.height) - .orElse(blocksCache.get(ByteArrayWrapper(id)).map(_.header.height)) - .orElse(getHeightByHeaderIdDB(id)) + def getHeightByHeaderId(id: ModifierId): Option[Int] = + headersCache + .get(ByteArrayWrapper(id)) + .map(_.height) + .orElse(blocksCache.get(ByteArrayWrapper(id)).map(_.header.height)) + .orElse(getHeightByHeaderIdDB(id)) def isBestBlockDefined: Boolean = getBestBlockId.map(id => blocksCache.contains(ByteArrayWrapper(id))).isDefined || @@ -110,33 +134,39 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore isModifierDefined(id) def getBestHeaderIdAtHeight(h: Int): Option[ModifierId] = getBestHeaderIdAtHeightDB(h) - def headerIdsAtHeight(height: Int): Seq[ModifierId] = headerIdsAtHeightDB(height) - .getOrElse(Seq.empty[ModifierId]) + def headerIdsAtHeight(height: Int): Seq[ModifierId] = + headerIdsAtHeightDB(height) + .getOrElse(Seq.empty[ModifierId]) - def modifierBytesById(id: ModifierId): Option[Array[Byte]] = headersCache - .get(ByteArrayWrapper(id)).map(h => HeaderProtoSerializer.toProto(h).toByteArray) - .orElse(blocksCache.get(ByteArrayWrapper(id)).map(b => BlockProtoSerializer.toProto(b).toByteArray)) - .orElse(modifierBytesByIdDB(id)) + def modifierBytesById(id: ModifierId): Option[Array[Byte]] = + headersCache + .get(ByteArrayWrapper(id)) + .map(h => HeaderProtoSerializer.toProto(h).toByteArray) + .orElse(blocksCache.get(ByteArrayWrapper(id)).map(b => BlockProtoSerializer.toProto(b).toByteArray)) + .orElse(modifierBytesByIdDB(id)) - def lastHeaders(count: Int): HeaderChain = getBestHeader - .map(bestHeader => headerChainBack(count, bestHeader, _ => false)) - .getOrElse(HeaderChain.empty) + def lastHeaders(count: Int): HeaderChain = + getBestHeader + .map(bestHeader => headerChainBack(count, bestHeader, _ => false)) + .getOrElse(HeaderChain.empty) - def getHeaderIds(count: Int, offset: Int = 0): Seq[ModifierId] = (offset until (count + offset)) - .flatMap(getBestHeaderIdAtHeight) + def getHeaderIds(count: Int, offset: Int = 0): Seq[ModifierId] = + (offset until (count + offset)) + .flatMap(getBestHeaderIdAtHeight) def payloadsIdsToDownload(howMany: Int, excluding: HashSet[ModifierId]): Seq[ModifierId] = { @tailrec def continuation(height: Int, acc: Seq[ModifierId]): Seq[ModifierId] = if (acc.lengthCompare(howMany) >= 0) acc else if (height > lastAvailableManifestHeight && fastSyncInProgress.fastSyncVal) acc - else getBestHeaderIdAtHeight(height).flatMap(getHeaderById) match { - case Some(h) if !excluding.exists(_.sameElements(h.payloadId)) && !isBlockDefined(h) => - continuation(height + 1, acc :+ h.payloadId) - case Some(_) => - continuation(height + 1, acc) - case None => - acc - } + else + getBestHeaderIdAtHeight(height).flatMap(getHeaderById) match { + case Some(h) if !excluding.exists(_.sameElements(h.payloadId)) && !isBlockDefined(h) => + continuation(height + 1, acc :+ h.payloadId) + case Some(_) => + continuation(height + 1, acc) + case None => + acc + } (for { bestBlockId <- getBestBlockId @@ -166,41 +196,50 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore } else none def headerChainBack(limit: Int, startHeader: Header, until: Header => Boolean): HeaderChain = { - @tailrec def loop(header: Header, acc: Seq[Header]): Seq[Header] = { + @tailrec def loop(header: Header, acc: Seq[Header]): Seq[Header] = if (acc.length == limit || until(header)) acc - else getHeaderById(header.parentId) match { - case Some(parent: Header) => loop(parent, acc :+ parent) - case None if acc.contains(header) => acc - case _ => acc :+ header - } - } + else + getHeaderById(header.parentId) match { + case Some(parent: Header) => loop(parent, acc :+ parent) + case None if acc.contains(header) => acc + case _ => acc :+ header + } if (getBestHeaderId.isEmpty || (limit == 0)) HeaderChain(Seq.empty) else HeaderChain(loop(startHeader, Seq(startHeader)).reverse) } - @tailrec final def loopHeightDown(height: Int, p: ModifierId => Boolean): Option[Header] = headerIdsAtHeight(height) - .find(p) - .flatMap(getHeaderById) match { - case h@Some(_) => h + @tailrec final def loopHeightDown(height: Int, p: ModifierId => Boolean): Option[Header] = + headerIdsAtHeight(height) + .find(p) + .flatMap(getHeaderById) match { + case h @ Some(_) => h case None if height > settings.constants.GenesisHeight => loopHeightDown(height - 1, p) - case n@None => n - } + case n @ None => n + } def requiredDifficultyAfter(parent: Header): Either[HistoryApiError, Difficulty] = { - val requiredHeights: Seq[Height] = PowLinearController.getHeightsForRetargetingAt(Height @@ (parent.height + 1), - settings.constants.EpochLength, settings.constants.RetargetingEpochsQty) + val requiredHeights: Seq[Height] = PowLinearController.getHeightsForRetargetingAt( + Height @@ (parent.height + 1), + settings.constants.EpochLength, + settings.constants.RetargetingEpochsQty + ) for { - _ <- Either.cond(requiredHeights.lastOption.contains(parent.height), (), - HistoryApiError("Incorrect heights sequence in requiredDifficultyAfter function")) + _ <- Either.cond(requiredHeights.lastOption.contains(parent.height), + (), + HistoryApiError("Incorrect heights sequence in requiredDifficultyAfter function")) chain = headerChainBack(requiredHeights.max - requiredHeights.min + 1, parent, (_: Header) => false) requiredHeaders = (requiredHeights.min to requiredHeights.max) .zip(chain.headers) .filter(p => requiredHeights.contains(p._1)) - _ <- Either.cond(requiredHeights.length == requiredHeaders.length, (), - HistoryApiError(s"Missed headers: $requiredHeights != ${requiredHeaders.map(_._1)}")) - } yield PowLinearController.getDifficulty(requiredHeaders, settings.constants.EpochLength, - settings.constants.DesiredBlockInterval, settings.constants.InitialDifficulty) + _ <- Either.cond(requiredHeights.length == requiredHeaders.length, + (), + HistoryApiError(s"Missed headers: $requiredHeights != ${requiredHeaders.map(_._1)}")) + } yield + PowLinearController.getDifficulty(requiredHeaders, + settings.constants.EpochLength, + settings.constants.DesiredBlockInterval, + settings.constants.InitialDifficulty) } def syncInfo: SyncInfo = lastSyncInfo @@ -219,7 +258,7 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore case Some(id) if si.lastHeaderIds.exists(_ sameElements id) => Older /* Other history is empty, or our history contains last id from other history */ case Some(_) if si.lastHeaderIds.isEmpty || si.lastHeaderIds.lastOption.exists(isHeaderDefined) => Younger - case Some(_) => + case Some(_) => //Our history contains some ids from other history if (si.lastHeaderIds.exists(isHeaderDefined)) Fork //Unknown comparison result @@ -238,7 +277,8 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore startId <- headerIdsAtHeight(heightFrom).headOption startHeader <- getHeaderById(startId) } yield headerChainBack(size, startHeader, _ => false)) match { - case Some(value) if value.headers.exists(_.height == settings.constants.GenesisHeight) => value.headers.map(_.id) + case Some(value) if value.headers.exists(_.height == settings.constants.GenesisHeight) => + value.headers.map(_.id) case _ => Seq.empty } } else { @@ -246,19 +286,18 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore (for { lastHeaderInOurBestChain <- ids.view.reverse.find(m => isInBestChain(m)) theirHeight <- heightOf(lastHeaderInOurBestChain) - heightFrom = Math.min(getBestHeaderHeight, theirHeight + size) + heightFrom = Math.min(getBestHeaderHeight, theirHeight + size) startId <- headerIdsAtHeight(heightFrom).headOption startHeader <- getHeaderById(startId) - } yield headerChainBack(size, startHeader, h => h.parentId sameElements lastHeaderInOurBestChain) - .headers + } yield + headerChainBack(size, startHeader, h => h.parentId sameElements lastHeaderInOurBestChain).headers .map(_.id)) match { - case Some(value) => value - case None => Seq.empty + case Some(value) => value + case None => Seq.empty } } - def commonBlockThenSuffixes(header1: Header, - header2: Header): (HeaderChain, HeaderChain) = { + def commonBlockThenSuffixes(header1: Header, header2: Header): (HeaderChain, HeaderChain) = { val heightDelta: Int = Math.max(header1.height - header2.height, 0) @scala.annotation.tailrec @@ -284,13 +323,13 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore loop(2, HeaderChain(Seq(header2))) } - def getChainToHeader(fromHeaderOpt: Option[Header], - toHeader: Header): (Option[ModifierId], HeaderChain) = fromHeaderOpt match { - case Some(h1) => - val (prevChain, newChain) = commonBlockThenSuffixes(h1, toHeader) - (prevChain.headOption.map(_.id), newChain.tail) - case None => (None, headerChainBack(toHeader.height + 1, toHeader, _ => false)) - } + def getChainToHeader(fromHeaderOpt: Option[Header], toHeader: Header): (Option[ModifierId], HeaderChain) = + fromHeaderOpt match { + case Some(h1) => + val (prevChain, newChain) = commonBlockThenSuffixes(h1, toHeader) + (prevChain.headOption.map(_.id), newChain.tail) + case None => (None, headerChainBack(toHeader.height + 1, toHeader, _ => false)) + } def isNewHeader(header: Header): Boolean = timeProvider.estimatedTime - header.timestamp < @@ -298,18 +337,16 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore def isHeadersChainSynced: Boolean = isHeadersChainSyncedVar - def continuationHeaderChains(header: Header, - filterCond: Header => Boolean): Seq[Seq[Header]] = { + def continuationHeaderChains(header: Header, filterCond: Header => Boolean): Seq[Seq[Header]] = { @tailrec def loop(currentHeight: Int, acc: Seq[Seq[Header]]): Seq[Seq[Header]] = { - val nextHeightHeaders: Seq[Header] = headerIdsAtHeight(currentHeight + 1) - .view + val nextHeightHeaders: Seq[Header] = headerIdsAtHeight(currentHeight + 1).view .flatMap(getHeaderById) .filter(filterCond) .toList if (nextHeightHeaders.isEmpty) acc.map(_.reverse) else { - val updatedChains: Seq[Seq[Header]] = nextHeightHeaders.flatMap(h => - acc.find(chain => chain.nonEmpty && (h.parentId sameElements chain.head.id)).map(h +: _) + val updatedChains: Seq[Seq[Header]] = nextHeightHeaders.flatMap( + h => acc.find(chain => chain.nonEmpty && (h.parentId sameElements chain.head.id)).map(h +: _) ) val nonUpdatedChains: Seq[Seq[Header]] = acc.filter(chain => !nextHeightHeaders.exists(_.parentId sameElements chain.head.id)) @@ -345,7 +382,7 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore logger.debug(s"Should add ${Algos.encode(b.id)} to block cache") val newBlocksIdsAtBlockHeight = blocksCacheIndexes.getOrElse(b.header.height, Seq.empty[ModifierId]) :+ b.id blocksCacheIndexes = blocksCacheIndexes + (b.header.height -> newBlocksIdsAtBlockHeight) - blocksCache = blocksCache + (ByteArrayWrapper(b.id) -> b) + blocksCache = blocksCache + (ByteArrayWrapper(b.id) -> b) // cleanup cache if necessary if (blocksCacheIndexes.size > settings.constants.MaxRollbackDepth) { blocksCacheIndexes.get(getBestBlockHeight - settings.constants.MaxRollbackDepth).foreach { blocksIds => @@ -358,4 +395,4 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore logger.debug(s"headersCache size: ${blocksCache.size}") logger.debug(s"headersCacheIndexes size: ${blocksCacheIndexes.size}") } -} \ No newline at end of file +} diff --git a/src/main/scala/encry/view/history/HistoryDBApi.scala b/src/main/scala/encry/view/history/HistoryDBApi.scala index 55427766a8..3e972e0910 100644 --- a/src/main/scala/encry/view/history/HistoryDBApi.scala +++ b/src/main/scala/encry/view/history/HistoryDBApi.scala @@ -5,9 +5,9 @@ import com.typesafe.scalalogging.StrictLogging import encry.settings.EncryAppSettings import encry.storage.VersionalStorage.StorageKey import encry.view.history.storage.HistoryStorage -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{Height, ModifierId, ModifierTypeId} +import org.encryfoundation.common.utils.TaggedTypes.{ Height, ModifierId, ModifierTypeId } import scorex.crypto.hash.Digest32 import scala.reflect.ClassTag @@ -23,67 +23,80 @@ trait HistoryDBApi extends StrictLogging { lazy val BestBlockKey: StorageKey = StorageKey @@ Array.fill(settings.constants.DigestLength)(-1: Byte) - private def getModifierById[T: ClassTag](id: ModifierId): Option[T] = historyStorage - .modifierById(id) - .collect { case m: T => m } + private def getModifierById[T: ClassTag](id: ModifierId): Option[T] = + historyStorage + .modifierById(id) + .collect { case m: T => m } - def getHeightByHeaderIdDB(id: ModifierId): Option[Int] = historyStorage - .get(headerHeightKey(id)) - .map(Ints.fromByteArray) + def getHeightByHeaderIdDB(id: ModifierId): Option[Int] = + historyStorage + .get(headerHeightKey(id)) + .map(Ints.fromByteArray) - def getHeaderByIdDB(id: ModifierId): Option[Header] = getModifierById[Header](id) + def getHeaderByIdDB(id: ModifierId): Option[Header] = getModifierById[Header](id) def getPayloadByIdDB(pId: ModifierId): Option[Payload] = getModifierById[Payload](pId) - def getBlockByHeaderDB(header: Header): Option[Block] = getModifierById[Payload](header.payloadId) - .map(payload => Block(header, payload)) - def getBlockByHeaderIdDB(id: ModifierId): Option[Block] = getHeaderByIdDB(id) - .flatMap(h => getModifierById[Payload](h.payloadId).map(p => Block(h, p))) + def getBlockByHeaderDB(header: Header): Option[Block] = + getModifierById[Payload](header.payloadId) + .map(payload => Block(header, payload)) + def getBlockByHeaderIdDB(id: ModifierId): Option[Block] = + getHeaderByIdDB(id) + .flatMap(h => getModifierById[Payload](h.payloadId).map(p => Block(h, p))) def getBestHeaderId: Option[ModifierId] = historyStorage.get(BestHeaderKey).map(ModifierId @@ _) - def getBestHeaderDB: Option[Header] = getBestHeaderId.flatMap(getHeaderByIdDB) - def getBestHeaderHeightDB: Int = getBestHeaderId - .flatMap(getHeightByHeaderIdDB) - .getOrElse(settings.constants.PreGenesisHeight) + def getBestHeaderDB: Option[Header] = getBestHeaderId.flatMap(getHeaderByIdDB) + def getBestHeaderHeightDB: Int = + getBestHeaderId + .flatMap(getHeightByHeaderIdDB) + .getOrElse(settings.constants.PreGenesisHeight) def getBestBlockId: Option[ModifierId] = historyStorage.get(BestBlockKey).map(ModifierId @@ _) - def getBestBlockDB: Option[Block] = getBestBlockId.flatMap(getBlockByHeaderIdDB) - def getBestBlockHeightDB: Int = getBestBlockId - .flatMap(getHeightByHeaderIdDB) - .getOrElse(settings.constants.PreGenesisHeight) + def getBestBlockDB: Option[Block] = getBestBlockId.flatMap(getBlockByHeaderIdDB) + def getBestBlockHeightDB: Int = + getBestBlockId + .flatMap(getHeightByHeaderIdDB) + .getOrElse(settings.constants.PreGenesisHeight) def modifierBytesByIdDB(id: ModifierId): Option[Array[Byte]] = historyStorage.modifiersBytesById(id) def isModifierDefined(id: ModifierId): Boolean = historyStorage.containsMod(id) //todo probably rewrite with indexes collection - def lastBestBlockHeightRelevantToBestChain(probablyAt: Int): Option[Int] = (for { - headerId <- getBestHeaderIdAtHeightDB(probablyAt) - header <- getHeaderByIdDB(headerId) if isModifierDefined(header.payloadId) - } yield header.height).orElse(lastBestBlockHeightRelevantToBestChain(probablyAt - 1)) + def lastBestBlockHeightRelevantToBestChain(probablyAt: Int): Option[Int] = + (for { + headerId <- getBestHeaderIdAtHeightDB(probablyAt) + header <- getHeaderByIdDB(headerId) if isModifierDefined(header.payloadId) + } yield header.height).orElse(lastBestBlockHeightRelevantToBestChain(probablyAt - 1)) - def headerIdsAtHeightDB(height: Int): Option[Seq[ModifierId]] = historyStorage - .get(heightIdsKey(height)) - .map(_.grouped(32).map(ModifierId @@ _).toSeq) + def headerIdsAtHeightDB(height: Int): Option[Seq[ModifierId]] = + historyStorage + .get(heightIdsKey(height)) + .map(_.grouped(32).map(ModifierId @@ _).toSeq) def getBestHeaderIdAtHeightDB(h: Int): Option[ModifierId] = headerIdsAtHeightDB(h).flatMap(_.headOption) def getBestHeaderAtHeightDB(h: Int): Option[Header] = getBestHeaderIdAtHeightDB(h).flatMap(getHeaderByIdDB) - def isInBestChain(h: Header): Boolean = getBestHeaderIdAtHeightDB(h.height) - .exists(_.sameElements(h.id)) + def isInBestChain(h: Header): Boolean = + getBestHeaderIdAtHeightDB(h.height) + .exists(_.sameElements(h.id)) - def isInBestChain(id: ModifierId): Boolean = heightOf(id) - .flatMap(getBestHeaderIdAtHeightDB) - .exists(_.sameElements(id)) + def isInBestChain(id: ModifierId): Boolean = + heightOf(id) + .flatMap(getBestHeaderIdAtHeightDB) + .exists(_.sameElements(id)) - def getBestHeadersChainScore: BigInt = getBestHeaderId.flatMap(scoreOf).getOrElse(BigInt(0)) //todo ?.getOrElse(BigInt(0))? + def getBestHeadersChainScore: BigInt = + getBestHeaderId.flatMap(scoreOf).getOrElse(BigInt(0)) //todo ?.getOrElse(BigInt(0))? - def scoreOf(id: ModifierId): Option[BigInt] = historyStorage - .get(headerScoreKey(id)) - .map(d => BigInt(d)) + def scoreOf(id: ModifierId): Option[BigInt] = + historyStorage + .get(headerScoreKey(id)) + .map(d => BigInt(d)) - def heightOf(id: ModifierId): Option[Height] = historyStorage - .get(headerHeightKey(id)) - .map(d => Height @@ Ints.fromByteArray(d)) + def heightOf(id: ModifierId): Option[Height] = + historyStorage + .get(headerHeightKey(id)) + .map(d => Height @@ Ints.fromByteArray(d)) def heightIdsKey(height: Int): StorageKey = StorageKey @@ Algos.hash(Ints.toByteArray(height)).untag(Digest32) @@ -93,4 +106,4 @@ trait HistoryDBApi extends StrictLogging { StorageKey @@ Algos.hash("height".getBytes(Algos.charset) ++ id).untag(Digest32) def validityKey(id: Array[Byte]): StorageKey = StorageKey @@ Algos.hash("validity".getBytes(Algos.charset) ++ id).untag(Digest32) -} \ No newline at end of file +} diff --git a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala index 0741251609..a3848f7871 100644 --- a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala +++ b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala @@ -5,14 +5,17 @@ import encry.consensus.EquihashPowScheme import encry.view.history.ValidationError.FatalValidationError._ import encry.view.history.ValidationError.NonFatalValidationError._ import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{Header, Payload} -import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, ModifierId} +import org.encryfoundation.common.modifiers.history.{ Header, Payload } +import org.encryfoundation.common.utils.TaggedTypes.{ Difficulty, ModifierId } import org.encryfoundation.common.validation.ModifierSemanticValidity trait HistoryModifiersValidator extends HistoryApi { - lazy val powScheme: EquihashPowScheme = EquihashPowScheme(settings.constants.n, settings.constants.k, settings.constants.Version, - settings.constants.PreGenesisHeight, settings.constants.MaxTarget) + lazy val powScheme: EquihashPowScheme = EquihashPowScheme(settings.constants.n, + settings.constants.k, + settings.constants.Version, + settings.constants.PreGenesisHeight, + settings.constants.MaxTarget) def testApplicable(modifier: PersistentModifier): Either[ValidationError, PersistentModifier] = (modifier match { @@ -20,77 +23,137 @@ trait HistoryModifiersValidator extends HistoryApi { case payload: Payload => validatePayload(payload) case mod => UnknownModifierFatalError(s"Modifier $mod has incorrect type.").asLeft[PersistentModifier] }) match { - case l@Left(value) => logger.info(s"Validation result for ${modifier.encodedId} failed cause $value"); l - case r@Right(_) => r + case l @ Left(value) => logger.info(s"Validation result for ${modifier.encodedId} failed cause $value"); l + case r @ Right(_) => r } private def validateHeader(h: Header): Either[ValidationError, Header] = if (h.isGenesis) genesisBlockHeaderValidator(h) - else getHeaderById(h.parentId) - .map(p => headerValidator(h, p)) - .getOrElse(HeaderNonFatalValidationError(s"Header's ${h.encodedId} parent doesn't contain in history").asLeft[Header]) + else + getHeaderById(h.parentId) + .map(p => headerValidator(h, p)) + .getOrElse( + HeaderNonFatalValidationError(s"Header's ${h.encodedId} parent doesn't contain in history").asLeft[Header] + ) - private def validatePayload(mod: Payload): Either[ValidationError, PersistentModifier] = getHeaderById(mod.headerId) - .map(header => payloadValidator(mod, header, blockDownloadProcessor.minimalBlockHeight)) - .getOrElse(PayloadNonFatalValidationError(s"Header for ${mod.encodedId} doesn't contain in history").asLeft[PersistentModifier]) + private def validatePayload(mod: Payload): Either[ValidationError, PersistentModifier] = + getHeaderById(mod.headerId) + .map(header => payloadValidator(mod, header, blockDownloadProcessor.minimalBlockHeight)) + .getOrElse( + PayloadNonFatalValidationError(s"Header for ${mod.encodedId} doesn't contain in history") + .asLeft[PersistentModifier] + ) private def realDifficulty(h: Header): Difficulty = Difficulty !@@ powScheme.realDifficulty(h) - private def isSemanticallyValid(modifierId: ModifierId): ModifierSemanticValidity = historyStorage - .get(validityKey(modifierId)) match { + private def isSemanticallyValid(modifierId: ModifierId): ModifierSemanticValidity = + historyStorage + .get(validityKey(modifierId)) match { case Some(mod) if mod.headOption.contains(1.toByte) => ModifierSemanticValidity.Valid case Some(mod) if mod.headOption.contains(0.toByte) => ModifierSemanticValidity.Invalid case None if isModifierDefined(modifierId) => ModifierSemanticValidity.Unknown case None => ModifierSemanticValidity.Absent - case mod => logger.error(s"Incorrect validity status: $mod") - ModifierSemanticValidity.Absent - } + case mod => + logger.error(s"Incorrect validity status: $mod") + ModifierSemanticValidity.Absent + } - private def genesisBlockHeaderValidator(h: Header): Either[ValidationError, Header] = for { - _ <- Either.cond(h.parentId.sameElements(Header.GenesisParentId), (), - GenesisBlockFatalValidationError(s"Genesis block with header ${h.encodedId} should has genesis parent id")) - _ <- Either.cond(getBestHeaderId.isEmpty, (), - GenesisBlockFatalValidationError(s"Genesis block with header ${h.encodedId} appended to non-empty history")) - _ <- Either.cond(h.height == settings.constants.GenesisHeight, (), - GenesisBlockFatalValidationError(s"Height of genesis block with header ${h.encodedId} is incorrect")) - } yield h + private def genesisBlockHeaderValidator(h: Header): Either[ValidationError, Header] = + for { + _ <- Either.cond( + h.parentId.sameElements(Header.GenesisParentId), + (), + GenesisBlockFatalValidationError(s"Genesis block with header ${h.encodedId} should has genesis parent id") + ) + _ <- Either.cond( + getBestHeaderId.isEmpty, + (), + GenesisBlockFatalValidationError(s"Genesis block with header ${h.encodedId} appended to non-empty history") + ) + _ <- Either.cond( + h.height == settings.constants.GenesisHeight, + (), + GenesisBlockFatalValidationError(s"Height of genesis block with header ${h.encodedId} is incorrect") + ) + } yield h - private def headerValidator(h: Header, parent: Header): Either[ValidationError, Header] = for { - _ <- Either.cond(h.timestamp > parent.timestamp, (), - HeaderFatalValidationError(s"Header ${h.encodedId} has timestamp ${h.timestamp}" + - s" less than parent's ${parent.timestamp}")) - _ <- Either.cond(h.height == parent.height + 1, (), - HeaderFatalValidationError(s"Header ${h.encodedId} has height ${h.height}" + - s" not greater by 1 than parent's ${parent.height}")) - _ <- Either.cond(!historyStorage.containsMod(h.id), (), - HeaderFatalValidationError(s"Header ${h.encodedId} is already in history")) - _ <- Either.cond(realDifficulty(h) >= h.requiredDifficulty, (), - HeaderFatalValidationError(s"Incorrect real difficulty in header ${h.encodedId}")) - _ <- Either.cond(requiredDifficultyAfter(parent).exists(_ <= h.difficulty), (), - HeaderFatalValidationError(s"Incorrect required difficulty in header ${h.encodedId}")) - _ <- Either.cond(heightOf(h.parentId).exists(h => getBestHeaderHeight - h < settings.constants.MaxRollbackDepth), (), - HeaderFatalValidationError(s"Header ${h.encodedId} has height greater than max roll back depth")) - powSchemeValidationResult = powScheme.verify(h) - _ <- Either.cond(powSchemeValidationResult.isRight, (), - HeaderFatalValidationError(s"Wrong proof-of-work solution in header ${h.encodedId}" + - s" caused: $powSchemeValidationResult")) - _ <- Either.cond(isSemanticallyValid(h.parentId) != ModifierSemanticValidity.Invalid, (), - HeaderFatalValidationError(s"Header ${h.encodedId} is semantically invalid")) - _ <- Either.cond(h.timestamp - timeProvider.estimatedTime <= settings.constants.MaxTimeDrift, (), - HeaderNonFatalValidationError(s"Header ${h.encodedId} with timestamp ${h.timestamp}" + - s" is too far in future from now ${timeProvider.estimatedTime}")) - } yield h + private def headerValidator(h: Header, parent: Header): Either[ValidationError, Header] = + for { + _ <- Either.cond( + h.timestamp > parent.timestamp, + (), + HeaderFatalValidationError( + s"Header ${h.encodedId} has timestamp ${h.timestamp}" + + s" less than parent's ${parent.timestamp}" + ) + ) + _ <- Either.cond( + h.height == parent.height + 1, + (), + HeaderFatalValidationError( + s"Header ${h.encodedId} has height ${h.height}" + + s" not greater by 1 than parent's ${parent.height}" + ) + ) + _ <- Either.cond(!historyStorage.containsMod(h.id), + (), + HeaderFatalValidationError(s"Header ${h.encodedId} is already in history")) + _ <- Either.cond(realDifficulty(h) >= h.requiredDifficulty, + (), + HeaderFatalValidationError(s"Incorrect real difficulty in header ${h.encodedId}")) + _ <- Either.cond(requiredDifficultyAfter(parent).exists(_ <= h.difficulty), + (), + HeaderFatalValidationError(s"Incorrect required difficulty in header ${h.encodedId}")) + _ <- Either.cond( + heightOf(h.parentId).exists(h => getBestHeaderHeight - h < settings.constants.MaxRollbackDepth), + (), + HeaderFatalValidationError(s"Header ${h.encodedId} has height greater than max roll back depth") + ) + powSchemeValidationResult = powScheme.verify(h) + _ <- Either.cond( + powSchemeValidationResult.isRight, + (), + HeaderFatalValidationError( + s"Wrong proof-of-work solution in header ${h.encodedId}" + + s" caused: $powSchemeValidationResult" + ) + ) + _ <- Either.cond(isSemanticallyValid(h.parentId) != ModifierSemanticValidity.Invalid, + (), + HeaderFatalValidationError(s"Header ${h.encodedId} is semantically invalid")) + _ <- Either.cond( + h.timestamp - timeProvider.estimatedTime <= settings.constants.MaxTimeDrift, + (), + HeaderNonFatalValidationError( + s"Header ${h.encodedId} with timestamp ${h.timestamp}" + + s" is too far in future from now ${timeProvider.estimatedTime}" + ) + ) + } yield h private def payloadValidator(m: PersistentModifier, header: Header, - minimalHeight: Int): Either[ValidationError, PersistentModifier] = for { - _ <- Either.cond(!historyStorage.containsMod(m.id), (), - PayloadFatalValidationError(s"Modifier ${m.encodedId} is already in history")) - _ <- Either.cond(header.isRelated(m), (), - PayloadFatalValidationError(s"Modifier ${m.encodedId} does not corresponds to header ${header.encodedId}")) - _ <- Either.cond(isSemanticallyValid(header.id) != ModifierSemanticValidity.Invalid, (), - PayloadFatalValidationError(s"Header ${header.encodedId} for modifier ${m.encodedId} is semantically invalid")) - _ <- Either.cond(header.height >= minimalHeight, (), - PayloadNonFatalValidationError(s"Too old modifier ${m.encodedId}: ${header.height} < $minimalHeight")) - } yield m -} \ No newline at end of file + minimalHeight: Int): Either[ValidationError, PersistentModifier] = + for { + _ <- Either.cond(!historyStorage.containsMod(m.id), + (), + PayloadFatalValidationError(s"Modifier ${m.encodedId} is already in history")) + _ <- Either.cond( + header.isRelated(m), + (), + PayloadFatalValidationError(s"Modifier ${m.encodedId} does not corresponds to header ${header.encodedId}") + ) + _ <- Either.cond( + isSemanticallyValid(header.id) != ModifierSemanticValidity.Invalid, + (), + PayloadFatalValidationError( + s"Header ${header.encodedId} for modifier ${m.encodedId} is semantically invalid" + ) + ) + _ <- Either.cond( + header.height >= minimalHeight, + (), + PayloadNonFatalValidationError(s"Too old modifier ${m.encodedId}: ${header.height} < $minimalHeight") + ) + } yield m +} diff --git a/src/main/scala/encry/view/history/ValidationError.scala b/src/main/scala/encry/view/history/ValidationError.scala index bcb5bd0397..73ac6faab5 100644 --- a/src/main/scala/encry/view/history/ValidationError.scala +++ b/src/main/scala/encry/view/history/ValidationError.scala @@ -16,4 +16,4 @@ object ValidationError { case class PayloadNonFatalValidationError(error: String) extends NonFatalValidationError } case class HistoryApiError(error: String) extends ValidationError -} \ No newline at end of file +} From e16af0483c9ddaa659ea244756fd19a2d2629ba4 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 14 Jan 2020 16:23:57 +0300 Subject: [PATCH 02/10] improved history dependency graph with cake pattern --- .../view/history/BlockDownloadProcessor.scala | 18 +- .../view/history/FastSyncProcessor.scala | 12 +- .../scala/encry/view/history/History.scala | 87 +++--- .../scala/encry/view/history/HistoryApi.scala | 257 ++++++++---------- .../encry/view/history/HistoryDBApi.scala | 95 +++---- .../history/HistoryHeadersProcessor.scala | 4 +- .../history/HistoryModifiersValidator.scala | 195 +++++-------- .../history/HistoryPayloadsProcessor.scala | 6 +- .../encry/view/history/ValidationError.scala | 2 +- 9 files changed, 276 insertions(+), 400 deletions(-) diff --git a/src/main/scala/encry/view/history/BlockDownloadProcessor.scala b/src/main/scala/encry/view/history/BlockDownloadProcessor.scala index 5a6a520b72..94d4867ade 100644 --- a/src/main/scala/encry/view/history/BlockDownloadProcessor.scala +++ b/src/main/scala/encry/view/history/BlockDownloadProcessor.scala @@ -5,7 +5,7 @@ import org.encryfoundation.common.modifiers.history.Header import org.encryfoundation.common.utils.constants.Constants /** Class that keeps and calculates minimal height for full blocks starting from which we need to download these full - * blocks from the network and keep them in our history. */ + * blocks from the network and keep them in our history. */ case class BlockDownloadProcessor(nodeSettings: NodeSettings, constants: Constants) { private[history] var minimalBlockHeightVar: Int = Int.MaxValue @@ -13,26 +13,26 @@ case class BlockDownloadProcessor(nodeSettings: NodeSettings, constants: Constan def updateMinimalBlockHeightVar(height: Int): Unit = minimalBlockHeightVar = height /** Start height to download full blocks. - * Int.MaxValue when headers chain is not synchronized with the network and no full blocks download needed */ + * Int.MaxValue when headers chain is not synchronized with the network and no full blocks download needed */ def minimalBlockHeight: Int = minimalBlockHeightVar /** Update minimal full block height - * - * @param header - header of new best full block - * @return minimal height to process best full block */ + * + * @param header - header of new best full block + * @return minimal height to process best full block */ def updateBestBlock(header: Header): Int = { minimalBlockHeightVar = minimalBlockHeightAfter(header) minimalBlockHeightVar } - private def minimalBlockHeightAfter(header: Header): Int = + private def minimalBlockHeightAfter(header: Header): Int = { if (minimalBlockHeightVar == Int.MaxValue) { // just synced with the headers chain - determine first full block to apply if (nodeSettings.blocksToKeep < 0) constants.GenesisHeight // keep all blocks in history // TODO: start with the height of UTXO snapshot applied. Start from genesis until this is implemented // Start from config.blocksToKeep blocks back else Math.max(constants.GenesisHeight, header.height - nodeSettings.blocksToKeep + 1) - } else if (nodeSettings.blocksToKeep >= 0) - Math.max(header.height - nodeSettings.blocksToKeep + 1, minimalBlockHeightVar) + } else if (nodeSettings.blocksToKeep >= 0) Math.max(header.height - nodeSettings.blocksToKeep + 1, minimalBlockHeightVar) else constants.GenesisHeight -} + } +} \ No newline at end of file diff --git a/src/main/scala/encry/view/history/FastSyncProcessor.scala b/src/main/scala/encry/view/history/FastSyncProcessor.scala index d5d6ff2a5d..d25dca9ab1 100644 --- a/src/main/scala/encry/view/history/FastSyncProcessor.scala +++ b/src/main/scala/encry/view/history/FastSyncProcessor.scala @@ -2,10 +2,10 @@ package encry.view.history import cats.syntax.option.none import encry.consensus.HistoryConsensus.ProgressInfo -import encry.storage.VersionalStorage.{ StorageKey, StorageValue, StorageVersion } +import encry.storage.VersionalStorage.{StorageKey, StorageValue, StorageVersion} import org.encryfoundation.common.modifiers.history.Payload -trait FastSyncProcessor extends HistoryApi { +trait FastSyncProcessor { historyApi: HistoryApi => def processPayload(payload: Payload): ProgressInfo = { val startTime: Long = System.currentTimeMillis() @@ -18,11 +18,9 @@ trait FastSyncProcessor extends HistoryApi { StorageVersion @@ validityKey(block.payload.id).untag(StorageKey), List(block.header.id, block.payload.id).map(id => validityKey(id) -> StorageValue @@ Array(1.toByte)) ) - logger.info( - s"Finished processing block ${block.encodedId}. " + - s"Processing time is ${(System.currentTimeMillis() - startTime) / 1000} s" - ) + logger.info(s"Finished processing block ${block.encodedId}. " + + s"Processing time is ${(System.currentTimeMillis() - startTime) / 1000} s") } ProgressInfo(none, Seq.empty, Seq.empty, none) } -} +} \ No newline at end of file diff --git a/src/main/scala/encry/view/history/History.scala b/src/main/scala/encry/view/history/History.scala index dee83c6bf1..ee3c068451 100644 --- a/src/main/scala/encry/view/history/History.scala +++ b/src/main/scala/encry/view/history/History.scala @@ -1,27 +1,27 @@ package encry.view.history import java.io.File -import cats.syntax.either._ import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus.ProgressInfo import encry.settings._ import encry.storage.VersionalStorage -import encry.storage.VersionalStorage.{ StorageKey, StorageValue, StorageVersion } +import encry.storage.VersionalStorage.{StorageKey, StorageValue, StorageVersion} import encry.storage.iodb.versionalIODB.IODBHistoryWrapper -import encry.storage.levelDb.versionalLevelDB.{ LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion } +import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} import encry.utils.NetworkTimeProvider import encry.view.history.storage.HistoryStorage import io.iohk.iodb.LSMStore import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId import org.iq80.leveldb.Options +import cats.syntax.either._ /** - * History implementation. It is processing persistent modifiers generated locally or received from the network. + * History implementation. It is processing persistent modifiers generated locally or received from the network. **/ -trait History extends HistoryModifiersValidator with AutoCloseable { +trait History extends HistoryModifiersValidator with HistoryApi with AutoCloseable { var isFullChainSynced: Boolean @@ -29,7 +29,7 @@ trait History extends HistoryModifiersValidator with AutoCloseable { def append(modifier: PersistentModifier): Either[Throwable, (History, ProgressInfo)] = { logger.info(s"Trying to append modifier ${Algos.encode(modifier.id)} of type ${modifier.modifierTypeId} to history") Either.catchNonFatal(modifier match { - case header: Header => + case header: Header => logger.info(s"Append header ${header.encodedId} at height ${header.height} to history") (this, processHeader(header)) case payload: Payload => (this, processPayload(payload)) @@ -48,28 +48,22 @@ trait History extends HistoryModifiersValidator with AutoCloseable { } /** - * Marks modifier and all modifiers in child chains as invalid - * - * @param modifier that is invalid against the State - * @return ProgressInfo with next modifier to try to apply - */ + * Marks modifier and all modifiers in child chains as invalid + * + * @param modifier that is invalid against the State + * @return ProgressInfo with next modifier to try to apply + */ def reportModifierIsInvalid(modifier: PersistentModifier): (History, ProgressInfo) = { logger.info(s"Modifier ${modifier.encodedId} of type ${modifier.modifierTypeId} is marked as invalid") correspondingHeader(modifier) match { case Some(invalidatedHeader) => val invalidatedHeaders: Seq[Header] = continuationHeaderChains(invalidatedHeader, _ => true).flatten.distinct val validityRow: List[(StorageKey, StorageValue)] = invalidatedHeaders - .flatMap( - h => - Seq(h.id, h.payloadId) - .map(id => validityKey(id) -> StorageValue @@ Array(0.toByte)) - ) - .toList + .flatMap(h => Seq(h.id, h.payloadId) + .map(id => validityKey(id) -> StorageValue @@ Array(0.toByte))).toList logger.info(s"Going to invalidate ${invalidatedHeader.encodedId} and ${invalidatedHeaders.map(_.encodedId)}") - val bestHeaderIsInvalidated: Boolean = - getBestHeaderId.exists(id => invalidatedHeaders.exists(_.id sameElements id)) - val bestFullIsInvalidated: Boolean = - getBestBlockId.exists(id => invalidatedHeaders.exists(_.id sameElements id)) + val bestHeaderIsInvalidated: Boolean = getBestHeaderId.exists(id => invalidatedHeaders.exists(_.id sameElements id)) + val bestFullIsInvalidated: Boolean = getBestBlockId.exists(id => invalidatedHeaders.exists(_.id sameElements id)) (bestHeaderIsInvalidated, bestFullIsInvalidated) match { case (false, false) => // Modifiers from best header and best full chain are not involved, no rollback and links change required. @@ -91,20 +85,17 @@ trait History extends HistoryModifiersValidator with AutoCloseable { this -> ProgressInfo(None, Seq.empty, Seq.empty, None) } else { val invalidatedChain: Seq[Block] = getBestBlock.toSeq - .flatMap( - f => headerChainBack(getBestBlockHeight + 1, f.header, h => !invalidatedHeaders.contains(h)).headers - ) + .flatMap(f => headerChainBack(getBestBlockHeight + 1, f.header, h => !invalidatedHeaders.contains(h)).headers) .flatMap(h => getBlockByHeader(h)) .ensuring(_.lengthCompare(1) > 0, "invalidatedChain should contain at least bestFullBlock and parent") val branchPoint: Block = invalidatedChain.head val validChain: Seq[Block] = - continuationHeaderChains(branchPoint.header, - h => getBlockByHeader(h).isDefined && !invalidatedHeaders.contains(h)) + continuationHeaderChains(branchPoint.header, h => getBlockByHeader(h).isDefined && !invalidatedHeaders.contains(h)) .maxBy(chain => scoreOf(chain.last.id).getOrElse(BigInt(0))) .flatMap(h => getBlockByHeader(h)) val changedLinks: Seq[(StorageKey, StorageValue)] = List( - BestBlockKey -> StorageValue @@ validChain.last.id.untag(ModifierId), + BestBlockKey -> StorageValue @@ validChain.last.id.untag(ModifierId), BestHeaderKey -> StorageValue @@ newBestHeader.id.untag(ModifierId) ) val toInsert: List[(StorageKey, StorageValue)] = validityRow ++ changedLinks @@ -123,22 +114,21 @@ trait History extends HistoryModifiersValidator with AutoCloseable { } /** - * Marks modifier as valid - * - * @param modifier that is valid against the State - * @return ProgressInfo with next modifier to try to apply - */ + * Marks modifier as valid + * + * @param modifier that is valid against the State + * @return ProgressInfo with next modifier to try to apply + */ def reportModifierIsValid(modifier: PersistentModifier): History = { logger.info(s"Modifier ${modifier.encodedId} of type ${modifier.modifierTypeId} is marked as valid ") modifier match { case block: Block => val nonMarkedIds: Seq[ModifierId] = Seq(block.header.id, block.payload.id) .filter(id => historyStorage.get(validityKey(id)).isEmpty) - if (nonMarkedIds.nonEmpty) - historyStorage.insert( - StorageVersion @@ validityKey(nonMarkedIds.head).untag(StorageKey), - nonMarkedIds.map(id => validityKey(id) -> StorageValue @@ Array(1.toByte)).toList - ) + if (nonMarkedIds.nonEmpty) historyStorage.insert( + StorageVersion @@ validityKey(nonMarkedIds.head).untag(StorageKey), + nonMarkedIds.map(id => validityKey(id) -> StorageValue @@ Array(1.toByte)).toList + ) this case _ => historyStorage.insert( @@ -175,8 +165,8 @@ object History extends StrictLogging { case VersionalStorage.IODB => logger.info("Init history with iodb storage") val historyObjectsDir: File = getHistoryObjectsDir(settingsEncry) - val indexStore: LSMStore = new LSMStore(historyIndexDir, keepVersions = 0) - val objectsStore: LSMStore = new LSMStore(historyObjectsDir, keepVersions = 0) + val indexStore: LSMStore = new LSMStore(historyIndexDir, keepVersions = 0) + val objectsStore: LSMStore = new LSMStore(historyObjectsDir, keepVersions = 0) IODBHistoryWrapper(indexStore, objectsStore) case VersionalStorage.LevelDB => logger.info("Init history with levelDB storage") @@ -185,17 +175,18 @@ object History extends StrictLogging { } if (settingsEncry.snapshotSettings.enableFastSynchronization && !settingsEncry.node.offlineGeneration) new History with HistoryHeadersProcessor with FastSyncProcessor { - override val settings: EncryAppSettings = settingsEncry - override var isFullChainSynced: Boolean = settings.node.offlineGeneration - override val historyStorage: HistoryStorage = HistoryStorage(vldbInit) + override val settings: EncryAppSettings = settingsEncry + override var isFullChainSynced: Boolean = settings.node.offlineGeneration + override val historyStorage: HistoryStorage = HistoryStorage(vldbInit) override val timeProvider: NetworkTimeProvider = new NetworkTimeProvider(settingsEncry.ntp) - } else + } + else new History with HistoryHeadersProcessor with HistoryPayloadsProcessor { - override val settings: EncryAppSettings = settingsEncry - override var isFullChainSynced: Boolean = settings.node.offlineGeneration - override val historyStorage: HistoryStorage = HistoryStorage(vldbInit) + override val settings: EncryAppSettings = settingsEncry + override var isFullChainSynced: Boolean = settings.node.offlineGeneration + override val historyStorage: HistoryStorage = HistoryStorage(vldbInit) override val timeProvider: NetworkTimeProvider = new NetworkTimeProvider(settingsEncry.ntp) } } -} +} \ No newline at end of file diff --git a/src/main/scala/encry/view/history/HistoryApi.scala b/src/main/scala/encry/view/history/HistoryApi.scala index d0160bc314..02d87fa74b 100644 --- a/src/main/scala/encry/view/history/HistoryApi.scala +++ b/src/main/scala/encry/view/history/HistoryApi.scala @@ -11,7 +11,7 @@ import io.iohk.iodb.ByteArrayWrapper import org.encryfoundation.common.modifiers.history._ import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ Difficulty, Height, ModifierId, ModifierTypeId } +import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, Height, ModifierId, ModifierTypeId} import scala.annotation.tailrec import scala.collection.immutable.HashSet @@ -41,85 +41,61 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore lazy val fastSyncInProgress: FastSyncProcessor = FastSyncProcessor(settings) - def getHeaderById(id: ModifierId): Option[Header] = + def getHeaderById(id: ModifierId): Option[Header] = headersCache + .get(ByteArrayWrapper(id)) + .orElse(blocksCache.get(ByteArrayWrapper(id)).map(_.header)) + .orElse(getHeaderByIdDB(id)) + + def getBlockByHeaderId(id: ModifierId): Option[Block] = + blocksCache + .get(ByteArrayWrapper(id)) + .orElse(headersCache.get(ByteArrayWrapper(id)) + .flatMap(h => getPayloadByIdDB(h.payloadId).map(p => Block(h, p)))) + .orElse(getBlockByHeaderIdDB(id)) + + def getBlockByHeader(header: Header): Option[Block] = blocksCache + .get(ByteArrayWrapper(header.id)) + .orElse(getPayloadByIdDB(header.payloadId).map(p => Block(header, p))) + + def getBestHeader: Option[Header] = getBestHeaderId.flatMap(id => headersCache .get(ByteArrayWrapper(id)) .orElse(blocksCache.get(ByteArrayWrapper(id)).map(_.header)) .orElse(getHeaderByIdDB(id)) + ) - def getBlockByHeaderId(id: ModifierId): Option[Block] = - blocksCache - .get(ByteArrayWrapper(id)) - .orElse( - headersCache - .get(ByteArrayWrapper(id)) - .flatMap(h => getPayloadByIdDB(h.payloadId).map(p => Block(h, p))) - ) + def getBestHeaderHeight: Int = getBestHeaderId.flatMap(id => + headersCache.get(ByteArrayWrapper(id)).map(_.height) + .orElse(blocksCache.get(ByteArrayWrapper(id)).map(_.header.height)) + .orElse(getHeightByHeaderId(id)) + ).getOrElse(settings.constants.PreGenesisHeight) + + def getBestBlock: Option[Block] = getBestBlockId.flatMap(id => + blocksCache.get(ByteArrayWrapper(id)) .orElse(getBlockByHeaderIdDB(id)) + ) - def getBlockByHeader(header: Header): Option[Block] = - blocksCache - .get(ByteArrayWrapper(header.id)) - .orElse(getPayloadByIdDB(header.payloadId).map(p => Block(header, p))) - - def getBestHeader: Option[Header] = - getBestHeaderId.flatMap( - id => - headersCache - .get(ByteArrayWrapper(id)) - .orElse(blocksCache.get(ByteArrayWrapper(id)).map(_.header)) - .orElse(getHeaderByIdDB(id)) - ) - - def getBestHeaderHeight: Int = - getBestHeaderId - .flatMap( - id => - headersCache - .get(ByteArrayWrapper(id)) - .map(_.height) - .orElse(blocksCache.get(ByteArrayWrapper(id)).map(_.header.height)) - .orElse(getHeightByHeaderId(id)) - ) - .getOrElse(settings.constants.PreGenesisHeight) - - def getBestBlock: Option[Block] = - getBestBlockId.flatMap( - id => - blocksCache - .get(ByteArrayWrapper(id)) - .orElse(getBlockByHeaderIdDB(id)) - ) - - def getBestBlockHeight: Int = - getBestBlockId - .flatMap(id => blocksCache.get(ByteArrayWrapper(id)).map(_.header.height).orElse(getHeightByHeaderId(id))) - .getOrElse(settings.constants.PreGenesisHeight) - - def getHeaderOfBestBlock: Option[Header] = - getBestBlockId.flatMap( - id => - headersCache - .get(ByteArrayWrapper(id)) - .orElse(blocksCache.get(ByteArrayWrapper(id)).map(_.header)) - .orElse(getHeaderByIdDB(id)) - ) + def getBestBlockHeight: Int = getBestBlockId + .flatMap(id => blocksCache.get(ByteArrayWrapper(id)).map(_.header.height).orElse(getHeightByHeaderId(id))) + .getOrElse(settings.constants.PreGenesisHeight) + + def getHeaderOfBestBlock: Option[Header] = getBestBlockId.flatMap(id => + headersCache.get(ByteArrayWrapper(id)) + .orElse(blocksCache.get(ByteArrayWrapper(id)).map(_.header)) + .orElse(getHeaderByIdDB(id)) + ) def getBestHeaderAtHeight(h: Int): Option[Header] = getBestHeaderAtHeightDB(h) - def getBlockByPayload(payload: Payload): Option[Block] = - headersCache - .get(ByteArrayWrapper(payload.headerId)) - .map(h => Block(h, payload)) - .orElse(blocksCache.get(ByteArrayWrapper(payload.headerId))) - .orElse(getHeaderById(payload.headerId).flatMap(h => Some(Block(h, payload)))) + def getBlockByPayload(payload: Payload): Option[Block] = headersCache + .get(ByteArrayWrapper(payload.headerId)).map(h => Block(h, payload)) + .orElse(blocksCache.get(ByteArrayWrapper(payload.headerId))) + .orElse(getHeaderById(payload.headerId).flatMap(h => Some(Block(h, payload)))) - def getHeightByHeaderId(id: ModifierId): Option[Int] = - headersCache - .get(ByteArrayWrapper(id)) - .map(_.height) - .orElse(blocksCache.get(ByteArrayWrapper(id)).map(_.header.height)) - .orElse(getHeightByHeaderIdDB(id)) + def getHeightByHeaderId(id: ModifierId): Option[Int] = headersCache + .get(ByteArrayWrapper(id)).map(_.height) + .orElse(blocksCache.get(ByteArrayWrapper(id)).map(_.header.height)) + .orElse(getHeightByHeaderIdDB(id)) def isBestBlockDefined: Boolean = getBestBlockId.map(id => blocksCache.contains(ByteArrayWrapper(id))).isDefined || @@ -134,39 +110,33 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore isModifierDefined(id) def getBestHeaderIdAtHeight(h: Int): Option[ModifierId] = getBestHeaderIdAtHeightDB(h) - def headerIdsAtHeight(height: Int): Seq[ModifierId] = - headerIdsAtHeightDB(height) - .getOrElse(Seq.empty[ModifierId]) + def headerIdsAtHeight(height: Int): Seq[ModifierId] = headerIdsAtHeightDB(height) + .getOrElse(Seq.empty[ModifierId]) - def modifierBytesById(id: ModifierId): Option[Array[Byte]] = - headersCache - .get(ByteArrayWrapper(id)) - .map(h => HeaderProtoSerializer.toProto(h).toByteArray) - .orElse(blocksCache.get(ByteArrayWrapper(id)).map(b => BlockProtoSerializer.toProto(b).toByteArray)) - .orElse(modifierBytesByIdDB(id)) + def modifierBytesById(id: ModifierId): Option[Array[Byte]] = headersCache + .get(ByteArrayWrapper(id)).map(h => HeaderProtoSerializer.toProto(h).toByteArray) + .orElse(blocksCache.get(ByteArrayWrapper(id)).map(b => BlockProtoSerializer.toProto(b).toByteArray)) + .orElse(modifierBytesByIdDB(id)) - def lastHeaders(count: Int): HeaderChain = - getBestHeader - .map(bestHeader => headerChainBack(count, bestHeader, _ => false)) - .getOrElse(HeaderChain.empty) + def lastHeaders(count: Int): HeaderChain = getBestHeader + .map(bestHeader => headerChainBack(count, bestHeader, _ => false)) + .getOrElse(HeaderChain.empty) - def getHeaderIds(count: Int, offset: Int = 0): Seq[ModifierId] = - (offset until (count + offset)) - .flatMap(getBestHeaderIdAtHeight) + def getHeaderIds(count: Int, offset: Int = 0): Seq[ModifierId] = (offset until (count + offset)) + .flatMap(getBestHeaderIdAtHeight) def payloadsIdsToDownload(howMany: Int, excluding: HashSet[ModifierId]): Seq[ModifierId] = { @tailrec def continuation(height: Int, acc: Seq[ModifierId]): Seq[ModifierId] = if (acc.lengthCompare(howMany) >= 0) acc else if (height > lastAvailableManifestHeight && fastSyncInProgress.fastSyncVal) acc - else - getBestHeaderIdAtHeight(height).flatMap(getHeaderById) match { - case Some(h) if !excluding.exists(_.sameElements(h.payloadId)) && !isBlockDefined(h) => - continuation(height + 1, acc :+ h.payloadId) - case Some(_) => - continuation(height + 1, acc) - case None => - acc - } + else getBestHeaderIdAtHeight(height).flatMap(getHeaderById) match { + case Some(h) if !excluding.exists(_.sameElements(h.payloadId)) && !isBlockDefined(h) => + continuation(height + 1, acc :+ h.payloadId) + case Some(_) => + continuation(height + 1, acc) + case None => + acc + } (for { bestBlockId <- getBestBlockId @@ -186,7 +156,7 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore } def toDownload(header: Header): Option[(ModifierTypeId, ModifierId)] = - // Already synced and header is not too far back. Download required modifiers + // Already synced and header is not too far back. Download required modifiers if (header.height >= blockDownloadProcessor.minimalBlockHeight) (Payload.modifierTypeId -> header.payloadId).some // Headers chain is synced after this header. Start downloading full blocks else if (!isHeadersChainSynced && isNewHeader(header)) { @@ -196,50 +166,41 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore } else none def headerChainBack(limit: Int, startHeader: Header, until: Header => Boolean): HeaderChain = { - @tailrec def loop(header: Header, acc: Seq[Header]): Seq[Header] = + @tailrec def loop(header: Header, acc: Seq[Header]): Seq[Header] = { if (acc.length == limit || until(header)) acc - else - getHeaderById(header.parentId) match { - case Some(parent: Header) => loop(parent, acc :+ parent) - case None if acc.contains(header) => acc - case _ => acc :+ header - } + else getHeaderById(header.parentId) match { + case Some(parent: Header) => loop(parent, acc :+ parent) + case None if acc.contains(header) => acc + case _ => acc :+ header + } + } if (getBestHeaderId.isEmpty || (limit == 0)) HeaderChain(Seq.empty) else HeaderChain(loop(startHeader, Seq(startHeader)).reverse) } - @tailrec final def loopHeightDown(height: Int, p: ModifierId => Boolean): Option[Header] = - headerIdsAtHeight(height) - .find(p) - .flatMap(getHeaderById) match { - case h @ Some(_) => h - case None if height > settings.constants.GenesisHeight => loopHeightDown(height - 1, p) - case n @ None => n - } + @tailrec final def loopHeightDown(height: Int, p: ModifierId => Boolean): Option[Header] = headerIdsAtHeight(height) + .find(p) + .flatMap(getHeaderById) match { + case h@Some(_) => h + case None if height > settings.constants.GenesisHeight => loopHeightDown(height - 1, p) + case n@None => n + } def requiredDifficultyAfter(parent: Header): Either[HistoryApiError, Difficulty] = { - val requiredHeights: Seq[Height] = PowLinearController.getHeightsForRetargetingAt( - Height @@ (parent.height + 1), - settings.constants.EpochLength, - settings.constants.RetargetingEpochsQty - ) + val requiredHeights: Seq[Height] = PowLinearController.getHeightsForRetargetingAt(Height @@ (parent.height + 1), + settings.constants.EpochLength, settings.constants.RetargetingEpochsQty) for { - _ <- Either.cond(requiredHeights.lastOption.contains(parent.height), - (), - HistoryApiError("Incorrect heights sequence in requiredDifficultyAfter function")) + _ <- Either.cond(requiredHeights.lastOption.contains(parent.height), (), + HistoryApiError("Incorrect heights sequence in requiredDifficultyAfter function")) chain = headerChainBack(requiredHeights.max - requiredHeights.min + 1, parent, (_: Header) => false) requiredHeaders = (requiredHeights.min to requiredHeights.max) .zip(chain.headers) .filter(p => requiredHeights.contains(p._1)) - _ <- Either.cond(requiredHeights.length == requiredHeaders.length, - (), - HistoryApiError(s"Missed headers: $requiredHeights != ${requiredHeaders.map(_._1)}")) - } yield - PowLinearController.getDifficulty(requiredHeaders, - settings.constants.EpochLength, - settings.constants.DesiredBlockInterval, - settings.constants.InitialDifficulty) + _ <- Either.cond(requiredHeights.length == requiredHeaders.length, (), + HistoryApiError(s"Missed headers: $requiredHeights != ${requiredHeaders.map(_._1)}")) + } yield PowLinearController.getDifficulty(requiredHeaders, settings.constants.EpochLength, + settings.constants.DesiredBlockInterval, settings.constants.InitialDifficulty) } def syncInfo: SyncInfo = lastSyncInfo @@ -258,7 +219,7 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore case Some(id) if si.lastHeaderIds.exists(_ sameElements id) => Older /* Other history is empty, or our history contains last id from other history */ case Some(_) if si.lastHeaderIds.isEmpty || si.lastHeaderIds.lastOption.exists(isHeaderDefined) => Younger - case Some(_) => + case Some(_) => //Our history contains some ids from other history if (si.lastHeaderIds.exists(isHeaderDefined)) Fork //Unknown comparison result @@ -277,8 +238,7 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore startId <- headerIdsAtHeight(heightFrom).headOption startHeader <- getHeaderById(startId) } yield headerChainBack(size, startHeader, _ => false)) match { - case Some(value) if value.headers.exists(_.height == settings.constants.GenesisHeight) => - value.headers.map(_.id) + case Some(value) if value.headers.exists(_.height == settings.constants.GenesisHeight) => value.headers.map(_.id) case _ => Seq.empty } } else { @@ -286,18 +246,19 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore (for { lastHeaderInOurBestChain <- ids.view.reverse.find(m => isInBestChain(m)) theirHeight <- heightOf(lastHeaderInOurBestChain) - heightFrom = Math.min(getBestHeaderHeight, theirHeight + size) + heightFrom = Math.min(getBestHeaderHeight, theirHeight + size) startId <- headerIdsAtHeight(heightFrom).headOption startHeader <- getHeaderById(startId) - } yield - headerChainBack(size, startHeader, h => h.parentId sameElements lastHeaderInOurBestChain).headers - .map(_.id)) match { + } yield headerChainBack(size, startHeader, h => h.parentId sameElements lastHeaderInOurBestChain) + .headers + .map(_.id)) match { case Some(value) => value case None => Seq.empty } } - def commonBlockThenSuffixes(header1: Header, header2: Header): (HeaderChain, HeaderChain) = { + def commonBlockThenSuffixes(header1: Header, + header2: Header): (HeaderChain, HeaderChain) = { val heightDelta: Int = Math.max(header1.height - header2.height, 0) @scala.annotation.tailrec @@ -323,13 +284,13 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore loop(2, HeaderChain(Seq(header2))) } - def getChainToHeader(fromHeaderOpt: Option[Header], toHeader: Header): (Option[ModifierId], HeaderChain) = - fromHeaderOpt match { - case Some(h1) => - val (prevChain, newChain) = commonBlockThenSuffixes(h1, toHeader) - (prevChain.headOption.map(_.id), newChain.tail) - case None => (None, headerChainBack(toHeader.height + 1, toHeader, _ => false)) - } + def getChainToHeader(fromHeaderOpt: Option[Header], + toHeader: Header): (Option[ModifierId], HeaderChain) = fromHeaderOpt match { + case Some(h1) => + val (prevChain, newChain) = commonBlockThenSuffixes(h1, toHeader) + (prevChain.headOption.map(_.id), newChain.tail) + case None => (None, headerChainBack(toHeader.height + 1, toHeader, _ => false)) + } def isNewHeader(header: Header): Boolean = timeProvider.estimatedTime - header.timestamp < @@ -337,16 +298,18 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore def isHeadersChainSynced: Boolean = isHeadersChainSyncedVar - def continuationHeaderChains(header: Header, filterCond: Header => Boolean): Seq[Seq[Header]] = { + def continuationHeaderChains(header: Header, + filterCond: Header => Boolean): Seq[Seq[Header]] = { @tailrec def loop(currentHeight: Int, acc: Seq[Seq[Header]]): Seq[Seq[Header]] = { - val nextHeightHeaders: Seq[Header] = headerIdsAtHeight(currentHeight + 1).view + val nextHeightHeaders: Seq[Header] = headerIdsAtHeight(currentHeight + 1) + .view .flatMap(getHeaderById) .filter(filterCond) .toList if (nextHeightHeaders.isEmpty) acc.map(_.reverse) else { - val updatedChains: Seq[Seq[Header]] = nextHeightHeaders.flatMap( - h => acc.find(chain => chain.nonEmpty && (h.parentId sameElements chain.head.id)).map(h +: _) + val updatedChains: Seq[Seq[Header]] = nextHeightHeaders.flatMap(h => + acc.find(chain => chain.nonEmpty && (h.parentId sameElements chain.head.id)).map(h +: _) ) val nonUpdatedChains: Seq[Seq[Header]] = acc.filter(chain => !nextHeightHeaders.exists(_.parentId sameElements chain.head.id)) @@ -382,7 +345,7 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore logger.debug(s"Should add ${Algos.encode(b.id)} to block cache") val newBlocksIdsAtBlockHeight = blocksCacheIndexes.getOrElse(b.header.height, Seq.empty[ModifierId]) :+ b.id blocksCacheIndexes = blocksCacheIndexes + (b.header.height -> newBlocksIdsAtBlockHeight) - blocksCache = blocksCache + (ByteArrayWrapper(b.id) -> b) + blocksCache = blocksCache + (ByteArrayWrapper(b.id) -> b) // cleanup cache if necessary if (blocksCacheIndexes.size > settings.constants.MaxRollbackDepth) { blocksCacheIndexes.get(getBestBlockHeight - settings.constants.MaxRollbackDepth).foreach { blocksIds => @@ -395,4 +358,4 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore logger.debug(s"headersCache size: ${blocksCache.size}") logger.debug(s"headersCacheIndexes size: ${blocksCacheIndexes.size}") } -} +} \ No newline at end of file diff --git a/src/main/scala/encry/view/history/HistoryDBApi.scala b/src/main/scala/encry/view/history/HistoryDBApi.scala index 3e972e0910..55427766a8 100644 --- a/src/main/scala/encry/view/history/HistoryDBApi.scala +++ b/src/main/scala/encry/view/history/HistoryDBApi.scala @@ -5,9 +5,9 @@ import com.typesafe.scalalogging.StrictLogging import encry.settings.EncryAppSettings import encry.storage.VersionalStorage.StorageKey import encry.view.history.storage.HistoryStorage -import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ Height, ModifierId, ModifierTypeId } +import org.encryfoundation.common.utils.TaggedTypes.{Height, ModifierId, ModifierTypeId} import scorex.crypto.hash.Digest32 import scala.reflect.ClassTag @@ -23,80 +23,67 @@ trait HistoryDBApi extends StrictLogging { lazy val BestBlockKey: StorageKey = StorageKey @@ Array.fill(settings.constants.DigestLength)(-1: Byte) - private def getModifierById[T: ClassTag](id: ModifierId): Option[T] = - historyStorage - .modifierById(id) - .collect { case m: T => m } + private def getModifierById[T: ClassTag](id: ModifierId): Option[T] = historyStorage + .modifierById(id) + .collect { case m: T => m } - def getHeightByHeaderIdDB(id: ModifierId): Option[Int] = - historyStorage - .get(headerHeightKey(id)) - .map(Ints.fromByteArray) + def getHeightByHeaderIdDB(id: ModifierId): Option[Int] = historyStorage + .get(headerHeightKey(id)) + .map(Ints.fromByteArray) - def getHeaderByIdDB(id: ModifierId): Option[Header] = getModifierById[Header](id) + def getHeaderByIdDB(id: ModifierId): Option[Header] = getModifierById[Header](id) def getPayloadByIdDB(pId: ModifierId): Option[Payload] = getModifierById[Payload](pId) - def getBlockByHeaderDB(header: Header): Option[Block] = - getModifierById[Payload](header.payloadId) - .map(payload => Block(header, payload)) - def getBlockByHeaderIdDB(id: ModifierId): Option[Block] = - getHeaderByIdDB(id) - .flatMap(h => getModifierById[Payload](h.payloadId).map(p => Block(h, p))) + def getBlockByHeaderDB(header: Header): Option[Block] = getModifierById[Payload](header.payloadId) + .map(payload => Block(header, payload)) + def getBlockByHeaderIdDB(id: ModifierId): Option[Block] = getHeaderByIdDB(id) + .flatMap(h => getModifierById[Payload](h.payloadId).map(p => Block(h, p))) def getBestHeaderId: Option[ModifierId] = historyStorage.get(BestHeaderKey).map(ModifierId @@ _) - def getBestHeaderDB: Option[Header] = getBestHeaderId.flatMap(getHeaderByIdDB) - def getBestHeaderHeightDB: Int = - getBestHeaderId - .flatMap(getHeightByHeaderIdDB) - .getOrElse(settings.constants.PreGenesisHeight) + def getBestHeaderDB: Option[Header] = getBestHeaderId.flatMap(getHeaderByIdDB) + def getBestHeaderHeightDB: Int = getBestHeaderId + .flatMap(getHeightByHeaderIdDB) + .getOrElse(settings.constants.PreGenesisHeight) def getBestBlockId: Option[ModifierId] = historyStorage.get(BestBlockKey).map(ModifierId @@ _) - def getBestBlockDB: Option[Block] = getBestBlockId.flatMap(getBlockByHeaderIdDB) - def getBestBlockHeightDB: Int = - getBestBlockId - .flatMap(getHeightByHeaderIdDB) - .getOrElse(settings.constants.PreGenesisHeight) + def getBestBlockDB: Option[Block] = getBestBlockId.flatMap(getBlockByHeaderIdDB) + def getBestBlockHeightDB: Int = getBestBlockId + .flatMap(getHeightByHeaderIdDB) + .getOrElse(settings.constants.PreGenesisHeight) def modifierBytesByIdDB(id: ModifierId): Option[Array[Byte]] = historyStorage.modifiersBytesById(id) def isModifierDefined(id: ModifierId): Boolean = historyStorage.containsMod(id) //todo probably rewrite with indexes collection - def lastBestBlockHeightRelevantToBestChain(probablyAt: Int): Option[Int] = - (for { - headerId <- getBestHeaderIdAtHeightDB(probablyAt) - header <- getHeaderByIdDB(headerId) if isModifierDefined(header.payloadId) - } yield header.height).orElse(lastBestBlockHeightRelevantToBestChain(probablyAt - 1)) + def lastBestBlockHeightRelevantToBestChain(probablyAt: Int): Option[Int] = (for { + headerId <- getBestHeaderIdAtHeightDB(probablyAt) + header <- getHeaderByIdDB(headerId) if isModifierDefined(header.payloadId) + } yield header.height).orElse(lastBestBlockHeightRelevantToBestChain(probablyAt - 1)) - def headerIdsAtHeightDB(height: Int): Option[Seq[ModifierId]] = - historyStorage - .get(heightIdsKey(height)) - .map(_.grouped(32).map(ModifierId @@ _).toSeq) + def headerIdsAtHeightDB(height: Int): Option[Seq[ModifierId]] = historyStorage + .get(heightIdsKey(height)) + .map(_.grouped(32).map(ModifierId @@ _).toSeq) def getBestHeaderIdAtHeightDB(h: Int): Option[ModifierId] = headerIdsAtHeightDB(h).flatMap(_.headOption) def getBestHeaderAtHeightDB(h: Int): Option[Header] = getBestHeaderIdAtHeightDB(h).flatMap(getHeaderByIdDB) - def isInBestChain(h: Header): Boolean = - getBestHeaderIdAtHeightDB(h.height) - .exists(_.sameElements(h.id)) + def isInBestChain(h: Header): Boolean = getBestHeaderIdAtHeightDB(h.height) + .exists(_.sameElements(h.id)) - def isInBestChain(id: ModifierId): Boolean = - heightOf(id) - .flatMap(getBestHeaderIdAtHeightDB) - .exists(_.sameElements(id)) + def isInBestChain(id: ModifierId): Boolean = heightOf(id) + .flatMap(getBestHeaderIdAtHeightDB) + .exists(_.sameElements(id)) - def getBestHeadersChainScore: BigInt = - getBestHeaderId.flatMap(scoreOf).getOrElse(BigInt(0)) //todo ?.getOrElse(BigInt(0))? + def getBestHeadersChainScore: BigInt = getBestHeaderId.flatMap(scoreOf).getOrElse(BigInt(0)) //todo ?.getOrElse(BigInt(0))? - def scoreOf(id: ModifierId): Option[BigInt] = - historyStorage - .get(headerScoreKey(id)) - .map(d => BigInt(d)) + def scoreOf(id: ModifierId): Option[BigInt] = historyStorage + .get(headerScoreKey(id)) + .map(d => BigInt(d)) - def heightOf(id: ModifierId): Option[Height] = - historyStorage - .get(headerHeightKey(id)) - .map(d => Height @@ Ints.fromByteArray(d)) + def heightOf(id: ModifierId): Option[Height] = historyStorage + .get(headerHeightKey(id)) + .map(d => Height @@ Ints.fromByteArray(d)) def heightIdsKey(height: Int): StorageKey = StorageKey @@ Algos.hash(Ints.toByteArray(height)).untag(Digest32) @@ -106,4 +93,4 @@ trait HistoryDBApi extends StrictLogging { StorageKey @@ Algos.hash("height".getBytes(Algos.charset) ++ id).untag(Digest32) def validityKey(id: Array[Byte]): StorageKey = StorageKey @@ Algos.hash("validity".getBytes(Algos.charset) ++ id).untag(Digest32) -} +} \ No newline at end of file diff --git a/src/main/scala/encry/view/history/HistoryHeadersProcessor.scala b/src/main/scala/encry/view/history/HistoryHeadersProcessor.scala index b8e414cce8..b530efaced 100644 --- a/src/main/scala/encry/view/history/HistoryHeadersProcessor.scala +++ b/src/main/scala/encry/view/history/HistoryHeadersProcessor.scala @@ -9,7 +9,7 @@ import encry.storage.VersionalStorage.{ StorageKey, StorageValue } import org.encryfoundation.common.modifiers.history.Header import org.encryfoundation.common.utils.TaggedTypes.{ Difficulty, ModifierId } -trait HistoryHeadersProcessor extends HistoryApi { +trait HistoryHeadersProcessor { historyApi: HistoryApi => def processHeader(h: Header): ProgressInfo = getHeaderInfoUpdate(h) match { case dataToUpdate: Seq[_] if dataToUpdate.nonEmpty => @@ -77,4 +77,4 @@ trait HistoryHeadersProcessor extends HistoryApi { logger.info(s"New orphaned header ${h.encodedId} at height ${h.height} with score $score") Seq(heightIdsKey(h.height) -> StorageValue @@ (headerIdsAtHeight(h.height) :+ h.id).flatten.toArray) } -} +} \ No newline at end of file diff --git a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala index a3848f7871..bee13e90a9 100644 --- a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala +++ b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala @@ -5,17 +5,14 @@ import encry.consensus.EquihashPowScheme import encry.view.history.ValidationError.FatalValidationError._ import encry.view.history.ValidationError.NonFatalValidationError._ import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{ Header, Payload } -import org.encryfoundation.common.utils.TaggedTypes.{ Difficulty, ModifierId } +import org.encryfoundation.common.modifiers.history.{Header, Payload} +import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, ModifierId} import org.encryfoundation.common.validation.ModifierSemanticValidity -trait HistoryModifiersValidator extends HistoryApi { +trait HistoryModifiersValidator { historyApi: HistoryApi => - lazy val powScheme: EquihashPowScheme = EquihashPowScheme(settings.constants.n, - settings.constants.k, - settings.constants.Version, - settings.constants.PreGenesisHeight, - settings.constants.MaxTarget) + lazy val powScheme: EquihashPowScheme = EquihashPowScheme(settings.constants.n, settings.constants.k, settings.constants.Version, + settings.constants.PreGenesisHeight, settings.constants.MaxTarget) def testApplicable(modifier: PersistentModifier): Either[ValidationError, PersistentModifier] = (modifier match { @@ -23,137 +20,77 @@ trait HistoryModifiersValidator extends HistoryApi { case payload: Payload => validatePayload(payload) case mod => UnknownModifierFatalError(s"Modifier $mod has incorrect type.").asLeft[PersistentModifier] }) match { - case l @ Left(value) => logger.info(s"Validation result for ${modifier.encodedId} failed cause $value"); l - case r @ Right(_) => r + case l@Left(value) => logger.info(s"Validation result for ${modifier.encodedId} failed cause $value"); l + case r@Right(_) => r } private def validateHeader(h: Header): Either[ValidationError, Header] = if (h.isGenesis) genesisBlockHeaderValidator(h) - else - getHeaderById(h.parentId) - .map(p => headerValidator(h, p)) - .getOrElse( - HeaderNonFatalValidationError(s"Header's ${h.encodedId} parent doesn't contain in history").asLeft[Header] - ) + else getHeaderById(h.parentId) + .map(p => headerValidator(h, p)) + .getOrElse(HeaderNonFatalValidationError(s"Header's ${h.encodedId} parent doesn't contain in history").asLeft[Header]) - private def validatePayload(mod: Payload): Either[ValidationError, PersistentModifier] = - getHeaderById(mod.headerId) - .map(header => payloadValidator(mod, header, blockDownloadProcessor.minimalBlockHeight)) - .getOrElse( - PayloadNonFatalValidationError(s"Header for ${mod.encodedId} doesn't contain in history") - .asLeft[PersistentModifier] - ) + private def validatePayload(mod: Payload): Either[ValidationError, PersistentModifier] = getHeaderById(mod.headerId) + .map(header => payloadValidator(mod, header, blockDownloadProcessor.minimalBlockHeight)) + .getOrElse(PayloadNonFatalValidationError(s"Header for ${mod.encodedId} doesn't contain in history").asLeft[PersistentModifier]) private def realDifficulty(h: Header): Difficulty = Difficulty !@@ powScheme.realDifficulty(h) - private def isSemanticallyValid(modifierId: ModifierId): ModifierSemanticValidity = - historyStorage - .get(validityKey(modifierId)) match { - case Some(mod) if mod.headOption.contains(1.toByte) => ModifierSemanticValidity.Valid - case Some(mod) if mod.headOption.contains(0.toByte) => ModifierSemanticValidity.Invalid - case None if isModifierDefined(modifierId) => ModifierSemanticValidity.Unknown - case None => ModifierSemanticValidity.Absent - case mod => - logger.error(s"Incorrect validity status: $mod") - ModifierSemanticValidity.Absent - } + private def isSemanticallyValid(modifierId: ModifierId): ModifierSemanticValidity = historyStorage + .get(validityKey(modifierId)) match { + case Some(mod) if mod.headOption.contains(1.toByte) => ModifierSemanticValidity.Valid + case Some(mod) if mod.headOption.contains(0.toByte) => ModifierSemanticValidity.Invalid + case None if isModifierDefined(modifierId) => ModifierSemanticValidity.Unknown + case None => ModifierSemanticValidity.Absent + case mod => logger.error(s"Incorrect validity status: $mod") + ModifierSemanticValidity.Absent + } - private def genesisBlockHeaderValidator(h: Header): Either[ValidationError, Header] = - for { - _ <- Either.cond( - h.parentId.sameElements(Header.GenesisParentId), - (), - GenesisBlockFatalValidationError(s"Genesis block with header ${h.encodedId} should has genesis parent id") - ) - _ <- Either.cond( - getBestHeaderId.isEmpty, - (), - GenesisBlockFatalValidationError(s"Genesis block with header ${h.encodedId} appended to non-empty history") - ) - _ <- Either.cond( - h.height == settings.constants.GenesisHeight, - (), - GenesisBlockFatalValidationError(s"Height of genesis block with header ${h.encodedId} is incorrect") - ) - } yield h + private def genesisBlockHeaderValidator(h: Header): Either[ValidationError, Header] = for { + _ <- Either.cond(h.parentId.sameElements(Header.GenesisParentId), (), + GenesisBlockFatalValidationError(s"Genesis block with header ${h.encodedId} should has genesis parent id")) + _ <- Either.cond(getBestHeaderId.isEmpty, (), + GenesisBlockFatalValidationError(s"Genesis block with header ${h.encodedId} appended to non-empty history")) + _ <- Either.cond(h.height == settings.constants.GenesisHeight, (), + GenesisBlockFatalValidationError(s"Height of genesis block with header ${h.encodedId} is incorrect")) + } yield h - private def headerValidator(h: Header, parent: Header): Either[ValidationError, Header] = - for { - _ <- Either.cond( - h.timestamp > parent.timestamp, - (), - HeaderFatalValidationError( - s"Header ${h.encodedId} has timestamp ${h.timestamp}" + - s" less than parent's ${parent.timestamp}" - ) - ) - _ <- Either.cond( - h.height == parent.height + 1, - (), - HeaderFatalValidationError( - s"Header ${h.encodedId} has height ${h.height}" + - s" not greater by 1 than parent's ${parent.height}" - ) - ) - _ <- Either.cond(!historyStorage.containsMod(h.id), - (), - HeaderFatalValidationError(s"Header ${h.encodedId} is already in history")) - _ <- Either.cond(realDifficulty(h) >= h.requiredDifficulty, - (), - HeaderFatalValidationError(s"Incorrect real difficulty in header ${h.encodedId}")) - _ <- Either.cond(requiredDifficultyAfter(parent).exists(_ <= h.difficulty), - (), - HeaderFatalValidationError(s"Incorrect required difficulty in header ${h.encodedId}")) - _ <- Either.cond( - heightOf(h.parentId).exists(h => getBestHeaderHeight - h < settings.constants.MaxRollbackDepth), - (), - HeaderFatalValidationError(s"Header ${h.encodedId} has height greater than max roll back depth") - ) - powSchemeValidationResult = powScheme.verify(h) - _ <- Either.cond( - powSchemeValidationResult.isRight, - (), - HeaderFatalValidationError( - s"Wrong proof-of-work solution in header ${h.encodedId}" + - s" caused: $powSchemeValidationResult" - ) - ) - _ <- Either.cond(isSemanticallyValid(h.parentId) != ModifierSemanticValidity.Invalid, - (), - HeaderFatalValidationError(s"Header ${h.encodedId} is semantically invalid")) - _ <- Either.cond( - h.timestamp - timeProvider.estimatedTime <= settings.constants.MaxTimeDrift, - (), - HeaderNonFatalValidationError( - s"Header ${h.encodedId} with timestamp ${h.timestamp}" + - s" is too far in future from now ${timeProvider.estimatedTime}" - ) - ) - } yield h + private def headerValidator(h: Header, parent: Header): Either[ValidationError, Header] = for { + _ <- Either.cond(h.timestamp > parent.timestamp, (), + HeaderFatalValidationError(s"Header ${h.encodedId} has timestamp ${h.timestamp}" + + s" less than parent's ${parent.timestamp}")) + _ <- Either.cond(h.height == parent.height + 1, (), + HeaderFatalValidationError(s"Header ${h.encodedId} has height ${h.height}" + + s" not greater by 1 than parent's ${parent.height}")) + _ <- Either.cond(!historyStorage.containsMod(h.id), (), + HeaderFatalValidationError(s"Header ${h.encodedId} is already in history")) + _ <- Either.cond(realDifficulty(h) >= h.requiredDifficulty, (), + HeaderFatalValidationError(s"Incorrect real difficulty in header ${h.encodedId}")) + _ <- Either.cond(requiredDifficultyAfter(parent).exists(_ <= h.difficulty), (), + HeaderFatalValidationError(s"Incorrect required difficulty in header ${h.encodedId}")) + _ <- Either.cond(heightOf(h.parentId).exists(h => getBestHeaderHeight - h < settings.constants.MaxRollbackDepth), (), + HeaderFatalValidationError(s"Header ${h.encodedId} has height greater than max roll back depth")) + powSchemeValidationResult = powScheme.verify(h) + _ <- Either.cond(powSchemeValidationResult.isRight, (), + HeaderFatalValidationError(s"Wrong proof-of-work solution in header ${h.encodedId}" + + s" caused: $powSchemeValidationResult")) + _ <- Either.cond(isSemanticallyValid(h.parentId) != ModifierSemanticValidity.Invalid, (), + HeaderFatalValidationError(s"Header ${h.encodedId} is semantically invalid")) + _ <- Either.cond(h.timestamp - timeProvider.estimatedTime <= settings.constants.MaxTimeDrift, (), + HeaderNonFatalValidationError(s"Header ${h.encodedId} with timestamp ${h.timestamp}" + + s" is too far in future from now ${timeProvider.estimatedTime}")) + } yield h private def payloadValidator(m: PersistentModifier, header: Header, - minimalHeight: Int): Either[ValidationError, PersistentModifier] = - for { - _ <- Either.cond(!historyStorage.containsMod(m.id), - (), - PayloadFatalValidationError(s"Modifier ${m.encodedId} is already in history")) - _ <- Either.cond( - header.isRelated(m), - (), - PayloadFatalValidationError(s"Modifier ${m.encodedId} does not corresponds to header ${header.encodedId}") - ) - _ <- Either.cond( - isSemanticallyValid(header.id) != ModifierSemanticValidity.Invalid, - (), - PayloadFatalValidationError( - s"Header ${header.encodedId} for modifier ${m.encodedId} is semantically invalid" - ) - ) - _ <- Either.cond( - header.height >= minimalHeight, - (), - PayloadNonFatalValidationError(s"Too old modifier ${m.encodedId}: ${header.height} < $minimalHeight") - ) - } yield m -} + minimalHeight: Int): Either[ValidationError, PersistentModifier] = for { + _ <- Either.cond(!historyStorage.containsMod(m.id), (), + PayloadFatalValidationError(s"Modifier ${m.encodedId} is already in history")) + _ <- Either.cond(header.isRelated(m), (), + PayloadFatalValidationError(s"Modifier ${m.encodedId} does not corresponds to header ${header.encodedId}")) + _ <- Either.cond(isSemanticallyValid(header.id) != ModifierSemanticValidity.Invalid, (), + PayloadFatalValidationError(s"Header ${header.encodedId} for modifier ${m.encodedId} is semantically invalid")) + _ <- Either.cond(header.height >= minimalHeight, (), + PayloadNonFatalValidationError(s"Too old modifier ${m.encodedId}: ${header.height} < $minimalHeight")) + } yield m +} \ No newline at end of file diff --git a/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala b/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala index ecdc4be798..7e149e41f6 100644 --- a/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala +++ b/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala @@ -8,7 +8,7 @@ import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.utils.TaggedTypes.{ Height, ModifierId } -trait HistoryPayloadsProcessor extends HistoryApi { +trait HistoryPayloadsProcessor { historyApi: HistoryApi => def processPayload(payload: Payload): ProgressInfo = getBlockByPayload(payload).flatMap { block => @@ -66,7 +66,7 @@ trait HistoryPayloadsProcessor extends HistoryApi { scoreOf(fullBlock.id) .flatMap(fbScore => getBestHeaderId.flatMap(id => scoreOf(id).map(_ < fbScore))) .getOrElse(false) - ) + ) val updatedHeadersAtHeightIds = newChain.headers.map(header => updatedBestHeaderAtHeightRaw(header.id, Height @@ header.height)).toList updateStorage(fullBlock.payload, newBestHeader.id, updateBestHeader, updatedHeadersAtHeightIds) @@ -135,4 +135,4 @@ trait HistoryPayloadsProcessor extends HistoryApi { private def isValidFirstBlock(header: Header): Boolean = header.height == blockDownloadProcessor.minimalBlockHeight && getBestBlockId.isEmpty -} +} \ No newline at end of file diff --git a/src/main/scala/encry/view/history/ValidationError.scala b/src/main/scala/encry/view/history/ValidationError.scala index 73ac6faab5..bcb5bd0397 100644 --- a/src/main/scala/encry/view/history/ValidationError.scala +++ b/src/main/scala/encry/view/history/ValidationError.scala @@ -16,4 +16,4 @@ object ValidationError { case class PayloadNonFatalValidationError(error: String) extends NonFatalValidationError } case class HistoryApiError(error: String) extends ValidationError -} +} \ No newline at end of file From f8a6664b8d2643ae0200a8f101c2eb435306299a Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 14 Jan 2020 16:26:37 +0300 Subject: [PATCH 03/10] cleaned up code --- src/main/scala/encry/view/history/HistoryApi.scala | 8 ++++---- .../encry/view/history/HistoryModifiersValidator.scala | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/scala/encry/view/history/HistoryApi.scala b/src/main/scala/encry/view/history/HistoryApi.scala index 02d87fa74b..fef24a520f 100644 --- a/src/main/scala/encry/view/history/HistoryApi.scala +++ b/src/main/scala/encry/view/history/HistoryApi.scala @@ -182,9 +182,9 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore @tailrec final def loopHeightDown(height: Int, p: ModifierId => Boolean): Option[Header] = headerIdsAtHeight(height) .find(p) .flatMap(getHeaderById) match { - case h@Some(_) => h + case h@Some(_) => h case None if height > settings.constants.GenesisHeight => loopHeightDown(height - 1, p) - case n@None => n + case n@None => n } def requiredDifficultyAfter(parent: Header): Either[HistoryApiError, Difficulty] = { @@ -252,8 +252,8 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore } yield headerChainBack(size, startHeader, h => h.parentId sameElements lastHeaderInOurBestChain) .headers .map(_.id)) match { - case Some(value) => value - case None => Seq.empty + case Some(value) => value + case None => Seq.empty } } diff --git a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala index bee13e90a9..abb2bd17b4 100644 --- a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala +++ b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala @@ -43,7 +43,7 @@ trait HistoryModifiersValidator { historyApi: HistoryApi => case None if isModifierDefined(modifierId) => ModifierSemanticValidity.Unknown case None => ModifierSemanticValidity.Absent case mod => logger.error(s"Incorrect validity status: $mod") - ModifierSemanticValidity.Absent + ModifierSemanticValidity.Absent } private def genesisBlockHeaderValidator(h: Header): Either[ValidationError, Header] = for { From 763dc5f1784a36c364a1dc4f38d67af6ac3a8833 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 14 Jan 2020 16:28:52 +0300 Subject: [PATCH 04/10] cleaned up code --- src/main/scala/encry/view/history/HistoryApi.scala | 14 +++++++------- .../view/history/HistoryModifiersValidator.scala | 12 ++++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/main/scala/encry/view/history/HistoryApi.scala b/src/main/scala/encry/view/history/HistoryApi.scala index fef24a520f..018d0c4262 100644 --- a/src/main/scala/encry/view/history/HistoryApi.scala +++ b/src/main/scala/encry/view/history/HistoryApi.scala @@ -182,9 +182,9 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore @tailrec final def loopHeightDown(height: Int, p: ModifierId => Boolean): Option[Header] = headerIdsAtHeight(height) .find(p) .flatMap(getHeaderById) match { - case h@Some(_) => h - case None if height > settings.constants.GenesisHeight => loopHeightDown(height - 1, p) - case n@None => n + case h@Some(_) => h + case None if height > settings.constants.GenesisHeight => loopHeightDown(height - 1, p) + case n@None => n } def requiredDifficultyAfter(parent: Header): Either[HistoryApiError, Difficulty] = { @@ -250,10 +250,10 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore startId <- headerIdsAtHeight(heightFrom).headOption startHeader <- getHeaderById(startId) } yield headerChainBack(size, startHeader, h => h.parentId sameElements lastHeaderInOurBestChain) - .headers - .map(_.id)) match { - case Some(value) => value - case None => Seq.empty + .headers + .map(_.id)) match { + case Some(value) => value + case None => Seq.empty } } diff --git a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala index abb2bd17b4..3907fc6e4e 100644 --- a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala +++ b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala @@ -38,12 +38,12 @@ trait HistoryModifiersValidator { historyApi: HistoryApi => private def isSemanticallyValid(modifierId: ModifierId): ModifierSemanticValidity = historyStorage .get(validityKey(modifierId)) match { - case Some(mod) if mod.headOption.contains(1.toByte) => ModifierSemanticValidity.Valid - case Some(mod) if mod.headOption.contains(0.toByte) => ModifierSemanticValidity.Invalid - case None if isModifierDefined(modifierId) => ModifierSemanticValidity.Unknown - case None => ModifierSemanticValidity.Absent - case mod => logger.error(s"Incorrect validity status: $mod") - ModifierSemanticValidity.Absent + case Some(mod) if mod.headOption.contains(1.toByte) => ModifierSemanticValidity.Valid + case Some(mod) if mod.headOption.contains(0.toByte) => ModifierSemanticValidity.Invalid + case None if isModifierDefined(modifierId) => ModifierSemanticValidity.Unknown + case None => ModifierSemanticValidity.Absent + case mod => logger.error(s"Incorrect validity status: $mod") + ModifierSemanticValidity.Absent } private def genesisBlockHeaderValidator(h: Header): Either[ValidationError, Header] = for { From b7b652be0c1bddf9e6b7221a10c96429a17259b6 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 14 Jan 2020 16:40:58 +0300 Subject: [PATCH 05/10] refactoring --- src/main/scala/encry/view/history/History.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/encry/view/history/History.scala b/src/main/scala/encry/view/history/History.scala index ee3c068451..9280f3114c 100644 --- a/src/main/scala/encry/view/history/History.scala +++ b/src/main/scala/encry/view/history/History.scala @@ -21,7 +21,7 @@ import cats.syntax.either._ /** * History implementation. It is processing persistent modifiers generated locally or received from the network. **/ -trait History extends HistoryModifiersValidator with HistoryApi with AutoCloseable { +trait History extends HistoryApi with HistoryModifiersValidator with AutoCloseable { var isFullChainSynced: Boolean From c4f9be521270886bceb419d8cc07252b19cf9341 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 21 Jan 2020 16:02:18 +0300 Subject: [PATCH 06/10] new changes in frame --- .../scala/encry/view/NodeViewHolder.scala | 5 +- .../view/history/FastSyncProcessor.scala | 26 --- .../scala/encry/view/history/History.scala | 29 ++- .../scala/encry/view/history/HistoryApi.scala | 132 ++++--------- .../encry/view/history/HistoryDBApi.scala | 133 ++++++------- .../history/HistoryHeadersProcessor.scala | 80 -------- .../history/HistoryModifiersValidator.scala | 7 +- .../history/HistoryPayloadsProcessor.scala | 138 -------------- .../HeaderDefaultProcessorComponent.scala | 101 ++++++++++ .../HistoryHeaderProcessorComponent.scala | 13 ++ .../HistoryPayloadProcessorComponent.scala | 17 ++ .../PayloadFastSyncProcessorComponent.scala | 70 +++++++ .../PayloadNormalProcessorComponent.scala | 179 ++++++++++++++++++ .../encry/modifiers/InstanceFactory.scala | 5 +- 14 files changed, 514 insertions(+), 421 deletions(-) delete mode 100644 src/main/scala/encry/view/history/FastSyncProcessor.scala delete mode 100644 src/main/scala/encry/view/history/HistoryHeadersProcessor.scala delete mode 100644 src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala create mode 100644 src/main/scala/encry/view/history/processors/HeaderDefaultProcessorComponent.scala create mode 100644 src/main/scala/encry/view/history/processors/HistoryHeaderProcessorComponent.scala create mode 100644 src/main/scala/encry/view/history/processors/HistoryPayloadProcessorComponent.scala create mode 100644 src/main/scala/encry/view/history/processors/PayloadFastSyncProcessorComponent.scala create mode 100644 src/main/scala/encry/view/history/processors/PayloadNormalProcessorComponent.scala diff --git a/src/main/scala/encry/view/NodeViewHolder.scala b/src/main/scala/encry/view/NodeViewHolder.scala index 357b431ba2..31431e1466 100644 --- a/src/main/scala/encry/view/NodeViewHolder.scala +++ b/src/main/scala/encry/view/NodeViewHolder.scala @@ -23,8 +23,9 @@ import encry.view.NodeViewHolder.ReceivableMessages._ import encry.view.NodeViewHolder._ import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId import encry.view.fast.sync.SnapshotHolder._ +import encry.view.history.processors.{HeaderDefaultProcessorComponent, PayloadNormalProcessorComponent} import encry.view.history.storage.HistoryStorage -import encry.view.history.{History, HistoryHeadersProcessor, HistoryPayloadsProcessor} +import encry.view.history.{History, HistoryApi} import encry.view.mempool.MemoryPool.RolledBackTransactions import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree @@ -96,7 +97,7 @@ class NodeViewHolder(memoryPoolRef: ActorRef, FileUtils.deleteDirectory(new File(s"${encrySettings.directory}/keysTmp")) FileUtils.deleteDirectory(new File(s"${encrySettings.directory}/walletTmp")) logger.info(s"Updated best block in fast sync mod. Updated state height.") - val newHistory = new History with HistoryHeadersProcessor with HistoryPayloadsProcessor { + val newHistory = new History with HeaderDefaultProcessorComponent with PayloadNormalProcessorComponent { override val settings: EncryAppSettings = encrySettings override var isFullChainSynced: Boolean = settings.node.offlineGeneration override val timeProvider: NetworkTimeProvider = EncryApp.timeProvider diff --git a/src/main/scala/encry/view/history/FastSyncProcessor.scala b/src/main/scala/encry/view/history/FastSyncProcessor.scala deleted file mode 100644 index d25dca9ab1..0000000000 --- a/src/main/scala/encry/view/history/FastSyncProcessor.scala +++ /dev/null @@ -1,26 +0,0 @@ -package encry.view.history - -import cats.syntax.option.none -import encry.consensus.HistoryConsensus.ProgressInfo -import encry.storage.VersionalStorage.{StorageKey, StorageValue, StorageVersion} -import org.encryfoundation.common.modifiers.history.Payload - -trait FastSyncProcessor { historyApi: HistoryApi => - - def processPayload(payload: Payload): ProgressInfo = { - val startTime: Long = System.currentTimeMillis() - getBlockByPayload(payload).foreach { block => - logger.info(s"processPayloadFastSync") - historyStorage.bulkInsert(payload.id, Seq(BestBlockKey -> payload.headerId), Seq(payload)) - blockDownloadProcessor.updateBestBlock(block.header) - logger.info(s"BlockDownloadProcessor updated block at height ${block.header.height} successfully") - historyStorage.insert( - StorageVersion @@ validityKey(block.payload.id).untag(StorageKey), - List(block.header.id, block.payload.id).map(id => validityKey(id) -> StorageValue @@ Array(1.toByte)) - ) - logger.info(s"Finished processing block ${block.encodedId}. " + - s"Processing time is ${(System.currentTimeMillis() - startTime) / 1000} s") - } - ProgressInfo(none, Seq.empty, Seq.empty, none) - } -} \ No newline at end of file diff --git a/src/main/scala/encry/view/history/History.scala b/src/main/scala/encry/view/history/History.scala index 9280f3114c..830259434d 100644 --- a/src/main/scala/encry/view/history/History.scala +++ b/src/main/scala/encry/view/history/History.scala @@ -1,6 +1,7 @@ package encry.view.history import java.io.File + import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus.ProgressInfo import encry.settings._ @@ -17,29 +18,27 @@ import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId import org.iq80.leveldb.Options import cats.syntax.either._ +import encry.view.history.processors._ /** * History implementation. It is processing persistent modifiers generated locally or received from the network. **/ -trait History extends HistoryApi with HistoryModifiersValidator with AutoCloseable { +trait History extends HistoryModifiersValidator with HistoryApi with AutoCloseable { + this: HistoryHeaderProcessorComponent with HistoryPayloadProcessorComponent => var isFullChainSynced: Boolean /** Appends modifier to the history if it is applicable. */ - def append(modifier: PersistentModifier): Either[Throwable, (History, ProgressInfo)] = { + final def append(modifier: PersistentModifier): Either[Throwable, (History, ProgressInfo)] = { logger.info(s"Trying to append modifier ${Algos.encode(modifier.id)} of type ${modifier.modifierTypeId} to history") Either.catchNonFatal(modifier match { case header: Header => logger.info(s"Append header ${header.encodedId} at height ${header.height} to history") - (this, processHeader(header)) - case payload: Payload => (this, processPayload(payload)) + (this, headerProcessor.processHeader(header)) + case payload: Payload => (this, payloadProcessor.processPayload(payload)) }) } - def processHeader(h: Header): ProgressInfo - - def processPayload(payload: Payload): ProgressInfo - /** @return header, that corresponds to modifier */ private def correspondingHeader(modifier: PersistentModifier): Option[Header] = modifier match { case header: Header => Some(header) @@ -53,7 +52,7 @@ trait History extends HistoryApi with HistoryModifiersValidator with AutoClosea * @param modifier that is invalid against the State * @return ProgressInfo with next modifier to try to apply */ - def reportModifierIsInvalid(modifier: PersistentModifier): (History, ProgressInfo) = { + final def reportModifierIsInvalid(modifier: PersistentModifier): (History, ProgressInfo) = { logger.info(s"Modifier ${modifier.encodedId} of type ${modifier.modifierTypeId} is marked as invalid") correspondingHeader(modifier) match { case Some(invalidatedHeader) => @@ -119,7 +118,7 @@ trait History extends HistoryApi with HistoryModifiersValidator with AutoClosea * @param modifier that is valid against the State * @return ProgressInfo with next modifier to try to apply */ - def reportModifierIsValid(modifier: PersistentModifier): History = { + final def reportModifierIsValid(modifier: PersistentModifier): History = { logger.info(s"Modifier ${modifier.encodedId} of type ${modifier.modifierTypeId} is marked as valid ") modifier match { case block: Block => @@ -139,9 +138,9 @@ trait History extends HistoryApi with HistoryModifiersValidator with AutoClosea } } - override def close(): Unit = historyStorage.close() + override final protected[view] def close(): Unit = historyStorage.close() - def closeStorage(): Unit = historyStorage.close() + final protected[view] def closeStorage(): Unit = historyStorage.close() } object History extends StrictLogging { @@ -158,7 +157,7 @@ object History extends StrictLogging { dir } - def readOrGenerate(settingsEncry: EncryAppSettings, ntp: NetworkTimeProvider): History = { + def readOrGenerate(settingsEncry: EncryAppSettings, ntp: NetworkTimeProvider): History with HistoryApi = { val historyIndexDir: File = getHistoryIndexDir(settingsEncry) //Check what kind of storage in settings: val vldbInit = settingsEncry.storage.history match { @@ -174,14 +173,14 @@ object History extends StrictLogging { VLDBWrapper(VersionalLevelDBCompanion(levelDBInit, settingsEncry.levelDB)) } if (settingsEncry.snapshotSettings.enableFastSynchronization && !settingsEncry.node.offlineGeneration) - new History with HistoryHeadersProcessor with FastSyncProcessor { + new History with HeaderDefaultProcessorComponent with PayloadFastSyncProcessorComponent { override val settings: EncryAppSettings = settingsEncry override var isFullChainSynced: Boolean = settings.node.offlineGeneration override val historyStorage: HistoryStorage = HistoryStorage(vldbInit) override val timeProvider: NetworkTimeProvider = new NetworkTimeProvider(settingsEncry.ntp) } else - new History with HistoryHeadersProcessor with HistoryPayloadsProcessor { + new History with HeaderDefaultProcessorComponent with PayloadNormalProcessorComponent { override val settings: EncryAppSettings = settingsEncry override var isFullChainSynced: Boolean = settings.node.offlineGeneration override val historyStorage: HistoryStorage = HistoryStorage(vldbInit) diff --git a/src/main/scala/encry/view/history/HistoryApi.scala b/src/main/scala/encry/view/history/HistoryApi.scala index 018d0c4262..cb49fc887a 100644 --- a/src/main/scala/encry/view/history/HistoryApi.scala +++ b/src/main/scala/encry/view/history/HistoryApi.scala @@ -1,6 +1,5 @@ package encry.view.history -import cats.syntax.option._ import encry.consensus.HistoryConsensus._ import encry.consensus._ import encry.modifiers.history._ @@ -11,7 +10,7 @@ import io.iohk.iodb.ByteArrayWrapper import org.encryfoundation.common.modifiers.history._ import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, Height, ModifierId, ModifierTypeId} +import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, Height, ModifierId} import scala.annotation.tailrec import scala.collection.immutable.HashSet @@ -19,153 +18,108 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore val timeProvider: NetworkTimeProvider - var headersCacheIndexes: Map[Int, Seq[ModifierId]] = Map.empty[Int, Seq[ModifierId]] + private var headersCacheIndexes: Map[Int, Seq[ModifierId]] = Map.empty[Int, Seq[ModifierId]] - var headersCache: Map[ByteArrayWrapper, Header] = Map.empty[ByteArrayWrapper, Header] + private var headersCache: Map[ByteArrayWrapper, Header] = Map.empty[ByteArrayWrapper, Header] - var blocksCacheIndexes: Map[Int, Seq[ModifierId]] = Map.empty[Int, Seq[ModifierId]] + private var blocksCacheIndexes: Map[Int, Seq[ModifierId]] = Map.empty[Int, Seq[ModifierId]] - var blocksCache: Map[ByteArrayWrapper, Block] = Map.empty[ByteArrayWrapper, Block] + private var blocksCache: Map[ByteArrayWrapper, Block] = Map.empty[ByteArrayWrapper, Block] private var lastSyncInfo: SyncInfo = SyncInfo(Seq.empty[ModifierId]) lazy val blockDownloadProcessor: BlockDownloadProcessor = BlockDownloadProcessor(settings.node, settings.constants) - var isHeadersChainSyncedVar: Boolean = false + final var isHeadersChainSyncedVar: Boolean = false final case class FastSyncProcessor(localSettings: EncryAppSettings) { var fastSyncVal: Boolean = settings.snapshotSettings.enableFastSynchronization && !settings.node.offlineGeneration } - var lastAvailableManifestHeight: Int = 0 + final var lastAvailableManifestHeight: Int = 0 - lazy val fastSyncInProgress: FastSyncProcessor = FastSyncProcessor(settings) + final lazy val fastSyncInProgress: FastSyncProcessor = FastSyncProcessor(settings) - def getHeaderById(id: ModifierId): Option[Header] = headersCache + def payloadsIdsToDownload(howMany: Int, excluding: HashSet[ModifierId]): Seq[ModifierId] + + final def getHeaderById(id: ModifierId): Option[Header] = headersCache .get(ByteArrayWrapper(id)) .orElse(blocksCache.get(ByteArrayWrapper(id)).map(_.header)) .orElse(getHeaderByIdDB(id)) - def getBlockByHeaderId(id: ModifierId): Option[Block] = - blocksCache - .get(ByteArrayWrapper(id)) - .orElse(headersCache.get(ByteArrayWrapper(id)) - .flatMap(h => getPayloadByIdDB(h.payloadId).map(p => Block(h, p)))) - .orElse(getBlockByHeaderIdDB(id)) - - def getBlockByHeader(header: Header): Option[Block] = blocksCache + final def getBlockByHeader(header: Header): Option[Block] = blocksCache .get(ByteArrayWrapper(header.id)) .orElse(getPayloadByIdDB(header.payloadId).map(p => Block(header, p))) - def getBestHeader: Option[Header] = getBestHeaderId.flatMap(id => + final def getBestHeader: Option[Header] = getBestHeaderId.flatMap(id => headersCache .get(ByteArrayWrapper(id)) .orElse(blocksCache.get(ByteArrayWrapper(id)).map(_.header)) .orElse(getHeaderByIdDB(id)) ) - def getBestHeaderHeight: Int = getBestHeaderId.flatMap(id => + final def getBestHeaderHeight: Int = getBestHeaderId.flatMap(id => headersCache.get(ByteArrayWrapper(id)).map(_.height) .orElse(blocksCache.get(ByteArrayWrapper(id)).map(_.header.height)) .orElse(getHeightByHeaderId(id)) ).getOrElse(settings.constants.PreGenesisHeight) - def getBestBlock: Option[Block] = getBestBlockId.flatMap(id => + final def getBestBlock: Option[Block] = getBestBlockId.flatMap(id => blocksCache.get(ByteArrayWrapper(id)) .orElse(getBlockByHeaderIdDB(id)) ) - def getBestBlockHeight: Int = getBestBlockId + final def getBestBlockHeight: Int = getBestBlockId .flatMap(id => blocksCache.get(ByteArrayWrapper(id)).map(_.header.height).orElse(getHeightByHeaderId(id))) .getOrElse(settings.constants.PreGenesisHeight) - def getHeaderOfBestBlock: Option[Header] = getBestBlockId.flatMap(id => + final def getHeaderOfBestBlock: Option[Header] = getBestBlockId.flatMap(id => headersCache.get(ByteArrayWrapper(id)) .orElse(blocksCache.get(ByteArrayWrapper(id)).map(_.header)) .orElse(getHeaderByIdDB(id)) ) - def getBestHeaderAtHeight(h: Int): Option[Header] = getBestHeaderAtHeightDB(h) + final def getBestHeaderAtHeight(h: Int): Option[Header] = getBestHeaderAtHeightDB(h) - def getBlockByPayload(payload: Payload): Option[Block] = headersCache + final def getBlockByPayload(payload: Payload): Option[Block] = headersCache .get(ByteArrayWrapper(payload.headerId)).map(h => Block(h, payload)) .orElse(blocksCache.get(ByteArrayWrapper(payload.headerId))) .orElse(getHeaderById(payload.headerId).flatMap(h => Some(Block(h, payload)))) - def getHeightByHeaderId(id: ModifierId): Option[Int] = headersCache + final def getHeightByHeaderId(id: ModifierId): Option[Int] = headersCache .get(ByteArrayWrapper(id)).map(_.height) .orElse(blocksCache.get(ByteArrayWrapper(id)).map(_.header.height)) .orElse(getHeightByHeaderIdDB(id)) - def isBestBlockDefined: Boolean = + final def isBestBlockDefined: Boolean = getBestBlockId.map(id => blocksCache.contains(ByteArrayWrapper(id))).isDefined || getHeaderOfBestBlock.map(h => isModifierDefined(h.payloadId)).isDefined - def isBlockDefined(header: Header): Boolean = + final def isBlockDefined(header: Header): Boolean = blocksCache.get(ByteArrayWrapper(header.id)).isDefined || isModifierDefined(header.payloadId) - def isHeaderDefined(id: ModifierId): Boolean = + final def isHeaderDefined(id: ModifierId): Boolean = headersCache.get(ByteArrayWrapper(id)).isDefined || blocksCache.get(ByteArrayWrapper(id)).isDefined || isModifierDefined(id) - def getBestHeaderIdAtHeight(h: Int): Option[ModifierId] = getBestHeaderIdAtHeightDB(h) - def headerIdsAtHeight(height: Int): Seq[ModifierId] = headerIdsAtHeightDB(height) + final def getBestHeaderIdAtHeight(h: Int): Option[ModifierId] = getBestHeaderIdAtHeightDB(h) + final def headerIdsAtHeight(height: Int): Seq[ModifierId] = headerIdsAtHeightDB(height) .getOrElse(Seq.empty[ModifierId]) - def modifierBytesById(id: ModifierId): Option[Array[Byte]] = headersCache + final def modifierBytesById(id: ModifierId): Option[Array[Byte]] = headersCache .get(ByteArrayWrapper(id)).map(h => HeaderProtoSerializer.toProto(h).toByteArray) .orElse(blocksCache.get(ByteArrayWrapper(id)).map(b => BlockProtoSerializer.toProto(b).toByteArray)) .orElse(modifierBytesByIdDB(id)) - def lastHeaders(count: Int): HeaderChain = getBestHeader + final def lastHeaders(count: Int): HeaderChain = getBestHeader .map(bestHeader => headerChainBack(count, bestHeader, _ => false)) .getOrElse(HeaderChain.empty) - def getHeaderIds(count: Int, offset: Int = 0): Seq[ModifierId] = (offset until (count + offset)) + final def getHeaderIds(count: Int, offset: Int = 0): Seq[ModifierId] = (offset until (count + offset)) .flatMap(getBestHeaderIdAtHeight) - def payloadsIdsToDownload(howMany: Int, excluding: HashSet[ModifierId]): Seq[ModifierId] = { - @tailrec def continuation(height: Int, acc: Seq[ModifierId]): Seq[ModifierId] = - if (acc.lengthCompare(howMany) >= 0) acc - else if (height > lastAvailableManifestHeight && fastSyncInProgress.fastSyncVal) acc - else getBestHeaderIdAtHeight(height).flatMap(getHeaderById) match { - case Some(h) if !excluding.exists(_.sameElements(h.payloadId)) && !isBlockDefined(h) => - continuation(height + 1, acc :+ h.payloadId) - case Some(_) => - continuation(height + 1, acc) - case None => - acc - } - - (for { - bestBlockId <- getBestBlockId - headerLinkedToBestBlock <- getHeaderById(bestBlockId) - } yield headerLinkedToBestBlock) match { - case _ if !isHeadersChainSynced => - Seq.empty - case Some(header) if isInBestChain(header) => - continuation(header.height + 1, Seq.empty) - case Some(header) => - lastBestBlockHeightRelevantToBestChain(header.height) - .map(height => continuation(height + 1, Seq.empty)) - .getOrElse(continuation(blockDownloadProcessor.minimalBlockHeightVar, Seq.empty)) - case None => - continuation(blockDownloadProcessor.minimalBlockHeightVar, Seq.empty) - } - } - - def toDownload(header: Header): Option[(ModifierTypeId, ModifierId)] = - // Already synced and header is not too far back. Download required modifiers - if (header.height >= blockDownloadProcessor.minimalBlockHeight) (Payload.modifierTypeId -> header.payloadId).some - // Headers chain is synced after this header. Start downloading full blocks - else if (!isHeadersChainSynced && isNewHeader(header)) { - isHeadersChainSyncedVar = true - blockDownloadProcessor.updateBestBlock(header) - none - } else none - - def headerChainBack(limit: Int, startHeader: Header, until: Header => Boolean): HeaderChain = { + final def headerChainBack(limit: Int, startHeader: Header, until: Header => Boolean): HeaderChain = { @tailrec def loop(header: Header, acc: Seq[Header]): Seq[Header] = { if (acc.length == limit || until(header)) acc else getHeaderById(header.parentId) match { @@ -187,7 +141,7 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore case n@None => n } - def requiredDifficultyAfter(parent: Header): Either[HistoryApiError, Difficulty] = { + final def requiredDifficultyAfter(parent: Header): Either[HistoryApiError, Difficulty] = { val requiredHeights: Seq[Height] = PowLinearController.getHeightsForRetargetingAt(Height @@ (parent.height + 1), settings.constants.EpochLength, settings.constants.RetargetingEpochsQty) for { @@ -203,16 +157,16 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore settings.constants.DesiredBlockInterval, settings.constants.InitialDifficulty) } - def syncInfo: SyncInfo = lastSyncInfo + final def syncInfo: SyncInfo = lastSyncInfo - def updateIdsForSyncInfo(): Unit = + final protected[view] def updateIdsForSyncInfo(): Unit = lastSyncInfo = SyncInfo(getBestHeader.map { header: Header => ((header.height - settings.network.maxInvObjects + 1) to header.height).flatMap { height: Int => headerIdsAtHeight(height).headOption }.toList }.getOrElse(List.empty)) - def compare(si: SyncInfo): HistoryComparisonResult = lastSyncInfo.lastHeaderIds.lastOption match { + final def compare(si: SyncInfo): HistoryComparisonResult = lastSyncInfo.lastHeaderIds.lastOption match { //Our best header is the same as other history best header case Some(id) if si.lastHeaderIds.lastOption.exists(_ sameElements id) => Equal //Our best header is in other history best chain, but not at the last position @@ -230,7 +184,7 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore case None => Older } - def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] = + final def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] = if (getBestHeaderId.isEmpty) info.startingPoints.map(_._2) else if (info.lastHeaderIds.isEmpty) { val heightFrom: Int = Math.min(getBestHeaderHeight, size - 1) @@ -257,8 +211,8 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore } } - def commonBlockThenSuffixes(header1: Header, - header2: Header): (HeaderChain, HeaderChain) = { + final def commonBlockThenSuffixes(header1: Header, + header2: Header): (HeaderChain, HeaderChain) = { val heightDelta: Int = Math.max(header1.height - header2.height, 0) @scala.annotation.tailrec @@ -284,7 +238,7 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore loop(2, HeaderChain(Seq(header2))) } - def getChainToHeader(fromHeaderOpt: Option[Header], + final def getChainToHeader(fromHeaderOpt: Option[Header], toHeader: Header): (Option[ModifierId], HeaderChain) = fromHeaderOpt match { case Some(h1) => val (prevChain, newChain) = commonBlockThenSuffixes(h1, toHeader) @@ -292,13 +246,9 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore case None => (None, headerChainBack(toHeader.height + 1, toHeader, _ => false)) } - def isNewHeader(header: Header): Boolean = - timeProvider.estimatedTime - header.timestamp < - settings.constants.DesiredBlockInterval.toMillis * settings.constants.NewHeaderTimeMultiplier - - def isHeadersChainSynced: Boolean = isHeadersChainSyncedVar + final def isHeadersChainSynced: Boolean = isHeadersChainSyncedVar - def continuationHeaderChains(header: Header, + final def continuationHeaderChains(header: Header, filterCond: Header => Boolean): Seq[Seq[Header]] = { @tailrec def loop(currentHeight: Int, acc: Seq[Seq[Header]]): Seq[Seq[Header]] = { val nextHeightHeaders: Seq[Header] = headerIdsAtHeight(currentHeight + 1) @@ -321,7 +271,7 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore loop(header.height, Seq(Seq(header))) } - def addHeaderToCacheIfNecessary(h: Header): Unit = + protected[history] final def addHeaderToCacheIfNecessary(h: Header): Unit = if (h.height >= getBestHeaderHeight - settings.constants.MaxRollbackDepth) { logger.debug(s"Should add ${Algos.encode(h.id)} to header cache") val newHeadersIdsAtHeaderHeight = headersCacheIndexes.getOrElse(h.height, Seq.empty[ModifierId]) :+ h.id @@ -340,7 +290,7 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore logger.debug(s"headersCacheIndexes size: ${headersCacheIndexes.size}") } - def addBlockToCacheIfNecessary(b: Block): Unit = + protected[history] final def addBlockToCacheIfNecessary(b: Block): Unit = if (!blocksCache.contains(ByteArrayWrapper(b.id)) && (b.header.height >= getBestBlockHeight - settings.constants.MaxRollbackDepth)) { logger.debug(s"Should add ${Algos.encode(b.id)} to block cache") val newBlocksIdsAtBlockHeight = blocksCacheIndexes.getOrElse(b.header.height, Seq.empty[ModifierId]) :+ b.id diff --git a/src/main/scala/encry/view/history/HistoryDBApi.scala b/src/main/scala/encry/view/history/HistoryDBApi.scala index 55427766a8..82fa6eb1f9 100644 --- a/src/main/scala/encry/view/history/HistoryDBApi.scala +++ b/src/main/scala/encry/view/history/HistoryDBApi.scala @@ -5,9 +5,9 @@ import com.typesafe.scalalogging.StrictLogging import encry.settings.EncryAppSettings import encry.storage.VersionalStorage.StorageKey import encry.view.history.storage.HistoryStorage -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{Height, ModifierId, ModifierTypeId} +import org.encryfoundation.common.utils.TaggedTypes.{ Height, ModifierId, ModifierTypeId } import scorex.crypto.hash.Digest32 import scala.reflect.ClassTag @@ -16,81 +16,86 @@ trait HistoryDBApi extends StrictLogging { val settings: EncryAppSettings - val historyStorage: HistoryStorage + protected[view] val historyStorage: HistoryStorage - lazy val BestHeaderKey: StorageKey = + protected[history] final lazy val BestHeaderKey: StorageKey = StorageKey @@ Array.fill(settings.constants.DigestLength)(Header.modifierTypeId.untag(ModifierTypeId)) - lazy val BestBlockKey: StorageKey = + protected[history] final lazy val BestBlockKey: StorageKey = StorageKey @@ Array.fill(settings.constants.DigestLength)(-1: Byte) - private def getModifierById[T: ClassTag](id: ModifierId): Option[T] = historyStorage - .modifierById(id) - .collect { case m: T => m } + private def getModifierById[T: ClassTag](id: ModifierId): Option[T] = + historyStorage + .modifierById(id) + .collect { case m: T => m } - def getHeightByHeaderIdDB(id: ModifierId): Option[Int] = historyStorage - .get(headerHeightKey(id)) - .map(Ints.fromByteArray) + final def getHeightByHeaderIdDB(id: ModifierId): Option[Int] = + historyStorage + .get(headerHeightKey(id)) + .map(Ints.fromByteArray) - def getHeaderByIdDB(id: ModifierId): Option[Header] = getModifierById[Header](id) - def getPayloadByIdDB(pId: ModifierId): Option[Payload] = getModifierById[Payload](pId) - def getBlockByHeaderDB(header: Header): Option[Block] = getModifierById[Payload](header.payloadId) - .map(payload => Block(header, payload)) - def getBlockByHeaderIdDB(id: ModifierId): Option[Block] = getHeaderByIdDB(id) - .flatMap(h => getModifierById[Payload](h.payloadId).map(p => Block(h, p))) + final def getHeaderByIdDB(id: ModifierId): Option[Header] = getModifierById[Header](id) + final def getPayloadByIdDB(pId: ModifierId): Option[Payload] = getModifierById[Payload](pId) + final def getBlockByHeaderIdDB(id: ModifierId): Option[Block] = + getHeaderByIdDB(id) + .flatMap(h => getModifierById[Payload](h.payloadId).map(p => Block(h, p))) - def getBestHeaderId: Option[ModifierId] = historyStorage.get(BestHeaderKey).map(ModifierId @@ _) - def getBestHeaderDB: Option[Header] = getBestHeaderId.flatMap(getHeaderByIdDB) - def getBestHeaderHeightDB: Int = getBestHeaderId - .flatMap(getHeightByHeaderIdDB) - .getOrElse(settings.constants.PreGenesisHeight) + final def getBestHeaderId: Option[ModifierId] = + historyStorage.get(BestHeaderKey).map(ModifierId @@ _) - def getBestBlockId: Option[ModifierId] = historyStorage.get(BestBlockKey).map(ModifierId @@ _) - def getBestBlockDB: Option[Block] = getBestBlockId.flatMap(getBlockByHeaderIdDB) - def getBestBlockHeightDB: Int = getBestBlockId - .flatMap(getHeightByHeaderIdDB) - .getOrElse(settings.constants.PreGenesisHeight) + final def getBestBlockId: Option[ModifierId] = + historyStorage.get(BestBlockKey).map(ModifierId @@ _) - def modifierBytesByIdDB(id: ModifierId): Option[Array[Byte]] = historyStorage.modifiersBytesById(id) + final def modifierBytesByIdDB(id: ModifierId): Option[Array[Byte]] = + historyStorage.modifiersBytesById(id) - def isModifierDefined(id: ModifierId): Boolean = historyStorage.containsMod(id) + final def isModifierDefined(id: ModifierId): Boolean = historyStorage.containsMod(id) //todo probably rewrite with indexes collection - def lastBestBlockHeightRelevantToBestChain(probablyAt: Int): Option[Int] = (for { - headerId <- getBestHeaderIdAtHeightDB(probablyAt) - header <- getHeaderByIdDB(headerId) if isModifierDefined(header.payloadId) - } yield header.height).orElse(lastBestBlockHeightRelevantToBestChain(probablyAt - 1)) - - def headerIdsAtHeightDB(height: Int): Option[Seq[ModifierId]] = historyStorage - .get(heightIdsKey(height)) - .map(_.grouped(32).map(ModifierId @@ _).toSeq) - - def getBestHeaderIdAtHeightDB(h: Int): Option[ModifierId] = headerIdsAtHeightDB(h).flatMap(_.headOption) - - def getBestHeaderAtHeightDB(h: Int): Option[Header] = getBestHeaderIdAtHeightDB(h).flatMap(getHeaderByIdDB) - - def isInBestChain(h: Header): Boolean = getBestHeaderIdAtHeightDB(h.height) - .exists(_.sameElements(h.id)) - - def isInBestChain(id: ModifierId): Boolean = heightOf(id) - .flatMap(getBestHeaderIdAtHeightDB) - .exists(_.sameElements(id)) - - def getBestHeadersChainScore: BigInt = getBestHeaderId.flatMap(scoreOf).getOrElse(BigInt(0)) //todo ?.getOrElse(BigInt(0))? - - def scoreOf(id: ModifierId): Option[BigInt] = historyStorage - .get(headerScoreKey(id)) - .map(d => BigInt(d)) - - def heightOf(id: ModifierId): Option[Height] = historyStorage - .get(headerHeightKey(id)) - .map(d => Height @@ Ints.fromByteArray(d)) - - def heightIdsKey(height: Int): StorageKey = + final def lastBestBlockHeightRelevantToBestChain(probablyAt: Int): Option[Int] = + (for { + headerId <- getBestHeaderIdAtHeightDB(probablyAt) + header <- getHeaderByIdDB(headerId) if isModifierDefined(header.payloadId) + } yield header.height).orElse(lastBestBlockHeightRelevantToBestChain(probablyAt - 1)) + + final def headerIdsAtHeightDB(height: Int): Option[Seq[ModifierId]] = + historyStorage + .get(heightIdsKey(height)) + .map(_.grouped(32).map(ModifierId @@ _).toSeq) + + final def getBestHeaderIdAtHeightDB(h: Int): Option[ModifierId] = + headerIdsAtHeightDB(h).flatMap(_.headOption) + + final def getBestHeaderAtHeightDB(h: Int): Option[Header] = + getBestHeaderIdAtHeightDB(h).flatMap(getHeaderByIdDB) + + final def isInBestChain(h: Header): Boolean = + getBestHeaderIdAtHeightDB(h.height) + .exists(_.sameElements(h.id)) + + final def isInBestChain(id: ModifierId): Boolean = + heightOf(id) + .flatMap(getBestHeaderIdAtHeightDB) + .exists(_.sameElements(id)) + + final def getBestHeadersChainScore: BigInt = + getBestHeaderId.flatMap(scoreOf).getOrElse(BigInt(0)) //todo ?.getOrElse(BigInt(0))? + + final def scoreOf(id: ModifierId): Option[BigInt] = + historyStorage + .get(headerScoreKey(id)) + .map(d => BigInt(d)) + + final def heightOf(id: ModifierId): Option[Height] = + historyStorage + .get(headerHeightKey(id)) + .map(d => Height @@ Ints.fromByteArray(d)) + + final def heightIdsKey(height: Int): StorageKey = StorageKey @@ Algos.hash(Ints.toByteArray(height)).untag(Digest32) - def headerScoreKey(id: ModifierId): StorageKey = + final def headerScoreKey(id: ModifierId): StorageKey = StorageKey @@ Algos.hash("score".getBytes(Algos.charset) ++ id).untag(Digest32) - def headerHeightKey(id: ModifierId): StorageKey = + final def headerHeightKey(id: ModifierId): StorageKey = StorageKey @@ Algos.hash("height".getBytes(Algos.charset) ++ id).untag(Digest32) - def validityKey(id: Array[Byte]): StorageKey = + final def validityKey(id: Array[Byte]): StorageKey = StorageKey @@ Algos.hash("validity".getBytes(Algos.charset) ++ id).untag(Digest32) -} \ No newline at end of file +} diff --git a/src/main/scala/encry/view/history/HistoryHeadersProcessor.scala b/src/main/scala/encry/view/history/HistoryHeadersProcessor.scala deleted file mode 100644 index b530efaced..0000000000 --- a/src/main/scala/encry/view/history/HistoryHeadersProcessor.scala +++ /dev/null @@ -1,80 +0,0 @@ -package encry.view.history - -import cats.syntax.option.none -import com.google.common.primitives.Ints -import encry.EncryApp.forceStopApplication -import encry.consensus.ConsensusSchemeReaders -import encry.consensus.HistoryConsensus.ProgressInfo -import encry.storage.VersionalStorage.{ StorageKey, StorageValue } -import org.encryfoundation.common.modifiers.history.Header -import org.encryfoundation.common.utils.TaggedTypes.{ Difficulty, ModifierId } - -trait HistoryHeadersProcessor { historyApi: HistoryApi => - - def processHeader(h: Header): ProgressInfo = getHeaderInfoUpdate(h) match { - case dataToUpdate: Seq[_] if dataToUpdate.nonEmpty => - historyStorage.bulkInsert(h.id, dataToUpdate, Seq(h)) - getBestHeaderId match { - case Some(bestHeaderId) => - ProgressInfo(none, Seq.empty, if (!bestHeaderId.sameElements(h.id)) Seq.empty else Seq(h), toDownload(h)) - case _ => - forceStopApplication(errorMessage = "Should always have best header after header application") - } - case _ => ProgressInfo(none, Seq.empty, Seq.empty, none) - } - - private def getHeaderInfoUpdate(header: Header): Seq[(StorageKey, StorageValue)] = { - addHeaderToCacheIfNecessary(header) - if (header.isGenesis) { - logger.info(s"Initialize header chain with genesis header ${header.encodedId}") - Seq( - BestHeaderKey -> StorageValue @@ header.id, - heightIdsKey(settings.constants.GenesisHeight) -> StorageValue @@ header.id, - headerHeightKey(header.id) -> StorageValue @@ Ints.toByteArray(settings.constants.GenesisHeight), - headerScoreKey(header.id) -> StorageValue @@ header.difficulty.toByteArray - ) - } else - scoreOf(header.parentId).map { parentScore => - logger.info(s"getHeaderInfoUpdate for header $header") - val score: Difficulty = - Difficulty @@ (parentScore + ConsensusSchemeReaders.consensusScheme.realDifficulty(header)) - val bestHeaderHeight: Int = getBestHeaderHeight - val bestHeadersChainScore: BigInt = getBestHeadersChainScore - val bestRow: Seq[(StorageKey, StorageValue)] = - if ((header.height > bestHeaderHeight) || (header.height == bestHeaderHeight && score > bestHeadersChainScore)) - Seq(BestHeaderKey -> StorageValue @@ header.id.untag(ModifierId)) - else Seq.empty - val scoreRow: (StorageKey, StorageValue) = headerScoreKey(header.id) -> StorageValue @@ score.toByteArray - val heightRow: (StorageKey, StorageValue) = - headerHeightKey(header.id) -> StorageValue @@ Ints.toByteArray(header.height) - val headerIdsRow: Seq[(StorageKey, StorageValue)] = - if ((header.height > bestHeaderHeight) || (header.height == bestHeaderHeight && score > bestHeadersChainScore)) - bestBlockHeaderIdsRow(header, score) - else orphanedBlockHeaderIdsRow(header, score) - Seq(scoreRow, heightRow) ++ bestRow ++ headerIdsRow - }.getOrElse(Seq.empty) - } - - private def bestBlockHeaderIdsRow(h: Header, score: Difficulty): Seq[(StorageKey, StorageValue)] = { - logger.info(s"New best header ${h.encodedId} with score: $score at height ${h.height}") - val self: (StorageKey, StorageValue) = - heightIdsKey(h.height) -> - StorageValue @@ (Seq(h.id) ++ headerIdsAtHeight(h.height).filterNot(_ sameElements h.id)).flatten.toArray - val forkHeaders: Seq[(StorageKey, StorageValue)] = getHeaderById(h.parentId).toList.view - .flatMap(headerChainBack(h.height, _, h => isInBestChain(h)).headers) - .filterNot(isInBestChain) - .map( - header => - heightIdsKey(header.height) -> - StorageValue @@ (Seq(header.id) ++ - headerIdsAtHeight(header.height).filterNot(_ sameElements header.id)).flatten.toArray - ) - .toList - forkHeaders :+ self - } - - private def orphanedBlockHeaderIdsRow(h: Header, score: Difficulty): Seq[(StorageKey, StorageValue)] = { - logger.info(s"New orphaned header ${h.encodedId} at height ${h.height} with score $score") - Seq(heightIdsKey(h.height) -> StorageValue @@ (headerIdsAtHeight(h.height) :+ h.id).flatten.toArray) - } -} \ No newline at end of file diff --git a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala index 3907fc6e4e..8234e8f841 100644 --- a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala +++ b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala @@ -9,12 +9,13 @@ import org.encryfoundation.common.modifiers.history.{Header, Payload} import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, ModifierId} import org.encryfoundation.common.validation.ModifierSemanticValidity -trait HistoryModifiersValidator { historyApi: HistoryApi => +trait HistoryModifiersValidator { + historyApi: HistoryApi => - lazy val powScheme: EquihashPowScheme = EquihashPowScheme(settings.constants.n, settings.constants.k, settings.constants.Version, + private lazy val powScheme: EquihashPowScheme = EquihashPowScheme(settings.constants.n, settings.constants.k, settings.constants.Version, settings.constants.PreGenesisHeight, settings.constants.MaxTarget) - def testApplicable(modifier: PersistentModifier): Either[ValidationError, PersistentModifier] = + final def testApplicable(modifier: PersistentModifier): Either[ValidationError, PersistentModifier] = (modifier match { case header: Header => validateHeader(header) case payload: Payload => validatePayload(payload) diff --git a/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala b/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala deleted file mode 100644 index 7e149e41f6..0000000000 --- a/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala +++ /dev/null @@ -1,138 +0,0 @@ -package encry.view.history - -import cats.syntax.either._ -import cats.syntax.option._ -import encry.consensus.HistoryConsensus.ProgressInfo -import encry.modifiers.history.HeaderChain -import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } -import org.encryfoundation.common.utils.TaggedTypes.{ Height, ModifierId } - -trait HistoryPayloadsProcessor { historyApi: HistoryApi => - - def processPayload(payload: Payload): ProgressInfo = - getBlockByPayload(payload).flatMap { block => - logger.info(s"proc block ${block.header.encodedId}!") - processBlock(block).some - }.getOrElse(putToHistory(payload)) - - private def processBlock(blockToProcess: Block): ProgressInfo = { - logger.info( - s"Starting processing block to history ||${blockToProcess.encodedId}||${blockToProcess.header.height}||" - ) - val bestFullChain: Seq[Block] = calculateBestFullChain(blockToProcess) - addBlockToCacheIfNecessary(blockToProcess) - bestFullChain.lastOption.map(_.header) match { - case Some(header) if isValidFirstBlock(blockToProcess.header) => - processValidFirstBlock(blockToProcess, header, bestFullChain) - case Some(header) if isBestBlockDefined && isBetterChain(header.id) => - processBetterChain(blockToProcess, header, Seq.empty, settings.node.blocksToKeep) - case Some(_) => - nonBestBlock(blockToProcess) - case None => - logger.debug(s"Best full chain is empty. Returning empty progress info") - ProgressInfo(none, Seq.empty, Seq.empty, none) - } - } - - private def processValidFirstBlock(fullBlock: Block, - newBestHeader: Header, - newBestChain: Seq[Block]): ProgressInfo = { - logger.info(s"Appending ${fullBlock.encodedId} as a valid first block with height ${fullBlock.header.height}") - updateStorage(fullBlock.payload, newBestHeader.id) - ProgressInfo(none, Seq.empty, newBestChain, none) - } - - private def processBetterChain(fullBlock: Block, - newBestHeader: Header, - newBestChain: Seq[Block], - blocksToKeep: Int): ProgressInfo = - getHeaderOfBestBlock.map { header => - val (prevChain: HeaderChain, newChain: HeaderChain) = commonBlockThenSuffixes(header, newBestHeader) - val toRemove: Seq[Block] = prevChain.tail.headers - .flatMap(getBlockByHeader) - val toApply: Seq[Block] = newChain.tail.headers - .flatMap(h => if (h == fullBlock.header) fullBlock.some else getBlockByHeader(h)) - toApply.foreach(addBlockToCacheIfNecessary) - if (toApply.lengthCompare(newChain.length - 1) != 0) nonBestBlock(fullBlock) - else { - //application of this block leads to full chain with higher score - logger.info(s"Appending ${fullBlock.encodedId}|${fullBlock.header.height} as a better chain") - val branchPoint: Option[ModifierId] = toRemove.headOption.map(_ => prevChain.head.id) - val bestHeaderHeight: Int = getBestHeaderHeight - val updateBestHeader: Boolean = - (fullBlock.header.height > bestHeaderHeight) || ( - (fullBlock.header.height == bestHeaderHeight) && - scoreOf(fullBlock.id) - .flatMap(fbScore => getBestHeaderId.flatMap(id => scoreOf(id).map(_ < fbScore))) - .getOrElse(false) - ) - val updatedHeadersAtHeightIds = - newChain.headers.map(header => updatedBestHeaderAtHeightRaw(header.id, Height @@ header.height)).toList - updateStorage(fullBlock.payload, newBestHeader.id, updateBestHeader, updatedHeadersAtHeightIds) - if (blocksToKeep >= 0) { - val lastKept: Int = blockDownloadProcessor.updateBestBlock(fullBlock.header) - val bestHeight: Int = toApply.lastOption.map(_.header.height).getOrElse(0) - val diff: Int = bestHeight - header.height - clipBlockDataAt(((lastKept - diff) until lastKept).filter(_ >= 0)) - } - ProgressInfo(branchPoint, toRemove, toApply, none) - } - }.getOrElse(ProgressInfo(none, Seq.empty, Seq.empty, none)) - - private def nonBestBlock(fullBlock: Block): ProgressInfo = { - //Orphaned block or full chain is not initialized yet - logger.info(s"Process block to history ${fullBlock.encodedId}||${fullBlock.header.height}||") - historyStorage.bulkInsert(fullBlock.payload.id, Seq.empty, Seq(fullBlock.payload)) - ProgressInfo(none, Seq.empty, Seq.empty, none) - } - - private def updatedBestHeaderAtHeightRaw(headerId: ModifierId, height: Height): (Array[Byte], Array[Byte]) = - heightIdsKey(height) -> - (Seq(headerId) ++ - headerIdsAtHeight(height).filterNot(_ sameElements headerId)).flatten.toArray - - private def putToHistory(payload: Payload): ProgressInfo = { - historyStorage.insertObjects(Seq(payload)) - ProgressInfo(none, Seq.empty, Seq.empty, none) - } - - private def isBetterChain(id: ModifierId): Boolean = - (for { - bestFullBlockId <- getBestBlockId - heightOfThisHeader <- getHeightByHeaderId(id) - prevBestScore <- scoreOf(bestFullBlockId) - score <- scoreOf(id) - bestBlockHeight = getBestBlockHeight - } yield (bestBlockHeight < heightOfThisHeader) || (bestBlockHeight == heightOfThisHeader && score > prevBestScore)) - .getOrElse(false) - - private def calculateBestFullChain(block: Block): Seq[Block] = { - val continuations: Seq[Seq[Header]] = continuationHeaderChains(block.header, h => isBlockDefined(h)).map(_.tail) - logger.debug(s"continuations: ${continuations.map(seq => s"Seq contains: ${seq.length}").mkString(",")}") - val chains: Seq[Seq[Block]] = continuations.map(_.filter(isBlockDefined).flatMap(getBlockByHeader)) - logger.debug(s"Chains: ${chains.map(chain => s"chain contain: ${chain.length}").mkString(",")}") - chains.map(c => block +: c).maxBy(c => scoreOf(c.last.id).get) - } - - private def clipBlockDataAt(heights: Seq[Int]): Either[Throwable, Unit] = Either.catchNonFatal { - val toRemove: Seq[ModifierId] = heights - .flatMap(headerIdsAtHeight) - .flatMap(getHeaderById) - .map(_.payloadId) - historyStorage.removeObjects(toRemove) - } - - private def updateStorage(newModRow: PersistentModifier, - bestFullHeaderId: ModifierId, - updateHeaderInfo: Boolean = false, - additionalIndexes: List[(Array[Byte], Array[Byte])] = List.empty): Unit = { - val indicesToInsert: Seq[(Array[Byte], Array[Byte])] = - if (updateHeaderInfo) Seq(BestBlockKey -> bestFullHeaderId, BestHeaderKey -> bestFullHeaderId) - else Seq(BestBlockKey -> bestFullHeaderId) - historyStorage.bulkInsert(newModRow.id, indicesToInsert ++ additionalIndexes, Seq(newModRow)) - } - - private def isValidFirstBlock(header: Header): Boolean = - header.height == blockDownloadProcessor.minimalBlockHeight && getBestBlockId.isEmpty -} \ No newline at end of file diff --git a/src/main/scala/encry/view/history/processors/HeaderDefaultProcessorComponent.scala b/src/main/scala/encry/view/history/processors/HeaderDefaultProcessorComponent.scala new file mode 100644 index 0000000000..c2cef75f12 --- /dev/null +++ b/src/main/scala/encry/view/history/processors/HeaderDefaultProcessorComponent.scala @@ -0,0 +1,101 @@ +package encry.view.history.processors + +import cats.syntax.option._ +import com.google.common.primitives.Ints +import encry.EncryApp.forceStopApplication +import encry.consensus.HistoryConsensus.ProgressInfo +import encry.consensus.{ConsensusSchemeReaders, HistoryConsensus} +import encry.storage.VersionalStorage.{StorageKey, StorageValue} +import encry.view.history.HistoryApi +import org.encryfoundation.common.modifiers.history.{Header, Payload} +import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, ModifierId, ModifierTypeId} + +trait HeaderDefaultProcessorComponent extends HistoryHeaderProcessorComponent { + this: HistoryApi => + + override val headerProcessor: HeaderProcessor = new HeaderDefaultProcessor + + class HeaderDefaultProcessor extends HeaderProcessor { + override def processHeader(h: Header): HistoryConsensus.ProgressInfo = getHeaderInfoUpdate(h) match { + case dataToUpdate: Seq[_] if dataToUpdate.nonEmpty => + historyStorage.bulkInsert(h.id, dataToUpdate, Seq(h)) + getBestHeaderId match { + case Some(bestHeaderId) => + ProgressInfo(none, Seq.empty, if (!bestHeaderId.sameElements(h.id)) Seq.empty else Seq(h), toDownload(h)) + case _ => + forceStopApplication(errorMessage = "Should always have best header after header application") + } + case _ => ProgressInfo(none, Seq.empty, Seq.empty, none) + } + + private def getHeaderInfoUpdate(header: Header): Seq[(StorageKey, StorageValue)] = { + addHeaderToCacheIfNecessary(header) + if (header.isGenesis) { + logger.info(s"Initialize header chain with genesis header ${header.encodedId}") + Seq( + BestHeaderKey -> StorageValue @@ header.id, + heightIdsKey(settings.constants.GenesisHeight) -> StorageValue @@ header.id, + headerHeightKey(header.id) -> StorageValue @@ Ints.toByteArray(settings.constants.GenesisHeight), + headerScoreKey(header.id) -> StorageValue @@ header.difficulty.toByteArray + ) + } else + scoreOf(header.parentId).map { parentScore => + logger.info(s"getHeaderInfoUpdate for header $header") + val score: Difficulty = + Difficulty @@ (parentScore + ConsensusSchemeReaders.consensusScheme.realDifficulty(header)) + val bestHeaderHeight: Int = getBestHeaderHeight + val bestHeadersChainScore: BigInt = getBestHeadersChainScore + val bestRow: Seq[(StorageKey, StorageValue)] = + if ((header.height > bestHeaderHeight) || (header.height == bestHeaderHeight && score > bestHeadersChainScore)) + Seq(BestHeaderKey -> StorageValue @@ header.id.untag(ModifierId)) + else Seq.empty + val scoreRow: (StorageKey, StorageValue) = headerScoreKey(header.id) -> StorageValue @@ score.toByteArray + val heightRow: (StorageKey, StorageValue) = + headerHeightKey(header.id) -> StorageValue @@ Ints.toByteArray(header.height) + val headerIdsRow: Seq[(StorageKey, StorageValue)] = + if ((header.height > bestHeaderHeight) || (header.height == bestHeaderHeight && score > bestHeadersChainScore)) + bestBlockHeaderIdsRow(header, score) + else orphanedBlockHeaderIdsRow(header, score) + Seq(scoreRow, heightRow) ++ bestRow ++ headerIdsRow + }.getOrElse(Seq.empty) + } + + private def bestBlockHeaderIdsRow(h: Header, score: Difficulty): Seq[(StorageKey, StorageValue)] = { + logger.info(s"New best header ${h.encodedId} with score: $score at height ${h.height}") + val self: (StorageKey, StorageValue) = + heightIdsKey(h.height) -> + StorageValue @@ (Seq(h.id) ++ headerIdsAtHeight(h.height).filterNot(_ sameElements h.id)).flatten.toArray + val forkHeaders: Seq[(StorageKey, StorageValue)] = getHeaderById(h.parentId).toList.view + .flatMap(headerChainBack(h.height, _, h => isInBestChain(h)).headers) + .filterNot(isInBestChain) + .map( + header => + heightIdsKey(header.height) -> + StorageValue @@ (Seq(header.id) ++ + headerIdsAtHeight(header.height).filterNot(_ sameElements header.id)).flatten.toArray + ) + .toList + forkHeaders :+ self + } + + private def orphanedBlockHeaderIdsRow(h: Header, score: Difficulty): Seq[(StorageKey, StorageValue)] = { + logger.info(s"New orphaned header ${h.encodedId} at height ${h.height} with score $score") + Seq(heightIdsKey(h.height) -> StorageValue @@ (headerIdsAtHeight(h.height) :+ h.id).flatten.toArray) + } + + private def toDownload(header: Header): Option[(ModifierTypeId, ModifierId)] = + // Already synced and header is not too far back. Download required modifiers + if (header.height >= blockDownloadProcessor.minimalBlockHeight) (Payload.modifierTypeId -> header.payloadId).some + // Headers chain is synced after this header. Start downloading full blocks + else if (!isHeadersChainSynced && isNewHeader(header)) { + isHeadersChainSyncedVar = true + blockDownloadProcessor.updateBestBlock(header) + none + } else none + + private def isNewHeader(header: Header): Boolean = + timeProvider.estimatedTime - header.timestamp < + settings.constants.DesiredBlockInterval.toMillis * settings.constants.NewHeaderTimeMultiplier + } + +} diff --git a/src/main/scala/encry/view/history/processors/HistoryHeaderProcessorComponent.scala b/src/main/scala/encry/view/history/processors/HistoryHeaderProcessorComponent.scala new file mode 100644 index 0000000000..239403b643 --- /dev/null +++ b/src/main/scala/encry/view/history/processors/HistoryHeaderProcessorComponent.scala @@ -0,0 +1,13 @@ +package encry.view.history.processors + +import encry.consensus.HistoryConsensus.ProgressInfo +import org.encryfoundation.common.modifiers.history.Header + +trait HistoryHeaderProcessorComponent { + + val headerProcessor: HeaderProcessor + + trait HeaderProcessor { + def processHeader(h: Header): ProgressInfo + } +} diff --git a/src/main/scala/encry/view/history/processors/HistoryPayloadProcessorComponent.scala b/src/main/scala/encry/view/history/processors/HistoryPayloadProcessorComponent.scala new file mode 100644 index 0000000000..3ee41735c6 --- /dev/null +++ b/src/main/scala/encry/view/history/processors/HistoryPayloadProcessorComponent.scala @@ -0,0 +1,17 @@ +package encry.view.history.processors + +import encry.consensus.HistoryConsensus.ProgressInfo +import org.encryfoundation.common.modifiers.history.Payload +import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import scala.collection.immutable.HashSet + +trait HistoryPayloadProcessorComponent { + + val payloadProcessor: PayloadProcessor + + def payloadsIdsToDownload(howMany: Int, excluding: HashSet[ModifierId]): Seq[ModifierId] + + trait PayloadProcessor { + def processPayload(payload: Payload): ProgressInfo + } +} diff --git a/src/main/scala/encry/view/history/processors/PayloadFastSyncProcessorComponent.scala b/src/main/scala/encry/view/history/processors/PayloadFastSyncProcessorComponent.scala new file mode 100644 index 0000000000..90e2373f76 --- /dev/null +++ b/src/main/scala/encry/view/history/processors/PayloadFastSyncProcessorComponent.scala @@ -0,0 +1,70 @@ +package encry.view.history.processors + +import cats.syntax.option.none +import encry.consensus.HistoryConsensus +import encry.consensus.HistoryConsensus.ProgressInfo +import encry.storage.VersionalStorage.{ StorageKey, StorageValue, StorageVersion } +import encry.view.history.HistoryApi +import org.encryfoundation.common.modifiers.history.Payload +import org.encryfoundation.common.utils.TaggedTypes.ModifierId + +import scala.annotation.tailrec +import scala.collection.immutable.HashSet + +trait PayloadFastSyncProcessorComponent extends HistoryPayloadProcessorComponent { + this: HistoryApi => + + override val payloadProcessor = new PayloadFastSyncProcessor + + def payloadsIdsToDownload(howMany: Int, excluding: HashSet[ModifierId]): Seq[ModifierId] = { + @tailrec def continuation(height: Int, acc: Seq[ModifierId]): Seq[ModifierId] = + if (acc.lengthCompare(howMany) >= 0) acc + else if (height > lastAvailableManifestHeight && fastSyncInProgress.fastSyncVal) acc + else + getBestHeaderIdAtHeight(height).flatMap(getHeaderById) match { + case Some(h) if !excluding.exists(_.sameElements(h.payloadId)) && !isBlockDefined(h) => + continuation(height + 1, acc :+ h.payloadId) + case Some(_) => + continuation(height + 1, acc) + case None => + acc + } + + (for { + bestBlockId <- getBestBlockId + headerLinkedToBestBlock <- getHeaderById(bestBlockId) + } yield headerLinkedToBestBlock) match { + case _ if !isHeadersChainSynced => + Seq.empty + case Some(header) if isInBestChain(header) => + continuation(header.height + 1, Seq.empty) + case Some(header) => + lastBestBlockHeightRelevantToBestChain(header.height) + .map(height => continuation(height + 1, Seq.empty)) + .getOrElse(continuation(blockDownloadProcessor.minimalBlockHeightVar, Seq.empty)) + case None => + continuation(blockDownloadProcessor.minimalBlockHeightVar, Seq.empty) + } + } + + class PayloadFastSyncProcessor extends PayloadProcessor { + override def processPayload(payload: Payload): HistoryConsensus.ProgressInfo = { + val startTime: Long = System.currentTimeMillis() + getBlockByPayload(payload).foreach { block => + logger.info(s"processPayloadFastSync") + historyStorage.bulkInsert(payload.id, Seq(BestBlockKey -> payload.headerId), Seq(payload)) + blockDownloadProcessor.updateBestBlock(block.header) + logger.info(s"BlockDownloadProcessor updated block at height ${block.header.height} successfully") + historyStorage.insert( + StorageVersion @@ validityKey(block.payload.id).untag(StorageKey), + List(block.header.id, block.payload.id).map(id => validityKey(id) -> StorageValue @@ Array(1.toByte)) + ) + logger.info( + s"Finished processing block ${block.encodedId}. " + + s"Processing time is ${(System.currentTimeMillis() - startTime) / 1000} s" + ) + } + ProgressInfo(none, Seq.empty, Seq.empty, none) + } + } +} diff --git a/src/main/scala/encry/view/history/processors/PayloadNormalProcessorComponent.scala b/src/main/scala/encry/view/history/processors/PayloadNormalProcessorComponent.scala new file mode 100644 index 0000000000..2d98e8784d --- /dev/null +++ b/src/main/scala/encry/view/history/processors/PayloadNormalProcessorComponent.scala @@ -0,0 +1,179 @@ +package encry.view.history.processors + +import cats.syntax.option._ +import cats.syntax.either._ +import encry.consensus.HistoryConsensus +import encry.consensus.HistoryConsensus.ProgressInfo +import encry.modifiers.history.HeaderChain +import encry.view.history.HistoryApi +import org.encryfoundation.common.modifiers.PersistentModifier +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.utils.TaggedTypes.{ Height, ModifierId } + +import scala.annotation.tailrec +import scala.collection.immutable.HashSet + +trait PayloadNormalProcessorComponent extends HistoryPayloadProcessorComponent { + this: HistoryApi => + + override val payloadProcessor: PayloadProcessor = new PayloadNormalProcessor + + def payloadsIdsToDownload(howMany: Int, excluding: HashSet[ModifierId]): Seq[ModifierId] = { + @tailrec def continuation(height: Int, acc: Seq[ModifierId]): Seq[ModifierId] = + if (acc.lengthCompare(howMany) >= 0) acc + else + getBestHeaderIdAtHeight(height).flatMap(getHeaderById) match { + case Some(h) if !excluding.exists(_.sameElements(h.payloadId)) && !isBlockDefined(h) => + continuation(height + 1, acc :+ h.payloadId) + case Some(_) => + continuation(height + 1, acc) + case None => + acc + } + + (for { + bestBlockId <- getBestBlockId + headerLinkedToBestBlock <- getHeaderById(bestBlockId) + } yield headerLinkedToBestBlock) match { + case _ if !isHeadersChainSynced => + Seq.empty + case Some(header) if isInBestChain(header) => + continuation(header.height + 1, Seq.empty) + case Some(header) => + lastBestBlockHeightRelevantToBestChain(header.height) + .map(height => continuation(height + 1, Seq.empty)) + .getOrElse(continuation(blockDownloadProcessor.minimalBlockHeightVar, Seq.empty)) + case None => + continuation(blockDownloadProcessor.minimalBlockHeightVar, Seq.empty) + } + } + + class PayloadNormalProcessor extends PayloadProcessor { + override def processPayload(payload: Payload): HistoryConsensus.ProgressInfo = + getBlockByPayload(payload).flatMap { block => + logger.info(s"proc block ${block.header.encodedId}!") + processBlock(block).some + }.getOrElse(putToHistory(payload)) + + private def processBlock(blockToProcess: Block): ProgressInfo = { + logger.info( + s"Starting processing block to history ||${blockToProcess.encodedId}||${blockToProcess.header.height}||" + ) + val bestFullChain: Seq[Block] = calculateBestFullChain(blockToProcess) + addBlockToCacheIfNecessary(blockToProcess) + bestFullChain.lastOption.map(_.header) match { + case Some(header) if isValidFirstBlock(blockToProcess.header) => + processValidFirstBlock(blockToProcess, header, bestFullChain) + case Some(header) if isBestBlockDefined && isBetterChain(header.id) => + processBetterChain(blockToProcess, header, Seq.empty, settings.node.blocksToKeep) + case Some(_) => + nonBestBlock(blockToProcess) + case None => + logger.debug(s"Best full chain is empty. Returning empty progress info") + ProgressInfo(none, Seq.empty, Seq.empty, none) + } + } + + private def processValidFirstBlock(fullBlock: Block, + newBestHeader: Header, + newBestChain: Seq[Block]): ProgressInfo = { + logger.info(s"Appending ${fullBlock.encodedId} as a valid first block with height ${fullBlock.header.height}") + updateStorage(fullBlock.payload, newBestHeader.id) + ProgressInfo(none, Seq.empty, newBestChain, none) + } + + private def processBetterChain(fullBlock: Block, + newBestHeader: Header, + newBestChain: Seq[Block], + blocksToKeep: Int): ProgressInfo = + getHeaderOfBestBlock.map { header => + val (prevChain: HeaderChain, newChain: HeaderChain) = commonBlockThenSuffixes(header, newBestHeader) + val toRemove: Seq[Block] = prevChain.tail.headers + .flatMap(getBlockByHeader) + val toApply: Seq[Block] = newChain.tail.headers + .flatMap(h => if (h == fullBlock.header) fullBlock.some else getBlockByHeader(h)) + toApply.foreach(addBlockToCacheIfNecessary) + if (toApply.lengthCompare(newChain.length - 1) != 0) nonBestBlock(fullBlock) + else { + //application of this block leads to full chain with higher score + logger.info(s"Appending ${fullBlock.encodedId}|${fullBlock.header.height} as a better chain") + val branchPoint: Option[ModifierId] = toRemove.headOption.map(_ => prevChain.head.id) + val bestHeaderHeight: Int = getBestHeaderHeight + val updateBestHeader: Boolean = + (fullBlock.header.height > bestHeaderHeight) || ( + (fullBlock.header.height == bestHeaderHeight) && + scoreOf(fullBlock.id) + .flatMap(fbScore => getBestHeaderId.flatMap(id => scoreOf(id).map(_ < fbScore))) + .getOrElse(false) + ) + val updatedHeadersAtHeightIds = + newChain.headers.map(header => updatedBestHeaderAtHeightRaw(header.id, Height @@ header.height)).toList + updateStorage(fullBlock.payload, newBestHeader.id, updateBestHeader, updatedHeadersAtHeightIds) + if (blocksToKeep >= 0) { + val lastKept: Int = blockDownloadProcessor.updateBestBlock(fullBlock.header) + val bestHeight: Int = toApply.lastOption.map(_.header.height).getOrElse(0) + val diff: Int = bestHeight - header.height + clipBlockDataAt(((lastKept - diff) until lastKept).filter(_ >= 0)) + } + ProgressInfo(branchPoint, toRemove, toApply, none) + } + }.getOrElse(ProgressInfo(none, Seq.empty, Seq.empty, none)) + + private def nonBestBlock(fullBlock: Block): ProgressInfo = { + //Orphaned block or full chain is not initialized yet + logger.info(s"Process block to history ${fullBlock.encodedId}||${fullBlock.header.height}||") + historyStorage.bulkInsert(fullBlock.payload.id, Seq.empty, Seq(fullBlock.payload)) + ProgressInfo(none, Seq.empty, Seq.empty, none) + } + + private def updatedBestHeaderAtHeightRaw(headerId: ModifierId, height: Height): (Array[Byte], Array[Byte]) = + heightIdsKey(height) -> + (Seq(headerId) ++ + headerIdsAtHeight(height).filterNot(_ sameElements headerId)).flatten.toArray + + private def putToHistory(payload: Payload): ProgressInfo = { + historyStorage.insertObjects(Seq(payload)) + ProgressInfo(none, Seq.empty, Seq.empty, none) + } + + private def isBetterChain(id: ModifierId): Boolean = + (for { + bestFullBlockId <- getBestBlockId + heightOfThisHeader <- getHeightByHeaderId(id) + prevBestScore <- scoreOf(bestFullBlockId) + score <- scoreOf(id) + bestBlockHeight = getBestBlockHeight + } yield + (bestBlockHeight < heightOfThisHeader) || (bestBlockHeight == heightOfThisHeader && score > prevBestScore)) + .getOrElse(false) + + private def calculateBestFullChain(block: Block): Seq[Block] = { + val continuations: Seq[Seq[Header]] = continuationHeaderChains(block.header, h => isBlockDefined(h)).map(_.tail) + logger.debug(s"continuations: ${continuations.map(seq => s"Seq contains: ${seq.length}").mkString(",")}") + val chains: Seq[Seq[Block]] = continuations.map(_.filter(isBlockDefined).flatMap(getBlockByHeader)) + logger.debug(s"Chains: ${chains.map(chain => s"chain contain: ${chain.length}").mkString(",")}") + chains.map(c => block +: c).maxBy(c => scoreOf(c.last.id).get) + } + + private def clipBlockDataAt(heights: Seq[Int]): Either[Throwable, Unit] = Either.catchNonFatal { + val toRemove: Seq[ModifierId] = heights + .flatMap(headerIdsAtHeight) + .flatMap(getHeaderById) + .map(_.payloadId) + historyStorage.removeObjects(toRemove) + } + + private def updateStorage(newModRow: PersistentModifier, + bestFullHeaderId: ModifierId, + updateHeaderInfo: Boolean = false, + additionalIndexes: List[(Array[Byte], Array[Byte])] = List.empty): Unit = { + val indicesToInsert: Seq[(Array[Byte], Array[Byte])] = + if (updateHeaderInfo) Seq(BestBlockKey -> bestFullHeaderId, BestHeaderKey -> bestFullHeaderId) + else Seq(BestBlockKey -> bestFullHeaderId) + historyStorage.bulkInsert(newModRow.id, indicesToInsert ++ additionalIndexes, Seq(newModRow)) + } + + private def isValidFirstBlock(header: Header): Boolean = + header.height == blockDownloadProcessor.minimalBlockHeight && getBestBlockId.isEmpty + } +} diff --git a/src/test/scala/encry/modifiers/InstanceFactory.scala b/src/test/scala/encry/modifiers/InstanceFactory.scala index 1636d72238..ea044f3fad 100755 --- a/src/test/scala/encry/modifiers/InstanceFactory.scala +++ b/src/test/scala/encry/modifiers/InstanceFactory.scala @@ -6,7 +6,8 @@ import encry.modifiers.state.Keys import encry.settings.{EncryAppSettings, NodeSettings} import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} import encry.utils.{EncryGenerator, FileHelper, NetworkTimeProvider, TestHelper} -import encry.view.history.{History, HistoryHeadersProcessor, HistoryPayloadsProcessor} +import encry.view.history.processors.{HeaderDefaultProcessorComponent, PayloadNormalProcessorComponent} +import encry.view.history.History import encry.view.history.storage.HistoryStorage import io.iohk.iodb.LSMStore import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} @@ -229,7 +230,7 @@ trait InstanceFactory extends Keys with EncryGenerator { val ntp: NetworkTimeProvider = new NetworkTimeProvider(settingsEncry.ntp) - new History with HistoryHeadersProcessor with HistoryPayloadsProcessor { + new History with HeaderDefaultProcessorComponent with PayloadNormalProcessorComponent { override val settings: EncryAppSettings = settingsEncry override var isFullChainSynced = settings.node.offlineGeneration override val historyStorage: HistoryStorage = storage From 1ae8c3cb1e0986efdd1e478608587bc0e908165c Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 21 Jan 2020 16:13:36 +0300 Subject: [PATCH 07/10] renamed processors --- src/main/scala/encry/view/NodeViewHolder.scala | 4 ++-- src/main/scala/encry/view/history/History.scala | 4 ++-- ...omponent.scala => HeaderFullChainProcessorComponent.scala} | 2 +- ...mponent.scala => PayloadFullChainProcessorComponent.scala} | 2 +- src/test/scala/encry/modifiers/InstanceFactory.scala | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) rename src/main/scala/encry/view/history/processors/{HeaderDefaultProcessorComponent.scala => HeaderFullChainProcessorComponent.scala} (98%) rename src/main/scala/encry/view/history/processors/{PayloadNormalProcessorComponent.scala => PayloadFullChainProcessorComponent.scala} (99%) diff --git a/src/main/scala/encry/view/NodeViewHolder.scala b/src/main/scala/encry/view/NodeViewHolder.scala index 31431e1466..be9b5133e9 100644 --- a/src/main/scala/encry/view/NodeViewHolder.scala +++ b/src/main/scala/encry/view/NodeViewHolder.scala @@ -23,7 +23,7 @@ import encry.view.NodeViewHolder.ReceivableMessages._ import encry.view.NodeViewHolder._ import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId import encry.view.fast.sync.SnapshotHolder._ -import encry.view.history.processors.{HeaderDefaultProcessorComponent, PayloadNormalProcessorComponent} +import encry.view.history.processors.{HeaderFullChainProcessorComponent, PayloadFullChainProcessorComponent} import encry.view.history.storage.HistoryStorage import encry.view.history.{History, HistoryApi} import encry.view.mempool.MemoryPool.RolledBackTransactions @@ -97,7 +97,7 @@ class NodeViewHolder(memoryPoolRef: ActorRef, FileUtils.deleteDirectory(new File(s"${encrySettings.directory}/keysTmp")) FileUtils.deleteDirectory(new File(s"${encrySettings.directory}/walletTmp")) logger.info(s"Updated best block in fast sync mod. Updated state height.") - val newHistory = new History with HeaderDefaultProcessorComponent with PayloadNormalProcessorComponent { + val newHistory = new History with HeaderFullChainProcessorComponent with PayloadFullChainProcessorComponent { override val settings: EncryAppSettings = encrySettings override var isFullChainSynced: Boolean = settings.node.offlineGeneration override val timeProvider: NetworkTimeProvider = EncryApp.timeProvider diff --git a/src/main/scala/encry/view/history/History.scala b/src/main/scala/encry/view/history/History.scala index 830259434d..148ea51797 100644 --- a/src/main/scala/encry/view/history/History.scala +++ b/src/main/scala/encry/view/history/History.scala @@ -173,14 +173,14 @@ object History extends StrictLogging { VLDBWrapper(VersionalLevelDBCompanion(levelDBInit, settingsEncry.levelDB)) } if (settingsEncry.snapshotSettings.enableFastSynchronization && !settingsEncry.node.offlineGeneration) - new History with HeaderDefaultProcessorComponent with PayloadFastSyncProcessorComponent { + new History with HeaderFullChainProcessorComponent with PayloadFastSyncProcessorComponent { override val settings: EncryAppSettings = settingsEncry override var isFullChainSynced: Boolean = settings.node.offlineGeneration override val historyStorage: HistoryStorage = HistoryStorage(vldbInit) override val timeProvider: NetworkTimeProvider = new NetworkTimeProvider(settingsEncry.ntp) } else - new History with HeaderDefaultProcessorComponent with PayloadNormalProcessorComponent { + new History with HeaderFullChainProcessorComponent with PayloadFullChainProcessorComponent { override val settings: EncryAppSettings = settingsEncry override var isFullChainSynced: Boolean = settings.node.offlineGeneration override val historyStorage: HistoryStorage = HistoryStorage(vldbInit) diff --git a/src/main/scala/encry/view/history/processors/HeaderDefaultProcessorComponent.scala b/src/main/scala/encry/view/history/processors/HeaderFullChainProcessorComponent.scala similarity index 98% rename from src/main/scala/encry/view/history/processors/HeaderDefaultProcessorComponent.scala rename to src/main/scala/encry/view/history/processors/HeaderFullChainProcessorComponent.scala index c2cef75f12..e6752d21ea 100644 --- a/src/main/scala/encry/view/history/processors/HeaderDefaultProcessorComponent.scala +++ b/src/main/scala/encry/view/history/processors/HeaderFullChainProcessorComponent.scala @@ -10,7 +10,7 @@ import encry.view.history.HistoryApi import org.encryfoundation.common.modifiers.history.{Header, Payload} import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, ModifierId, ModifierTypeId} -trait HeaderDefaultProcessorComponent extends HistoryHeaderProcessorComponent { +trait HeaderFullChainProcessorComponent extends HistoryHeaderProcessorComponent { this: HistoryApi => override val headerProcessor: HeaderProcessor = new HeaderDefaultProcessor diff --git a/src/main/scala/encry/view/history/processors/PayloadNormalProcessorComponent.scala b/src/main/scala/encry/view/history/processors/PayloadFullChainProcessorComponent.scala similarity index 99% rename from src/main/scala/encry/view/history/processors/PayloadNormalProcessorComponent.scala rename to src/main/scala/encry/view/history/processors/PayloadFullChainProcessorComponent.scala index 2d98e8784d..ef7030740a 100644 --- a/src/main/scala/encry/view/history/processors/PayloadNormalProcessorComponent.scala +++ b/src/main/scala/encry/view/history/processors/PayloadFullChainProcessorComponent.scala @@ -13,7 +13,7 @@ import org.encryfoundation.common.utils.TaggedTypes.{ Height, ModifierId } import scala.annotation.tailrec import scala.collection.immutable.HashSet -trait PayloadNormalProcessorComponent extends HistoryPayloadProcessorComponent { +trait PayloadFullChainProcessorComponent extends HistoryPayloadProcessorComponent { this: HistoryApi => override val payloadProcessor: PayloadProcessor = new PayloadNormalProcessor diff --git a/src/test/scala/encry/modifiers/InstanceFactory.scala b/src/test/scala/encry/modifiers/InstanceFactory.scala index ea044f3fad..29d7a2f230 100755 --- a/src/test/scala/encry/modifiers/InstanceFactory.scala +++ b/src/test/scala/encry/modifiers/InstanceFactory.scala @@ -6,7 +6,7 @@ import encry.modifiers.state.Keys import encry.settings.{EncryAppSettings, NodeSettings} import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} import encry.utils.{EncryGenerator, FileHelper, NetworkTimeProvider, TestHelper} -import encry.view.history.processors.{HeaderDefaultProcessorComponent, PayloadNormalProcessorComponent} +import encry.view.history.processors.{HeaderFullChainProcessorComponent, PayloadFullChainProcessorComponent} import encry.view.history.History import encry.view.history.storage.HistoryStorage import io.iohk.iodb.LSMStore @@ -230,7 +230,7 @@ trait InstanceFactory extends Keys with EncryGenerator { val ntp: NetworkTimeProvider = new NetworkTimeProvider(settingsEncry.ntp) - new History with HeaderDefaultProcessorComponent with PayloadNormalProcessorComponent { + new History with HeaderFullChainProcessorComponent with PayloadFullChainProcessorComponent { override val settings: EncryAppSettings = settingsEncry override var isFullChainSynced = settings.node.offlineGeneration override val historyStorage: HistoryStorage = storage From 4080efa1e5ee5dfe2e18e19c804b53ee8ce3a155 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 21 Jan 2020 16:14:10 +0300 Subject: [PATCH 08/10] renamed processors --- .../history/processors/PayloadFullChainProcessorComponent.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/scala/encry/view/history/processors/PayloadFullChainProcessorComponent.scala b/src/main/scala/encry/view/history/processors/PayloadFullChainProcessorComponent.scala index ef7030740a..306d69bc0c 100644 --- a/src/main/scala/encry/view/history/processors/PayloadFullChainProcessorComponent.scala +++ b/src/main/scala/encry/view/history/processors/PayloadFullChainProcessorComponent.scala @@ -9,7 +9,6 @@ import encry.view.history.HistoryApi import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.utils.TaggedTypes.{ Height, ModifierId } - import scala.annotation.tailrec import scala.collection.immutable.HashSet From cc2b4a6d79b1e3ba795a070e510f1a5bf0cd63b3 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 21 Jan 2020 16:24:01 +0300 Subject: [PATCH 09/10] minor fix --- .../scala/encry/view/history/HistoryModifiersValidator.scala | 3 +-- .../history/processors/HeaderFullChainProcessorComponent.scala | 2 -- .../history/processors/HistoryHeaderProcessorComponent.scala | 3 ++- .../history/processors/HistoryPayloadProcessorComponent.scala | 3 ++- .../history/processors/PayloadFastSyncProcessorComponent.scala | 2 -- .../processors/PayloadFullChainProcessorComponent.scala | 2 -- 6 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala index 8234e8f841..2e1a76182f 100644 --- a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala +++ b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala @@ -9,8 +9,7 @@ import org.encryfoundation.common.modifiers.history.{Header, Payload} import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, ModifierId} import org.encryfoundation.common.validation.ModifierSemanticValidity -trait HistoryModifiersValidator { - historyApi: HistoryApi => +trait HistoryModifiersValidator extends HistoryApi { private lazy val powScheme: EquihashPowScheme = EquihashPowScheme(settings.constants.n, settings.constants.k, settings.constants.Version, settings.constants.PreGenesisHeight, settings.constants.MaxTarget) diff --git a/src/main/scala/encry/view/history/processors/HeaderFullChainProcessorComponent.scala b/src/main/scala/encry/view/history/processors/HeaderFullChainProcessorComponent.scala index e6752d21ea..debdf0b18e 100644 --- a/src/main/scala/encry/view/history/processors/HeaderFullChainProcessorComponent.scala +++ b/src/main/scala/encry/view/history/processors/HeaderFullChainProcessorComponent.scala @@ -6,12 +6,10 @@ import encry.EncryApp.forceStopApplication import encry.consensus.HistoryConsensus.ProgressInfo import encry.consensus.{ConsensusSchemeReaders, HistoryConsensus} import encry.storage.VersionalStorage.{StorageKey, StorageValue} -import encry.view.history.HistoryApi import org.encryfoundation.common.modifiers.history.{Header, Payload} import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, ModifierId, ModifierTypeId} trait HeaderFullChainProcessorComponent extends HistoryHeaderProcessorComponent { - this: HistoryApi => override val headerProcessor: HeaderProcessor = new HeaderDefaultProcessor diff --git a/src/main/scala/encry/view/history/processors/HistoryHeaderProcessorComponent.scala b/src/main/scala/encry/view/history/processors/HistoryHeaderProcessorComponent.scala index 239403b643..a6bd911d48 100644 --- a/src/main/scala/encry/view/history/processors/HistoryHeaderProcessorComponent.scala +++ b/src/main/scala/encry/view/history/processors/HistoryHeaderProcessorComponent.scala @@ -1,9 +1,10 @@ package encry.view.history.processors import encry.consensus.HistoryConsensus.ProgressInfo +import encry.view.history.HistoryApi import org.encryfoundation.common.modifiers.history.Header -trait HistoryHeaderProcessorComponent { +trait HistoryHeaderProcessorComponent extends HistoryApi { val headerProcessor: HeaderProcessor diff --git a/src/main/scala/encry/view/history/processors/HistoryPayloadProcessorComponent.scala b/src/main/scala/encry/view/history/processors/HistoryPayloadProcessorComponent.scala index 3ee41735c6..2e73ef7360 100644 --- a/src/main/scala/encry/view/history/processors/HistoryPayloadProcessorComponent.scala +++ b/src/main/scala/encry/view/history/processors/HistoryPayloadProcessorComponent.scala @@ -1,11 +1,12 @@ package encry.view.history.processors import encry.consensus.HistoryConsensus.ProgressInfo +import encry.view.history.HistoryApi import org.encryfoundation.common.modifiers.history.Payload import org.encryfoundation.common.utils.TaggedTypes.ModifierId import scala.collection.immutable.HashSet -trait HistoryPayloadProcessorComponent { +trait HistoryPayloadProcessorComponent extends HistoryApi { val payloadProcessor: PayloadProcessor diff --git a/src/main/scala/encry/view/history/processors/PayloadFastSyncProcessorComponent.scala b/src/main/scala/encry/view/history/processors/PayloadFastSyncProcessorComponent.scala index 90e2373f76..e60eaebca4 100644 --- a/src/main/scala/encry/view/history/processors/PayloadFastSyncProcessorComponent.scala +++ b/src/main/scala/encry/view/history/processors/PayloadFastSyncProcessorComponent.scala @@ -7,12 +7,10 @@ import encry.storage.VersionalStorage.{ StorageKey, StorageValue, StorageVersion import encry.view.history.HistoryApi import org.encryfoundation.common.modifiers.history.Payload import org.encryfoundation.common.utils.TaggedTypes.ModifierId - import scala.annotation.tailrec import scala.collection.immutable.HashSet trait PayloadFastSyncProcessorComponent extends HistoryPayloadProcessorComponent { - this: HistoryApi => override val payloadProcessor = new PayloadFastSyncProcessor diff --git a/src/main/scala/encry/view/history/processors/PayloadFullChainProcessorComponent.scala b/src/main/scala/encry/view/history/processors/PayloadFullChainProcessorComponent.scala index 306d69bc0c..c63ea48103 100644 --- a/src/main/scala/encry/view/history/processors/PayloadFullChainProcessorComponent.scala +++ b/src/main/scala/encry/view/history/processors/PayloadFullChainProcessorComponent.scala @@ -5,7 +5,6 @@ import cats.syntax.either._ import encry.consensus.HistoryConsensus import encry.consensus.HistoryConsensus.ProgressInfo import encry.modifiers.history.HeaderChain -import encry.view.history.HistoryApi import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.utils.TaggedTypes.{ Height, ModifierId } @@ -13,7 +12,6 @@ import scala.annotation.tailrec import scala.collection.immutable.HashSet trait PayloadFullChainProcessorComponent extends HistoryPayloadProcessorComponent { - this: HistoryApi => override val payloadProcessor: PayloadProcessor = new PayloadNormalProcessor From 4f3251bc3be2276a5b403afbd812b3ba886dd73a Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 21 Jan 2020 17:29:06 +0300 Subject: [PATCH 10/10] minor fix --- .../processors/HeaderFullChainProcessorComponent.scala | 10 +++++----- .../processors/HistoryPayloadProcessorComponent.scala | 4 +++- .../processors/PayloadFastSyncProcessorComponent.scala | 1 - 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/main/scala/encry/view/history/processors/HeaderFullChainProcessorComponent.scala b/src/main/scala/encry/view/history/processors/HeaderFullChainProcessorComponent.scala index debdf0b18e..3fe4842998 100644 --- a/src/main/scala/encry/view/history/processors/HeaderFullChainProcessorComponent.scala +++ b/src/main/scala/encry/view/history/processors/HeaderFullChainProcessorComponent.scala @@ -4,10 +4,10 @@ import cats.syntax.option._ import com.google.common.primitives.Ints import encry.EncryApp.forceStopApplication import encry.consensus.HistoryConsensus.ProgressInfo -import encry.consensus.{ConsensusSchemeReaders, HistoryConsensus} -import encry.storage.VersionalStorage.{StorageKey, StorageValue} -import org.encryfoundation.common.modifiers.history.{Header, Payload} -import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, ModifierId, ModifierTypeId} +import encry.consensus.{ ConsensusSchemeReaders, HistoryConsensus } +import encry.storage.VersionalStorage.{ StorageKey, StorageValue } +import org.encryfoundation.common.modifiers.history.{ Header, Payload } +import org.encryfoundation.common.utils.TaggedTypes.{ Difficulty, ModifierId, ModifierTypeId } trait HeaderFullChainProcessorComponent extends HistoryHeaderProcessorComponent { @@ -82,7 +82,7 @@ trait HeaderFullChainProcessorComponent extends HistoryHeaderProcessorComponent } private def toDownload(header: Header): Option[(ModifierTypeId, ModifierId)] = - // Already synced and header is not too far back. Download required modifiers + // Already synced and header is not too far back. Download required modifiers if (header.height >= blockDownloadProcessor.minimalBlockHeight) (Payload.modifierTypeId -> header.payloadId).some // Headers chain is synced after this header. Start downloading full blocks else if (!isHeadersChainSynced && isNewHeader(header)) { diff --git a/src/main/scala/encry/view/history/processors/HistoryPayloadProcessorComponent.scala b/src/main/scala/encry/view/history/processors/HistoryPayloadProcessorComponent.scala index 2e73ef7360..a3db102fc0 100644 --- a/src/main/scala/encry/view/history/processors/HistoryPayloadProcessorComponent.scala +++ b/src/main/scala/encry/view/history/processors/HistoryPayloadProcessorComponent.scala @@ -6,7 +6,9 @@ import org.encryfoundation.common.modifiers.history.Payload import org.encryfoundation.common.utils.TaggedTypes.ModifierId import scala.collection.immutable.HashSet -trait HistoryPayloadProcessorComponent extends HistoryApi { +trait HistoryPayloadProcessorComponent + extends HistoryApi +{ val payloadProcessor: PayloadProcessor diff --git a/src/main/scala/encry/view/history/processors/PayloadFastSyncProcessorComponent.scala b/src/main/scala/encry/view/history/processors/PayloadFastSyncProcessorComponent.scala index e60eaebca4..bf38c6ff69 100644 --- a/src/main/scala/encry/view/history/processors/PayloadFastSyncProcessorComponent.scala +++ b/src/main/scala/encry/view/history/processors/PayloadFastSyncProcessorComponent.scala @@ -4,7 +4,6 @@ import cats.syntax.option.none import encry.consensus.HistoryConsensus import encry.consensus.HistoryConsensus.ProgressInfo import encry.storage.VersionalStorage.{ StorageKey, StorageValue, StorageVersion } -import encry.view.history.HistoryApi import org.encryfoundation.common.modifiers.history.Payload import org.encryfoundation.common.utils.TaggedTypes.ModifierId import scala.annotation.tailrec