diff --git a/do-release.sh b/do-release.sh index 2010f7a372..c930ca1eab 100755 --- a/do-release.sh +++ b/do-release.sh @@ -36,30 +36,26 @@ if [[ $conf_version != $jar_version ]]; then exit 1 fi -openapi_files=("openapi.yaml" "openapi-ai.yaml") +openapi_path="src/main/resources/api/openapi.yaml" +version_line=$(grep 'version:' "$openapi_path") -for file in "${openapi_files[@]}"; do - openapi_path="src/main/resources/api/$file" - version_line=$(grep 'version:' "$openapi_path") - - if [[ -z $version_line ]]; then - echo "Error: Version line not found in $openapi_path" - exit 1 - fi +if [[ -z $version_line ]]; then + echo "Error: Version line not found in $openapi_path" + exit 1 +fi - actual_version=$(echo $version_line | awk -F '"' '{print $2}') +actual_version=$(echo $version_line | awk -F '"' '{print $2}') - if [[ -z $actual_version ]]; then - echo "Error: Version not found in $openapi_path" - exit 1 - fi +if [[ -z $actual_version ]]; then + echo "Error: Version not found in $openapi_path" + exit 1 +fi - if [[ $actual_version != $jar_version ]]; then - echo "Version mismatch in $openapi_path: ($actual_version) != jar ($jar_version)." - echo "Removing jar $jar" - rm "$jar" - exit 1 - fi -done +if [[ $actual_version != $jar_version ]]; then + echo "Version mismatch in $openapi_path: ($actual_version) != jar ($jar_version)." + echo "Removing jar $jar" + rm "$jar" + exit 1 +fi echo "do-release completed successfully" diff --git a/ergo-core/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala b/ergo-core/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala index d28f6f2418..8b7d4aa299 100644 --- a/ergo-core/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala +++ b/ergo-core/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala @@ -45,6 +45,14 @@ trait ApiCodecs extends JsonCodecs { fromTry(validationResult.toTry) } + private def fromTryAt[T](value: Try[T], cursor: ACursor): Decoder.Result[T] = value match { + case Success(result) => Right(result) + case Failure(e) => Left(DecodingFailure.fromThrowable(e, cursor.history)) + } + + private def decodeBase16(value: String, cursor: ACursor): Decoder.Result[Array[Byte]] = + fromTryAt(Base16.decode(value), cursor) + implicit val leafDataEncoder: Encoder[LeafData] = Encoder.instance(xs => Base16.encode(xs).asJson) implicit val digestEncoder: Encoder[Digest] = Encoder.instance(x => Base16.encode(x).asJson) @@ -338,11 +346,14 @@ trait ApiCodecs extends JsonCodecs { pubkey <- c.downField("pubkey").as[SigmaLeaf] proof <- c.downField("proof").as[String] position <- c.downField("position").as[NodePosition] + challengeBytes <- decodeBase16(challenge, c.downField("challenge")) + proofBytes <- decodeBase16(proof, c.downField("proof")) + proofTree <- fromTryAt(Try(SigSerializer.parseAndComputeChallenges(pubkey, proofBytes)(null)), c.downField("proof")) } yield RealSecretProof( pubkey, - Challenge @@ Base16.decode(challenge).get.toColl, - SigSerializer.parseAndComputeChallenges(pubkey, Base16.decode(proof).get)(null), + Challenge @@ challengeBytes.toColl, + proofTree, position ) case h: String if h == "proofSimulated" => @@ -351,11 +362,14 @@ trait ApiCodecs extends JsonCodecs { pubkey <- c.downField("pubkey").as[SigmaLeaf] proof <- c.downField("proof").as[String] position <- c.downField("position").as[NodePosition] + challengeBytes <- decodeBase16(challenge, c.downField("challenge")) + proofBytes <- decodeBase16(proof, c.downField("proof")) + proofTree <- fromTryAt(Try(SigSerializer.parseAndComputeChallenges(pubkey, proofBytes)(null)), c.downField("proof")) } yield SimulatedSecretProof( pubkey, - Challenge @@ Base16.decode(challenge).get.toColl, - SigSerializer.parseAndComputeChallenges(pubkey, Base16.decode(proof).get)(null), + Challenge @@ challengeBytes.toColl, + proofTree, position ) case _ => diff --git a/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/NipopowAlgos.scala b/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/NipopowAlgos.scala index a441cfe9ff..0d76a9815e 100644 --- a/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/NipopowAlgos.scala +++ b/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/NipopowAlgos.scala @@ -75,6 +75,8 @@ class NipopowAlgos(val chainSettings: ChainSettings) { Int.MaxValue } + def hasValidPow(header: Header): Boolean = powScheme.validate(header).isSuccess + /** * Computes best score of a given chain. * The score value depends on number of ยต-superblocks in the given chain. @@ -96,6 +98,8 @@ class NipopowAlgos(val chainSettings: ChainSettings) { * end function */ def bestArg(chain: Seq[Header])(m: Int): Int = { + require(m >= 1, s"$m < 1") + @scala.annotation.tailrec def loop(level: Int, acc: Seq[(Int, Int)] = Seq.empty): Seq[(Int, Int)] = if (level == 0) { @@ -130,7 +134,6 @@ class NipopowAlgos(val chainSettings: ChainSettings) { val k = params.k val m = params.m - require(params.k >= 1, s"$k < 1") require(chain.lengthCompare(k + m) >= 0, s"Can not prove chain of size < ${k + m}") require(chain.head.header.isGenesis, "Can not prove non-anchored chain") diff --git a/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/NipopowProof.scala b/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/NipopowProof.scala index c963877ada..b922aca7c0 100644 --- a/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/NipopowProof.scala +++ b/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/NipopowProof.scala @@ -72,7 +72,12 @@ case class NipopowProof(popowAlgos: NipopowAlgos, * @return true if the proof is valid */ lazy val isValid: Boolean = { - this.hasValidConnections && this.hasValidHeights && this.hasValidProofs && this.hasValidDifficultyHeaders + PoPowParams.isValid(m, k) && + this.hasValidConnections && + this.hasValidHeights && + this.hasValidProofs && + this.hasValidDifficultyHeaders && + this.hasValidPow } /** @@ -155,6 +160,8 @@ case class NipopowProof(popowAlgos: NipopowAlgos, suffixHead.checkInterlinksProof() } + lazy val hasValidPow: Boolean = headersChain.forall(popowAlgos.hasValidPow) + } object NipopowProof { diff --git a/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/PoPowParams.scala b/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/PoPowParams.scala index 93aacc7339..659fe3c8e8 100644 --- a/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/PoPowParams.scala +++ b/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/PoPowParams.scala @@ -1,5 +1,7 @@ package org.ergoplatform.modifiers.history.popow +import scala.util.Try + /** * NiPoPoW proof params from the KMZ17 paper * @@ -12,5 +14,15 @@ package org.ergoplatform.modifiers.history.popow * to the block header) * */ -case class PoPowParams(m: Int, k: Int, continuous: Boolean) +final class PoPowParams private (val m: Int, val k: Int, val continuous: Boolean, val minChainLength: Int) + +object PoPowParams { + def isValid(m: Int, k: Int): Boolean = + m >= 1 && k >= 1 && m.toLong + k.toLong <= Int.MaxValue + + def apply(m: Int, k: Int, continuous: Boolean): Try[PoPowParams] = Try { + require(isValid(m, k), s"Invalid NiPoPoW parameters: m=$m, k=$k") + new PoPowParams(m, k, continuous, m + k) + } +} diff --git a/ergo-core/src/test/scala/org/ergoplatform/serialization/JsonSerializationCoreSpec.scala b/ergo-core/src/test/scala/org/ergoplatform/serialization/JsonSerializationCoreSpec.scala index 360cb1c52e..f6355d081a 100644 --- a/ergo-core/src/test/scala/org/ergoplatform/serialization/JsonSerializationCoreSpec.scala +++ b/ergo-core/src/test/scala/org/ergoplatform/serialization/JsonSerializationCoreSpec.scala @@ -1,7 +1,7 @@ package org.ergoplatform.serialization import io.circe.syntax._ -import io.circe.ACursor +import io.circe.{ACursor, Json} import org.ergoplatform.ErgoBox import org.ergoplatform.ErgoBox.{AdditionalRegisters, NonMandatoryRegisterId} import org.ergoplatform.http.api.ApiCodecs @@ -14,6 +14,9 @@ import org.ergoplatform.wallet.Constants.ScanId import org.ergoplatform.wallet.boxes.TrackedBox import cats.syntax.either._ import sigma.ast.{ErgoTree, EvaluatedValue, SType} +import sigmastate.interpreter.SecretProven + +import scala.util.Try class JsonSerializationCoreSpec extends ErgoCorePropertyTest with ApiCodecs { @@ -61,6 +64,26 @@ class JsonSerializationCoreSpec extends ErgoCorePropertyTest } } + property("secret proof decoder should reject invalid hex without throwing") { + val pubkey = proveDlogGen.sample.get + val pubkeyJson = Json.obj("op" -> pubkey.opCode.toByte.asJson, "h" -> pubkey.value.asJson) + + Seq("zz" -> "00", "00" -> "zz").foreach { case (challenge, proof) => + val json = Json.obj( + "hint" -> "proofReal".asJson, + "challenge" -> challenge.asJson, + "pubkey" -> pubkeyJson, + "proof" -> proof.asJson, + "position" -> "0".asJson + ) + + val decoded = Try(json.as[SecretProven]) + decoded.isSuccess shouldBe true + decoded.get.isLeft shouldBe true + decoded.get.left.get.message.toLowerCase should include ("hex") + } + } + private def checkTrackedBox(c: ACursor, b: TrackedBox)(implicit opts: Detalization) = { c.downField("spent").as[Boolean] shouldBe Right(b.spendingStatus.spent) c.downField("onchain").as[Boolean] shouldBe Right(b.chainStatus.onChain) diff --git a/src/it/scala/org/ergoplatform/it/OpenApiSpec.scala b/src/it/scala/org/ergoplatform/it/OpenApiSpec.scala index 1a3106217d..68653494c4 100644 --- a/src/it/scala/org/ergoplatform/it/OpenApiSpec.scala +++ b/src/it/scala/org/ergoplatform/it/OpenApiSpec.scala @@ -25,6 +25,9 @@ class OpenApiSpec extends AnyFlatSpec with IntegrationSuite { .withFallback(nodeSeedConfigs.head) .withFallback(allowLocalConfig) + // `lazy` so the container is only started when a test actually touches `node`. + // The single test below is currently `ignore`d (the openapi-checker image is gone), + // so without `lazy` we would start and tear down a node for nothing. lazy val node: Node = docker.startDevNetNode(offlineGeneratingPeer).get def renderTemplate(template: String, varMapping: Map[String, String]): String = diff --git a/src/main/resources/.well-known/ai-plugin.json b/src/main/resources/.well-known/ai-plugin.json deleted file mode 100644 index 5100ce83dd..0000000000 --- a/src/main/resources/.well-known/ai-plugin.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "schema_version": "v1", - "name_for_human": "Ergo Node Plugin (no auth)", - "name_for_model": "ergonode", - "description_for_human": "Plugin for interacting with Ergo node.", - "description_for_model": "Specification of Ergo Node API for ChatGPT plugin.\n The following endpoints supported \n - /blocks/chainSlice - Get headers in a specified range of heights\n - /info - Get the basic information about the status of Ergo Node. \n - /transactions/unconfirmed/byTransactionId - Get unconfirmed transaction from the mempool\n - /transactions/poolHistogram - Get histogram (waittime, (n_trans, sum(fee)) for transactions in mempool.\n - /peers/connected - Get a list of current connected peers\n - /peers/blacklisted - Get a list of blacklisted peers\n - /utils/address - Check address validity\n - /blockchain/indexedHeight - Get current indexed block height. (The indexer has processed all blocks up to this height.)\n - /blockchain/transaction/byId - Retrieve a transaction by its id\n - /blockchain/transaction/byAddress - Retrieve a list of transactions by their associated address\n - /blockchain/box/byId - Retrieve a box by its id\n - /blockchain/box/byAddress - Retrieve boxes by their associated \n - /blockchain/box/unspent/byAddress - Retrieve unspent boxes by their associated address\n - /blockchain/token/byId - Retrieve minting information about a token\n - /blockchain/balanceForAddress - Retrieve balance information of an Ergo address.", - "auth": { - "type": "none" - }, - "api": { - "type": "openapi", - "url": "http://localhost:9053/openapi.yaml", - "is_user_authenticated": false - }, - "logo_url": "https://cryptologos.cc/logos/ergo-erg-logo.png", - "contact_email": "team@ergoplatform.org", - "legal_info_url": "https://ergoplatform.org/en/legal/" -} diff --git a/src/main/resources/api/openapi-ai.yaml b/src/main/resources/api/openapi-ai.yaml deleted file mode 100644 index a92d54ed84..0000000000 --- a/src/main/resources/api/openapi-ai.yaml +++ /dev/null @@ -1,1489 +0,0 @@ -openapi: "3.0.2" - -info: - version: "6.0.3" - title: Ergo Node API - description: Specification of Ergo Node API for ChatGPT plugin. - The following endpoints supported - - /blocks/chainSlice - Get headers in a specified range of heights - - /info - Get the basic information about the status of Ergo Node. - - /transactions/unconfirmed/byTransactionId - Get unconfirmed transaction from the mempool - - /transactions/poolHistogram - Get histogram (waittime, (n_trans, sum(fee)) for transactions in mempool. - - /blockchain/indexedHeight - Get current indexed block height. (The indexer has processed all blocks up to this height.) - - /blockchain/transaction/byId - Retrieve a transaction by its id - - /blockchain/transaction/byAddress - Retrieve a list of transactions by their associated address - - /blockchain/box/byId - Retrieve a box by its id - - /blockchain/box/byAddress - Retrieve boxes by their associated - - /blockchain/box/unspent/byAddress - Retrieve unspent boxes by their associated address - - /blockchain/token/byId - Retrieve minting information about a token - - /blockchain/balanceForAddress - Retrieve balance information of an Ergo address. - -servers: - - url: http://localhost:9052 - description: Ergo full node API (testnet). - - url: http://localhost:9053 - description: Ergo full node API (mainnet). - -components: - schemas: - # Objects - ErgoTransactionInput: - type: object - required: - - boxId - - spendingProof - properties: - boxId: - $ref: '#/components/schemas/TransactionBoxId' - spendingProof: - $ref: '#/components/schemas/SpendingProof' - - ErgoTransactionDataInput: - type: object - required: - - boxId - properties: - boxId: - $ref: '#/components/schemas/TransactionBoxId' - - SpendingProof: - description: Spending proof for transaction input - type: object - required: - - proofBytes - - extension - properties: - proofBytes: - $ref: '#/components/schemas/SpendingProofBytes' - extension: - type: object - description: Variables to be put into context - additionalProperties: - $ref: '#/components/schemas/SValue' - example: - '1': 'a2aed72ff1b139f35d1ad2938cb44c9848a34d4dcfd6d8ab717ebde40a7304f2541cf628ffc8b5c496e6161eba3f169c6dd440704b1719e0' - - ErgoTransactionOutput: - type: object - required: - - value - - ergoTree - - additionalRegisters - - creationHeight - properties: - boxId: - $ref: '#/components/schemas/TransactionBoxId' - value: - description: Amount of Ergo token - type: integer - format: int64 - minimum: 0 - example: 147 - ergoTree: - $ref: '#/components/schemas/ErgoTree' - creationHeight: - description: Height the output was created at - type: integer - format: int32 - example: 9149 - assets: - description: Assets list in the transaction - type: array - items: - $ref: '#/components/schemas/Asset' - additionalRegisters: - $ref: '#/components/schemas/Registers' - transactionId: - $ref: '#/components/schemas/TransactionId' - index: - description: Index in the transaction outputs - type: integer - format: int32 - - BalanceInfo: - type: object - description: Represents a balance information (e.g. for an address) - required: - - nanoErgs - - tokens - properties: - nanoErgs: - type: integer - format: int64 - description: Balance of nanoERGs - tokens: - type: array - description: List of assets (aks tokens) with balances - items: - type: object - properties: - tokenId: - $ref: '#/components/schemas/ModifierId' - description: Identifier of the asset (aka token) - amount: - type: integer - format: int64 - description: Amount of the asset (aka token) - decimals: - type: integer - description: Number of decimals of the token - name: - type: string - description: Name of the token, if any - - IndexedErgoBox: - type: object - description: Box indexed with extra information - required: - - box - - confirmationsNum - - address - - creationTransaction - - spendingTransaction - - spendingHeight - - inclusionHeight - - spent - - globalIndex - properties: - box: - $ref: '#/components/schemas/ErgoTransactionOutput' - confirmationsNum: - description: Number of confirmations, if the box is included into the blockchain - type: integer - format: int32 - minimum: 0 - example: 147 - nullable: true - address: - $ref: '#/components/schemas/ErgoAddress' - creationTransaction: - description: Transaction which created the box - $ref: '#/components/schemas/ModifierId' - spendingTransaction: - description: Transaction which created the box - nullable: true - $ref: '#/components/schemas/ModifierId' - spendingHeight: - description: The height the box was spent at - type: integer - format: int32 - minimum: 0 - example: 147 - nullable: true - inclusionHeight: - description: The height the transaction containing the box was included in a block at - type: integer - format: int32 - minimum: 0 - example: 147 - spent: - description: A flag signalling whether the box was spent - type: boolean - example: false - globalIndex: - description: Global index of the output in the blockchain - type: integer - format: int64 - minimum: 0 - example: 83927 - - IndexedToken: - type: object - description: Token indexed with extra information - required: - - id - - boxId - - emissionAmount - - name - - description - - decimals - properties: - id: - description: Id of the token - $ref: '#/components/schemas/ModifierId' - boxId: - description: Id of the box that created the token - $ref: '#/components/schemas/ModifierId' - emissionAmount: - description: The total supply of the token - type: integer - format: int64 - minimum: 1 - example: 3500000 - name: - description: The name of the token - type: string - description: - description: The description of the token - type: string - decimals: - description: The number of decimals the token supports - type: integer - format: int32 - minimum: 0 - example: 8 - - ErgoTransaction: - type: object - description: ErgoTransaction is an atomic operation which changes UTXO state. - required: - - inputs - - dataInputs - - outputs - properties: - id: - description: Id of the transaction - $ref: '#/components/schemas/TransactionId' - inputs: - description: Inputs, that will be spent by this transaction - type: array - items: - $ref: '#/components/schemas/ErgoTransactionInput' - dataInputs: - description: Read-only inputs, that are not going to be spent by transaction. - type: array - items: - $ref: '#/components/schemas/ErgoTransactionDataInput' - outputs: - description: Outputs of the transaction, i.e. box candidates to be created by this transaction. - type: array - items: - $ref: '#/components/schemas/ErgoTransactionOutput' - size: - description: Size of ErgoTransaction in bytes - type: integer - format: int32 - - IndexedErgoTransaction: - type: object - description: Transaction indexed with extra information - required: - - id - - inputs - - dataInputs - - outputs - - inclusionHeight - - numConfirmations - - blockId - - timestamp - - index - - globalIndex - - size - properties: - id: - $ref: '#/components/schemas/TransactionId' - inputs: - description: Transaction inputs - type: array - items: - $ref: '#/components/schemas/ErgoTransactionInput' - dataInputs: - description: Transaction data inputs - type: array - items: - $ref: '#/components/schemas/ErgoTransactionDataInput' - outputs: - description: Transaction outputs - type: array - items: - $ref: '#/components/schemas/ErgoTransactionOutput' - inclusionHeight: - description: Height of a block the transaction was included in - type: integer - format: int32 - example: 20998 - numConfirmations: - description: Number of transaction confirmations - type: integer - format: int32 - example: 20998 - blockId: - description: Id of the block the transaction was included in - allOf: - - $ref: '#/components/schemas/ModifierId' - timestamp: - $ref: '#/components/schemas/Timestamp' - index: - description: index of the transaction in the block it was included in - type: integer - format: int32 - example: 3 - globalIndex: - description: Global index of the transaction in the blockchain - type: integer - format: int64 - example: 3565445 - size: - description: Size in bytes - type: integer - format: int32 - - ErgoAddress: - description: Encoded Ergo Address - type: string - example: '3WwbzW6u8hKWBcL1W7kNVMr25s2UHfSBnYtwSHvrRQt7DdPuoXrt' - - FullBlock: - description: Block with header and transactions - type: object - required: - - header - - blockTransactions - - adProofs - - extension - - size - properties: - header: - $ref: '#/components/schemas/BlockHeader' - blockTransactions: - $ref: '#/components/schemas/BlockTransactions' - adProofs: - $ref: '#/components/schemas/BlockADProofs' - extension: - $ref: '#/components/schemas/Extension' - size: - description: Size in bytes - type: integer - format: int32 - - PowSolutions: - description: An object containing all components of pow solution - type: object - required: - - pk - - w - - n - - d - properties: - pk: - type: string - description: Base16-encoded public key - example: '0350e25cee8562697d55275c96bb01b34228f9bd68fd9933f2a25ff195526864f5' - w: - type: string - example: '0366ea253123dfdb8d6d9ca2cb9ea98629e8f34015b1e4ba942b1d88badfcc6a12' - n: - type: string - example: '0000000000000000' - d: - type: number - example: 987654321 - - BlockHeader: - description: Header of a block. - It authenticates link to a previous block, other block sections - (transactions, UTXO set transformation proofs, extension), UTXO set, votes for blockchain parameters - to be changed and proof-of-work related data. - type: object - required: - - id - - timestamp - - version - - adProofsRoot - - stateRoot - - transactionsRoot - - nBits - - extensionHash - - powSolutions - - height - - difficulty - - parentId - - votes - properties: - id: - description: Block id - $ref: '#/components/schemas/ModifierId' - timestamp: - description: Block generation time reported by a miner - $ref: '#/components/schemas/Timestamp' - version: - description: Protocol version used to generate the block - $ref: '#/components/schemas/Version' - adProofsRoot: - description: Digest of UTXO set transformation proofs - $ref: '#/components/schemas/Digest32' - stateRoot: - description: AVL+ tree digest of UTXO set (after the block is applied) - $ref: '#/components/schemas/ADDigest' - transactionsRoot: - description: Merkle tree digest of transactions in the block (BlockTransactions section) - $ref: '#/components/schemas/Digest32' - nBits: - description: Proof-of-work target (difficulty encoded) - type: integer - format: int64 - minimum: 0 - example: 19857408 - extensionHash: - description: Merkle tree digest of the extension section of the block - $ref: '#/components/schemas/Digest32' - powSolutions: - description: Solution for the proof-of-work puzzle - $ref: '#/components/schemas/PowSolutions' - height: - description: Height of the block (genesis block height == 1) - type: integer - format: int32 - minimum: 0 - example: 667 - difficulty: - type: string - example: '9575989248' - parentId: - $ref: '#/components/schemas/ModifierId' - votes: - description: Votes for changing system parameters - $ref: '#/components/schemas/Votes' - size: - description: Size of the header in bytes - type: integer - format: int32 - extensionId: - description: Hash of the extension section of the block == hash(modifier type id, header id, extensionHash) - $ref: '#/components/schemas/ModifierId' - transactionsId: - description: Hash of the transactions section of the block == hash(modifier type id, header id, transactionsRoot) - $ref: '#/components/schemas/ModifierId' - adProofsId: - description: Hash of the UTXO set transformation proofs section of the block == hash(modifier type id, header id, adProofsRoot) - $ref: '#/components/schemas/ModifierId' - - BlockTransactions: - description: Section of a block which contains transactions. - type: object - required: - - headerId - - transactions - - size - properties: - headerId: - description: Identifier of a header of a corresponding block - $ref: '#/components/schemas/ModifierId' - transactions: - description: Transactions of the block - $ref: '#/components/schemas/Transactions' - size: - description: Size in bytes of all block transactions - type: integer - format: int32 - - BlockADProofs: - type: object - required: - - headerId - - proofBytes - - digest - - size - properties: - headerId: - description: Identifier of a header of the block which contains the proofs - $ref: '#/components/schemas/ModifierId' - proofBytes: - description: Serialized bytes of the authenticated dictionary proof - $ref: '#/components/schemas/SerializedAdProof' - digest: - description: Hash of the proofBytes - $ref: '#/components/schemas/Digest32' - size: - description: Size in bytes - type: integer - format: int32 - - Extension: - description: Section of a block which contains extension data. - type: object - required: - - headerId - - digest - - fields - properties: - headerId: - description: Identifier of a header of a corresponding block - $ref: '#/components/schemas/ModifierId' - digest: - description: Root hash (aka digest) merkelized list of key-value records - $ref: '#/components/schemas/Digest32' - fields: - description: List of key-value records - type: array - nullable: true - items: - $ref: '#/components/schemas/KeyValueItem' - - KeyValueItem: - description: Key-value record represented as a pair of hex strings in an array. - type: array - items: - $ref: '#/components/schemas/HexString' - - Peer: - type: object - required: - - address - properties: - address: - type: string - example: '127.0.0.1:5673' - restApiUrl: - type: string - example: 'https://example.com' - nullable: true - name: - type: string - example: mynode - nullable: true - lastSeen: - $ref: '#/components/schemas/Timestamp' - connectionType: - type: string - nullable: true - enum: - - Incoming - - Outgoing - - BlacklistedPeers: - type: object - required: - - addresses - properties: - addresses: - type: array - items: - type: string - description: Blacklisted node address - - NodeInfo: - description: Data container for /info API request output. - Contains information about node's state and configuration. - Contains data about best block, best header, state, etc. - Best block is the block with the maximum height. - type: object - required: - - name - - appVersion - - fullHeight - - headersHeight - - maxPeerHeight - - bestFullHeaderId - - previousFullHeaderId - - bestHeaderId - - headersScore - - fullBlocksScore - - stateRoot - - stateType - - stateVersion - - isMining - - peersCount - - unconfirmedCount - - difficulty - - currentTime - - launchTime - - genesisBlockId - - parameters - properties: - name: - description: Node's (peer) self-chosen name from config - type: string - example: my-node-1 - appVersion: - description: Node's application version - type: string - example: 0.0.1 - fullHeight: - type: integer - format: int32 - description: Height of the best block known to the node. - Can be 'null' if state is empty (no full block is applied since node launch) - minimum: 0 - example: 667 - nullable: true - headersHeight: - type: integer - format: int32 - description: The height of the best header (i.e. the one with the maximum height). - Can be 'null' if state is empty (no header applied since node launch) - minimum: 0 - example: 667 - nullable: true - maxPeerHeight: - type: integer - format: int32 - description: Maximum block height of connected peers. - Can be 'null' if state is empty (no peer connected since node launch) - minimum: 0 - example: 706162 - nullable: true - bestFullHeaderId: - type: string - description: Best full-block id (header id of such block). - Can be 'null' if no full block is applied since node launch. - nullable: true - allOf: - - $ref: '#/components/schemas/ModifierId' - previousFullHeaderId: - type: string - description: Header id of the parent block of the best full-block (i.e. previous block in the blockchain). - Can be 'null' if no full block is applied since node launch - nullable: true - allOf: - - $ref: '#/components/schemas/ModifierId' - bestHeaderId: - type: string - description: Best header ID (hex representation). - Can be 'null' if no header applied since node launch. - nullable: true - allOf: - - $ref: '#/components/schemas/ModifierId' - stateRoot: - type: string - nullable: true - description: Current UTXO set digest. - Can be 'null' if state is empty (no full block is applied since node launch) - example: 'dab9da11fc216660e974842cc3b7705e62ebb9e0bf5ff78e53f9cd40abadd117' - stateType: - description: Whether the node is storing UTXO set or only its digest. - Equals `digest` if only digest is stored, `utxo` if full UTXO set is stored. - type: string - enum: - - digest - - utxo - stateVersion: - description: Id of a block where UTXO set digest is taken from. - Can be 'null' if no full block is applied since node launch. - type: string - example: 'fab9da11fc216660e974842cc3b7705e62ebb9e0bf5ff78e53f9cd40abadd117' - nullable: true - isMining: - description: Whether the node is mining (i.e. generating blocks). - type: boolean - example: true - peersCount: - type: integer - description: Number of peers this node is connected with. - format: int32 - minimum: 0 - example: 327 - unconfirmedCount: - description: Number of unconfirmed transactions in the mempool. - type: integer - format: int32 - minimum: 0 - maximum: 10000 - example: 327 - difficulty: - type: integer - minimum: 0 - nullable: true - example: 667 - description: Difficulty on current bestFullHeaderId. - Can be 'null' if no full block is applied since node launch. - Difficulty is a BigInt integer. - currentTime: - type: integer - description: Current internal node time - allOf: - - $ref: '#/components/schemas/Timestamp' - launchTime: - type: integer - description: When the node was launched (in Java time format, UNIX time * 1000). - allOf: - - $ref: '#/components/schemas/Timestamp' - headersScore: - type: integer - description: Cumulative difficulty of best headers-chain. - Can be 'null' if no headers is applied since node launch. headersScore is a BigInt integer. - nullable: true - fullBlocksScore: - type: integer - description: Cumulative difficulty of best full blocks chain. - Can be 'null' if no full block is applied since node launch. fullBlocksScore is a BigInt integer. - nullable: true - genesisBlockId: - type: string - description: Header id of genesis block. Can be 'null' if genesis blocks is not produced yet. - nullable: true - allOf: - - $ref: '#/components/schemas/ModifierId' - parameters: - type: object - description: System parameters which could be readjusted via collective miners decision. - $ref: '#/components/schemas/Parameters' - eip27Supported: - type: boolean - example: true - description: Whether EIP-27 locked in - restApiUrl: - type: string - example: 'https://example.com' - description: Publicly accessible url of node which exposes restApi in firewall - - Parameters: - description: System parameters which could be readjusted via collective miners decision. - type: object - required: - - height - - blockVersion - - storageFeeFactor - - minValuePerByte - - maxBlockSize - - maxBlockCost - - tokenAccessCost - - inputCost - - dataInputCost - - outputCost - properties: - height: - type: integer - format: int32 - description: Height when current parameters were considered(not actual height). Can be '0' if state is empty - minimum: 0 - example: 667 - nullable: false - storageFeeFactor: - type: integer - format: int32 - description: Storage fee coefficient (per byte per storage period ~4 years) - minimum: 0 - example: 100000 - nullable: false - minValuePerByte: - type: integer - format: int32 - description: Minimum value per byte of an output - minimum: 0 - example: 360 - nullable: false - maxBlockSize: - type: integer - format: int32 - description: Maximum block size (in bytes) - minimum: 0 - example: 1048576 - nullable: false - maxBlockCost: - type: integer - format: int32 - description: Maximum cumulative computational cost of input scripts in block transactions - minimum: 0 - example: 104876 - nullable: false - blockVersion: - $ref: '#/components/schemas/Version' - nullable: false - tokenAccessCost: - type: integer - format: int32 - description: Validation cost of a single token - minimum: 0 - example: 100 - nullable: false - inputCost: - type: integer - format: int32 - description: Validation cost per one transaction input - minimum: 0 - example: 100 - nullable: false - dataInputCost: - type: integer - format: int32 - description: Validation cost per one data input - minimum: 0 - example: 100 - nullable: false - outputCost: - type: integer - format: int32 - description: Validation cost per one transaction output - minimum: 0 - example: 100 - nullable: false - - Version: - description: Ergo blockchain protocol version - type: integer - format: int8 - example: 2 - - TransactionBoxId: - description: Base16-encoded transaction box id bytes. Should be 32 bytes long - type: string - format: base16 - example: '1ab9da11fc216660e974842cc3b7705e62ebb9e0bf5ff78e53f9cd40abadd117' - - TransactionId: - description: Base16-encoded transaction id bytes - type: string - format: base16 - example: '2ab9da11fc216660e974842cc3b7705e62ebb9e0bf5ff78e53f9cd40abadd117' - - ErgoTree: - description: Base16-encoded ergo tree bytes - type: string - format: base16 - example: '0008cd0336100ef59ced80ba5f89c4178ebd57b6c1dd0f3d135ee1db9f62fc634d637041' - - Transactions: - description: List of ErgoTransaction objects - type: array - items: - $ref: '#/components/schemas/ErgoTransaction' - - FeeHistogramBin: - description: Fee histogram bin - type: object - properties: - nTxns: - type: integer - format: int32 - totalFee: - type: integer - format: int64 - - FeeHistogram: - description: Fee histogram for transactions in mempool - type: array - items: - $ref: '#/components/schemas/FeeHistogramBin' - - Asset: - description: Token detail in the transaction - type: object - required: - - tokenId - - amount - properties: - tokenId: - $ref: '#/components/schemas/Digest32' - amount: - description: Amount of the token - type: integer - format: int64 - example: 1000 - - Registers: - description: Ergo box registers - type: object - additionalProperties: - $ref: '#/components/schemas/SValue' - example: - R4: '100204a00b08cd0336100ef59ced80ba5f89c4178ebd57b6c1dd0f3d135ee1db9f62fc634d637041ea02d192a39a8cc7a70173007301' - - SValue: - description: Base-16 encoded serialized Sigma-state value - type: string - format: base16 - example: '100204a00b08cd0336100ef59ced80ba5f89c4178ebd57b6c1dd0f3d135ee1db9f62fc634d637041ea02d192a39a8cc7a70173007301' - - ModifierId: - description: Base16-encoded 32 byte modifier id - type: string - format: base16 - example: '3ab9da11fc216660e974842cc3b7705e62ebb9e0bf5ff78e53f9cd40abadd117' - - Digest32: - description: Base16-encoded 32 byte digest - type: string - format: base16 - example: '4ab9da11fc216660e974842cc3b7705e62ebb9e0bf5ff78e53f9cd40abadd117' - - HexString: - description: Base16-encoded bytes - type: string - format: base16 - example: '4ab9da11fc216660e974842cc3b7705e62ebb9e0bf5ff78e53f9cd40abadd117' - - ADDigest: - description: Base16-encoded 33 byte digest - digest with extra byte with tree height - type: string - format: base16 - example: '333ab9da11fc216660e974842cc3b7705e62ebb9e0bf5ff78e53f9cd40abadd117' - - SerializedAdProof: - description: Base16-encoded ad proofs - type: string - format: base16 - example: '3ab9da11fc216660e974842cc3b7705e62ebb9e0bf5ff78e53f9cd40abadd1173ab9da11fc216660e974842cc3b7705e62ebb9e0bf5ff78e53f9cd40abadd1173ab9da11fc216660e974842cc3b7705e62ebb9e0bf5ff78e53f9cd40abadd117' - - SpendingProofBytes: - description: Base16-encoded spending proofs - type: string - format: base16 - example: '4ab9da11fc216660e974842cc3b7705e62ebb9e0bf5ff78e53f9cd40abadd1173ab9da11fc216660e974842cc3b7705e62ebb9e0bf5ff78e53f9cd40abadd1173ab9da11fc216660e974842cc3b7705e62ebb9e0bf5ff78e53f9cd40abadd117' - - Timestamp: - description: Basic timestamp definition - type: integer - format: int64 - example: 1524143059077 - - AddressValidity: - description: Validity status of Ergo address - type: object - required: - - address - - isValid - properties: - address: - $ref: '#/components/schemas/ErgoAddress' - isValid: - type: boolean - error: - type: string - - ApiError: - description: Error response from API - type: object - required: - - error - - reason - - detail - properties: - error: - type: integer - description: Error code - example: 500 - reason: - type: string - description: Error message explaining the reason of the error - example: 'Internal server error' - detail: - type: string - nullable: true - description: Detailed error description - -paths: - /blocks/chainSlice: - get: - description: Get headers in a specified range of heights - operationId: getChainSlice - tags: - - blocks - parameters: - - in: query - name: fromHeight - required: false - description: Min header height (start of the range) - schema: - type: integer - format: int32 - default: 0 - - in: query - name: toHeight - required: false - description: Max header height of the range (last header height then omitted) - schema: - type: integer - format: int32 - default: -1 - responses: - '200': - description: Array of headers - content: - application/json: - schema: - type: array - description: Array of headers - items: - $ref: '#/components/schemas/BlockHeader' - default: - description: Error - content: - application/json: - schema: - $ref: '#/components/schemas/ApiError' - - /info: - get: - description: Get the basic information about the status of Ergo Node. - operationId: getNodeInfo - tags: - - info - responses: - '200': - description: Node info object - content: - application/json: - schema: - $ref: '#/components/schemas/NodeInfo' - default: - description: Error - content: - application/json: - schema: - $ref: '#/components/schemas/ApiError' - - /transactions/unconfirmed/byTransactionId/{txId}: - parameters: - - in: path - name: txId - required: true - description: ID of a transaction to return - schema: - type: string - get: - description: Get unconfirmed transaction from the mempool - operationId: getUnconfirmedTransactionById - tags: - - transactions - responses: - '200': - description: Ergo transaction - content: - application/json: - schema: - $ref: '#/components/schemas/ErgoTransaction' - default: - description: Error - content: - application/json: - schema: - $ref: '#/components/schemas/ApiError' - - /transactions/poolHistogram: - parameters: - - in: query - name: bins - required: false - description: The number of bins in histogram - schema: - type: integer - format: int32 - minimum: 1 - default: 10 - - in: query - name: maxtime - required: false - description: Maximal wait time in milliseconds - schema: - type: integer - format: int64 - minimum: 0 - default: 60000 - get: - description: Get histogram (waittime, (n_trans, sum(fee)) for transactions in mempool. - It contains "bins"+1 bins, where i-th elements corresponds to transaction with wait time [i*maxtime/bins, (i+1)*maxtime/bins), - and last bin corresponds to the transactions with wait time >= maxtime. - operationId: getFeeHistogram - tags: - - transactions - responses: - '200': - description: Array with fee histogram - content: - application/json: - schema: - $ref: '#/components/schemas/FeeHistogram' - default: - description: Error - content: - application/json: - schema: - $ref: '#/components/schemas/ApiError' - - /peers/connected: - get: - description: Get a list of current connected peers - operationId: getConnectedPeers - tags: - - peers - responses: - '200': - description: Array of peer objects - content: - application/json: - schema: - type: array - items: - $ref: '#/components/schemas/Peer' - default: - description: Error - content: - application/json: - schema: - $ref: '#/components/schemas/ApiError' - - /peers/blacklisted: - get: - description: Get a list of blacklisted peers - operationId: getBlacklistedPeers - tags: - - peers - responses: - '200': - description: Array of the addresses - content: - application/json: - schema: - $ref: '#/components/schemas/BlacklistedPeers' - - default: - description: Error - content: - application/json: - schema: - $ref: '#/components/schemas/ApiError' - - /utils/address/{address}: - get: - description: Check address validity - operationId: CheckAddressValidityWithGet - tags: - - utils - parameters: - - in: path - name: address - required: true - description: address to check - schema: - $ref: '#/components/schemas/ErgoAddress' - responses: - '200': - description: Address validity with validation error - content: - application/json: - schema: - $ref: '#/components/schemas/AddressValidity' - default: - description: Error - content: - application/json: - schema: - $ref: '#/components/schemas/ApiError' - - /blockchain/indexedHeight: - get: - description: Get current indexed block height. (The indexer has processed all blocks up to this height.) - operationId: getIndexedHeight - tags: - - blockchain - responses: - '200': - description: height of the indexer and full height - content: - application/json: - schema: - properties: - indexedHeight: - type: integer - default: 0 - description: number of blocks indexed - fullHeight: - type: integer - description: number of all known blocks - - /blockchain/transaction/byId/{txId}: - get: - description: Retrieve a transaction by its id - operationId: getTxById - tags: - - blockchain - parameters: - - in: path - name: txId - required: true - description: id of the wanted transaction - schema: - type: string - responses: - '200': - description: transaction with wanted id - content: - application/json: - schema: - $ref: '#/components/schemas/IndexedErgoTransaction' - '404': - description: Transaction with this id doesn't exist - content: - application/json: - schema: - $ref: '#/components/schemas/ApiError' - default: - description: Error - content: - application/json: - schema: - $ref: '#/components/schemas/ApiError' - - /blockchain/transaction/byAddress/{address}: - post: - description: Retrieve a list of transactions by their associated address - operationId: getTxsByAddress - tags: - - blockchain - parameters: - - in: path - name: address - required: true - description: address to retrieve associated transactions - schema: - $ref: '#/components/schemas/ErgoAddress' - - in: query - name: offset - required: false - description: amount of elements to skip from the start - schema: - type: integer - format: int32 - default: 0 - - in: query - name: limit - required: false - description: amount of elements to retrieve - schema: - type: integer - format: int32 - default: 5 - responses: - '200': - description: transactions associated with the given address - content: - application/json: - schema: - type: object - properties: - items: - type: array - description: Array of transactions - items: - $ref: '#/components/schemas/IndexedErgoTransaction' - total: - type: integer - description: Total count of retrieved transactions - '404': - description: No transactions found for wanted address - content: - application/json: - schema: - $ref: '#/components/schemas/ApiError' - default: - description: Error - content: - application/json: - schema: - $ref: '#/components/schemas/ApiError' - - /blockchain/box/byId/{boxId}: - get: - description: Retrieve a box by its id - operationId: getBoxById - tags: - - blockchain - parameters: - - in: path - name: boxId - required: true - description: id of the wanted box - schema: - type: string - responses: - '200': - description: box with wanted id - content: - application/json: - schema: - $ref: '#/components/schemas/IndexedErgoBox' - '404': - description: No box found with wanted id - content: - application/json: - schema: - $ref: '#/components/schemas/ApiError' - default: - description: Error - content: - application/json: - schema: - $ref: '#/components/schemas/ApiError' - - /blockchain/box/byAddress/{address}: - post: - description: Retrieve boxes by their associated address - operationId: getBoxesByAddress - tags: - - blockchain - parameters: - - in: path - name: address - required: true - description: address to retrieve boxes for - schema: - $ref: '#/components/schemas/ErgoAddress' - - in: query - name: offset - required: false - description: amount of elements to skip from the start - schema: - type: integer - format: int32 - default: 0 - - in: query - name: limit - required: false - description: amount of elements to retrieve - schema: - type: integer - format: int32 - default: 5 - responses: - '200': - description: boxes associated with wanted address - content: - application/json: - schema: - type: object - properties: - items: - type: array - description: Array of boxes - items: - $ref: '#/components/schemas/IndexedErgoBox' - total: - type: integer - description: Total number of retreived boxes - '404': - description: No boxes found for wanted address - content: - application/json: - schema: - $ref: '#/components/schemas/ApiError' - default: - description: Error - content: - application/json: - schema: - $ref: '#/components/schemas/ApiError' - - /blockchain/box/unspent/byAddress/{address}: - get: - description: Retrieve unspent boxes by their associated address - operationId: getBoxesByAddressUnspent - tags: - - blockchain - parameters: - - in: path - name: address - required: true - description: address to retrieve boxes for - schema: - $ref: '#/components/schemas/ErgoAddress' - - in: query - name: offset - required: false - description: amount of elements to skip from the start - schema: - type: integer - format: int32 - default: 0 - - in: query - name: limit - required: false - description: amount of elements to retrieve - schema: - type: integer - format: int32 - default: 5 - - in: query - name: sortDirection - required: false - description: desc = new boxes first ; asc = old boxes first - schema: - type: string - default: desc - - in: query - name: excludeMempoolSpent - required: false - description: if true exclude spent inputs from mempool - schema: - type: boolean - default: false - responses: - '200': - description: unspent boxes associated with wanted address - content: - application/json: - schema: - type: array - description: Array of boxes - items: - $ref: '#/components/schemas/IndexedErgoBox' - '404': - description: No unspent boxes found for wanted address - content: - application/json: - schema: - $ref: '#/components/schemas/ApiError' - default: - description: Error - content: - application/json: - schema: - $ref: '#/components/schemas/ApiError' - - /blockchain/token/byId/{tokenId}: - get: - description: Retrieve minting information about a token - operationId: getTokenById - tags: - - blockchain - parameters: - - in: path - name: tokenId - required: true - description: id of the wanted token - schema: - type: string - responses: - '200': - description: token with wanted id - content: - application/json: - schema: - $ref: '#/components/schemas/IndexedToken' - '404': - description: No token found with wanted id - content: - application/json: - schema: - $ref: '#/components/schemas/ApiError' - default: - description: Error - content: - application/json: - schema: - $ref: '#/components/schemas/ApiError' - - /blockchain/balanceForAddress/{address}: - get: - description: Retrieve balance information of an Ergo address. - Separately return confirmed and unconfirmed balance information. - operationId: getBalanceForAddress - tags: - - blockchain - parameters: - - in: path - name: address - required: true - description: address to retrieve balance information for - schema: - $ref: '#/components/schemas/ErgoAddress' - responses: - '200': - description: balance information - content: - application/json: - schema: - type: object - properties: - confirmed: - description: confirmed balance of the address - $ref: '#/components/schemas/BalanceInfo' - unconfirmed: - description: unconfirmed balance of the address - $ref: '#/components/schemas/BalanceInfo' - default: - description: Error - content: - application/json: - schema: - $ref: '#/components/schemas/ApiError' diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml index 1e8451367e..5a5f8fc75e 100644 --- a/src/main/resources/api/openapi.yaml +++ b/src/main/resources/api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.2" info: - version: "6.0.3" + version: "6.0.5" title: Ergo Node API description: API docs for Ergo Node. Models are shared between all Ergo products contact: @@ -1258,12 +1258,12 @@ components: description: commitment to secret along with secret (!) randomness allOf: # Combines the Commitment and the inline model - $ref: '#/components/schemas/Commitment' - type: object + - type: object required: - secret properties: - secret: - type: string + secret: + type: string SecretProven: type: object @@ -5132,8 +5132,6 @@ paths: /mining/candidate: get: - security: - - ApiKeyAuth: [api_key] summary: Request block candidate operationId: miningRequestBlockCandidate tags: @@ -5210,8 +5208,6 @@ paths: /mining/rewardAddress: get: - security: - - ApiKeyAuth: [api_key] summary: Read miner reward address operationId: miningReadMinerRewardAddress tags: @@ -5232,8 +5228,6 @@ paths: /mining/rewardPublicKey: get: - security: - - ApiKeyAuth: [api_key] summary: Read public key associated with miner rewards operationId: miningReadMinerRewardPubkey tags: @@ -5254,8 +5248,6 @@ paths: /mining/solution: post: - security: - - ApiKeyAuth: [api_key] summary: Submit solution for current candidate operationId: miningSubmitSolution tags: @@ -5540,8 +5532,6 @@ paths: /script/p2sAddress: post: - security: - - ApiKeyAuth: [api_key] summary: Create P2SAddress from Sigma source operationId: scriptP2SAddress tags: @@ -5574,8 +5564,6 @@ paths: /script/p2shAddress: post: - security: - - ApiKeyAuth: [api_key] summary: Create P2SHAddress from Sigma source operationId: scriptP2SHAddress tags: diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index b6b684f1bd..bc8db13f0e 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -446,7 +446,7 @@ scorex { nodeName = "ergo-node" # Network protocol version to be sent in handshakes - appVersion = 6.0.3 + appVersion = 6.0.5 # Network agent name. May contain information about client code # stack, starting from core code-base up to the end graphical interface. diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 2929ac6ade..d2f0932a66 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -30,6 +30,10 @@ + + + diff --git a/src/main/scala/org/ergoplatform/http/ErgoHttpService.scala b/src/main/scala/org/ergoplatform/http/ErgoHttpService.scala index c331e005e2..eb99d040a2 100644 --- a/src/main/scala/org/ergoplatform/http/ErgoHttpService.scala +++ b/src/main/scala/org/ergoplatform/http/ErgoHttpService.scala @@ -8,6 +8,7 @@ import akka.http.scaladsl.server.Directive0 import akka.http.scaladsl.server.directives.RouteDirectives import scorex.core.api.http.{ApiErrorHandler, ApiRejectionHandler, ApiRoute, CorsHandler} import akka.http.scaladsl.model.headers._ +import scorex.util.ScorexLogging import scala.collection.immutable @@ -15,7 +16,7 @@ final case class ErgoHttpService( apiRoutes: Seq[ApiRoute], swaggerRoute: SwaggerRoute, panelRoute: NodePanelRoute -)(implicit val system: ActorSystem) extends CorsHandler { +)(implicit val system: ActorSystem) extends CorsHandler with ScorexLogging { def rejectionHandler: RejectionHandler = ApiRejectionHandler.rejectionHandler @@ -36,15 +37,41 @@ final case class ErgoHttpService( super.respondWithHeaders(corsResponseHeaders) } + /** + * Logs every query served by the node's HTTP interface: method, relative URI (path and query + * string), response status and how long it took. + * + * Bodies are deliberately not logged, as requests carry secrets (a mnemonic on + * `/wallet/restore`, a password on `/wallet/unlock`, and so on) and responses can be large. + * + * Off by default, since the root logger is at INFO. To switch it on, add to `logback.xml`: + * {{{ + * + * }}} + * When it is off, the message is never built: `log.debug` is a macro guarded by `isDebugEnabled`. + */ + private val logQueries: Directive0 = + extractRequest.flatMap { request => + val startTime = System.currentTimeMillis() + mapResponse { response => + val elapsedMs = System.currentTimeMillis() - startTime + log.debug(s"${request.method.value} ${request.uri.toRelative} - " + + s"${response.status.intValue()} in $elapsedMs ms") + response + } + } + val compositeRoute: Route = - handleRejections(rejectionHandler) { - handleExceptions(exceptionHandler) { - corsHandler { - apiR ~ - apiSpecR ~ - swaggerRoute.route ~ - panelRoute.route ~ - redirectToSwaggerR + logQueries { + handleRejections(rejectionHandler) { + handleExceptions(exceptionHandler) { + corsHandler { + apiR ~ + apiSpecR ~ + swaggerRoute.route ~ + panelRoute.route ~ + redirectToSwaggerR + } } } } diff --git a/src/main/scala/org/ergoplatform/http/api/BlocksApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlocksApiRoute.scala index e57898b8fc..1dfee3c570 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlocksApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlocksApiRoute.scala @@ -150,7 +150,7 @@ case class BlocksApiRoute(viewHolderRef: ActorRef, readersHolder: ActorRef, ergo def getChainSliceR: Route = (pathPrefix("chainSlice") & chainPagination) { (fromHeight, toHeight) => if (toHeight < fromHeight) { BadRequest("toHeight < fromHeight") - } else if (fromHeight - toHeight > MaxHeaders) { + } else if (toHeight.toLong - fromHeight.toLong > MaxHeaders) { BadRequest(s"No more than $MaxHeaders headers can be requested") } else { ApiResponse(getChainSlice(fromHeight, toHeight)) diff --git a/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala index 50162b59f9..2670c2a7a3 100644 --- a/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala @@ -7,7 +7,7 @@ import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnconfirmedTransacti import org.ergoplatform.nodeView.ErgoReadersHolder.{GetReaders, Readers} import org.ergoplatform.nodeView.mempool.ErgoMemPoolReader import org.ergoplatform.nodeView.state.{ErgoStateReader, UtxoStateReader} -import org.ergoplatform.settings.{Algos, ErgoSettings} +import org.ergoplatform.settings.{Algos, Constants, ErgoSettings} import scorex.core.api.http.ApiRoute import scorex.util.{bytesToId, ModifierId} import akka.pattern.ask @@ -34,10 +34,19 @@ trait ErgoBaseApiRoute extends ApiRoute with ApiCodecs { val modifierIdGet: Directive1[ModifierId] = parameters("id".as[String]) .flatMap(handleModifierId) + private def parseModifierId(value: String): Try[ModifierId] = + Algos.decode(value).flatMap { bytes => + if (bytes.length == Constants.ModifierIdSize) { + Success(bytesToId(bytes)) + } else { + Failure(new IllegalArgumentException("Wrong modifierId length")) + } + } + private def handleModifierId(value: String): Directive1[ModifierId] = { - Algos.decode(value) match { - case Success(bytes) => provide(bytesToId(bytes)) - case _ => reject(ValidationRejection("Wrong modifierId format")) + parseModifierId(value) match { + case Success(id) => provide(id) + case _ => reject(ValidationRejection("Wrong modifierId format")) } } @@ -45,9 +54,9 @@ trait ErgoBaseApiRoute extends ApiRoute with ApiCodecs { val acc = collection.mutable.Buffer.empty[ModifierId] val err = collection.mutable.Buffer.empty[String] for (value <- values) { - Algos.decode(value) match { - case Success(bytes) => acc += bytesToId(bytes) - case Failure(e) => err += e.getMessage + parseModifierId(value) match { + case Success(id) => acc += id + case Failure(e) => err += e.getMessage } } if (err.nonEmpty) { diff --git a/src/main/scala/org/ergoplatform/http/api/InfoApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/InfoApiRoute.scala index 701e9f2dfb..43b54378d1 100644 --- a/src/main/scala/org/ergoplatform/http/api/InfoApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/InfoApiRoute.scala @@ -1,7 +1,6 @@ package org.ergoplatform.http.api import akka.actor.{ActorRef, ActorRefFactory} -import akka.http.scaladsl.model.ContentTypes import akka.http.scaladsl.server.Route import akka.pattern.ask import io.circe.syntax._ @@ -28,12 +27,6 @@ case class InfoApiRoute(statsCollector: ActorRef, "lastMemPoolUpdateTime" -> nodeInfo.lastMemPoolUpdateTime.asJson )) }) - } ~ - (path(".well-known" / "ai-plugin.json") & get) { - getFromResource(".well-known/ai-plugin.json", ContentTypes.`application/json`) - } ~ - (path("openapi.yaml") & get) { - getFromResource("api/openapi-ai.yaml", ContentTypes.`text/plain(UTF-8)`) } } diff --git a/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala index abc407338d..90b6caa07f 100644 --- a/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/ScanApiRoute.scala @@ -13,7 +13,6 @@ import scala.util.{Failure, Success} import ScanEntities._ import org.ergoplatform.ErgoBox.R1 import org.ergoplatform.http.api.ApiError.BadRequest -import org.ergoplatform.wallet.Constants.ScanId import sigma.ast.ByteArrayConstant import sigma.serialization.ErgoTreeSerializer @@ -66,21 +65,23 @@ case class ScanApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSettings) def unspentR: Route = (path("unspentBoxes" / IntNumber) & get & boxParams) { (scanIdInt, minConfNum, maxConfNum, minHeight, maxHeight, limit, offset) => - val scanId = ScanId @@ scanIdInt.toShort - val considerUnconfirmed = minConfNum == -1 - withWallet(_.scanUnspentBoxes(scanId, considerUnconfirmed, minHeight, maxHeight).map { - _.filter(boxConfirmationFilter(_, minConfNum, maxConfNum)) - .slice(offset, offset + limit) - }) + withScanId(scanIdInt) { scanId => + val considerUnconfirmed = minConfNum == -1 + withWallet(_.scanUnspentBoxes(scanId, considerUnconfirmed, minHeight, maxHeight).map { + _.filter(boxConfirmationFilter(_, minConfNum, maxConfNum)) + .slice(offset, offset + limit) + }) + } } def spentR: Route = (path("spentBoxes" / IntNumber) & get & boxParams) { (scanIdInt, minConfNum, maxConfNum, minHeight, maxHeight, limit, offset) => - val scanId = ScanId @@ scanIdInt.toShort - withWallet(_.scanSpentBoxes(scanId).map { - _.filter(boxConfirmationHeightFilter(_, minConfNum, maxConfNum, minHeight, maxHeight)) - .slice(offset, offset + limit) - }) + withScanId(scanIdInt) { scanId => + withWallet(_.scanSpentBoxes(scanId).map { + _.filter(boxConfirmationHeightFilter(_, minConfNum, maxConfNum, minHeight, maxHeight)) + .slice(offset, offset + limit) + }) + } } def stopTrackingR: Route = (path("stopTracking") & post & entity(as[ScanIdBoxId])) { scanIdBoxId => diff --git a/src/main/scala/org/ergoplatform/http/api/TransactionsApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/TransactionsApiRoute.scala index 38135ec7b7..95ec88a5b8 100644 --- a/src/main/scala/org/ergoplatform/http/api/TransactionsApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/TransactionsApiRoute.scala @@ -14,7 +14,7 @@ import org.ergoplatform.nodeView.ErgoReadersHolder.{GetReaders, Readers} import org.ergoplatform.nodeView.mempool.ErgoMemPoolReader import org.ergoplatform.nodeView.mempool.HistogramStats.getFeeHistogram import org.ergoplatform.nodeView.state.{ErgoStateReader, UtxoStateReader} -import org.ergoplatform.settings.{Algos, ErgoSettings, RESTApiSettings} +import org.ergoplatform.settings.{Algos, Constants, ErgoSettings, RESTApiSettings} import scorex.core.api.http.ApiResponse import scorex.crypto.authds.ADKey import scorex.util.encode.Base16 @@ -37,11 +37,11 @@ case class TransactionsApiRoute(readersHolder: ActorRef, val boxId: Directive1[BoxId] = pathPrefix(Segment).flatMap(handleBoxId) private def handleBoxId(value: String): Directive1[BoxId] = { - ADKey @@ Base16.decode(value) match { - case Success(boxId) => - provide(boxId) + Base16.decode(value) match { + case Success(boxId) if boxId.length == Constants.ModifierIdSize => + provide(ADKey @@ boxId) case _ => - reject(ValidationRejection(s"boxId $value is invalid, it should be hex string")) + reject(ValidationRejection(s"boxId $value is invalid, it should be 64 chars long hex string")) } } @@ -49,7 +49,7 @@ case class TransactionsApiRoute(readersHolder: ActorRef, private def handleTokenId(value: String): Directive1[TokenId] = { Algos.decode(value) match { - case Success(tokenId) => + case Success(tokenId) if tokenId.length == Constants.ModifierIdSize => provide(tokenId.toTokenId) case _ => reject(ValidationRejection(s"tokenId $value is invalid, it should be 64 chars long hex string")) diff --git a/src/main/scala/org/ergoplatform/http/api/WalletApiOperations.scala b/src/main/scala/org/ergoplatform/http/api/WalletApiOperations.scala index 9beb7ecabf..d4bc6d11af 100644 --- a/src/main/scala/org/ergoplatform/http/api/WalletApiOperations.scala +++ b/src/main/scala/org/ergoplatform/http/api/WalletApiOperations.scala @@ -4,11 +4,14 @@ import akka.actor.ActorRef import akka.http.scaladsl.server.{Directive, Route, ValidationRejection} import akka.pattern.ask import io.circe.Encoder +import org.ergoplatform.http.api.ApiError.BadRequest import org.ergoplatform.nodeView.ErgoReadersHolder.{GetReaders, Readers} import org.ergoplatform.nodeView.wallet.{ErgoWalletReader, WalletBox} +import org.ergoplatform.wallet.Constants.ScanId import scorex.core.api.http.ApiResponse import scala.concurrent.Future +import scala.util.Try trait WalletApiOperations extends ErgoBaseApiRoute { @@ -75,4 +78,19 @@ trait WalletApiOperations extends ErgoBaseApiRoute { withWalletOp(op)(ApiResponse.apply[T]) } + protected def withScanId(scanId: Int)(route: ScanId => Route): Route = { + if (scanId < Short.MinValue || scanId > Short.MaxValue) { + BadRequest(s"scanId $scanId is outside Short range") + } else { + route(ScanId @@ scanId.toShort) + } + } + + protected def withScanId(scanId: String)(route: ScanId => Route): Route = { + Try(scanId.toShort).toOption match { + case Some(id) => route(ScanId @@ id) + case None => BadRequest(s"scanId $scanId is invalid") + } + } + } diff --git a/src/main/scala/org/ergoplatform/http/api/WalletApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/WalletApiRoute.scala index 1232e882fd..767c19daab 100644 --- a/src/main/scala/org/ergoplatform/http/api/WalletApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/WalletApiRoute.scala @@ -13,7 +13,6 @@ import org.ergoplatform.nodeView.wallet._ import org.ergoplatform.nodeView.wallet.requests._ import org.ergoplatform.settings.{ErgoSettings, RESTApiSettings} import org.ergoplatform.wallet.Constants -import org.ergoplatform.wallet.Constants.ScanId import org.ergoplatform.wallet.boxes.ErgoBoxSerializer import org.ergoplatform.http.api.ApiError.{BadRequest, NotExists} import scorex.core.api.http.ApiResponse @@ -367,23 +366,25 @@ case class WalletApiRoute(readersHolder: ActorRef, def getTransactionsByScanIdR: Route = (path("transactionsByScanId" / Segment) & get & txsByScanIdParams) { case (id, minHeight, maxHeight, minConfNum, maxConfNum, includeUnconfirmed) => - if ((minHeight > 0 || maxHeight < Int.MaxValue) && (minConfNum > 0 || maxConfNum < Int.MaxValue)) - BadRequest("Bad request: both heights and confirmations set") - else if (minHeight == 0 && maxHeight == Int.MaxValue && minConfNum == 0 && maxConfNum == Int.MaxValue) { - withWalletOp(_.transactionsByScanId(ScanId @@ id.toShort, includeUnconfirmed)) { - resp => ApiResponse(resp.result.asJson) + withScanId(id) { scanId => + if ((minHeight > 0 || maxHeight < Int.MaxValue) && (minConfNum > 0 || maxConfNum < Int.MaxValue)) + BadRequest("Bad request: both heights and confirmations set") + else if (minHeight == 0 && maxHeight == Int.MaxValue && minConfNum == 0 && maxConfNum == Int.MaxValue) { + withWalletOp(_.transactionsByScanId(scanId, includeUnconfirmed)) { + resp => ApiResponse(resp.result.asJson) + } } - } - else { - withWalletOp(_.filteredScanTransactions( - List(ScanId @@ id.toShort), - minHeight, - maxHeight, - minConfNum, - maxConfNum, - includeUnconfirmed) - ) { - resp => ApiResponse(resp.asJson) + else { + withWalletOp(_.filteredScanTransactions( + List(scanId), + minHeight, + maxHeight, + minConfNum, + maxConfNum, + includeUnconfirmed) + ) { + resp => ApiResponse(resp.asJson) + } } } } diff --git a/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala b/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala index 0a9b3888c0..947c11c53e 100644 --- a/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala +++ b/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala @@ -105,7 +105,8 @@ class CandidateGenerator( h, s, m, - avgGenTime = 1000.millis + avgGenTime = 1000.millis, + lastAppliedBlockTxs = None ) ) ) @@ -147,18 +148,21 @@ class CandidateGenerator( * When new block is applied, either one mined by us or received from peers isn't equal to our candidate's parent, * we need to generate new candidate and possibly also discard existing solution if it is also behind */ - case FullBlockApplied(header) => + case applied: FullBlockApplied => + val header = applied.header log.info( s"Preparing new candidate on getting new block at ${header.height}" ) + val stateWithAppliedTxs = + state.copy(lastAppliedBlockTxs = Some(header.id -> applied.txIds.toSet)) if (needNewCandidate(state.cachedCandidate, header)) { if (needNewSolution(state.solvedBlock, header.id)) - context.become(initialized(state.copy(cachedCandidate = None, cachedPreviousCandidate = None, solvedBlock = None))) + context.become(initialized(stateWithAppliedTxs.copy(cachedCandidate = None, cachedPreviousCandidate = None, solvedBlock = None))) else - context.become(initialized(state.copy(cachedCandidate = None, cachedPreviousCandidate = None))) + context.become(initialized(stateWithAppliedTxs.copy(cachedCandidate = None, cachedPreviousCandidate = None))) self ! GenerateCandidate(txsToInclude = Seq.empty, reply = false, forced = false) } else { - context.become(initialized(state)) + context.become(initialized(stateWithAppliedTxs)) } case gen @ GenerateCandidate(txsToInclude, reply, forced, optPk) => @@ -174,6 +178,7 @@ class CandidateGenerator( state.mpr, effectiveMinerPk, txsToInclude, + state.lastAppliedBlockTxs, ergoSettings ) match { case Some(Failure(ex)) => @@ -280,7 +285,8 @@ object CandidateGenerator extends ScorexLogging { hr: ErgoHistoryReader, sr: UtxoStateReader, mpr: ErgoMemPoolReader, - avgGenTime: FiniteDuration // approximation of average block generation time for more efficient retries + avgGenTime: FiniteDuration, // approximation of average block generation time for more efficient retries + lastAppliedBlockTxs: Option[(ModifierId, Set[ModifierId])] // header id and tx ids of the last applied block ) def apply( @@ -387,6 +393,38 @@ object CandidateGenerator extends ScorexLogging { private def inputsNotSpent(tx: ErgoTransaction, s: UtxoStateReader): Boolean = tx.inputs.forall(inp => s.boxById(inp.boxId).isDefined) + /** + * Checks that the best full block in the history corresponds to the state. + * Evaluated via live history storage reads, so re-checking it after candidate assembly + * detects a block applied concurrently with the assembly. + */ + def isChainSynced( + bestFullBlockIdOpt: Option[ModifierId], + stateContext: ErgoStateContext + ): Boolean = + bestFullBlockIdOpt == stateContext.lastHeaderOpt.map(_.id) + + /** + * Filters out from `poolTxs` transactions included into the last applied block + * (`lastAppliedBlockTxs`), if the block is still the best full block (`bestFullBlockIdOpt`). + * Such transactions are removed from the mempool by the node view holder itself on block + * application, so there is no need to validate them during candidate assembly (which logs + * misleading double-spending messages) nor to eliminate them via EliminateTransactions. + */ + def excludeAppliedTxs( + poolTxs: Seq[UnconfirmedTransaction], + lastAppliedBlockTxs: Option[(ModifierId, Set[ModifierId])], + bestFullBlockIdOpt: Option[ModifierId] + ): Seq[UnconfirmedTransaction] = { + lastAppliedBlockTxs match { + case Some((appliedHeaderId, appliedTxIds)) + if appliedTxIds.nonEmpty && bestFullBlockIdOpt.contains(appliedHeaderId) => + poolTxs.filterNot(tx => appliedTxIds.contains(tx.id)) + case _ => + poolTxs + } + } + /** * @return None if chain is not synced or Some of attempt to create candidate */ @@ -396,6 +434,7 @@ object CandidateGenerator extends ScorexLogging { m: ErgoMemPoolReader, pk: ProveDlog, txsToInclude: Seq[ErgoTransaction], + lastAppliedBlockTxs: Option[(ModifierId, Set[ModifierId])], ergoSettings: ErgoSettings ): Option[Try[(Candidate, EliminateTransactions)]] = { // mandatory transactions to include into next block taken from the previous candidate @@ -406,14 +445,16 @@ object CandidateGenerator extends ScorexLogging { val stateContext = s.stateContext - //only transactions valid from against the current utxo state we take from the mem pool - lazy val poolTransactions = m.getAllPrioritized + //only transactions valid from against the current utxo state we take from the mem pool, + //skipping transactions already included into the last applied block + lazy val poolTransactions = + excludeAppliedTxs(m.getAllPrioritized, lastAppliedBlockTxs, h.bestFullBlockOpt.map(_.id)) lazy val emissionTxOpt = CandidateGenerator.collectEmission(s, pk, stateContext) def chainSynced = - h.bestFullBlockOpt.map(_.id) == stateContext.lastHeaderOpt.map(_.id) + isChainSynced(h.bestFullBlockOpt.map(_.id), stateContext) def hasAnyMemPoolOrMinerTx = poolTransactions.nonEmpty || unspentTxsToInclude.nonEmpty || emissionTxOpt.nonEmpty @@ -436,18 +477,25 @@ object CandidateGenerator extends ScorexLogging { } else { ergoSettings.votingTargets.desiredUpdate } - Some( - createCandidate( - pk, - h, - desiredUpdate, - s, - poolTransactions, - emissionTxOpt, - unspentTxsToInclude, - ergoSettings - ) + val candidateAttempt = createCandidate( + pk, + h, + desiredUpdate, + s, + poolTransactions, + emissionTxOpt, + unspentTxsToInclude, + ergoSettings ) + if (!chainSynced) { + log.debug( + "Discarding block candidate as a new block was applied during its assembly, " + + "a new candidate will be generated on FullBlockApplied" + ) + None + } else { + Some(candidateAttempt) + } } } diff --git a/src/main/scala/org/ergoplatform/modifiers/history/popow/NipopowProverWithDbAlgs.scala b/src/main/scala/org/ergoplatform/modifiers/history/popow/NipopowProverWithDbAlgs.scala index d4bffa263d..94dc4924fa 100644 --- a/src/main/scala/org/ergoplatform/modifiers/history/popow/NipopowProverWithDbAlgs.scala +++ b/src/main/scala/org/ergoplatform/modifiers/history/popow/NipopowProverWithDbAlgs.scala @@ -29,7 +29,6 @@ object NipopowProverWithDbAlgs { val k = params.k val m = params.m - require(params.k >= 1, s"$k < 1") require(histReader.headersHeight >= k + m, s"Can not prove chain of size < ${k + m}") def linksWithIndexes(header: PoPowHeader): Seq[(ModifierId, Int)] = header.interlinks.tail.reverse.zipWithIndex diff --git a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala index 5d7ff03a56..ced63d1e8a 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala @@ -1429,7 +1429,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, } // Locally mined block applied - skip broadcast (already done via NewBlockMined) - case LocalBlockApplied(header) => + case LocalBlockApplied(header, _) => log.debug( s"Local block applied at height ${header.height}, " + s"header id: ${header.encodedId}, skipping broadcast" @@ -1440,7 +1440,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, processFirstTxProcessingCacheRecord() // resume cache processing // Peer-received block applied - broadcast to our peers - case RemoteBlockApplied(header) => + case RemoteBlockApplied(header, _) => if (header.isNew(2.hours)) { broadcastModifierInv(Header.modifierTypeId, header.id) header.sectionIds.foreach { case (mtId, id) => diff --git a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizerMessages.scala b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizerMessages.scala index a4b568cf4f..e7784ed7c3 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizerMessages.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizerMessages.scala @@ -105,9 +105,11 @@ object ErgoNodeViewSynchronizerMessages { * Use LocalBlockApplied or RemoteBlockApplied for specific cases. * * @param header - full block's header + * @param txIds - ids of transactions included into the full block applied */ sealed trait FullBlockApplied extends ModificationOutcome { def header: Header + def txIds: Seq[ModifierId] } object FullBlockApplied { @@ -119,13 +121,13 @@ object ErgoNodeViewSynchronizerMessages { * The block was already broadcast via NewBlockMined, so no additional * inv broadcast is needed. */ - case class LocalBlockApplied(header: Header) extends FullBlockApplied + case class LocalBlockApplied(header: Header, txIds: Seq[ModifierId]) extends FullBlockApplied /** * Signal sent when a peer-received full block is applied to state. * An inv broadcast should be sent to propagate the block to our peers. */ - case class RemoteBlockApplied(header: Header) extends FullBlockApplied + case class RemoteBlockApplied(header: Header, txIds: Seq[ModifierId]) extends FullBlockApplied /** * Signal sent by CandidateGenerator when a new block is mined locally. diff --git a/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala b/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala index 56764d821d..63722861e9 100644 --- a/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala +++ b/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala @@ -237,11 +237,12 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti case Success(stateAfterApply) => history.reportModifierIsValid(modToApply).map { newHis => if (modToApply.modifierTypeId == ErgoFullBlock.modifierTypeId) { - val header = modToApply.asInstanceOf[ErgoFullBlock].header + val fullBlock = modToApply.asInstanceOf[ErgoFullBlock] + val txIds = fullBlock.blockTransactions.transactions.map(_.id) val event = if (local) { - LocalBlockApplied(header) + LocalBlockApplied(fullBlock.header, txIds) } else { - RemoteBlockApplied(header) + RemoteBlockApplied(fullBlock.header, txIds) } context.system.eventStream.publish(event) } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 30da91cec4..89cfddefb4 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -376,7 +376,10 @@ trait ExtraIndexerBase extends Actor with Stash with ScorexLogging { log.info(s"Buffered block $height / $chainHeight [txs: ${txs.length}, boxes: $boxCount] (buffer: $modCount / $saveLimit)") val maxHeight = headerOpt.map(_.height).getOrElse(chainHeight) - newState.copy(caughtUp = newState.indexedHeight == maxHeight) + newState.copy( + caughtUp = newState.indexedHeight == maxHeight, + indexedHeaderId = headerOpt.map(_.id).orElse(history.bestHeaderIdAtHeight(height)) + ) } /** @@ -457,7 +460,12 @@ trait ExtraIndexerBase extends Actor with Stash with ScorexLogging { newState = newState.incrementBoxIndex // Save changes - newState = newState.copy(indexedHeight = height, rollbackTo = 0, caughtUp = true) + newState = newState.copy( + indexedHeight = height, + rollbackTo = 0, + caughtUp = state.caughtUp, + indexedHeaderId = history.bestHeaderIdAtHeight(height) + ) historyStorage.removeExtra(toRemove.toArray) saveProgress(newState) } catch { @@ -470,10 +478,18 @@ trait ExtraIndexerBase extends Actor with Stash with ScorexLogging { protected def loaded(state: IndexerState): Receive = { case Index() if !state.caughtUp && !state.rollbackInProgress => - val newState = index(state.incrementIndexedHeight) - if (modCount >= saveLimit) saveProgress(newState) - context.become(receive.orElse(loaded(newState))) - self ! Index() + val nextHeaderOpt = history.bestHeaderAtHeight(state.indexedHeight + 1) + val extendsIndexedTip = nextHeaderOpt.forall { header => + state.indexedHeight == 0 || state.indexedHeaderId.contains(header.parentId) + } + if (extendsIndexedTip) { + val newState = index(state.incrementIndexedHeight) + if (modCount >= saveLimit) saveProgress(newState) + context.become(receive.orElse(loaded(newState))) + self ! Index() + } else { + log.info("Deferring catch-up because the next header does not extend the indexed tip") + } case Index() if state.caughtUp => if (modCount > 0) saveProgress(state) @@ -483,16 +499,30 @@ trait ExtraIndexerBase extends Actor with Stash with ScorexLogging { // after the indexer caught up with the chain, stay up to date case FullBlockApplied(header: Header) if state.caughtUp && !state.rollbackInProgress => - if (header.height == state.indexedHeight + 1) { // applied block is next in line + val indexedTipStillBest = state.indexedHeight == 0 || + (chainHeight >= state.indexedHeight && state.indexedHeaderId.exists { indexedHeaderId => + history.bestHeaderIdAtHeight(state.indexedHeight).contains(indexedHeaderId) + }) + val isDirectSuccessor = header.height == state.indexedHeight + 1 && + (state.indexedHeight == 0 || state.indexedHeaderId.contains(header.parentId)) && + history.bestHeaderIdAtHeight(header.height).contains(header.id) + + if (isDirectSuccessor) { val newState: IndexerState = index(state.incrementIndexedHeight, Some(header)) saveProgress(newState) context.become(receive.orElse(loaded(newState))) caughtUpHook(header.height) - } else if (header.height > state.indexedHeight + 1) { // applied block is ahead of indexer + } else if (!indexedTipStillBest) { + log.info(s"Deferring block ${header.id} at height ${header.height} until rollback") + context.become(receive.orElse(loaded(state.copy(caughtUp = false)))) + } else if (header.height > state.indexedHeight + 1) { context.become(receive.orElse(loaded(state.copy(caughtUp = false)))) self ! Index() - } else // applied block has already been indexed, skipping duplicate + } else { log.warn(s"Skipping block ${header.id} applied at height ${header.height}, indexed height is ${state.indexedHeight}") + } + + case _: FullBlockApplied if state.rollbackInProgress => stash() case Rollback(branchPoint: ModifierId) => if (state.rollbackInProgress) { @@ -504,6 +534,10 @@ trait ExtraIndexerBase extends Actor with Stash with ScorexLogging { if (branchHeight < state.indexedHeight) { context.become(receive.orElse(loaded(state.copy(rollbackTo = branchHeight)))) self ! RemoveAfter(branchHeight) + } else if (!state.caughtUp) { + blockCache.clear() + readingUpTo = 0 + self ! Index() } case None => log.error(s"No rollback height found for $branchPoint") @@ -518,6 +552,7 @@ trait ExtraIndexerBase extends Actor with Stash with ScorexLogging { readingUpTo = 0 val newState = removeAfter(state, branchHeight) context.become(receive.orElse(loaded(newState))) + if (!newState.caughtUp && !newState.rollbackInProgress) self ! Index() caughtUpHook() log.info(s"Successfully rolled back indexes to $branchHeight") unstashAll() diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexerState.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexerState.scala index 30f8fc4eeb..1ebcd653f7 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexerState.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexerState.scala @@ -2,6 +2,7 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.nodeView.history.ErgoHistory import org.ergoplatform.nodeView.history.extra.ExtraIndexer._ +import scorex.util.ModifierId /** * An immutable state for extra indexer @@ -10,12 +11,14 @@ import org.ergoplatform.nodeView.history.extra.ExtraIndexer._ * @param globalBoxIndex - Indexed box count * @param rollbackTo - blockheight to rollback to, 0 if no rollback is in progress * @param caughtUp - flag to indicate if the indexer is caught up with the chain and is listening for updates + * @param indexedHeaderId - id of the last block represented by the extra index */ case class IndexerState(indexedHeight: Int, globalTxIndex: Long, globalBoxIndex: Long, rollbackTo: Int, - caughtUp: Boolean) { + caughtUp: Boolean, + indexedHeaderId: Option[ModifierId] = None) { def rollbackInProgress: Boolean = rollbackTo > 0 @@ -42,7 +45,8 @@ object IndexerState { globalTxIndex, globalBoxIndex, rollbackTo, - caughtUp = indexedHeight == history.fullBlockHeight + caughtUp = indexedHeight == history.fullBlockHeight, + indexedHeaderId = history.bestHeaderIdAtHeight(indexedHeight) ) } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/PopowProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/PopowProcessor.scala index 59922347a3..5e2ebd2183 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/PopowProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/PopowProcessor.scala @@ -107,8 +107,9 @@ trait PopowProcessor extends BasicReaders with ScorexLogging { * @return PoPow proof if success, Failure instance otherwise */ def popowProof(m: Int, k: Int, headerIdOpt: Option[ModifierId]): Try[NipopowProof] = { - val proofParams = PoPowParams(m, k, continuous = true) - NipopowProverWithDbAlgs.prove(historyReader, headerIdOpt = headerIdOpt, chainSettings)(proofParams) + PoPowParams(m, k, continuous = true).flatMap { proofParams => + NipopowProverWithDbAlgs.prove(historyReader, headerIdOpt = headerIdOpt, chainSettings)(proofParams) + } } /** diff --git a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala index 6e58782dfe..a919afb76a 100644 --- a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala +++ b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala @@ -323,7 +323,8 @@ class ErgoMemPool private[mempool](private[mempool] val pool: OrderedTxPool, case _ => None } - loop(waitMinutes = 0).getOrElse(settings.nodeSettings.minimalFeeAmount) + val recommendedFee = loop(waitMinutes = 0).getOrElse(settings.nodeSettings.minimalFeeAmount) + math.max(recommendedFee, settings.nodeSettings.minimalFeeAmount) } /** @@ -346,8 +347,9 @@ class ErgoMemPool private[mempool](private[mempool] val pool: OrderedTxPool, // Time since statistics measurement interval (needed to calculate average tx rate) val elapsed = System.currentTimeMillis() - stats.startMeasurement - if (stats.takenTxns != 0) { - elapsed * posInPool / stats.takenTxns + val cappedElapsed = math.max(0L, math.min(elapsed, MemPoolStatistics.measurementIntervalMsec.toLong)) + if (stats.takenTxns > 0) { + cappedElapsed * posInPool / stats.takenTxns } else { 0 } diff --git a/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala b/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala index 5b8cd7c8bd..2d5527ec7a 100644 --- a/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala +++ b/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala @@ -41,6 +41,32 @@ class OrderedTxPool(val orderedTransactions: TreeMap[WeightedTxId, UnconfirmedTr private val mempoolCapacity = settings.nodeSettings.mempoolCapacity + private def withoutTransaction(id: ModifierId): TreeMap[WeightedTxId, UnconfirmedTransaction] = { + // Keep healthy mutations logarithmic; scan by ID only after cardinality diverges. + transactionsRegistry.get(id) match { + case Some(wtx) if orderedTransactions.size == transactionsRegistry.size && + orderedTransactions.contains(wtx) => + orderedTransactions - wtx + case None if orderedTransactions.size == transactionsRegistry.size => + orderedTransactions + case _ => + orderedTransactions.filter { case (wtx, utx) => wtx.id != id && utx.id != id } + } + } + + private def hasUnregisteredTransaction(id: ModifierId): Boolean = + orderedTransactions.size != transactionsRegistry.size && + orderedTransactions.valuesIterator.exists(_.id == id) + + private def currentTransaction(id: ModifierId): Option[(WeightedTxId, UnconfirmedTransaction)] = + transactionsRegistry.get(id) + .flatMap(wtx => orderedTransactions.get(wtx).filter(_.id == id).map(wtx -> _)) + .orElse { + orderedTransactions.iterator.collectFirst { + case (wtx, utx) if wtx.id == id && utx.id == id => wtx -> utx + } + } + def size: Int = orderedTransactions.size def get(id: ModifierId): Option[UnconfirmedTransaction] = { @@ -69,17 +95,18 @@ class OrderedTxPool(val orderedTransactions: TreeMap[WeightedTxId, UnconfirmedTr val newPool = transactionsRegistry.get(tx.id) match { case Some(wtx) => + val currentWtx = currentTransaction(tx.id).map(_._1).getOrElse(wtx) new OrderedTxPool( - orderedTransactions.updated(wtx, unconfirmedTx), - transactionsRegistry, + withoutTransaction(tx.id).updated(currentWtx, unconfirmedTx), + transactionsRegistry.updated(tx.id, currentWtx), invalidatedTxIds, - outputs, - inputs + outputs ++ tx.outputs.map(_.id -> currentWtx), + inputs ++ tx.inputs.map(_.boxId -> currentWtx) ) case None => val wtx = weighted(tx, feeFactor) new OrderedTxPool( - orderedTransactions.updated(wtx, unconfirmedTx), + withoutTransaction(tx.id).updated(wtx, unconfirmedTx), transactionsRegistry.updated(wtx.id, wtx), invalidatedTxIds, outputs ++ tx.outputs.map(_.id -> wtx), @@ -107,7 +134,7 @@ class OrderedTxPool(val orderedTransactions: TreeMap[WeightedTxId, UnconfirmedTr transactionsRegistry.get(tx.id) match { case Some(wtx) if orderedTransactions.contains(wtx) => new OrderedTxPool( - orderedTransactions - wtx, + withoutTransaction(tx.id), transactionsRegistry - tx.id, invalidatedTxIds, outputs -- tx.outputs.map(_.id), @@ -116,7 +143,7 @@ class OrderedTxPool(val orderedTransactions: TreeMap[WeightedTxId, UnconfirmedTr case Some(_) => if (orderedTransactions.valuesIterator.exists(_.id == tx.id)) { new OrderedTxPool( - orderedTransactions.filter(_._2.id != tx.id), + withoutTransaction(tx.id), transactionsRegistry - tx.id, invalidatedTxIds, outputs -- tx.outputs.map(_.id), @@ -126,7 +153,17 @@ class OrderedTxPool(val orderedTransactions: TreeMap[WeightedTxId, UnconfirmedTr this } case None => - this + if (hasUnregisteredTransaction(tx.id)) { + new OrderedTxPool( + withoutTransaction(tx.id), + transactionsRegistry, + invalidatedTxIds, + outputs -- tx.outputs.map(_.id), + inputs -- tx.inputs.map(_.boxId) + ) + } else { + this + } } } @@ -140,7 +177,7 @@ class OrderedTxPool(val orderedTransactions: TreeMap[WeightedTxId, UnconfirmedTr transactionsRegistry.get(tx.id) match { case Some(wtx) if orderedTransactions.contains(wtx) => new OrderedTxPool( - orderedTransactions - wtx, + withoutTransaction(tx.id), transactionsRegistry - tx.id, invalidatedTxIds.put(tx.id), outputs -- tx.outputs.map(_.id), @@ -149,7 +186,7 @@ class OrderedTxPool(val orderedTransactions: TreeMap[WeightedTxId, UnconfirmedTr case Some(_) => if (orderedTransactions.valuesIterator.exists(utx => utx.id == tx.id)) { new OrderedTxPool( - orderedTransactions.filter(_._2.id != tx.id), + withoutTransaction(tx.id), transactionsRegistry - tx.id, invalidatedTxIds.put(tx.id), outputs -- tx.outputs.map(_.id), @@ -159,7 +196,17 @@ class OrderedTxPool(val orderedTransactions: TreeMap[WeightedTxId, UnconfirmedTr new OrderedTxPool(orderedTransactions, transactionsRegistry, invalidatedTxIds.put(tx.id), outputs, inputs) } case None => - new OrderedTxPool(orderedTransactions, transactionsRegistry, invalidatedTxIds.put(tx.id), outputs, inputs) + if (hasUnregisteredTransaction(tx.id)) { + new OrderedTxPool( + withoutTransaction(tx.id), + transactionsRegistry, + invalidatedTxIds.put(tx.id), + outputs -- tx.outputs.map(_.id), + inputs -- tx.inputs.map(_.boxId) + ) + } else { + new OrderedTxPool(orderedTransactions, transactionsRegistry, invalidatedTxIds.put(tx.id), outputs, inputs) + } } } @@ -211,17 +258,22 @@ class OrderedTxPool(val orderedTransactions: TreeMap[WeightedTxId, UnconfirmedTr val uniqueTxIds: Set[WeightedTxId] = tx.inputs.flatMap(input => this.outputs.get(input.boxId)).toSet val parentTxs = uniqueTxIds.flatMap(wtx => this.orderedTransactions.get(wtx).map(ut => wtx -> ut)) - parentTxs.foldLeft(this) { case (pool, (wtx, ut)) => - val parent = ut.transaction - val newWtx = WeightedTxId(wtx.id, wtx.weight + weight, wtx.feePerFactor, wtx.created) - val newPool = new OrderedTxPool( - pool.orderedTransactions - wtx + (newWtx -> ut), - pool.transactionsRegistry.updated(parent.id, newWtx), - invalidatedTxIds, - parent.outputs.foldLeft(pool.outputs)((newOutputs, box) => newOutputs.updated(box.id, newWtx)), - parent.inputs.foldLeft(pool.inputs)((newInputs, inp) => newInputs.updated(inp.boxId, newWtx)) - ) - newPool.updateFamily(parent, weight, startTime, depth + 1) + parentTxs.foldLeft(this) { case (pool, (snapshotWtx, _)) => + pool.currentTransaction(snapshotWtx.id) match { + case Some((wtx, ut)) => + val parent = ut.transaction + val newWtx = WeightedTxId(wtx.id, wtx.weight + weight, wtx.feePerFactor, wtx.created) + val newPool = new OrderedTxPool( + pool.withoutTransaction(parent.id).updated(newWtx, ut), + pool.transactionsRegistry.updated(parent.id, newWtx), + pool.invalidatedTxIds, + parent.outputs.foldLeft(pool.outputs)((newOutputs, box) => newOutputs.updated(box.id, newWtx)), + parent.inputs.foldLeft(pool.inputs)((newInputs, inp) => newInputs.updated(inp.boxId, newWtx)) + ) + newPool.updateFamily(parent, weight, startTime, depth + 1) + case None => + pool + } } } } diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSupport.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSupport.scala index 03b91e19ca..17e2a7e36b 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSupport.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSupport.scala @@ -324,7 +324,7 @@ trait ErgoWalletSupport extends ScorexLogging { require(outputs.forall(_.additionalTokens.forall(_._2 > 0)), "Non-positive asset value") val assetIssueBox = outputs - .zip(requests) + .zip(requestsWithoutBurnTokens) .filter(_._2.isInstanceOf[AssetIssueRequest]) .map(_._1) .headOption diff --git a/src/main/scala/scorex/core/network/NetworkController.scala b/src/main/scala/scorex/core/network/NetworkController.scala index 8db3500db9..0cec487f35 100644 --- a/src/main/scala/scorex/core/network/NetworkController.scala +++ b/src/main/scala/scorex/core/network/NetworkController.scala @@ -163,11 +163,11 @@ class NetworkController(ergoSettings: ErgoSettings, peerManagerRef ! PeerManager.ReceivableMessages.Penalize(peerAddress, penaltyType) case Blacklisted(peerAddress) => - connections.get(peerAddress).foreach { peer => - connections = connections.filterNot { case (address, _) => // clear all connections related to banned peer ip - Option(peer.connectionId.remoteAddress.getAddress).exists(Option(address.getAddress).contains(_)) - } - peer.handlerRef ! CloseConnection + Option(peerAddress.getAddress).foreach { blacklistedIp => + val peersToClose = connections.valuesIterator.filter { peer => + Option(peer.connectionId.remoteAddress.getAddress).contains(blacklistedIp) + }.toSeq + peersToClose.foreach(_.handlerRef ! CloseConnection) } } diff --git a/src/main/scala/scorex/core/network/PeerConnectionHandler.scala b/src/main/scala/scorex/core/network/PeerConnectionHandler.scala index 8c32487397..bd6a573bed 100644 --- a/src/main/scala/scorex/core/network/PeerConnectionHandler.scala +++ b/src/main/scala/scorex/core/network/PeerConnectionHandler.scala @@ -4,14 +4,19 @@ import akka.actor.{Actor, ActorRef, Cancellable, Props, SupervisorStrategy} import akka.io.Tcp import akka.io.Tcp._ import akka.util.{ByteString, CompactByteString} -import org.ergoplatform.network.{Handshake, HandshakeSerializer, PeerSpec, Version} import org.ergoplatform.network.Version.Eip37ForkVersion -import scorex.core.app.ScorexContext -import scorex.core.network.NetworkController.ReceivableMessages.{Handshaked, PenalizePeer} -import scorex.core.network.PeerConnectionHandler.ReceivableMessages +import org.ergoplatform.network.{Handshake, HandshakeSerializer, PeerSpec, Version} +import org.ergoplatform.network.message.MessageConstants.{ + ChecksumLength, + HeaderLength, + MaxMessageSize +} import org.ergoplatform.network.message.MessageSerializer import org.ergoplatform.network.peer.{PeerInfo, PenaltyType} import org.ergoplatform.settings.ScorexSettings +import scorex.core.app.ScorexContext +import scorex.core.network.NetworkController.ReceivableMessages.{Handshaked, PenalizePeer} +import scorex.core.network.PeerConnectionHandler.ReceivableMessages import scorex.util.ScorexLogging import scala.annotation.tailrec @@ -27,6 +32,7 @@ class PeerConnectionHandler(scorexSettings: ScorexSettings, extends Actor with ScorexLogging { import PeerConnectionHandler.ReceivableMessages._ + import PeerConnectionHandler.{MaxBufferedOutboundBytes, MaxBufferedOutboundMessages} private val networkSettings = scorexSettings.network private val connection = connectionDescription.connection @@ -48,6 +54,8 @@ class PeerConnectionHandler(scorexSettings: ScorexSettings, private var outMessagesBuffer: TreeMap[Long, ByteString] = TreeMap.empty + private var outMessagesBufferBytes: Long = 0L + private var outMessagesCounter: Long = 0 override def preStart: Unit = { @@ -179,7 +187,10 @@ class PeerConnectionHandler(scorexSettings: ScorexSettings, writeFirst() case ReceivableMessages.Ack(id) => - outMessagesBuffer -= id + outMessagesBuffer.get(id).foreach { msg => + outMessagesBuffer -= id + outMessagesBufferBytes -= msg.length + } if (outMessagesBuffer.nonEmpty){ writeFirst() } else { @@ -226,7 +237,22 @@ class PeerConnectionHandler(scorexSettings: ScorexSettings, } private def buffer(id: Long, msg: ByteString): Unit = { - outMessagesBuffer += id -> msg + val previousMessage = outMessagesBuffer.get(id) + val previousLength = previousMessage.fold(0)(_.length) + val candidateBytes = outMessagesBufferBytes - previousLength + msg.length + val candidateMessages = outMessagesBuffer.size + previousMessage.fold(1)(_ => 0) + if (candidateBytes > MaxBufferedOutboundBytes || + candidateMessages > MaxBufferedOutboundMessages) { + log.warn(s"Buffered outbound data for $connectionId would exceed its limit " + + s"($candidateMessages messages, $candidateBytes bytes), aborting the connection") + outMessagesBuffer = TreeMap.empty + outMessagesBufferBytes = 0L + connection ! Abort + context.stop(self) + } else { + outMessagesBuffer += id -> msg + outMessagesBufferBytes = candidateBytes + } } private def writeFirst(): Unit = { @@ -259,6 +285,14 @@ class PeerConnectionHandler(scorexSettings: ScorexSettings, object PeerConnectionHandler { + // Keep one maximum serialized frame per peer. Backpressured snapshot transfers + // retry instead of retaining their entire application-level in-flight window. + private[network] val MaxBufferedOutboundBytes: Long = + MaxMessageSize.toLong + HeaderLength + ChecksumLength + + // Independently bound collection overhead from small messages. + private[network] val MaxBufferedOutboundMessages: Int = 64 + object ReceivableMessages { case object HandshakeTimeout diff --git a/src/test/scala/org/ergoplatform/http/routes/BlocksApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/BlocksApiRouteSpec.scala index f0862a516a..81cc53bfb1 100644 --- a/src/test/scala/org/ergoplatform/http/routes/BlocksApiRouteSpec.scala +++ b/src/test/scala/org/ergoplatform/http/routes/BlocksApiRouteSpec.scala @@ -84,6 +84,12 @@ class BlocksApiRouteSpec } } + it should "reject chain slice ranges above the maximum headers limit" in { + Get(prefix + "/chainSlice?fromHeight=0&toHeight=16385") ~> route ~> check { + status shouldBe StatusCodes.BadRequest + } + } + it should "get block by header id" in { Get(prefix + "/" + headerIdString) ~> route ~> check { status shouldBe StatusCodes.OK diff --git a/src/test/scala/org/ergoplatform/http/routes/ErgoBaseApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/ErgoBaseApiRouteSpec.scala new file mode 100644 index 0000000000..d206f03e98 --- /dev/null +++ b/src/test/scala/org/ergoplatform/http/routes/ErgoBaseApiRouteSpec.scala @@ -0,0 +1,67 @@ +package org.ergoplatform.http.routes + +import akka.actor.ActorRefFactory +import akka.http.scaladsl.model.StatusCodes +import akka.http.scaladsl.server.Route +import akka.http.scaladsl.testkit.ScalatestRouteTest +import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport +import io.circe.syntax._ +import org.ergoplatform.http.api.ErgoBaseApiRoute +import org.ergoplatform.settings.RESTApiSettings +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.net.InetSocketAddress +import scala.concurrent.duration._ + +class ErgoBaseApiRouteSpec extends AnyFlatSpec + with Matchers + with ScalatestRouteTest + with FailFastCirceSupport { + + private val restApiSettings = + RESTApiSettings(new InetSocketAddress("localhost", 8080), None, None, 10.seconds, None) + + private class TestRoute(implicit val context: ActorRefFactory) extends ErgoBaseApiRoute { + override val settings: RESTApiSettings = restApiSettings + + override val route: Route = Route.seal { + pathPrefix("modifier") { + get { + modifierId { _ => + complete(StatusCodes.OK) + } + } + } ~ + path("modifiers") { + post { + modifierIds { _ => + complete(StatusCodes.OK) + } + } + } + } + } + + private val route = new TestRoute().route + + it should "reject modifier ids with invalid byte length" in { + val validModifierId = "00" * 32 + + Get(s"/modifier/$validModifierId") ~> route ~> check { + status shouldBe StatusCodes.OK + } + + Get("/modifier/00") ~> route ~> check { + status shouldBe StatusCodes.BadRequest + } + + Post("/modifiers", Seq(validModifierId).asJson) ~> route ~> check { + status shouldBe StatusCodes.OK + } + + Post("/modifiers", Seq("00").asJson) ~> route ~> check { + status shouldBe StatusCodes.BadRequest + } + } +} diff --git a/src/test/scala/org/ergoplatform/http/routes/ErgoHttpServiceSpec.scala b/src/test/scala/org/ergoplatform/http/routes/ErgoHttpServiceSpec.scala new file mode 100644 index 0000000000..13b730ecc8 --- /dev/null +++ b/src/test/scala/org/ergoplatform/http/routes/ErgoHttpServiceSpec.scala @@ -0,0 +1,112 @@ +package org.ergoplatform.http.routes + +import akka.http.scaladsl.model.StatusCodes +import akka.http.scaladsl.server.Route +import akka.http.scaladsl.testkit.ScalatestRouteTest +import ch.qos.logback.classic.spi.ILoggingEvent +import ch.qos.logback.classic.{Level, Logger => LogbackLogger} +import ch.qos.logback.core.read.ListAppender +import org.ergoplatform.http.api.EmissionApiRoute +import org.ergoplatform.http.{ErgoHttpService, NodePanelRoute, SwaggerRoute} +import org.ergoplatform.utils.Stubs +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.slf4j.LoggerFactory + +import scala.collection.JavaConverters._ + +class ErgoHttpServiceSpec extends AnyFlatSpec + with Matchers + with ScalatestRouteTest + with Stubs { + + import org.ergoplatform.utils.ErgoNodeTestConstants._ + + private val restApiSettings = settings.scorexSettings.restApi + + private val service = ErgoHttpService( + apiRoutes = Seq(EmissionApiRoute(settings)), + swaggerRoute = SwaggerRoute(restApiSettings, swaggerConfig = ""), + panelRoute = NodePanelRoute() + ) + + private val route: Route = service.compositeRoute + + private val serviceLogger: LogbackLogger = + LoggerFactory.getLogger(classOf[ErgoHttpService]).asInstanceOf[LogbackLogger] + + /** Runs `body` while capturing what the service logs at `level` */ + private def capturingLogs[T](level: Level)(body: => T): (T, Seq[String]) = { + val appender = new ListAppender[ILoggingEvent] + appender.start() + val previousLevel = serviceLogger.getLevel + serviceLogger.setLevel(level) + serviceLogger.addAppender(appender) + try { + val result = body + (result, appender.list.asScala.map(_.getFormattedMessage).toList) + } finally { + serviceLogger.detachAppender(appender) + serviceLogger.setLevel(previousLevel) + appender.stop() + } + } + + it should "log served queries at DEBUG level" in { + val (_, messages) = capturingLogs(Level.DEBUG) { + Get("/emission/at/100") ~> route ~> check { + status shouldBe StatusCodes.OK + } + } + + val logged = messages.filter(_.startsWith("GET /emission/at/100")) + logged.size shouldBe 1 + // method, uri, response status and elapsed time, and nothing else + logged.head should fullyMatch regex """GET /emission/at/100 - 200 in \d+ ms""" + } + + it should "log the query string, and log unmatched paths with the status they were rejected with" in { + val (rejectedStatus, messages) = capturingLogs(Level.DEBUG) { + Get("/emission/at/100?foo=bar") ~> route ~> check { + status shouldBe StatusCodes.OK + } + Get("/no/such/route") ~> route ~> check { + status.isSuccess() shouldBe false + status.intValue() + } + } + + messages.exists(_.startsWith("GET /emission/at/100?foo=bar - 200 in ")) shouldBe true + // rejections are turned into responses by the rejection handler, so they are logged too + messages.exists(_.startsWith(s"GET /no/such/route - $rejectedStatus in ")) shouldBe true + } + + it should "log nothing when the logger is not at DEBUG level" in { + val (_, messages) = capturingLogs(Level.INFO) { + Get("/emission/at/100") ~> route ~> check { + status shouldBe StatusCodes.OK + } + } + + messages shouldBe empty + } + + it should "not change the response when logging is enabled" in { + val body = capturingLogs(Level.DEBUG) { + Get("/emission/at/100") ~> route ~> check { + status shouldBe StatusCodes.OK + responseAs[String] + } + }._1 + + val bodyWithoutLogging = capturingLogs(Level.OFF) { + Get("/emission/at/100") ~> route ~> check { + status shouldBe StatusCodes.OK + responseAs[String] + } + }._1 + + body shouldBe bodyWithoutLogging + } + +} diff --git a/src/test/scala/org/ergoplatform/http/routes/MiningApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/MiningApiRouteSpec.scala index 2194d6c7aa..b480ff6025 100644 --- a/src/test/scala/org/ergoplatform/http/routes/MiningApiRouteSpec.scala +++ b/src/test/scala/org/ergoplatform/http/routes/MiningApiRouteSpec.scala @@ -1,7 +1,7 @@ package org.ergoplatform.http.routes import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.server.Route +import akka.http.scaladsl.server.{AuthorizationFailedRejection, Route} import akka.http.scaladsl.testkit.ScalatestRouteTest import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport import io.circe.Json @@ -32,6 +32,15 @@ class MiningApiRouteSpec val localSetting: ErgoSettings = settings.copy(nodeSettings = settings.nodeSettings.copy(useExternalMiner = true)) val route: Route = MiningApiRoute(minerRef, localSetting).route + val settingsWithAuth: ErgoSettings = localSetting.copy( + scorexSettings = localSetting.scorexSettings.copy( + restApi = localSetting.scorexSettings.restApi.copy( + apiKeyHash = Some("e1c7ef7b3b742c5ae8f52c24d2c5f5c5dccd9c23a41fa6e76e5a6b62c8f72a10") + ) + ) + ) + val routeWithAuth: Route = MiningApiRoute(minerRef, settingsWithAuth).route + val solution = AutolykosSolution(genECPoint.sample.get, genECPoint.sample.get, Array.fill(32)(9: Byte), BigInt(0)) // Valid compressed public key hex (33 bytes = 66 hex chars) - using a valid secp256k1 point @@ -68,6 +77,36 @@ class MiningApiRouteSpec } } + it should "return candidate without api_key when auth is enabled" in { + Get(prefix + "/candidate") ~> routeWithAuth ~> check { + status shouldBe StatusCodes.OK + } + } + + it should "return reward address without api_key when auth is enabled" in { + Get(prefix + "/rewardAddress") ~> routeWithAuth ~> check { + status shouldBe StatusCodes.OK + } + } + + it should "return reward public key without api_key when auth is enabled" in { + Get(prefix + "/rewardPublicKey") ~> routeWithAuth ~> check { + status shouldBe StatusCodes.OK + } + } + + it should "accept solution without api_key when auth is enabled" in { + Post(prefix + "/solution", solution.asJson) ~> routeWithAuth ~> check { + status shouldBe StatusCodes.OK + } + } + + it should "reject candidateWithTxs without api_key when auth is enabled" in { + Post(prefix + "/candidateWithTxs", Json.arr()) ~> routeWithAuth ~> check { + rejection shouldBe AuthorizationFailedRejection + } + } + it should "encode and decode MiningRequest correctly" in { val request = MiningRequest(Seq.empty, validPkHex) diff --git a/src/test/scala/org/ergoplatform/http/routes/NipopowApiRoutesSpec.scala b/src/test/scala/org/ergoplatform/http/routes/NipopowApiRoutesSpec.scala index cfc95a8337..205aedc0bf 100644 --- a/src/test/scala/org/ergoplatform/http/routes/NipopowApiRoutesSpec.scala +++ b/src/test/scala/org/ergoplatform/http/routes/NipopowApiRoutesSpec.scala @@ -35,6 +35,12 @@ class NipopowApiRoutesSpec extends AnyFlatSpec } } + it should "reject proof request when minimum and suffix length overflow" in { + Get(s"/nipopow/proof/${Int.MaxValue}/1") ~> route ~> check { + status shouldBe StatusCodes.BadRequest + } + } + it should "proof request with missing headerId" in { Get("/nipopow/proof/1/1/05bf63aa1ecfc9f4e3fadc993f87b33edb4d58e151c1891816d734dd5a0e2e09") ~> route ~> check { status shouldBe StatusCodes.BadRequest diff --git a/src/test/scala/org/ergoplatform/http/routes/ScanApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/ScanApiRouteSpec.scala index 4898defea9..d2d00ac71b 100644 --- a/src/test/scala/org/ergoplatform/http/routes/ScanApiRouteSpec.scala +++ b/src/test/scala/org/ergoplatform/http/routes/ScanApiRouteSpec.scala @@ -39,6 +39,7 @@ class ScanApiRouteSpec extends AnyFlatSpec val ergoSettings: ErgoSettings = ErgoSettingsReader.read( Args(userConfigPathOpt = Some("src/test/resources/application.conf"), networkTypeOpt = None)) val route: Route = ScanApiRoute(utxoReadersRef, ergoSettings).route + val sealedRoute: Route = Route.seal(route) private val predicate0 = ContainsScanningPredicate(ErgoBox.R4, ByteArrayConstant(Array(0: Byte, 1: Byte))) private val predicate1 = ContainsScanningPredicate(ErgoBox.R4, ByteArrayConstant(Array(1: Byte, 1: Byte))) @@ -101,6 +102,16 @@ class ScanApiRouteSpec extends AnyFlatSpec } } + it should "reject scan ids outside Short range" in { + Get(prefix + "/unspentBoxes/70000") ~> sealedRoute ~> check { + status shouldBe StatusCodes.BadRequest + } + + Get(prefix + "/spentBoxes/70000") ~> sealedRoute ~> check { + status shouldBe StatusCodes.BadRequest + } + } + it should "list unspent boxes for a scan with lower constraint" in { val minConfirmations = 15 val minInclusionHeight = 20 diff --git a/src/test/scala/org/ergoplatform/http/routes/ScriptApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/ScriptApiRouteSpec.scala index cb366b33b6..9461378d31 100644 --- a/src/test/scala/org/ergoplatform/http/routes/ScriptApiRouteSpec.scala +++ b/src/test/scala/org/ergoplatform/http/routes/ScriptApiRouteSpec.scala @@ -33,6 +33,15 @@ class ScriptApiRouteSpec extends AnyFlatSpec Args(userConfigPathOpt = Some("src/test/resources/application.conf"), networkTypeOpt = None)) val route: Route = ScriptApiRoute(digestReadersRef, settings).route + val settingsWithAuth: ErgoSettings = settings.copy( + scorexSettings = settings.scorexSettings.copy( + restApi = settings.scorexSettings.restApi.copy( + apiKeyHash = Some("e1c7ef7b3b742c5ae8f52c24d2c5f5c5dccd9c23a41fa6e76e5a6b62c8f72a10") + ) + ) + ) + val routeWithAuth: Route = ScriptApiRoute(digestReadersRef, settingsWithAuth).route + val scriptSource: String = """ |{ @@ -262,4 +271,16 @@ class ScriptApiRouteSpec extends AnyFlatSpec } } + it should "generate p2sAddress without api_key when auth is enabled" in { + Post(prefix + "/p2sAddress", Json.obj("source" -> scriptSource.asJson, "treeVersion" -> 0.asJson)) ~> routeWithAuth ~> check { + status shouldBe StatusCodes.OK + } + } + + it should "generate p2shAddress without api_key when auth is enabled" in { + Post(prefix + "/p2shAddress", Json.obj("source" -> scriptSource.asJson, "treeVersion" -> 0.asJson)) ~> routeWithAuth ~> check { + status shouldBe StatusCodes.OK + } + } + } diff --git a/src/test/scala/org/ergoplatform/http/routes/TransactionApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/TransactionApiRouteSpec.scala index 38dd7290b7..a9a06fc040 100644 --- a/src/test/scala/org/ergoplatform/http/routes/TransactionApiRouteSpec.scala +++ b/src/test/scala/org/ergoplatform/http/routes/TransactionApiRouteSpec.scala @@ -240,6 +240,23 @@ class TransactionApiRouteSpec extends AnyFlatSpec } } + it should "reject box and token ids with invalid byte length" in { + val shortId = "00" + val sealedRoute = Route.seal(route) + + Get(prefix + s"/unconfirmed/inputs/byBoxId/$shortId") ~> sealedRoute ~> check { + status shouldBe StatusCodes.BadRequest + } + + Get(prefix + s"/unconfirmed/outputs/byBoxId/$shortId") ~> sealedRoute ~> check { + status shouldBe StatusCodes.BadRequest + } + + Get(prefix + s"/unconfirmed/outputs/byTokenId/$shortId") ~> sealedRoute ~> check { + status shouldBe StatusCodes.BadRequest + } + } + it should "return unconfirmed outputs by exact same registers" in { val searchedRegs = Map( diff --git a/src/test/scala/org/ergoplatform/http/routes/WalletApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/WalletApiRouteSpec.scala index abbab0c452..1a96dd7557 100644 --- a/src/test/scala/org/ergoplatform/http/routes/WalletApiRouteSpec.scala +++ b/src/test/scala/org/ergoplatform/http/routes/WalletApiRouteSpec.scala @@ -41,6 +41,7 @@ class WalletApiRouteSpec extends AnyFlatSpec val ergoSettings: ErgoSettings = ErgoSettingsReader.read( Args(userConfigPathOpt = Some("src/test/resources/application.conf"), networkTypeOpt = None)) val route: Route = WalletApiRoute(digestReadersRef, nodeViewRef, settings).route + val sealedRoute: Route = Route.seal(route) val failingNodeViewRef = system.actorOf(NodeViewStub.failingProps()) val failingRoute: Route = WalletApiRoute(digestReadersRef, failingNodeViewRef, settings).route @@ -307,6 +308,16 @@ class WalletApiRouteSpec extends AnyFlatSpec } } + it should "reject invalid transactionsByScanId values" in { + Get(prefix + "/transactionsByScanId/not-a-number") ~> sealedRoute ~> check { + status shouldBe StatusCodes.BadRequest + } + + Get(prefix + "/transactionsByScanId/32768") ~> sealedRoute ~> check { + status shouldBe StatusCodes.BadRequest + } + } + it should "get lock status" in { Get(prefix + "/status") ~> route ~> check { status shouldBe StatusCodes.OK diff --git a/src/test/scala/org/ergoplatform/local/NipopowVerifierSpec.scala b/src/test/scala/org/ergoplatform/local/NipopowVerifierSpec.scala index 1216d77244..e4b8543b8d 100644 --- a/src/test/scala/org/ergoplatform/local/NipopowVerifierSpec.scala +++ b/src/test/scala/org/ergoplatform/local/NipopowVerifierSpec.scala @@ -1,5 +1,8 @@ package org.ergoplatform.local +import java.util.concurrent.{CountDownLatch, TimeUnit} +import java.util.concurrent.atomic.AtomicReference + import org.ergoplatform.modifiers.history.popow.{PoPowHeader, PoPowParams} import org.ergoplatform.modifiers.ErgoFullBlock import org.scalatest.matchers.should.Matchers @@ -11,7 +14,7 @@ class NipopowVerifierSpec extends AnyPropSpec with Matchers { import org.ergoplatform.utils.generators.ChainGenerator._ - private val poPowParams = PoPowParams(30, 30, continuous = false) + private val poPowParams = PoPowParams(30, 30, continuous = false).get val toPoPoWChain = (c: Seq[ErgoFullBlock]) => c.map(b => PoPowHeader.fromBlock(b).get) property("processes new proofs") { @@ -43,4 +46,49 @@ class NipopowVerifierSpec extends AnyPropSpec with Matchers { verifier.bestChain.last.id shouldBe longestProof.headersChain.last.id } } + + property("rejects proofs with invalid security parameters") { + val baseChain = genChain(100) + val params = PoPowParams(5, 5, continuous = false).get + val proof = nipopowAlgos.prove(toPoPoWChain(baseChain))(params).get + + Seq( + proof.copy(m = 0), + proof.copy(k = 0), + proof.copy(m = Int.MaxValue, k = 1) + ).foreach { invalidProof => + val proofBytes = invalidProof.serializer.toBytes(invalidProof) + val receivedProof = invalidProof.serializer.parseBytes(proofBytes) + receivedProof.isValid shouldBe false + + val verifier = new NipopowVerifier(Some(baseChain.head.id)) + verifier.process(receivedProof) shouldBe ValidationError + verifier.bestChain shouldBe empty + } + } + + property("returns when a duplicate invalid proof is processed") { + val baseChain = genChain(100) + val params = PoPowParams(5, 5, continuous = false).get + val invalidProof = nipopowAlgos.prove(toPoPoWChain(baseChain))(params).get.copy(m = 0) + val proofBytes = invalidProof.serializer.toBytes(invalidProof) + val receivedProof = invalidProof.serializer.parseBytes(proofBytes) + val verifier = new NipopowVerifier(Some(baseChain.head.id)) + + val firstResult = verifier.process(receivedProof) + val secondResult = new AtomicReference[NipopowProofVerificationResult]() + val completed = new CountDownLatch(1) + val worker = new Thread(new Runnable { + override def run(): Unit = + try secondResult.set(verifier.process(receivedProof)) + finally completed.countDown() + }) + worker.setDaemon(true) + worker.start() + + completed.await(2, TimeUnit.SECONDS) shouldBe true + firstResult shouldBe ValidationError + secondResult.get() shouldBe ValidationError + verifier.bestChain shouldBe empty + } } diff --git a/src/test/scala/org/ergoplatform/mining/CandidateGeneratorPropSpec.scala b/src/test/scala/org/ergoplatform/mining/CandidateGeneratorPropSpec.scala index 6b1a874a0e..64047e3f97 100644 --- a/src/test/scala/org/ergoplatform/mining/CandidateGeneratorPropSpec.scala +++ b/src/test/scala/org/ergoplatform/mining/CandidateGeneratorPropSpec.scala @@ -1,12 +1,14 @@ package org.ergoplatform.mining import org.ergoplatform.ErgoTreePredef +import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnconfirmedTransaction} import org.ergoplatform.nodeView.history.ErgoHistoryUtils._ import org.ergoplatform.nodeView.state.ErgoStateContext import org.ergoplatform.settings.MonetarySettings import org.ergoplatform.utils.{BoxUtils, ErgoCorePropertyTest, RandomWrapper} import org.ergoplatform.wallet.interpreter.ErgoInterpreter import org.scalacheck.Gen +import scorex.util.{ModifierId, bytesToId} import sigma.data.ProveDlog import scala.concurrent.duration._ @@ -280,6 +282,143 @@ class CandidateGeneratorPropSpec extends ErgoCorePropertyTest { } } + property("stale emission tx is invalidated when its box was spent by concurrently applied block") { + val us0 = createUtxoState(settings)._1 + us0.emissionBoxOpt should not be None + val emissionTx = + CandidateGenerator.collectEmission(us0, defaultMinerPk, emptyStateContext).toSeq.head + + val appliedBlock = validFullBlock(None, us0, Seq(emissionTx)) + val us1 = us0.applyModifier(appliedBlock, None)(_ => ()).get + + val h = appliedBlock.header + val upcomingContext = us1.stateContext.upcoming( + h.minerPk, + h.timestamp, + h.nBits, + h.votes, + emptyVSUpdate, + h.version + ) + + val (collected, invalid) = CandidateGenerator.collectTxs( + defaultMinerPk, + parameters.maxBlockCost, + parameters.maxBlockSize, + us1, + upcomingContext, + Seq(emissionTx) + ) + + collected shouldBe empty + invalid shouldBe Seq(emissionTx.id) + } + + property("mempool transactions spent by applied block are invalidated at next candidate assembly") { + val bh = boxesHolderGen.sample.get + val rnd = new RandomWrapper + val us0 = createUtxoState(bh, parameters) + val minValue = BoxUtils.sufficientAmount(parameters) + val inputs = bh.boxes.values.toIndexedSeq.filter(_.value >= minValue * 2).takeRight(10) + val mempoolTxs = + inputs.map(i => validTransactionFromBoxes(IndexedSeq(i), rnd, issueNew = false, feeProp)) + + val appliedBlock = validFullBlock(None, us0, mempoolTxs) + val us1 = us0.applyModifier(appliedBlock, None)(_ => ()).get + + val h = appliedBlock.header + val upcomingContext = us1.stateContext.upcoming( + h.minerPk, + h.timestamp, + h.nBits, + h.votes, + emptyVSUpdate, + h.version + ) + + val (collected, invalid) = CandidateGenerator.collectTxs( + defaultMinerPk, + parameters.maxBlockCost, + parameters.maxBlockSize, + us1, + upcomingContext, + mempoolTxs + ) + + collected shouldBe empty + invalid should contain theSameElementsAs mempoolTxs.map(_.id) + } + + property("zero-fee transactions are collected without creating fee transaction") { + val bh = boxesHolderGen.sample.get + val rnd = new RandomWrapper + val us = createUtxoState(bh, parameters) + val minValue = BoxUtils.sufficientAmount(parameters) + val inputs = bh.boxes.values.toIndexedSeq.filter(_.value >= minValue * 2).takeRight(5) + val zeroFeeTxs = inputs.map(i => validTransactionFromBoxes(IndexedSeq(i), rnd, issueNew = false)) + zeroFeeTxs should not be empty + + val h = validFullBlock(None, us, bh, rnd).header + val upcomingContext = us.stateContext.upcoming( + h.minerPk, + h.timestamp, + h.nBits, + h.votes, + emptyVSUpdate, + h.version + ) + + val (collected, invalid) = CandidateGenerator.collectTxs( + defaultMinerPk, + parameters.maxBlockCost, + parameters.maxBlockSize, + us, + upcomingContext, + zeroFeeTxs + ) + + invalid shouldBe empty + collected should contain theSameElementsAs zeroFeeTxs + } + + property("excludeAppliedTxs filters transactions of the applied best block only") { + val now = System.currentTimeMillis() + def utx(t: ErgoTransaction): UnconfirmedTransaction = + new UnconfirmedTransaction(t, None, now, now, None, None) + + val tx1 = validErgoTransactionGen.sample.get._2 + val tx2 = validErgoTransactionGen.sample.get._2 + val tx3 = validErgoTransactionGen.sample.get._2 + val pool = Seq(utx(tx1), utx(tx2), utx(tx3)) + val appliedId = bytesToId(Array.fill(32)(11.toByte)) + val otherId = bytesToId(Array.fill(32)(22.toByte)) + + CandidateGenerator.excludeAppliedTxs( + pool, + Some(appliedId -> Set(tx1.id, tx3.id)), + Some(appliedId) + ).map(_.id) shouldBe Seq(tx2.id) + + CandidateGenerator.excludeAppliedTxs( + pool, + Some(appliedId -> Set(tx1.id, tx3.id)), + Some(otherId) + ).map(_.id) shouldBe Seq(tx1.id, tx2.id, tx3.id) + + CandidateGenerator.excludeAppliedTxs(pool, None, Some(appliedId)) shouldBe pool + + CandidateGenerator.excludeAppliedTxs( + pool, + Some(appliedId -> Set.empty[ModifierId]), + Some(appliedId) + ) shouldBe pool + } + + property("isChainSynced compares best full block id with state context last header id") { + CandidateGenerator.isChainSynced(None, emptyStateContext) shouldBe true + CandidateGenerator.isChainSynced(Some(bytesToId(Array.fill(32)(33.toByte))), emptyStateContext) shouldBe false + } + property("it should calculate average block mining time from creation timestamps") { val timestamps1 = System.currentTimeMillis() val timestamps2 = timestamps1 + 100 diff --git a/src/test/scala/org/ergoplatform/mining/CandidateGeneratorSpec.scala b/src/test/scala/org/ergoplatform/mining/CandidateGeneratorSpec.scala index e9b4351c59..8eaae8fe35 100644 --- a/src/test/scala/org/ergoplatform/mining/CandidateGeneratorSpec.scala +++ b/src/test/scala/org/ergoplatform/mining/CandidateGeneratorSpec.scala @@ -1,6 +1,6 @@ package org.ergoplatform.mining -import akka.actor.{ActorRef, ActorSystem} +import akka.actor.{Actor, ActorRef, ActorSystem, Props} import akka.pattern.{StatusReply, ask} import akka.testkit.{TestKit, TestProbe} import akka.util.Timeout @@ -9,15 +9,21 @@ import org.ergoplatform.mining.CandidateGenerator.{Candidate, GenerateCandidate} import org.ergoplatform.modifiers.ErgoFullBlock import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnconfirmedTransaction, UnsignedErgoTransaction} -import org.ergoplatform.network.ErgoNodeViewSynchronizerMessages.FullBlockApplied -import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.LocallyGeneratedTransaction +import org.ergoplatform.network.ErgoNodeViewSynchronizerMessages.{ChangedMempool, FullBlockApplied, LocalBlockApplied} + +import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.{EliminateTransactions, LocallyGeneratedTransaction} import org.ergoplatform.nodeView.ErgoReadersHolder.{GetReaders, Readers} -import org.ergoplatform.nodeView.history.ErgoHistoryReader +import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader} +import org.ergoplatform.nodeView.mempool.ErgoMemPool import org.ergoplatform.nodeView.state.StateType +import org.ergoplatform.nodeView.wallet.ErgoWalletReader import org.ergoplatform.nodeView.{ErgoNodeViewRef, ErgoReadersHolderRef} import org.ergoplatform.settings.NetworkType.DevNet60 import org.ergoplatform.settings.{ErgoSettings, ErgoSettingsReader} import org.ergoplatform.utils.ErgoTestHelpers +import org.ergoplatform.utils.generators.ValidBlocksGenerators.{createUtxoState, validFullBlock, validTransactionsFromBoxHolder} +import org.ergoplatform.utils.generators.ChainGenerator.{applyChain, genHeaderChain} +import org.ergoplatform.utils.{HistoryTestHelpers, RandomWrapper} import org.ergoplatform.{ErgoBox, ErgoBoxCandidate, ErgoTreePredef, Input} import org.scalatest.concurrent.Eventually import org.scalatest.flatspec.AnyFlatSpec @@ -1151,4 +1157,151 @@ class CandidateGeneratorSpec extends AnyFlatSpec with Matchers with ErgoTestHelp system.terminate() } + private class FixedReadersHolder(readers: Readers) extends Actor { + override def receive: Receive = { + case GetReaders => sender() ! readers + } + } + + private def walletStub(implicit system: ActorSystem): ErgoWalletReader = new ErgoWalletReader { + val walletActor: ActorRef = system.deadLetters + } + + private def testSettings(directory: String): ErgoSettings = { + defaultSettings.copy( + directory = directory, + nodeSettings = defaultSettings.nodeSettings.copy( + blockCandidateGenerationInterval = 1.second + ) + ) + } + + private def historyWithBestFullBlock(blocks: Seq[ErgoFullBlock]): ErgoHistory = { + val h0 = HistoryTestHelpers.generateHistory( + verifyTransactions = true, + stateType = StateType.Utxo, + PoPoWBootstrap = false, + blocksToKeep = 100 + ) + val h1 = applyChain(h0, blocks) + val extraHeaders = genHeaderChain(2, h1, diffBitsOpt = None, useRealTs = false) + extraHeaders.headers.drop(h1.headersHeight).foldLeft(h1) { case (h, header) => + h.append(header).get._1 + } + } + + it should "exclude applied transactions from stale mempool and not eliminate them" in new TestKit( + ActorSystem() + ) { + + val testDir = s"${defaultSettings.directory}-a1-stale-${System.currentTimeMillis()}" + val settings = testSettings(testDir) + val viewHolderProbe = TestProbe() + val senderProbe = TestProbe() + + val (us0, bh0) = createUtxoState(settings) + val rnd = new RandomWrapper + + val (txs1, bh1) = validTransactionsFromBoxHolder(bh0, rnd) + txs1 should not be empty + val block1 = validFullBlock(None, us0, txs1) + val us1 = us0.applyModifier(block1, None)(_ => ()).get + + val (txs2, _) = validTransactionsFromBoxHolder(bh1, rnd) + txs2 should not be empty + val tx = txs2.head + val block2 = validFullBlock(Some(block1), us1, txs2) + val us2 = us1.applyModifier(block2, None)(_ => ()).get + + val history0 = HistoryTestHelpers.generateHistory( + verifyTransactions = true, + stateType = StateType.Utxo, + PoPoWBootstrap = false, + blocksToKeep = 100 + ) + val history2 = applyChain(history0, Seq(block1, block2)) + + val wallet = walletStub + val emptyMempool = ErgoMemPool.empty(settings) + val readers = Readers(history2, us2, emptyMempool, wallet) + + val readersHolderRef = system.actorOf(Props(new FixedReadersHolder(readers))) + val candidateGenerator = CandidateGenerator( + defaultMinerSecret.publicImage, + readersHolderRef, + viewHolderProbe.ref, + settings + ) + + // let the actor initialize and generate an initial candidate with the empty mempool + candidateGenerator.tell(GenerateCandidate(Seq.empty, reply = true, forced = false), senderProbe.ref) + senderProbe.expectMsgPF(candidateGenDelay) { + case StatusReply.Success(_: Candidate) => () + } + + candidateGenerator ! LocalBlockApplied(block2.header, Seq(tx.id)) + + val staleMempool = ErgoMemPool.empty(settings).put(Seq(UnconfirmedTransaction(tx, None))) + staleMempool.getAllPrioritized.map(_.id) should contain(tx.id) + candidateGenerator ! ChangedMempool(staleMempool) + + candidateGenerator.tell(GenerateCandidate(Seq.empty, reply = true, forced = true), senderProbe.ref) + + val candidate = senderProbe.expectMsgPF(candidateGenDelay) { + case StatusReply.Success(c: Candidate) => c + } + + candidate.candidateBlock.transactions should not be empty + candidate.candidateBlock.transactions.map(_.id) should not contain tx.id + + val eliminatedIds = viewHolderProbe.receiveWhile(500.millis) { + case e: EliminateTransactions => e + }.flatMap(_.ids) + eliminatedIds should not contain tx.id + + system.terminate() + } + + it should "discard candidate when history and state are out of sync" in new TestKit( + ActorSystem() + ) { + + val testDir = s"${defaultSettings.directory}-b1-sync-${System.currentTimeMillis()}" + val settings = testSettings(testDir) + val viewHolderProbe = TestProbe() + val senderProbe = TestProbe() + + val (us0, bh0) = createUtxoState(settings) + val rnd = new RandomWrapper + val (txs1, bh1) = validTransactionsFromBoxHolder(bh0, rnd) + txs1 should not be empty + + val block1 = validFullBlock(None, us0, txs1) + val us1 = us0.applyModifier(block1, None)(_ => ()).get + val history1 = historyWithBestFullBlock(Seq(block1)) + + val (txs2, _) = validTransactionsFromBoxHolder(bh1, rnd) + txs2 should not be empty + val block2 = validFullBlock(Some(block1), us1, txs2) + val us2 = us1.applyModifier(block2, None)(_ => ()).get + + val mempool = ErgoMemPool.empty(settings) + val wallet = walletStub + val readers = Readers(history1, us2, mempool, wallet) + + val readersHolderRef = system.actorOf(Props(new FixedReadersHolder(readers))) + val candidateGenerator = CandidateGenerator( + defaultMinerSecret.publicImage, + readersHolderRef, + viewHolderProbe.ref, + settings + ) + + candidateGenerator.tell(GenerateCandidate(Seq.empty, reply = true, forced = true), senderProbe.ref) + senderProbe.expectNoMessage(2.seconds) + viewHolderProbe.expectNoMessage(500.millis) + + system.terminate() + } + } diff --git a/src/test/scala/org/ergoplatform/modifiers/history/PoPowAlgosSpec.scala b/src/test/scala/org/ergoplatform/modifiers/history/PoPowAlgosSpec.scala index 66d903e79b..54c01f2ecb 100644 --- a/src/test/scala/org/ergoplatform/modifiers/history/PoPowAlgosSpec.scala +++ b/src/test/scala/org/ergoplatform/modifiers/history/PoPowAlgosSpec.scala @@ -1,5 +1,8 @@ package org.ergoplatform.modifiers.history +import java.util.concurrent.{CountDownLatch, TimeUnit} +import java.util.concurrent.atomic.AtomicReference + import org.ergoplatform.modifiers.history.popow.{NipopowAlgos, NipopowProof, PoPowHeader, PoPowParams} import org.ergoplatform.modifiers.ErgoFullBlock import org.scalacheck.Gen @@ -12,11 +15,45 @@ class PoPowAlgosSpec extends AnyPropSpec with Matchers { import org.ergoplatform.utils.generators.CoreObjectGenerators._ import org.ergoplatform.utils.ErgoCoreTestConstants._ - private val poPowParams = PoPowParams(30, 30, continuous = false) + private val poPowParams = PoPowParams(30, 30, continuous = false).get private val ChainLength = 10 private def toPoPoWChain = (c: Seq[ErgoFullBlock]) => c.map(b => PoPowHeader.fromBlock(b).get) + property("PoPowParams rejects invalid minimum chain lengths") { + PoPowParams.isValid(0, 1) shouldBe false + PoPowParams.isValid(1, 0) shouldBe false + PoPowParams.isValid(Int.MaxValue, 1) shouldBe false + + PoPowParams(0, 1, continuous = false) shouldBe 'failure + PoPowParams(1, 0, continuous = false) shouldBe 'failure + PoPowParams(Int.MaxValue, 1, continuous = false) shouldBe 'failure + + PoPowParams.isValid(Int.MaxValue - 1, 1) shouldBe true + PoPowParams(1, 1, continuous = false).get.minChainLength shouldBe 2 + } + + property("bestArg rejects a non-positive security parameter without looping") { + val algos = nipopowAlgos + val completed = new CountDownLatch(1) + val error = new AtomicReference[Throwable]() + val worker = new Thread(new Runnable { + override def run(): Unit = + try { + algos.bestArg(Seq.empty)(0) + } catch { + case t: Throwable => error.set(t) + } finally { + completed.countDown() + } + }) + worker.setDaemon(true) + worker.start() + + completed.await(2, TimeUnit.SECONDS) shouldBe true + error.get() shouldBe a[IllegalArgumentException] + } + property("updateInterlinks") { val chain = genChain(ChainLength) val genesis = chain.head @@ -144,7 +181,7 @@ class PoPowAlgosSpec extends AnyPropSpec with Matchers { } property("isBetterThan - a disconnected prefix chain should not win") { - val smallPoPowParams = PoPowParams(50, 1, continuous = false) + val smallPoPowParams = PoPowParams(50, 1, continuous = false).get val size = 100 val chain = toPoPoWChain(genChain(size)) val proof = nipopowAlgos.prove(chain)(smallPoPowParams).get @@ -158,7 +195,7 @@ class PoPowAlgosSpec extends AnyPropSpec with Matchers { } property("hasValidConnections - ensures a connected prefix chain") { - val smallPoPowParams = PoPowParams(5, 5, continuous = false) + val smallPoPowParams = PoPowParams(5, 5, continuous = false).get val sizes = Seq(100, 200) sizes.foreach { size => val chain = toPoPoWChain(genChain(size)) @@ -172,7 +209,7 @@ class PoPowAlgosSpec extends AnyPropSpec with Matchers { } property("hasValidConnections - ensures a connected suffix chain") { - val smallPoPowParams = PoPowParams(5, 5, continuous = false) + val smallPoPowParams = PoPowParams(5, 5, continuous = false).get val sizes = Seq(100, 200) sizes.foreach { size => diff --git a/src/test/scala/org/ergoplatform/modifiers/history/PoPowAlgosWithDBSpec.scala b/src/test/scala/org/ergoplatform/modifiers/history/PoPowAlgosWithDBSpec.scala index c81df767c2..f708e6e966 100644 --- a/src/test/scala/org/ergoplatform/modifiers/history/PoPowAlgosWithDBSpec.scala +++ b/src/test/scala/org/ergoplatform/modifiers/history/PoPowAlgosWithDBSpec.scala @@ -12,7 +12,7 @@ class PoPowAlgosWithDBSpec extends AnyPropSpec with Matchers { import org.ergoplatform.utils.generators.ChainGenerator._ property("proof(chain) is equivalent to proof(histReader)") { - val poPowParams = PoPowParams(m = 5, k = 6, continuous = false) + val poPowParams = PoPowParams(m = 5, k = 6, continuous = false).get val blocksChain = genChain(3000) val pchain = blocksChain.map(b => PoPowHeader.fromBlock(b).get) val proof0 = nipopowAlgos.prove(pchain)(poPowParams).get @@ -30,7 +30,7 @@ class PoPowAlgosWithDBSpec extends AnyPropSpec with Matchers { } property("proof(histReader) for a header in the past") { - val poPowParams = PoPowParams(5, 6, continuous = false) + val poPowParams = PoPowParams(5, 6, continuous = false).get val blocksChain = genChain(300) val at = 200 diff --git a/src/test/scala/org/ergoplatform/network/ErgoNodeViewSynchronizerSpecification.scala b/src/test/scala/org/ergoplatform/network/ErgoNodeViewSynchronizerSpecification.scala index 7a24884774..3e82649f70 100644 --- a/src/test/scala/org/ergoplatform/network/ErgoNodeViewSynchronizerSpecification.scala +++ b/src/test/scala/org/ergoplatform/network/ErgoNodeViewSynchronizerSpecification.scala @@ -227,6 +227,37 @@ class ErgoNodeViewSynchronizerSpecification extends AnyPropSpec } } + property("NodeViewSynchronizer: Message: InvSpec - header next to the best one is requested via RequestModifier") { + withFixture { ctx => + import ctx._ + deliveryTracker.reset() + + // header immediately following the best header the node has + // (history applied to the synchronizer contains the first 1000 headers of `chain`) + val nextHeader = chain.take(1001).last + deliveryTracker.status(nextHeader.id, Header.modifierTypeId, Seq.empty) shouldBe Unknown + + // a peer announces the header via an Inv message + val invData = InvData(Header.modifierTypeId, Seq(nextHeader.id)) + synchronizer ! Message(InvSpec, Left(InvSpec.toBytes(invData)), Some(peer)) + + // the synchronizer should reply to the peer with a RequestModifier message asking for the header + ncProbe.fishForMessage(3 seconds) { case m => + m match { + case stn: SendToNetwork if stn.message.spec.messageCode == RequestModifierSpec.messageCode => + val data = stn.message.data.get.asInstanceOf[InvData] + data.typeId == Header.modifierTypeId && data.ids == Seq(nextHeader.id) + case _ => false + } + } + + // and the header should be tracked as Requested + eventually { + deliveryTracker.status(nextHeader.id, Header.modifierTypeId, Seq.empty) shouldBe Requested + } + } + } + property("NodeViewSynchronizer: receiving valid header") { withFixture { ctx => import ctx._ @@ -972,7 +1003,7 @@ class ErgoNodeViewSynchronizerSpecification extends AnyPropSpec } // Second: LocalBlockApplied for same header should NOT send additional invs - synchronizerMockRef ! LocalBlockApplied(newBlock.header) + synchronizerMockRef ! LocalBlockApplied(newBlock.header, newBlock.transactions.map(_.id)) // Should receive no additional InvSpec messages ncProbe.expectNoMessage(1.second) @@ -1008,7 +1039,7 @@ class ErgoNodeViewSynchronizerSpecification extends AnyPropSpec } // LocalBlockApplied for same block should NOT broadcast again - synchronizerMockRef ! LocalBlockApplied(newBlock.header) + synchronizerMockRef ! LocalBlockApplied(newBlock.header, newBlock.transactions.map(_.id)) // Should receive no additional InvSpec messages ncProbe.expectNoMessage(1.second) @@ -1032,7 +1063,7 @@ class ErgoNodeViewSynchronizerSpecification extends AnyPropSpec val newBlock = statefulyValidFullBlock(wus) // Send RemoteBlockApplied to synchronizer - synchronizerMockRef ! RemoteBlockApplied(newBlock.header) + synchronizerMockRef ! RemoteBlockApplied(newBlock.header, newBlock.transactions.map(_.id)) // Expect 4 inv messages (1 header + 3 sections) val invMessages = (0 until 4).map { _ => @@ -1114,12 +1145,12 @@ class ErgoNodeViewSynchronizerSpecification extends AnyPropSpec val newBlock = statefulyValidFullBlock(wus) // Send LocalBlockApplied - should not broadcast but should perform cleanup - synchronizerMockRef ! LocalBlockApplied(newBlock.header) + synchronizerMockRef ! LocalBlockApplied(newBlock.header, newBlock.transactions.map(_.id)) ncProbe.expectNoMessage(500.millis) // Send RemoteBlockApplied - should broadcast (different block) val newBlock2 = statefulyValidFullBlock(wus) - synchronizerMockRef ! RemoteBlockApplied(newBlock2.header) + synchronizerMockRef ! RemoteBlockApplied(newBlock2.header, newBlock2.transactions.map(_.id)) // Expect 4 inv messages (1 header + 3 sections) val invMessages = (0 until 4).map { _ => diff --git a/src/test/scala/org/ergoplatform/nodeView/NodeViewSynchronizerTests.scala b/src/test/scala/org/ergoplatform/nodeView/NodeViewSynchronizerTests.scala index 0f393e819f..b81c49672b 100644 --- a/src/test/scala/org/ergoplatform/nodeView/NodeViewSynchronizerTests.scala +++ b/src/test/scala/org/ergoplatform/nodeView/NodeViewSynchronizerTests.scala @@ -105,7 +105,7 @@ trait NodeViewSynchronizerTests[ST <: ErgoState[ST]] extends AnyPropSpec property("NodeViewSynchronizer: SemanticallySuccessfulModifier") { withFixture { ctx => import ctx._ - node ! RemoteBlockApplied(mod.asInstanceOf[Header]) //todo: fix + node ! RemoteBlockApplied(mod.asInstanceOf[Header], Seq.empty) //todo: fix ncProbe.fishForMessage(3 seconds) { case m => m.isInstanceOf[SendToNetwork] } } } diff --git a/src/test/scala/org/ergoplatform/nodeView/history/PopowProcessorSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/PopowProcessorSpecification.scala index d9ace006c3..93e113a14b 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/PopowProcessorSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/PopowProcessorSpecification.scala @@ -1,13 +1,17 @@ package org.ergoplatform.nodeView.history +import org.ergoplatform.mining.AutolykosPowScheme import org.ergoplatform.modifiers.ErgoFullBlock import org.ergoplatform.modifiers.history.popow.PoPowHeader import org.ergoplatform.nodeView.state.StateType +import org.ergoplatform.settings.NipopowSettings import org.ergoplatform.utils.ErgoCorePropertyTest +import org.ergoplatform.wallet.utils.FileUtils import scorex.util.ModifierId -class PopowProcessorSpecification extends ErgoCorePropertyTest { +class PopowProcessorSpecification extends ErgoCorePropertyTest with FileUtils { import org.ergoplatform.utils.HistoryTestHelpers._ + import org.ergoplatform.utils.ErgoNodeTestConstants.{settings => baseSettings} import org.ergoplatform.utils.generators.ChainGenerator._ private def genHistory(genesisIdOpt: Option[ModifierId], popowBootstrap: Boolean) = @@ -15,6 +19,21 @@ class PopowProcessorSpecification extends ErgoCorePropertyTest { epochLength = 10000, useLastEpochs = 3, initialDiffOpt = None, genesisIdOpt) .ensuring(_.bestFullBlockOpt.isEmpty) + private def genRealPowHistory(genesisIdOpt: Option[ModifierId], + realPowScheme: AutolykosPowScheme): ErgoHistory = { + val realPowSettings = baseSettings.copy( + directory = createTempDir.getAbsolutePath, + chainSettings = baseSettings.chainSettings.copy(powScheme = realPowScheme, genesisId = genesisIdOpt), + nodeSettings = baseSettings.nodeSettings.copy( + stateType = StateType.Utxo, + verifyTransactions = true, + blocksToKeep = -1, + nipopowSettings = NipopowSettings(nipopowBootstrap = true, p2pNipopows = 1) + ) + ) + ErgoHistory.readOrGenerate(realPowSettings)(null).ensuring(_.bestFullBlockOpt.isEmpty) + } + val toPoPoWChain = (c: Seq[ErgoFullBlock]) => c.map(b => PoPowHeader.fromBlock(b).get) property("popow proof application") { @@ -32,4 +51,22 @@ class PopowProcessorSpecification extends ErgoCorePropertyTest { receiverHistory.bestHeaderOpt.get shouldBe senderHistory.bestHeaderOpt.get } + property("popow proof application rejects headers failing real Autolykos validation") { + val senderHistory = genHistory(None, popowBootstrap = false) + val senderChain = genChain(80, senderHistory) + applyChain(senderHistory, senderChain) + + val popowProofBytes = senderHistory.popowProofBytes().get + val realPowScheme = new AutolykosPowScheme(baseSettings.chainSettings.powScheme.k, baseSettings.chainSettings.powScheme.n) + val receiverHistory = genRealPowHistory(senderHistory.bestHeaderAtHeight(1).map(_.id), realPowScheme) + val popowProof = receiverHistory.nipopowSerializer.parseBytes(popowProofBytes) + + popowProof.headersChain.exists(h => realPowScheme.validate(h).isFailure) shouldBe true + + receiverHistory.headersHeight shouldBe 0 + receiverHistory.applyPopowProof(popowProof) + receiverHistory.headersHeight shouldBe 0 + receiverHistory.bestHeaderOpt shouldBe None + } + } diff --git a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala index 22ed2f00fd..eb6f6e0c5a 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala @@ -18,6 +18,7 @@ import spire.implicits.cfor import java.util.concurrent.locks.{Condition, ReentrantLock} import scala.collection.mutable +import scala.concurrent.duration.DurationInt import scala.reflect.ClassTag class ExtraIndexerSpecification extends ErgoCorePropertyTest { @@ -26,6 +27,7 @@ class ExtraIndexerSpecification extends ErgoCorePropertyTest { implicit val addressEncoder: ErgoAddressEncoder = settings.addressEncoder val initSettings: ErgoSettings = settings case class CreateDB(blockCount: Int) + case class ExtendDB(blockCount: Int) case class Reset() case class GenerateBetterChainTip() @@ -45,6 +47,12 @@ class ExtraIndexerSpecification extends ErgoCorePropertyTest { val done: Condition = lock.newCondition() val created: Condition = lock.newCondition() + def awaitCondition(condition: Condition): Unit = { + lock.lock() + try condition.await() + finally lock.unlock() + } + def manualIndex(limit: Int): (ID_LL, // address -> (erg,tokenSum) ID_LL, // template -> (spentBoxCount,unspentBoxCount) ID_LL, // tokenId -> (boxesCount,_) @@ -241,6 +249,31 @@ class ExtraIndexerSpecification extends ErgoCorePropertyTest { indexer ! Reset() } + property("skips a duplicate applied block without blocking later blocks") { + indexer ! CreateDB(HEIGHT) + indexer ! Index() + awaitCondition(done) + + indexer ! ExtendDB(HEIGHT + 2) + awaitCondition(created) + val firstHeader = history.typedModifierById[Header](history.bestHeaderIdAtHeight(HEIGHT + 1).get).get + val secondHeader = history.typedModifierById[Header](history.bestHeaderIdAtHeight(HEIGHT + 2).get).get + val blocks = (1 to HEIGHT + 2).map(height => history.bestBlockTransactionsAt(height).get) + val expectedTxCount = blocks.map(_.txs.size.toLong).sum + val expectedBoxCount = blocks.flatMap(_.txs).map(_.outputs.size.toLong).sum + indexer ! RemoteBlockApplied(firstHeader, history.getFullBlock(firstHeader).get.transactions.map(_.id)) + indexer ! RemoteBlockApplied(firstHeader, history.getFullBlock(firstHeader).get.transactions.map(_.id)) + indexer ! RemoteBlockApplied(secondHeader, history.getFullBlock(secondHeader).get.transactions.map(_.id)) + + org.ergoplatform.utils.untilTimeout(10.seconds, 50.millis) { + val state = IndexerState.fromHistory(_history) + state.indexedHeight shouldBe HEIGHT + 2 + state.globalTxIndex shouldBe expectedTxCount + state.globalBoxIndex shouldBe expectedBoxCount + } + indexer ! Reset() + } + property("transactions") { indexer ! CreateDB(HEIGHT) indexer ! Index() @@ -319,16 +352,15 @@ class ExtraIndexerSpecification extends ErgoCorePropertyTest { rollbackWithPattern("G-5;G-15;R-5;G-20;G-25;R-15;G-30;R-10;G-50;R-25") } - property("tokens dont disappear when rolling back with orphan block") { + property("indexes replacement blocks after rolling back an orphan block") { indexer ! CreateDB(HEIGHT) indexer ! Index() - lock.lock() - done.await() + awaitCondition(done) indexer ! GenerateBetterChainTip() lock.lock() created.await() val newBestHeaderOpt = history.typedModifierById[Header](history.headerIdsAtHeight(history.fullBlockHeight).last) - indexer ! RemoteBlockApplied(newBestHeaderOpt.get) // will be ignored + indexer ! RemoteBlockApplied(newBestHeaderOpt.get, Seq.empty) // will be ignored indexer ! CreateDB(HEIGHT + 1) lock.lock() created.await() diff --git a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerTestActor.scala b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerTestActor.scala index 41f801e0d8..c67e0c0c73 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerTestActor.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerTestActor.scala @@ -18,6 +18,7 @@ class ExtraIndexerTestActor(test: ExtraIndexerSpecification) extends ExtraIndexe override def receive: Receive = { case test.CreateDB(blockCount: Int) => createDB(blockCount) + case test.ExtendDB(blockCount: Int) => extendDB(blockCount) case test.Reset() => reset() case test.GenerateBetterChainTip() => GenerateBetterChainTip() } @@ -70,6 +71,14 @@ class ExtraIndexerTestActor(test: ExtraIndexerSpecification) extends ExtraIndexe test.lock.unlock() } + def extendDB(blockCount: Int): Unit = { + stateOpt = Some(ChainGenerator.generate(blockCount, dir, _history, stateOpt)) + test._history = _history + test.lock.lock() + test.created.signal() + test.lock.unlock() + } + def reset(): Unit = { stateOpt = None test._history = null @@ -84,7 +93,6 @@ class ExtraIndexerTestActor(test: ExtraIndexerSpecification) extends ExtraIndexe def GenerateBetterChainTip(): Unit = { stateOpt = Some(ChainGenerator.generateBetter(_history, stateOpt.get)) test._history = _history - context.become(receive.orElse(loaded(IndexerState.fromHistory(_history)))) test.lock.lock() test.created.signal() test.lock.unlock() diff --git a/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala b/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala index 1520a9f032..d35e22703d 100644 --- a/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala @@ -10,6 +10,7 @@ import org.ergoplatform.settings.{ErgoSettings, ErgoValidationSettingsUpdate, Pa import org.ergoplatform.utils.{ErgoTestHelpers, RandomWrapper} import org.scalatest.flatspec.AnyFlatSpec import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks +import scorex.crypto.authds.ADKey import scorex.util.encode.Base16 import sigma.ast.ByteArrayConstant import sigma.Colls @@ -29,6 +30,13 @@ class ErgoMemPoolSpec extends AnyFlatSpec import org.ergoplatform.utils.generators.ErgoCoreTransactionGenerators._ import org.ergoplatform.utils.generators.ValidBlocksGenerators._ + private def feeTx(inputSeed: Byte, fee: Long): ErgoTransaction = { + ErgoTransaction( + IndexedSeq(new Input(ADKey @@ Array.fill(32)(inputSeed), emptyProverResult)), + IndexedSeq(new ErgoBoxCandidate(fee, feeProp, creationHeight = 0)) + ) + } + it should "accept valid transaction" in { val (us, bh) = createUtxoState(settings) val genesis = validFullBlock(None, us, bh) @@ -486,6 +494,31 @@ class ErgoMemPoolSpec extends AnyFlatSpec pool.stats.takenTxns shouldBe (family_depth + 1) * txs.size } + it should "not recommend fee below node minimal fee" in { + val feeSettings = settings.copy(nodeSettings = settings.nodeSettings.copy(minimalFeeAmount = 1000000L)) + val minimalFee = feeSettings.nodeSettings.minimalFeeAmount + val now = System.currentTimeMillis() + val lowFeeHistogram = FeeHistogramBin(nTxns = 1, totalFee = minimalFee / 2) :: + List.fill(MemPoolStatistics.nHistogramBins - 1)(FeeHistogramBin(0, 0)) + val stats = MemPoolStatistics(now, takenTxns = 1, snapTime = now, histogram = lowFeeHistogram) + val pool = new ErgoMemPool(OrderedTxPool.empty(feeSettings), stats, SortingOption.FeePerByte)(feeSettings) + + pool.getRecommendedFee(expectedWaitTimeMinutes = 0, txSize = 1024) shouldBe minimalFee + } + + it should "not let idle uptime dominate expected wait time" in { + val feeSettings = settings.copy(nodeSettings = settings.nodeSettings.copy(minimalFeeAmount = 1000000L)) + val minimalFee = feeSettings.nodeSettings.minimalFeeAmount + val poolWithHigherFeeTx = ErgoMemPool.empty(feeSettings) + .put(UnconfirmedTransaction(feeTx(inputSeed = 1, fee = minimalFee * 100), None)) + val now = System.currentTimeMillis() + val staleMeasurementStart = now - 365L * 24 * 60 * 60 * 1000 + val staleStats = MemPoolStatistics(staleMeasurementStart, takenTxns = 1, snapTime = now) + val pool = new ErgoMemPool(poolWithHigherFeeTx.pool, staleStats, SortingOption.FeePerByte)(feeSettings) + + pool.getExpectedWaitTime(txFee = minimalFee, txSize = 1024) should be <= MemPoolStatistics.measurementIntervalMsec.toLong + } + it should "put not adding transaction twice" in { val pool = ErgoMemPool.empty(settings).pool val tx = invalidErgoTransactionGen.sample.get diff --git a/src/test/scala/org/ergoplatform/nodeView/mempool/OrderedTxPoolSpec.scala b/src/test/scala/org/ergoplatform/nodeView/mempool/OrderedTxPoolSpec.scala new file mode 100644 index 0000000000..1e40f6b8a9 --- /dev/null +++ b/src/test/scala/org/ergoplatform/nodeView/mempool/OrderedTxPoolSpec.scala @@ -0,0 +1,259 @@ +package org.ergoplatform.nodeView.mempool + +import org.ergoplatform.ErgoBox.BoxId +import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnconfirmedTransaction} +import org.ergoplatform.settings.Constants.TrueTree +import org.ergoplatform.utils.ErgoTestHelpers +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import scorex.crypto.authds.ADKey +import scorex.util.ModifierId + +class OrderedTxPoolSpec extends AnyFlatSpec with Matchers with ErgoTestHelpers { + import org.ergoplatform.utils.ErgoCoreTestConstants.emptyProverResult + import org.ergoplatform.utils.ErgoNodeTestConstants.settings + + private final class ReconvergentFixture( + val a: ErgoTransaction, + val b: ErgoTransaction, + val c: ErgoTransaction, + val d: ErgoTransaction, + val beforeD: ErgoMemPool, + val parentOrder: Seq[ModifierId] + ) + + private def deterministicRootId(nonce: Int): BoxId = { + val bytes = Array.fill[Byte](32)(0) + bytes(28) = (nonce >>> 24).toByte + bytes(29) = (nonce >>> 16).toByte + bytes(30) = (nonce >>> 8).toByte + bytes(31) = nonce.toByte + ADKey @@ bytes + } + + private def outputCandidates(plainCount: Int, plainValue: Long = 4000000L) = { + val feeProposition = settings.chainSettings.monetary.feeProposition + IndexedSeq.fill(plainCount)(new org.ergoplatform.ErgoBoxCandidate(plainValue, TrueTree, 0)) :+ + new org.ergoplatform.ErgoBoxCandidate(1000000L, feeProposition, 0) + } + + private def buildReconvergentFixture(nonce: Int): ReconvergentFixture = { + val a = ErgoTransaction( + IndexedSeq(new org.ergoplatform.Input(deterministicRootId(nonce), emptyProverResult)), + outputCandidates(3) + ) + val b = ErgoTransaction( + IndexedSeq(new org.ergoplatform.Input(a.outputs(0).id, emptyProverResult)), + outputCandidates(1, 3000000L) + ) + val c = ErgoTransaction( + IndexedSeq(new org.ergoplatform.Input(a.outputs(1).id, emptyProverResult)), + outputCandidates(1, 3000000L) + ) + val d = ErgoTransaction( + IndexedSeq( + new org.ergoplatform.Input(b.outputs(0).id, emptyProverResult), + new org.ergoplatform.Input(c.outputs(0).id, emptyProverResult), + new org.ergoplatform.Input(a.outputs(2).id, emptyProverResult) + ), + outputCandidates(1, 9000000L) + ) + + val beforeD = Seq(a, b, c).foldLeft(ErgoMemPool.empty(settings)) { + case (pool, tx) => pool.put(UnconfirmedTransaction(tx, None)) + } + val uniqueParentKeys = d.inputs.flatMap(input => beforeD.pool.outputs.get(input.boxId)).toSet + val parentOrder = uniqueParentKeys + .flatMap { wtx => + beforeD.pool.orderedTransactions.get(wtx).map(unconfirmed => wtx -> unconfirmed) + } + .toSeq + .map(_._1.id) + + new ReconvergentFixture(a, b, c, d, beforeD, parentOrder) + } + + private def fixtureWithSharedAncestorLast(): ReconvergentFixture = { + val fixture = (0 until 4096) + .iterator + .map(buildReconvergentFixture) + .find { candidate => + candidate.parentOrder.lastOption.contains(candidate.a.id) && + candidate.parentOrder.take(2).toSet == Set(candidate.b.id, candidate.c.id) + } + fixture.getOrElse(fail("No deterministic reconvergent fixture found")) + } + + private def orderedIds(pool: OrderedTxPool): Vector[ModifierId] = + pool.orderedTransactions.valuesIterator.map(_.id).toVector + + private def assertConsistent(pool: OrderedTxPool, expectedIds: Set[ModifierId]): Unit = { + val ids = orderedIds(pool) + ids.toSet shouldBe expectedIds + ids.distinct.size shouldBe ids.size + pool.orderedTransactions.size shouldBe expectedIds.size + pool.transactionsRegistry.keySet shouldBe expectedIds + pool.transactionsRegistry.foreach { case (id, key) => + pool.orderedTransactions.get(key).map(_.id) shouldBe Some(id) + } + val transactions = pool.orderedTransactions.valuesIterator.map(_.transaction).toVector + val expectedOutputIds = transactions.flatMap(_.outputs.map(_.id)).toSet + val expectedInputIds = transactions.flatMap(_.inputs.map(_.boxId)).toSet + pool.outputs.keySet shouldBe expectedOutputIds + pool.inputs.keySet shouldBe expectedInputIds + pool.orderedTransactions.foreach { case (key, unconfirmed) => + unconfirmed.transaction.outputs.foreach { output => + pool.outputs.get(output.id) shouldBe Some(key) + } + unconfirmed.transaction.inputs.foreach { input => + pool.inputs.get(input.boxId) shouldBe Some(key) + } + } + pool.outputs.valuesIterator.foreach { key => + pool.orderedTransactions.contains(key) shouldBe true + } + pool.inputs.valuesIterator.foreach { key => + pool.orderedTransactions.contains(key) shouldBe true + } + } + + private def withDuplicate(pool: OrderedTxPool, tx: ErgoTransaction): OrderedTxPool = { + val registeredKey = pool.transactionsRegistry(tx.id) + val unconfirmed = pool.orderedTransactions(registeredKey) + val duplicateKey = registeredKey.copy(weight = registeredKey.weight + 1L) + + new OrderedTxPool( + pool.orderedTransactions.updated(duplicateKey, unconfirmed), + pool.transactionsRegistry, + pool.invalidatedTxIds, + pool.outputs, + pool.inputs + )(settings) + } + + private def withoutRegistry(pool: OrderedTxPool, tx: ErgoTransaction): OrderedTxPool = { + new OrderedTxPool( + pool.orderedTransactions, + pool.transactionsRegistry - tx.id, + pool.invalidatedTxIds, + pool.outputs, + pool.inputs + )(settings) + } + + it should "keep indexes consistent when a transaction closes a reconvergent family" in { + val fixture = fixtureWithSharedAncestorLast() + val beforeIds = Set(fixture.a.id, fixture.b.id, fixture.c.id) + val beforeWeights = beforeIds.map { id => + id -> fixture.beforeD.pool.transactionsRegistry(id).weight + }.toMap + + fixture.parentOrder.last shouldBe fixture.a.id + fixture.parentOrder.take(2).toSet shouldBe Set(fixture.b.id, fixture.c.id) + fixture.d.inputs.map(_.boxId).distinct.size shouldBe fixture.d.inputs.size + assertConsistent(fixture.beforeD.pool, beforeIds) + + val afterD = fixture.beforeD.put(UnconfirmedTransaction(fixture.d, None)) + val expectedIds = beforeIds + fixture.d.id + val duplicateCounts = orderedIds(afterD.pool).groupBy(identity).mapValues(_.size) + val orderedKeys = afterD.pool.orderedTransactions.keysIterator + .map(key => key.id -> key.weight) + .toVector + + withClue(s"ordered keys=$orderedKeys, duplicate counts=$duplicateCounts") { + assertConsistent(afterD.pool, expectedIds) + val afterWeights = afterD.pool.transactionsRegistry.mapValues(_.weight) + val dWeight = afterWeights(fixture.d.id) + dWeight should be > (0L) + afterWeights(fixture.b.id) shouldBe beforeWeights(fixture.b.id) + dWeight + afterWeights(fixture.c.id) shouldBe beforeWeights(fixture.c.id) + dWeight + afterWeights(fixture.a.id) shouldBe beforeWeights(fixture.a.id) + 3L * dWeight + } + } + + + it should "heal a duplicated ancestor while propagating a new child" in { + val fixture = fixtureWithSharedAncestorLast() + val before = fixture.beforeD.pool + val ancestorWeight = before.transactionsRegistry(fixture.a.id).weight + val corrupted = withDuplicate(before, fixture.a) + + orderedIds(corrupted).count(_ == fixture.a.id) shouldBe 2 + val healed = corrupted.put(UnconfirmedTransaction(fixture.d, None), fixture.d.size) + + assertConsistent( + healed, + Set(fixture.a.id, fixture.b.id, fixture.c.id, fixture.d.id) + ) + val dWeight = healed.transactionsRegistry(fixture.d.id).weight + healed.transactionsRegistry(fixture.a.id).weight shouldBe ancestorWeight + 3L * dWeight + } + it should "self-heal duplicate keys without propagating family weight twice" in { + val fixture = fixtureWithSharedAncestorLast() + val before = fixture.beforeD.pool + val originalKey = before.transactionsRegistry(fixture.b.id) + val ancestorWeight = before.transactionsRegistry(fixture.a.id).weight + val corrupted = withDuplicate(before, fixture.b) + + orderedIds(corrupted).count(_ == fixture.b.id) shouldBe 2 + val healed = corrupted.put(UnconfirmedTransaction(fixture.b, None), fixture.b.size) + + assertConsistent(healed, Set(fixture.a.id, fixture.b.id, fixture.c.id)) + val healedKeys = healed.orderedTransactions.keysIterator + .filter(_.id == fixture.b.id) + .toVector + healedKeys.map(_.weight) shouldBe Vector(originalKey.weight) + healedKeys.map(_.created) shouldBe Vector(originalKey.created) + healed.transactionsRegistry(fixture.a.id).weight shouldBe ancestorWeight + } + + it should "purge duplicate keys and subtract family weight once on removal" in { + val fixture = fixtureWithSharedAncestorLast() + val before = fixture.beforeD.pool + val ancestorWeight = before.transactionsRegistry(fixture.a.id).weight + val childWeight = before.transactionsRegistry(fixture.b.id).weight + val siblingWeight = before.transactionsRegistry(fixture.c.id).weight + val corrupted = withDuplicate(before, fixture.b) + + orderedIds(corrupted).count(_ == fixture.b.id) shouldBe 2 + val removed = corrupted.remove(fixture.b) + + assertConsistent(removed, Set(fixture.a.id, fixture.c.id)) + removed.transactionsRegistry(fixture.a.id).weight shouldBe ancestorWeight - childWeight + removed.transactionsRegistry(fixture.c.id).weight shouldBe siblingWeight + } + + it should "purge duplicate keys and subtract family weight once on invalidation" in { + val fixture = fixtureWithSharedAncestorLast() + val before = fixture.beforeD.pool + val ancestorWeight = before.transactionsRegistry(fixture.a.id).weight + val childWeight = before.transactionsRegistry(fixture.b.id).weight + val siblingWeight = before.transactionsRegistry(fixture.c.id).weight + val corrupted = withDuplicate(before, fixture.b) + + orderedIds(corrupted).count(_ == fixture.b.id) shouldBe 2 + val invalidated = corrupted.invalidate(UnconfirmedTransaction(fixture.b, None)) + + assertConsistent(invalidated, Set(fixture.a.id, fixture.c.id)) + invalidated.transactionsRegistry(fixture.a.id).weight shouldBe ancestorWeight - childWeight + invalidated.transactionsRegistry(fixture.c.id).weight shouldBe siblingWeight + invalidated.isInvalidated(fixture.b.id) shouldBe true + } + + it should "purge unregistered orphan keys on removal and invalidation" in { + val fixture = fixtureWithSharedAncestorLast() + val before = fixture.beforeD.pool + val orphaned = withoutRegistry(before, fixture.b) + val expectedIds = Set(fixture.a.id, fixture.c.id) + + orphaned.transactionsRegistry should not contain fixture.b.id + orderedIds(orphaned).count(_ == fixture.b.id) shouldBe 1 + + val removed = orphaned.remove(fixture.b) + assertConsistent(removed, expectedIds) + + val invalidated = orphaned.invalidate(UnconfirmedTransaction(fixture.b, None)) + assertConsistent(invalidated, expectedIds) + invalidated.isInvalidated(fixture.b.id) shouldBe true + } +} diff --git a/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletServiceSpec.scala b/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletServiceSpec.scala index cc261eebc5..284952b6f4 100644 --- a/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletServiceSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletServiceSpec.scala @@ -7,7 +7,7 @@ import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnconfirmedTransacti import org.ergoplatform.nodeView.mempool.ErgoMemPoolReader import org.ergoplatform.nodeView.wallet.WalletScanLogic.ScanResults import org.ergoplatform.nodeView.wallet.persistence.{OffChainRegistry, WalletRegistry, WalletStorage} -import org.ergoplatform.nodeView.wallet.requests.{AssetIssueRequest, PaymentRequest} +import org.ergoplatform.nodeView.wallet.requests.{AssetIssueRequest, BurnTokensRequest, PaymentRequest} import org.ergoplatform.nodeView.wallet.scanning.{EqualsScanningPredicate, ScanRequest, ScanWalletInteraction} import org.ergoplatform.sdk.SecretString import org.ergoplatform.sdk.wallet.secrets.{DerivationPath, ExtendedSecretKey} @@ -28,6 +28,7 @@ import scorex.db.{LDBKVStore, LDBVersionedStore} import scorex.util.encode.Base16 import sigma.Extensions.ArrayOps import sigma.ast.{ByteArrayConstant, EvaluatedValue, FalseLeaf, SType} +import sigmastate.eval.Extensions._ import sigmastate.helpers.TestingHelpers.testBox import scala.collection.compat.immutable.ArraySeq @@ -274,6 +275,101 @@ class ErgoWalletServiceSpec } } + property("asset issuance should be independent of burn request order") { + withVersionedStore(2) { versionedStore => + withStore { store => + val wState = initialState(store, versionedStore) + val existingAssetAmount = 10L + val burnAmount = 3L + val issueAmount = 7L + val inputBoxes = boxesAvailable( + makeGenesisBlock(pks.head.pubkey, Seq(newAssetIdStub -> existingAssetAmount)), + pks.head.pubkey + ) + val existingTokenId = inputBoxes.flatMap(_.additionalTokens.toArray).head._1 + val encodedBoxes = inputBoxes.map(box => Base16.encode(ErgoBoxSerializer.toBytes(box))) + val burnRequest = BurnTokensRequest(Array(existingTokenId -> burnAmount)) + val paymentRequest = PaymentRequest(pks.head, 1000000L, Array.empty, Map.empty) + val issueRequest = AssetIssueRequest( + address = pks.head, + valueOpt = Some(10000000L), + amount = issueAmount, + name = "test-name", + description = "test-description", + decimals = 4, + registers = Option.empty + ) + val boxSelector = new ReplaceCompactCollectBoxSelector( + settings.walletSettings.maxInputs, + settings.walletSettings.optimalInputs, + None + ) + + val requestOrders = Seq( + Seq(burnRequest, issueRequest), + Seq(issueRequest, burnRequest) + ) ++ Seq(burnRequest, issueRequest, paymentRequest).permutations.toSeq + + requestOrders.foreach { requests => + val result = generateUnsignedTransaction( + wState, + boxSelector, + requests, + inputsRaw = encodedBoxes, + dataInputsRaw = Seq.empty + ) + val requestOrder = requests.map(_.getClass.getSimpleName).mkString(", ") + withClue(s"request order: $requestOrder; failure: ${result.failed.map(_.getMessage).toOption}") { + result.isSuccess shouldBe true + } + + val (tx, selectedInputs, _) = result.get + val issuedTokenId = selectedInputs.head.id.toTokenId + val issueOutputs = tx.outputCandidates.filter( + _.additionalTokens.toArray.exists { case (tokenId, _) => tokenId == issuedTokenId } + ) + issueOutputs should have size 1 + issueOutputs.head.value shouldBe issueRequest.valueOpt.get + issueOutputs.head.ergoTree shouldBe pks.head.script + issueOutputs.head.additionalTokens.toArray should contain(issuedTokenId -> issueAmount) + issueOutputs.head.additionalRegisters shouldBe Map( + ErgoBox.R4 -> ByteArrayConstant("test-name".getBytes("UTF-8")), + ErgoBox.R5 -> ByteArrayConstant("test-description".getBytes("UTF-8")), + ErgoBox.R6 -> ByteArrayConstant("4".getBytes("UTF-8")) + ) + + if (requests.contains(paymentRequest)) { + val paymentOutputs = tx.outputCandidates.filter(_.value == paymentRequest.value) + paymentOutputs should have size 1 + paymentOutputs.head.ergoTree shouldBe pks.head.script + paymentOutputs.head.additionalTokens.toArray shouldBe empty + paymentOutputs.head.additionalRegisters shouldBe empty + } + + selectedInputs + .flatMap(_.additionalTokens.toArray) + .collect { case (tokenId, amount) if tokenId == issuedTokenId => amount } + .sum shouldBe 0L + tx.outputCandidates + .flatMap(_.additionalTokens.toArray) + .collect { case (tokenId, amount) if tokenId == issuedTokenId => amount } + .sum shouldBe issueAmount + + val selectedExistingAmount = selectedInputs + .flatMap(_.additionalTokens.toArray) + .collect { case (tokenId, amount) if tokenId == existingTokenId => amount } + .sum + val outputExistingAmount = tx.outputCandidates + .flatMap(_.additionalTokens.toArray) + .collect { case (tokenId, amount) if tokenId == existingTokenId => amount } + .sum + selectedExistingAmount - outputExistingAmount shouldBe burnAmount + selectedInputs.map(_.value).sum shouldBe tx.outputCandidates.map(_.value).sum + } + } + } + } + property("it should process unlock using preEip3Derivation") { withVersionedStore(2) { versionedStore => withStore { store => diff --git a/src/test/scala/org/ergoplatform/settings/ErgoSettingsSpecification.scala b/src/test/scala/org/ergoplatform/settings/ErgoSettingsSpecification.scala index e7ecab3e24..b2df8ac1a1 100644 --- a/src/test/scala/org/ergoplatform/settings/ErgoSettingsSpecification.scala +++ b/src/test/scala/org/ergoplatform/settings/ErgoSettingsSpecification.scala @@ -5,7 +5,7 @@ import org.ergoplatform.nodeView.mempool.ErgoMemPoolUtils.SortingOption import org.ergoplatform.nodeView.state.StateType import org.ergoplatform.utils.ErgoCorePropertyTest -import java.net.{InetSocketAddress, URL} +import java.net.{InetSocketAddress, URI} import scala.concurrent.duration._ class ErgoSettingsSpecification extends ErgoCorePropertyTest { @@ -66,7 +66,7 @@ class ErgoSettingsSpecification extends ErgoCorePropertyTest { apiKeyHash = None, corsAllowedOrigin = Some("*"), timeout = 5.seconds, - publicUrl = Some(new URL("https://example.com:80")) + publicUrl = Some(URI.create("https://example.com:80").toURL) ) } @@ -165,7 +165,7 @@ class ErgoSettingsSpecification extends ErgoCorePropertyTest { "http://0.0.0.0", "http://example.com/foo/bar", "http://example.com?foo=bar" - ).map(new URL(_)) + ).map(s => URI.create(s).toURL) invalidUrls.forall(ErgoSettingsReader.invalidRestApiUrl) shouldBe true @@ -175,7 +175,7 @@ class ErgoSettingsSpecification extends ErgoCorePropertyTest { "http://example.com:80", "http://82.90.21.31", "http://82.90.21.31:80" - ).map(new URL(_)) + ).map(s => URI.create(s).toURL) validUrls.forall(url => !ErgoSettingsReader.invalidRestApiUrl(url)) shouldBe true } diff --git a/src/test/scala/org/ergoplatform/utils/generators/ErgoNodeGenerators.scala b/src/test/scala/org/ergoplatform/utils/generators/ErgoNodeGenerators.scala index b4043fe3e7..c4b03e7072 100644 --- a/src/test/scala/org/ergoplatform/utils/generators/ErgoNodeGenerators.scala +++ b/src/test/scala/org/ergoplatform/utils/generators/ErgoNodeGenerators.scala @@ -24,7 +24,7 @@ object ErgoNodeGenerators { } yield { val chain = genHeaderChain(m * mulM + k, diffBitsOpt = None, useRealTs = false) val popowChain = popowHeaderChain(chain) - val params = PoPowParams(m, k, continuous = false) + val params = PoPowParams(m, k, continuous = false).get nipopowAlgos.prove(popowChain)(params).get } } diff --git a/src/test/scala/scorex/core/network/NetworkControllerSpec.scala b/src/test/scala/scorex/core/network/NetworkControllerSpec.scala index 8bc287d6d9..1265b535ab 100644 --- a/src/test/scala/scorex/core/network/NetworkControllerSpec.scala +++ b/src/test/scala/scorex/core/network/NetworkControllerSpec.scala @@ -3,6 +3,7 @@ package scorex.core.network import akka.actor.ActorRef import akka.io.Tcp import akka.testkit.{TestActorRef, TestProbe} +import org.ergoplatform.network.ErgoNodeViewSynchronizerMessages.DisconnectedPeer import org.ergoplatform.network.message.MessageConstants.MessageCode import org.ergoplatform.network.peer.PeerInfo import org.ergoplatform.utils.ErgoCorePropertyTest @@ -25,6 +26,8 @@ class NetworkControllerSpec extends ErgoCorePropertyTest { val scorexContext: ScorexContext = ScorexContext(Seq.empty, None, None) + case class EstablishedConnection(connectionProbe: TestProbe, handlerRef: ActorRef) + def createController(maxConnections: Int): (TestActorRef[NetworkController], TestProbe, TestProbe) = { val peerManagerProbe = TestProbe("PeerManager") val tcpManagerProbe = TestProbe("TcpManager") @@ -56,6 +59,33 @@ class NetworkControllerSpec extends ErgoCorePropertyTest { peerManagerProbe: TestProbe, remoteAddress: InetSocketAddress ): InetSocketAddress = { + beginIncomingConnection(controller, peerManagerProbe, remoteAddress) + remoteAddress + } + + def establishIncomingConnectionWithHandler( + controller: TestActorRef[NetworkController], + peerManagerProbe: TestProbe, + remoteAddress: InetSocketAddress + ): EstablishedConnection = { + val connectionProbe = beginIncomingConnection( + controller, + peerManagerProbe, + remoteAddress + ) + + val handlerRef = connectionProbe.expectMsgType[Tcp.Register].handler + connectionProbe.expectMsg(Tcp.ResumeReading) + connectionProbe.expectMsgType[Tcp.Write] + + EstablishedConnection(connectionProbe, handlerRef) + } + + private def beginIncomingConnection( + controller: TestActorRef[NetworkController], + peerManagerProbe: TestProbe, + remoteAddress: InetSocketAddress + ): TestProbe = { val localAddress = settings.scorexSettings.network.bindAddress val connectionProbe = TestProbe("Connection") @@ -66,7 +96,7 @@ class NetworkControllerSpec extends ErgoCorePropertyTest { controller ! ConnectionConfirmed(ConnectionId(remoteAddress, localAddress, Incoming), handlerRef) } - remoteAddress + connectionProbe } def establishOutgoingConnection( @@ -191,6 +221,97 @@ class NetworkControllerSpec extends ErgoCorePropertyTest { } } + property("blacklisting should close exactly the live connections for the banned IP") { + withFixture { f => + implicit val system = f.system + val (controller, peerManagerProbe, _) = f.createController(maxConnections = 30) + val disconnectProbe = TestProbe("DisconnectedPeers") + f.system.eventStream.subscribe(disconnectProbe.ref, classOf[DisconnectedPeer]) + + val firstAddress = new InetSocketAddress("192.0.2.10", 9101) + val secondAddress = new InetSocketAddress("192.0.2.10", 9102) + val unrelatedAddress = new InetSocketAddress("198.51.100.20", 9201) + val first = f.establishIncomingConnectionWithHandler( + controller, + peerManagerProbe, + firstAddress + ) + val second = f.establishIncomingConnectionWithHandler( + controller, + peerManagerProbe, + secondAddress + ) + val unrelated = f.establishIncomingConnectionWithHandler( + controller, + peerManagerProbe, + unrelatedAddress + ) + + peerManagerProbe.send(controller, Blacklisted(firstAddress)) + + first.connectionProbe.expectMsg(Tcp.Abort) + second.connectionProbe.expectMsg(Tcp.Abort) + unrelated.connectionProbe.expectNoMessage(200.millis) + + val duplicateBeforeTermination = TestProbe("DuplicateBeforeTermination") + duplicateBeforeTermination.send( + controller, + Tcp.Connected(secondAddress, settings.scorexSettings.network.bindAddress) + ) + duplicateBeforeTermination.expectMsg(Tcp.Close) + + first.connectionProbe.watch(first.handlerRef) + second.connectionProbe.watch(second.handlerRef) + first.connectionProbe.send(first.handlerRef, Tcp.Aborted) + second.connectionProbe.send(second.handlerRef, Tcp.Aborted) + first.connectionProbe.expectTerminated(first.handlerRef) + second.connectionProbe.expectTerminated(second.handlerRef) + + val disconnectedAddresses = disconnectProbe.receiveN(2, 2.seconds).collect { + case DisconnectedPeer(peer) => peer.connectionId.remoteAddress + }.toSet + disconnectedAddresses shouldBe Set(firstAddress, secondAddress) + + val replacement = TestProbe("ReplacementConnection") + replacement.send( + controller, + Tcp.Connected(secondAddress, settings.scorexSettings.network.bindAddress) + ) + peerManagerProbe.expectMsgPF(1.second) { + case ConfirmConnection(connectionId, connectionRef) => + connectionId.remoteAddress shouldBe secondAddress + connectionRef shouldBe replacement.ref + } + + val unrelatedDuplicate = TestProbe("UnrelatedDuplicate") + unrelatedDuplicate.send( + controller, + Tcp.Connected(unrelatedAddress, settings.scorexSettings.network.bindAddress) + ) + unrelatedDuplicate.expectMsg(Tcp.Close) + } + } + + property("blacklisting should match by IP when the exact socket is absent") { + withFixture { f => + val (controller, peerManagerProbe, _) = f.createController(maxConnections = 30) + val siblingAddress = new InetSocketAddress("192.0.2.30", 9301) + val missingSocketAddress = new InetSocketAddress("192.0.2.30", 9399) + val sibling = f.establishIncomingConnectionWithHandler( + controller, + peerManagerProbe, + siblingAddress + ) + + peerManagerProbe.send(controller, Blacklisted(missingSocketAddress)) + + sibling.connectionProbe.expectMsg(Tcp.Abort) + sibling.connectionProbe.watch(sibling.handlerRef) + sibling.connectionProbe.send(sibling.handlerRef, Tcp.Aborted) + sibling.connectionProbe.expectTerminated(sibling.handlerRef) + } + } + property("outgoing connection should be accepted when total below maxConnections") { withFixture { f => val (controller, peerManagerProbe, tcpManagerProbe) = f.createController(maxConnections = 10) diff --git a/src/test/scala/scorex/core/network/PeerConnectionHandlerSpecification.scala b/src/test/scala/scorex/core/network/PeerConnectionHandlerSpecification.scala new file mode 100644 index 0000000000..f8319787f4 --- /dev/null +++ b/src/test/scala/scorex/core/network/PeerConnectionHandlerSpecification.scala @@ -0,0 +1,167 @@ +package scorex.core.network + +import akka.io.Tcp +import akka.testkit.{TestActorRef, TestProbe} +import akka.util.ByteString +import org.ergoplatform.network.message.{ + GetPeersSpec, + Message, + MessageSpec, + UtxoSnapshotChunkSpec +} +import org.ergoplatform.network.{Handshake, HandshakeSerializer} +import org.ergoplatform.utils.ErgoCorePropertyTest +import org.ergoplatform.utils.ErgoNodeTestConstants.{defaultPeerSpec, settings} +import scorex.core.app.ScorexContext +import scorex.testkit.utils.AkkaFixture + +import java.net.InetSocketAddress +import scala.concurrent.Await +import scala.concurrent.duration.{Duration, DurationInt} + +class PeerConnectionHandlerSpecification extends ErgoCorePropertyTest { + private final class ConnectedHandler(val connection: TestProbe, + val watcher: TestProbe, + val handler: TestActorRef[PeerConnectionHandler]) + + private def withConnectedHandler( + messageSpecs: Seq[MessageSpec[_]], + localPort: Int + )(test: ConnectedHandler => Unit): Unit = { + val fixture = new AkkaFixture + try { + implicit val system = fixture.system + implicit val ec = system.dispatcher + val connection = TestProbe("connection") + val controller = TestProbe("controller") + val localAddress = new InetSocketAddress("127.0.0.1", localPort) + val remoteAddress = new InetSocketAddress("127.0.0.1", localPort + 1) + val description = ConnectionDescription( + connection.ref, + ConnectionId(remoteAddress, localAddress, Incoming), + Some(localAddress), + Seq.empty + ) + val handler = TestActorRef(new PeerConnectionHandler( + settings.scorexSettings, + controller.ref, + ScorexContext(messageSpecs, None, None), + description + )) + + connection.expectMsgType[Tcp.Register] + connection.expectMsg(Tcp.ResumeReading) + connection.expectMsgType[Tcp.Write] + + val handshake = HandshakeSerializer.toBytes( + Handshake(defaultPeerSpec, System.currentTimeMillis()) + ) + connection.send(handler, Tcp.Received(ByteString(handshake))) + controller.expectMsgType[NetworkController.ReceivableMessages.Handshaked] + connection.expectMsg(Tcp.ResumeReading) + controller.watch(handler) + + test(new ConnectedHandler(connection, controller, handler)) + } finally { + Await.result(fixture.system.terminate(), Duration.Inf) + } + } + + property("abort before a fifth maximum snapshot frame is retained") { + withConnectedHandler(Seq(UtxoSnapshotChunkSpec), localPort = 9083) { fixture => + val chunkMessage = Message( + UtxoSnapshotChunkSpec, + Right(Array.fill[Byte](3999996)(1)), + None + ) + fixture.handler ! chunkMessage + val failedWrite = fixture.connection.expectMsgType[Tcp.Write] + failedWrite.data.length shouldEqual 4000013 + fixture.connection.send(fixture.handler, Tcp.CommandFailed(failedWrite)) + fixture.connection.expectMsg(Tcp.ResumeWriting) + + (2 to 4).foreach { id => + val write = Tcp.Write( + failedWrite.data, + PeerConnectionHandler.ReceivableMessages.Ack(id) + ) + fixture.connection.send(fixture.handler, Tcp.CommandFailed(write)) + fixture.connection.expectMsg(Tcp.ResumeWriting) + } + fixture.connection.expectNoMessage(200.millis) + + val overLimitWrite = Tcp.Write( + failedWrite.data, + PeerConnectionHandler.ReceivableMessages.Ack(5) + ) + fixture.connection.send( + fixture.handler, + Tcp.CommandFailed(overLimitWrite) + ) + fixture.connection.expectMsg(Tcp.ResumeWriting) + fixture.connection.expectMsg(1.second, Tcp.Abort) + fixture.watcher.expectTerminated(fixture.handler) + } + } + + property("abort before more than 64 outbound messages are buffered") { + withConnectedHandler(Seq(GetPeersSpec), localPort = 9093) { fixture => + val getPeersMessage = Message(GetPeersSpec, Right(()), None) + fixture.handler ! getPeersMessage + val failedWrite = fixture.connection.expectMsgType[Tcp.Write] + fixture.connection.send(fixture.handler, Tcp.CommandFailed(failedWrite)) + fixture.connection.expectMsg(Tcp.ResumeWriting) + + (1 until PeerConnectionHandler.MaxBufferedOutboundMessages) + .foreach(_ => fixture.handler ! getPeersMessage) + fixture.connection.expectNoMessage(200.millis) + + fixture.handler ! getPeersMessage + fixture.connection.expectMsg(1.second, Tcp.Abort) + fixture.watcher.expectTerminated(fixture.handler) + } + } + + property("account retried and acknowledged writes exactly") { + withConnectedHandler( + Seq(UtxoSnapshotChunkSpec), + localPort = 9103 + ) { fixture => + val chunkMessage = Message( + UtxoSnapshotChunkSpec, + Right(Array.fill[Byte](3999996)(1)), + None + ) + fixture.handler ! chunkMessage + val failedWrite = fixture.connection.expectMsgType[Tcp.Write] + failedWrite.data.length shouldEqual 4000013 + failedWrite.ack shouldEqual PeerConnectionHandler.ReceivableMessages.Ack(1) + + fixture.connection.send(fixture.handler, Tcp.CommandFailed(failedWrite)) + fixture.connection.expectMsg(Tcp.ResumeWriting) + + (2 to 4).foreach(_ => fixture.handler ! chunkMessage) + fixture.connection.expectNoMessage(200.millis) + + fixture.connection.send(fixture.handler, Tcp.WritingResumed) + val retriedWrite = fixture.connection.expectMsgType[Tcp.Write] + retriedWrite.data shouldEqual failedWrite.data + retriedWrite.ack shouldEqual PeerConnectionHandler.ReceivableMessages.Ack(1) + fixture.connection.send(fixture.handler, Tcp.CommandFailed(retriedWrite)) + fixture.connection.expectMsg(Tcp.ResumeWriting) + fixture.connection.expectNoMessage(200.millis) + + fixture.connection.send(fixture.handler, Tcp.WritingResumed) + val finalRetry = fixture.connection.expectMsgType[Tcp.Write] + finalRetry.data shouldEqual failedWrite.data + finalRetry.ack shouldEqual PeerConnectionHandler.ReceivableMessages.Ack(1) + fixture.connection.send( + fixture.handler, + PeerConnectionHandler.ReceivableMessages.Ack(1) + ) + + val nextWrite = fixture.connection.expectMsgType[Tcp.Write] + nextWrite.ack shouldEqual PeerConnectionHandler.ReceivableMessages.Ack(2) + } + } +}