diff --git a/etc/config/nodeop/aio/config.template.ini b/etc/config/nodeop/aio/config.template.ini index 5b2b223d3e..85cf30b637 100644 --- a/etc/config/nodeop/aio/config.template.ini +++ b/etc/config/nodeop/aio/config.template.ini @@ -46,5 +46,8 @@ p2p-server-address = 127.0.0.1:4444 max-clients = 150 # Trace API Plugin -trace-no-abis = true # Use to indicate that the RPC responses will not use ABIs. ( NOTE: REMOVE THIS LINE IF YOU WANT TO USE ABIs IN RPC RESPONSES) +# ABIs are captured automatically from observed setabi actions and lazy- +# fetched from chain state on first encounter -- no operator-supplied ABI +# files are required. trace-minimum-irreversible-history-blocks = -1 # Number of blocks to ensure are kept past LIB for retrieval before "slice" files can be automatically removed. A value of -1 indicates that automatic removal of "slice" files will be turned off. +# trace-max-block-range = 1000 # Max blocks scanned per get_actions / get_token_transfers request. Must be in [1, 10000]. Clients paginate by advancing block_num_start on each call. diff --git a/plugins/trace_api_plugin/examples/abis/sysio.abi b/plugins/trace_api_plugin/examples/abis/sysio.abi deleted file mode 100644 index 1f9bd87fbe..0000000000 --- a/plugins/trace_api_plugin/examples/abis/sysio.abi +++ /dev/null @@ -1,918 +0,0 @@ -{ - "____comment": "This file was generated with sysio-abigen. DO NOT EDIT ", - "version": "sysio::abi/1.2", - "types": [ - { - "new_type_name": "block_signing_authority", - "type": "variant_block_signing_authority_v0" - }, - { - "new_type_name": "blockchain_parameters_t", - "type": "blockchain_parameters" - } - ], - "structs": [ - { - "name": "abi_hash", - "base": "", - "fields": [ - { - "name": "owner", - "type": "name" - }, - { - "name": "hash", - "type": "checksum256" - } - ] - }, - { - "name": "activate", - "base": "", - "fields": [ - { - "name": "feature_digest", - "type": "checksum256" - } - ] - }, - { - "name": "authority", - "base": "", - "fields": [ - { - "name": "threshold", - "type": "uint32" - }, - { - "name": "keys", - "type": "key_weight[]" - }, - { - "name": "accounts", - "type": "permission_level_weight[]" - } - ] - }, - { - "name": "block_header", - "base": "", - "fields": [ - { - "name": "timestamp", - "type": "uint32" - }, - { - "name": "producer", - "type": "name" - }, - { - "name": "confirmed", - "type": "uint16" - }, - { - "name": "previous", - "type": "checksum256" - }, - { - "name": "transaction_mroot", - "type": "checksum256" - }, - { - "name": "action_mroot", - "type": "checksum256" - }, - { - "name": "schedule_version", - "type": "uint32" - }, - { - "name": "new_producers", - "type": "producer_schedule?" - } - ] - }, - { - "name": "block_info_record", - "base": "", - "fields": [ - { - "name": "version", - "type": "uint8" - }, - { - "name": "block_height", - "type": "uint32" - }, - { - "name": "block_timestamp", - "type": "time_point" - } - ] - }, - { - "name": "block_signing_authority_v0", - "base": "", - "fields": [ - { - "name": "threshold", - "type": "uint32" - }, - { - "name": "keys", - "type": "key_weight[]" - } - ] - }, - { - "name": "blockchain_parameters", - "base": "", - "fields": [ - { - "name": "max_block_net_usage", - "type": "uint64" - }, - { - "name": "target_block_net_usage_pct", - "type": "uint32" - }, - { - "name": "max_transaction_net_usage", - "type": "uint32" - }, - { - "name": "base_per_transaction_net_usage", - "type": "uint32" - }, - { - "name": "net_usage_leeway", - "type": "uint32" - }, - { - "name": "context_free_discount_net_usage_num", - "type": "uint32" - }, - { - "name": "context_free_discount_net_usage_den", - "type": "uint32" - }, - { - "name": "max_block_cpu_usage", - "type": "uint32" - }, - { - "name": "target_block_cpu_usage_pct", - "type": "uint32" - }, - { - "name": "max_transaction_cpu_usage", - "type": "uint32" - }, - { - "name": "min_transaction_cpu_usage", - "type": "uint32" - }, - { - "name": "max_transaction_lifetime", - "type": "uint32" - }, - { - "name": "deferred_trx_expiration_window", - "type": "uint32" - }, - { - "name": "max_transaction_delay", - "type": "uint32" - }, - { - "name": "max_inline_action_size", - "type": "uint32" - }, - { - "name": "max_inline_action_depth", - "type": "uint16" - }, - { - "name": "max_authority_depth", - "type": "uint16" - } - ] - }, - { - "name": "deleteauth", - "base": "", - "fields": [ - { - "name": "account", - "type": "name" - }, - { - "name": "permission", - "type": "name" - }, - { - "name": "authorized_by", - "type": "name$" - } - ] - }, - { - "name": "init", - "base": "", - "fields": [ - { - "name": "version", - "type": "varuint32" - }, - { - "name": "core", - "type": "symbol" - } - ] - }, - { - "name": "key_weight", - "base": "", - "fields": [ - { - "name": "key", - "type": "public_key" - }, - { - "name": "weight", - "type": "uint16" - } - ] - }, - { - "name": "limitauthchg", - "base": "", - "fields": [ - { - "name": "account", - "type": "name" - }, - { - "name": "allow_perms", - "type": "name[]" - }, - { - "name": "disallow_perms", - "type": "name[]" - } - ] - }, - { - "name": "linkauth", - "base": "", - "fields": [ - { - "name": "account", - "type": "name" - }, - { - "name": "code", - "type": "name" - }, - { - "name": "type", - "type": "name" - }, - { - "name": "requirement", - "type": "name" - }, - { - "name": "authorized_by", - "type": "name$" - } - ] - }, - { - "name": "newaccount", - "base": "", - "fields": [ - { - "name": "creator", - "type": "name" - }, - { - "name": "name", - "type": "name" - }, - { - "name": "owner", - "type": "authority" - }, - { - "name": "active", - "type": "authority" - } - ] - }, - { - "name": "onblock", - "base": "", - "fields": [ - { - "name": "header", - "type": "block_header" - } - ] - }, - { - "name": "permission_level", - "base": "", - "fields": [ - { - "name": "actor", - "type": "name" - }, - { - "name": "permission", - "type": "name" - } - ] - }, - { - "name": "permission_level_weight", - "base": "", - "fields": [ - { - "name": "permission", - "type": "permission_level" - }, - { - "name": "weight", - "type": "uint16" - } - ] - }, - { - "name": "producer_authority", - "base": "", - "fields": [ - { - "name": "producer_name", - "type": "name" - }, - { - "name": "authority", - "type": "block_signing_authority" - } - ] - }, - { - "name": "producer_info", - "base": "", - "fields": [ - { - "name": "owner", - "type": "name" - }, - { - "name": "producer_key", - "type": "public_key" - }, - { - "name": "is_active", - "type": "bool" - }, - { - "name": "url", - "type": "string" - }, - { - "name": "unpaid_blocks", - "type": "uint32" - }, - { - "name": "last_claim_time", - "type": "time_point" - }, - { - "name": "location", - "type": "uint16" - }, - { - "name": "producer_authority", - "type": "block_signing_authority" - } - ] - }, - { - "name": "producer_key", - "base": "", - "fields": [ - { - "name": "producer_name", - "type": "name" - }, - { - "name": "block_signing_key", - "type": "public_key" - } - ] - }, - { - "name": "producer_schedule", - "base": "", - "fields": [ - { - "name": "version", - "type": "uint32" - }, - { - "name": "producers", - "type": "producer_key[]" - } - ] - }, - { - "name": "regproducer", - "base": "", - "fields": [ - { - "name": "producer", - "type": "name" - }, - { - "name": "producer_key", - "type": "public_key" - }, - { - "name": "url", - "type": "string" - }, - { - "name": "location", - "type": "uint16" - } - ] - }, - { - "name": "regproducer2", - "base": "", - "fields": [ - { - "name": "producer", - "type": "name" - }, - { - "name": "producer_authority", - "type": "block_signing_authority" - }, - { - "name": "url", - "type": "string" - }, - { - "name": "location", - "type": "uint16" - } - ] - }, - { - "name": "rmvproducer", - "base": "", - "fields": [ - { - "name": "producer", - "type": "name" - } - ] - }, - { - "name": "setabi", - "base": "", - "fields": [ - { - "name": "account", - "type": "name" - }, - { - "name": "abi", - "type": "bytes" - }, - { - "name": "memo", - "type": "string$" - } - ] - }, - { - "name": "setacctcpu", - "base": "", - "fields": [ - { - "name": "account", - "type": "name" - }, - { - "name": "cpu_weight", - "type": "int64?" - } - ] - }, - { - "name": "setacctnet", - "base": "", - "fields": [ - { - "name": "account", - "type": "name" - }, - { - "name": "net_weight", - "type": "int64?" - } - ] - }, - { - "name": "setacctram", - "base": "", - "fields": [ - { - "name": "account", - "type": "name" - }, - { - "name": "ram_bytes", - "type": "int64?" - } - ] - }, - { - "name": "setalimits", - "base": "", - "fields": [ - { - "name": "account", - "type": "name" - }, - { - "name": "ram_bytes", - "type": "int64" - }, - { - "name": "net_weight", - "type": "int64" - }, - { - "name": "cpu_weight", - "type": "int64" - } - ] - }, - { - "name": "setcode", - "base": "", - "fields": [ - { - "name": "account", - "type": "name" - }, - { - "name": "vmtype", - "type": "uint8" - }, - { - "name": "vmversion", - "type": "uint8" - }, - { - "name": "code", - "type": "bytes" - }, - { - "name": "memo", - "type": "string$" - } - ] - }, - { - "name": "setparams", - "base": "", - "fields": [ - { - "name": "params", - "type": "blockchain_parameters_t" - } - ] - }, - { - "name": "setpriv", - "base": "", - "fields": [ - { - "name": "account", - "type": "name" - }, - { - "name": "is_priv", - "type": "uint8" - } - ] - }, - { - "name": "setprods", - "base": "", - "fields": [ - { - "name": "schedule", - "type": "producer_authority[]" - } - ] - }, - { - "name": "setram", - "base": "", - "fields": [ - { - "name": "max_ram_size", - "type": "uint64" - } - ] - }, - { - "name": "sysio_global_state", - "base": "blockchain_parameters", - "fields": [ - { - "name": "max_ram_size", - "type": "uint64" - }, - { - "name": "total_ram_bytes_reserved", - "type": "uint64" - }, - { - "name": "last_producer_schedule_update", - "type": "block_timestamp_type" - }, - { - "name": "last_pervote_bucket_fill", - "type": "time_point" - }, - { - "name": "total_unpaid_blocks", - "type": "uint32" - }, - { - "name": "total_activated_stake", - "type": "int64" - }, - { - "name": "thresh_activated_stake_time", - "type": "time_point" - }, - { - "name": "last_producer_schedule_size", - "type": "uint16" - } - ] - }, - { - "name": "unlinkauth", - "base": "", - "fields": [ - { - "name": "account", - "type": "name" - }, - { - "name": "code", - "type": "name" - }, - { - "name": "type", - "type": "name" - }, - { - "name": "authorized_by", - "type": "name$" - } - ] - }, - { - "name": "unregprod", - "base": "", - "fields": [ - { - "name": "producer", - "type": "name" - } - ] - }, - { - "name": "updateauth", - "base": "", - "fields": [ - { - "name": "account", - "type": "name" - }, - { - "name": "permission", - "type": "name" - }, - { - "name": "parent", - "type": "name" - }, - { - "name": "auth", - "type": "authority" - }, - { - "name": "authorized_by", - "type": "name$" - } - ] - }, - { - "name": "limit_auth_change", - "base": "", - "fields": [ - { - "name": "version", - "type": "uint8" - }, - { - "name": "account", - "type": "name" - }, - { - "name": "allow_perms", - "type": "name[]" - }, - { - "name": "disallow_perms", - "type": "name[]" - } - ] - } - ], - "actions": [ - { - "name": "activate", - "type": "activate", - "ricardian_contract": "" - }, - { - "name": "deleteauth", - "type": "deleteauth", - "ricardian_contract": "" - }, - { - "name": "init", - "type": "init", - "ricardian_contract": "" - }, - { - "name": "limitauthchg", - "type": "limitauthchg", - "ricardian_contract": "" - }, - { - "name": "linkauth", - "type": "linkauth", - "ricardian_contract": "" - }, - { - "name": "newaccount", - "type": "newaccount", - "ricardian_contract": "" - }, - { - "name": "onblock", - "type": "onblock", - "ricardian_contract": "" - }, - { - "name": "regproducer", - "type": "regproducer", - "ricardian_contract": "" - }, - { - "name": "regproducer2", - "type": "regproducer2", - "ricardian_contract": "" - }, - { - "name": "rmvproducer", - "type": "rmvproducer", - "ricardian_contract": "" - }, - { - "name": "setabi", - "type": "setabi", - "ricardian_contract": "" - }, - { - "name": "setacctcpu", - "type": "setacctcpu", - "ricardian_contract": "" - }, - { - "name": "setacctnet", - "type": "setacctnet", - "ricardian_contract": "" - }, - { - "name": "setacctram", - "type": "setacctram", - "ricardian_contract": "" - }, - { - "name": "setalimits", - "type": "setalimits", - "ricardian_contract": "" - }, - { - "name": "setcode", - "type": "setcode", - "ricardian_contract": "" - }, - { - "name": "setparams", - "type": "setparams", - "ricardian_contract": "" - }, - { - "name": "setpriv", - "type": "setpriv", - "ricardian_contract": "" - }, - { - "name": "setprods", - "type": "setprods", - "ricardian_contract": "" - }, - { - "name": "setram", - "type": "setram", - "ricardian_contract": "" - }, - { - "name": "unlinkauth", - "type": "unlinkauth", - "ricardian_contract": "" - }, - { - "name": "unregprod", - "type": "unregprod", - "ricardian_contract": "" - }, - { - "name": "updateauth", - "type": "updateauth", - "ricardian_contract": "" - } - ], - "tables": [ - { - "name": "abihash", - "type": "abi_hash", - "index_type": "i64", - "key_names": [], - "key_types": [] - }, - { - "name": "blockinfo", - "type": "block_info_record", - "index_type": "i64", - "key_names": [], - "key_types": [] - }, - { - "name": "global", - "type": "sysio_global_state", - "index_type": "i64", - "key_names": [], - "key_types": [] - }, - { - "name": "producers", - "type": "producer_info", - "index_type": "i64", - "key_names": [], - "key_types": [] - }, - { - "name": "limitauthchg", - "type": "limit_auth_change", - "index_type": "i64", - "key_names": [], - "key_types": [] - } - ], - "ricardian_clauses": [], - "variants": [ - { - "name": "variant_block_signing_authority_v0", - "types": ["block_signing_authority_v0"] - } - ], - "action_results": [] -} \ No newline at end of file diff --git a/plugins/trace_api_plugin/examples/abis/sysio.msig.abi b/plugins/trace_api_plugin/examples/abis/sysio.msig.abi deleted file mode 100644 index 36b6b12d36..0000000000 --- a/plugins/trace_api_plugin/examples/abis/sysio.msig.abi +++ /dev/null @@ -1,290 +0,0 @@ -{ - "version": "sysio::abi/1.2", - "types": [], - "structs": [{ - "name": "action", - "base": "", - "fields": [{ - "name": "account", - "type": "name" - },{ - "name": "name", - "type": "name" - },{ - "name": "authorization", - "type": "permission_level[]" - },{ - "name": "data", - "type": "bytes" - } - ] - },{ - "name": "approval", - "base": "", - "fields": [{ - "name": "level", - "type": "permission_level" - },{ - "name": "time", - "type": "time_point" - } - ] - },{ - "name": "approvals_info", - "base": "", - "fields": [{ - "name": "version", - "type": "uint8" - },{ - "name": "proposal_name", - "type": "name" - },{ - "name": "requested_approvals", - "type": "approval[]" - },{ - "name": "provided_approvals", - "type": "approval[]" - } - ] - },{ - "name": "approve", - "base": "", - "fields": [{ - "name": "proposer", - "type": "name" - },{ - "name": "proposal_name", - "type": "name" - },{ - "name": "level", - "type": "permission_level" - },{ - "name": "proposal_hash", - "type": "checksum256$" - } - ] - },{ - "name": "cancel", - "base": "", - "fields": [{ - "name": "proposer", - "type": "name" - },{ - "name": "proposal_name", - "type": "name" - },{ - "name": "canceler", - "type": "name" - } - ] - },{ - "name": "exec", - "base": "", - "fields": [{ - "name": "proposer", - "type": "name" - },{ - "name": "proposal_name", - "type": "name" - },{ - "name": "executer", - "type": "name" - } - ] - },{ - "name": "extension", - "base": "", - "fields": [{ - "name": "type", - "type": "uint16" - },{ - "name": "data", - "type": "bytes" - } - ] - },{ - "name": "invalidate", - "base": "", - "fields": [{ - "name": "account", - "type": "name" - } - ] - },{ - "name": "invalidation", - "base": "", - "fields": [{ - "name": "account", - "type": "name" - },{ - "name": "last_invalidation_time", - "type": "time_point" - } - ] - },{ - "name": "old_approvals_info", - "base": "", - "fields": [{ - "name": "proposal_name", - "type": "name" - },{ - "name": "requested_approvals", - "type": "permission_level[]" - },{ - "name": "provided_approvals", - "type": "permission_level[]" - } - ] - },{ - "name": "permission_level", - "base": "", - "fields": [{ - "name": "actor", - "type": "name" - },{ - "name": "permission", - "type": "name" - } - ] - },{ - "name": "proposal", - "base": "", - "fields": [{ - "name": "proposal_name", - "type": "name" - },{ - "name": "packed_transaction", - "type": "bytes" - },{ - "name": "earliest_exec_time", - "type": "time_point?$" - } - ] - },{ - "name": "propose", - "base": "", - "fields": [{ - "name": "proposer", - "type": "name" - },{ - "name": "proposal_name", - "type": "name" - },{ - "name": "requested", - "type": "permission_level[]" - },{ - "name": "trx", - "type": "transaction" - } - ] - },{ - "name": "transaction", - "base": "transaction_header", - "fields": [{ - "name": "context_free_actions", - "type": "action[]" - },{ - "name": "actions", - "type": "action[]" - },{ - "name": "transaction_extensions", - "type": "extension[]" - } - ] - },{ - "name": "transaction_header", - "base": "", - "fields": [{ - "name": "expiration", - "type": "time_point_sec" - },{ - "name": "ref_block_num", - "type": "uint16" - },{ - "name": "ref_block_prefix", - "type": "uint32" - },{ - "name": "max_net_usage_words", - "type": "varuint32" - },{ - "name": "max_cpu_usage_ms", - "type": "uint8" - },{ - "name": "delay_sec", - "type": "varuint32" - } - ] - },{ - "name": "unapprove", - "base": "", - "fields": [{ - "name": "proposer", - "type": "name" - },{ - "name": "proposal_name", - "type": "name" - },{ - "name": "level", - "type": "permission_level" - } - ] - } - ], - "actions": [{ - "name": "approve", - "type": "approve", - "ricardian_contract": "---\nspec_version: \"0.2.0\"\ntitle: Approve Proposed Transaction\nsummary: '{{nowrap level.actor}} approves the {{nowrap proposal_name}} proposal'\nicon: https://raw.githubusercontent.com/eosnetworkfoundation/eos-system-contracts/main/contracts/icons/multisig.png#4fb41d3cf02d0dd2d35a29308e93c2d826ec770d6bb520db668f530764be7153\n---\n\n{{level.actor}} approves the {{proposal_name}} proposal proposed by {{proposer}} with the {{level.permission}} permission of {{level.actor}}." - },{ - "name": "cancel", - "type": "cancel", - "ricardian_contract": "---\nspec_version: \"0.2.0\"\ntitle: Cancel Proposed Transaction\nsummary: '{{nowrap canceler}} cancels the {{nowrap proposal_name}} proposal'\nicon: https://raw.githubusercontent.com/eosnetworkfoundation/eos-system-contracts/main/contracts/icons/multisig.png#4fb41d3cf02d0dd2d35a29308e93c2d826ec770d6bb520db668f530764be7153\n---\n\n{{canceler}} cancels the {{proposal_name}} proposal submitted by {{proposer}}." - },{ - "name": "exec", - "type": "exec", - "ricardian_contract": "---\nspec_version: \"0.2.0\"\ntitle: Execute Proposed Transaction\nsummary: '{{nowrap executer}} executes the {{nowrap proposal_name}} proposal'\nicon: https://raw.githubusercontent.com/eosnetworkfoundation/eos-system-contracts/main/contracts/icons/multisig.png#4fb41d3cf02d0dd2d35a29308e93c2d826ec770d6bb520db668f530764be7153\n---\n\n{{executer}} executes the {{proposal_name}} proposal submitted by {{proposer}} if the minimum required approvals for the proposal have been secured." - },{ - "name": "invalidate", - "type": "invalidate", - "ricardian_contract": "---\nspec_version: \"0.2.0\"\ntitle: Invalidate All Approvals\nsummary: '{{nowrap account}} invalidates approvals on outstanding proposals'\nicon: https://raw.githubusercontent.com/eosnetworkfoundation/eos-system-contracts/main/contracts/icons/multisig.png#4fb41d3cf02d0dd2d35a29308e93c2d826ec770d6bb520db668f530764be7153\n---\n\n{{account}} invalidates all approvals on proposals which have not yet executed." - },{ - "name": "propose", - "type": "propose", - "ricardian_contract": "---\nspec_version: \"0.2.0\"\ntitle: Propose Transaction\nsummary: '{{nowrap proposer}} creates the {{nowrap proposal_name}}'\nicon: https://raw.githubusercontent.com/eosnetworkfoundation/eos-system-contracts/main/contracts/icons/multisig.png#4fb41d3cf02d0dd2d35a29308e93c2d826ec770d6bb520db668f530764be7153\n---\n\n{{proposer}} creates the {{proposal_name}} proposal for the following transaction:\n{{to_json trx}}\n\nThe proposal requests approvals from the following accounts at the specified permission levels:\n{{#each requested}}\n + {{this.permission}} permission of {{this.actor}}\n{{/each}}\n\nIf the proposed transaction is not executed prior to {{trx.expiration}}, the proposal will automatically expire." - },{ - "name": "unapprove", - "type": "unapprove", - "ricardian_contract": "---\nspec_version: \"0.2.0\"\ntitle: Unapprove Proposed Transaction\nsummary: '{{nowrap level.actor}} revokes the approval previously provided to {{nowrap proposal_name}} proposal'\nicon: https://raw.githubusercontent.com/eosnetworkfoundation/eos-system-contracts/main/contracts/icons/multisig.png#4fb41d3cf02d0dd2d35a29308e93c2d826ec770d6bb520db668f530764be7153\n---\n\n{{level.actor}} revokes the approval previously provided at their {{level.permission}} permission level from the {{proposal_name}} proposal proposed by {{proposer}}." - } - ], - "tables": [{ - "name": "approvals", - "index_type": "i64", - "key_names": [], - "key_types": [], - "type": "old_approvals_info" - },{ - "name": "approvals2", - "index_type": "i64", - "key_names": [], - "key_types": [], - "type": "approvals_info" - },{ - "name": "invals", - "index_type": "i64", - "key_names": [], - "key_types": [], - "type": "invalidation" - },{ - "name": "proposal", - "index_type": "i64", - "key_names": [], - "key_types": [], - "type": "proposal" - } - ], - "ricardian_clauses": [], - "error_messages": [], - "abi_extensions": [], - "variants": [], - "action_results": [] -} diff --git a/plugins/trace_api_plugin/examples/abis/sysio.token.abi b/plugins/trace_api_plugin/examples/abis/sysio.token.abi deleted file mode 100644 index da098d8d5c..0000000000 --- a/plugins/trace_api_plugin/examples/abis/sysio.token.abi +++ /dev/null @@ -1,184 +0,0 @@ -{ - "version": "sysio::abi/1.2", - "types": [], - "structs": [{ - "name": "account", - "base": "", - "fields": [{ - "name": "balance", - "type": "asset" - } - ] - },{ - "name": "close", - "base": "", - "fields": [{ - "name": "owner", - "type": "name" - },{ - "name": "symbol", - "type": "symbol" - } - ] - },{ - "name": "create", - "base": "", - "fields": [{ - "name": "issuer", - "type": "name" - },{ - "name": "maximum_supply", - "type": "asset" - } - ] - },{ - "name": "currency_stats", - "base": "", - "fields": [{ - "name": "supply", - "type": "asset" - },{ - "name": "max_supply", - "type": "asset" - },{ - "name": "issuer", - "type": "name" - } - ] - },{ - "name": "issue", - "base": "", - "fields": [{ - "name": "to", - "type": "name" - },{ - "name": "quantity", - "type": "asset" - },{ - "name": "memo", - "type": "string" - } - ] - },{ - "name": "issuefixed", - "base": "", - "fields": [{ - "name": "to", - "type": "name" - },{ - "name": "supply", - "type": "asset" - },{ - "name": "memo", - "type": "string" - } - ] - },{ - "name": "open", - "base": "", - "fields": [{ - "name": "owner", - "type": "name" - },{ - "name": "symbol", - "type": "symbol" - },{ - "name": "ram_payer", - "type": "name" - } - ] - },{ - "name": "retire", - "base": "", - "fields": [{ - "name": "quantity", - "type": "asset" - },{ - "name": "memo", - "type": "string" - } - ] - },{ - "name": "setmaxsupply", - "base": "", - "fields": [{ - "name": "issuer", - "type": "name" - },{ - "name": "maximum_supply", - "type": "asset" - } - ] - },{ - "name": "transfer", - "base": "", - "fields": [{ - "name": "from", - "type": "name" - },{ - "name": "to", - "type": "name" - },{ - "name": "quantity", - "type": "asset" - },{ - "name": "memo", - "type": "string" - } - ] - } - ], - "actions": [{ - "name": "close", - "type": "close", - "ricardian_contract": "---\nspec_version: \"0.2.0\"\ntitle: Close Token Balance\nsummary: 'Close {{nowrap owner}}’s zero quantity balance'\nicon: https://raw.githubusercontent.com/eosnetworkfoundation/eos-system-contracts/main/contracts/icons/token.png#207ff68b0406eaa56618b08bda81d6a0954543f36adc328ab3065f31a5c5d654\n---\n\n{{owner}} agrees to close their zero quantity balance for the {{symbol_to_symbol_code symbol}} token.\n\nRAM will be refunded to the RAM payer of the {{symbol_to_symbol_code symbol}} token balance for {{owner}}." - },{ - "name": "create", - "type": "create", - "ricardian_contract": "---\nspec_version: \"0.2.0\"\ntitle: Create New Token\nsummary: 'Create a new token'\nicon: https://raw.githubusercontent.com/eosnetworkfoundation/eos-system-contracts/main/contracts/icons/token.png#207ff68b0406eaa56618b08bda81d6a0954543f36adc328ab3065f31a5c5d654\n---\n\n{{$action.account}} agrees to create a new token with symbol {{asset_to_symbol_code maximum_supply}} to be managed by {{issuer}}.\n\nThis action will not result any any tokens being issued into circulation.\n\n{{issuer}} will be allowed to issue tokens into circulation, up to a maximum supply of {{maximum_supply}}.\n\nRAM will deducted from {{$action.account}}’s resources to create the necessary records." - },{ - "name": "issue", - "type": "issue", - "ricardian_contract": "---\nspec_version: \"0.2.0\"\ntitle: Issue Tokens into Circulation\nsummary: 'Issue {{nowrap quantity}} into circulation and transfer into {{nowrap to}}’s account'\nicon: https://raw.githubusercontent.com/eosnetworkfoundation/eos-system-contracts/main/contracts/icons/token.png#207ff68b0406eaa56618b08bda81d6a0954543f36adc328ab3065f31a5c5d654\n---\n\nThe token manager agrees to issue {{quantity}} into circulation, and transfer it into {{to}}’s account.\n\n{{#if memo}}There is a memo attached to the transfer stating:\n{{memo}}\n{{/if}}\n\nIf {{to}} does not have a balance for {{asset_to_symbol_code quantity}}, or the token manager does not have a balance for {{asset_to_symbol_code quantity}}, the token manager will be designated as the RAM payer of the {{asset_to_symbol_code quantity}} token balance for {{to}}. As a result, RAM will be deducted from the token manager’s resources to create the necessary records.\n\nThis action does not allow the total quantity to exceed the max allowed supply of the token." - },{ - "name": "issuefixed", - "type": "issuefixed", - "ricardian_contract": "---\nspec_version: \"0.2.0\"\ntitle: Issue Fixed Supply of Tokens into Circulation\nsummary: 'Issue up to {{nowrap supply}} supply into circulation and transfer into {{nowrap to}}’s account'\nicon: https://raw.githubusercontent.com/eosnetworkfoundation/eos-system-contracts/main/contracts/icons/token.png#207ff68b0406eaa56618b08bda81d6a0954543f36adc328ab3065f31a5c5d654\n---\n\nThe token manager agrees to issue tokens up to {{supply}} fixed supply into circulation, and transfer it into {{to}}’s account.\n\n{{#if memo}}There is a memo attached to the transfer stating:\n{{memo}}\n{{/if}}\n\nIf {{to}} does not have a balance for {{asset_to_symbol_code quantity}}, or the token manager does not have a balance for {{asset_to_symbol_code quantity}}, the token manager will be designated as the RAM payer of the {{asset_to_symbol_code quantity}} token balance for {{to}}. As a result, RAM will be deducted from the token manager’s resources to create the necessary records.\n\nThis action does not allow the total quantity to exceed the max allowed supply of the token." - },{ - "name": "open", - "type": "open", - "ricardian_contract": "---\nspec_version: \"0.2.0\"\ntitle: Open Token Balance\nsummary: 'Open a zero quantity balance for {{nowrap owner}}'\nicon: https://raw.githubusercontent.com/eosnetworkfoundation/eos-system-contracts/main/contracts/icons/token.png#207ff68b0406eaa56618b08bda81d6a0954543f36adc328ab3065f31a5c5d654\n---\n\n{{ram_payer}} agrees to establish a zero quantity balance for {{owner}} for the {{symbol_to_symbol_code symbol}} token.\n\nIf {{owner}} does not have a balance for {{symbol_to_symbol_code symbol}}, {{ram_payer}} will be designated as the RAM payer of the {{symbol_to_symbol_code symbol}} token balance for {{owner}}. As a result, RAM will be deducted from {{ram_payer}}’s resources to create the necessary records." - },{ - "name": "retire", - "type": "retire", - "ricardian_contract": "---\nspec_version: \"0.2.0\"\ntitle: Remove Tokens from Circulation\nsummary: 'Remove {{nowrap quantity}} from circulation'\nicon: https://raw.githubusercontent.com/eosnetworkfoundation/eos-system-contracts/main/contracts/icons/token.png#207ff68b0406eaa56618b08bda81d6a0954543f36adc328ab3065f31a5c5d654\n---\n\nThe token manager agrees to remove {{quantity}} from circulation, taken from their own account.\n\n{{#if memo}} There is a memo attached to the action stating:\n{{memo}}\n{{/if}}" - },{ - "name": "setmaxsupply", - "type": "setmaxsupply", - "ricardian_contract": "---\nspec_version: \"0.2.0\"\ntitle: Set Max Supply\nsummary: 'Set max supply for token'\nicon: https://raw.githubusercontent.com/eosnetworkfoundation/eos-system-contracts/main/contracts/icons/token.png#207ff68b0406eaa56618b08bda81d6a0954543f36adc328ab3065f31a5c5d654\n---\n\n{{issuer}} will be allowed to issue tokens into circulation, up to a maximum supply of {{maximum_supply}}.\n\nThis action will not result any any tokens being issued into circulation." - },{ - "name": "transfer", - "type": "transfer", - "ricardian_contract": "---\nspec_version: \"0.2.0\"\ntitle: Transfer Tokens\nsummary: 'Send {{nowrap quantity}} from {{nowrap from}} to {{nowrap to}}'\nicon: https://raw.githubusercontent.com/eosnetworkfoundation/eos-system-contracts/main/contracts/icons/transfer.png#5dfad0df72772ee1ccc155e670c1d124f5c5122f1d5027565df38b418042d1dd\n---\n\n{{from}} agrees to send {{quantity}} to {{to}}.\n\n{{#if memo}}There is a memo attached to the transfer stating:\n{{memo}}\n{{/if}}\n\nIf {{from}} is not already the RAM payer of their {{asset_to_symbol_code quantity}} token balance, {{from}} will be designated as such. As a result, RAM will be deducted from {{from}}’s resources to refund the original RAM payer.\n\nIf {{to}} does not have a balance for {{asset_to_symbol_code quantity}}, {{from}} will be designated as the RAM payer of the {{asset_to_symbol_code quantity}} token balance for {{to}}. As a result, RAM will be deducted from {{from}}’s resources to create the necessary records." - } - ], - "tables": [{ - "name": "accounts", - "index_type": "i64", - "key_names": [], - "key_types": [], - "type": "account" - },{ - "name": "stat", - "index_type": "i64", - "key_names": [], - "key_types": [], - "type": "currency_stats" - } - ], - "ricardian_clauses": [], - "error_messages": [], - "abi_extensions": [], - "variants": [], - "action_results": [] -} diff --git a/plugins/trace_api_plugin/examples/abis/sysio.wrap.abi b/plugins/trace_api_plugin/examples/abis/sysio.wrap.abi deleted file mode 100644 index 4326f6e57c..0000000000 --- a/plugins/trace_api_plugin/examples/abis/sysio.wrap.abi +++ /dev/null @@ -1,105 +0,0 @@ -{ - "version": "sysio::abi/1.2", - "types": [], - "structs": [{ - "name": "action", - "base": "", - "fields": [{ - "name": "account", - "type": "name" - },{ - "name": "name", - "type": "name" - },{ - "name": "authorization", - "type": "permission_level[]" - },{ - "name": "data", - "type": "bytes" - } - ] - },{ - "name": "exec", - "base": "", - "fields": [{ - "name": "executer", - "type": "name" - },{ - "name": "trx", - "type": "transaction" - } - ] - },{ - "name": "extension", - "base": "", - "fields": [{ - "name": "type", - "type": "uint16" - },{ - "name": "data", - "type": "bytes" - } - ] - },{ - "name": "permission_level", - "base": "", - "fields": [{ - "name": "actor", - "type": "name" - },{ - "name": "permission", - "type": "name" - } - ] - },{ - "name": "transaction", - "base": "transaction_header", - "fields": [{ - "name": "context_free_actions", - "type": "action[]" - },{ - "name": "actions", - "type": "action[]" - },{ - "name": "transaction_extensions", - "type": "extension[]" - } - ] - },{ - "name": "transaction_header", - "base": "", - "fields": [{ - "name": "expiration", - "type": "time_point_sec" - },{ - "name": "ref_block_num", - "type": "uint16" - },{ - "name": "ref_block_prefix", - "type": "uint32" - },{ - "name": "max_net_usage_words", - "type": "varuint32" - },{ - "name": "max_cpu_usage_ms", - "type": "uint8" - },{ - "name": "delay_sec", - "type": "varuint32" - } - ] - } - ], - "actions": [{ - "name": "exec", - "type": "exec", - "ricardian_contract": "---\nspec_version: \"0.2.0\"\ntitle: Privileged Execute\nsummary: '{{nowrap executer}} executes a transaction while bypassing authority checks'\nicon: https://raw.githubusercontent.com/eosnetworkfoundation/eos-system-contracts/main/contracts/icons/admin.png#9bf1cec664863bd6aaac0f814b235f8799fb02c850e9aa5da34e8a004bd6518e\n---\n\n{{executer}} executes the following transaction while bypassing authority checks:\n{{to_json trx}}\n\n{{$action.account}} must also authorize this action." - } - ], - "tables": [], - "ricardian_clauses": [], - "error_messages": [], - "abi_extensions": [], - "variants": [], - "action_results": [] -} diff --git a/plugins/trace_api_plugin/include/sysio/trace_api/abi_data_handler.hpp b/plugins/trace_api_plugin/include/sysio/trace_api/abi_data_handler.hpp index 5def5b8ac4..29ec02cf6b 100644 --- a/plugins/trace_api_plugin/include/sysio/trace_api/abi_data_handler.hpp +++ b/plugins/trace_api_plugin/include/sysio/trace_api/abi_data_handler.hpp @@ -1,6 +1,14 @@ #pragma once +#include +#include +#include +#include +#include +#include +#include #include +#include #include #include @@ -12,29 +20,72 @@ namespace sysio { namespace trace_api { /** - * Data Handler that uses sysio::chain::abi_serializer to decode data with a known set of ABI's - * Can be used directly as a Data_handler_provider OR shared between request_handlers using the - * ::shared_provider abstraction. + * Data Handler that uses sysio::chain::abi_serializer to decode action data. + * + * ABIs are resolved dynamically via an abi_lookup_fn callback, typically backed + * by the abi_log on disk. Given (account, action_global_sequence) it returns + * {effective_abi_global_seq, abi_bytes} -- the setabi record that was in effect + * at that action, or nullopt. + * + * A bounded LRU caches constructed abi_serializers by (account, effective_abi_global_seq) + * so bulk queries that span many actions sharing the same ABI version hit the same + * cache entry instead of rebuilding the serializer per action. + * + * Decode results flag success/failure via abi_data_handler::decode_status so the + * caller can emit a decode_error field on failure while still returning the raw + * hex payload. + * + * Can be used directly as a Data_handler_provider OR shared between request_handlers + * using the ::shared_provider abstraction. */ class abi_data_handler { public: - explicit abi_data_handler( exception_handler except_handler ) - :except_handler( std::move( except_handler ) ) - { - } + struct lookup_entry { + uint64_t effective_global_seq = 0; + std::vector abi_bytes; + }; + + /// Callback: (account, action_global_sequence) -> {effective_abi_global_seq, abi_bytes} + /// for the ABI version in effect at that action. Called on the HTTP thread; must be thread-safe. + using abi_lookup_fn = std::function(chain::name, uint64_t)>; + + enum class decode_status { + not_attempted, // no ABI available for this action + ok, // decoded successfully + failed // ABI was available but decoding threw + }; + + struct decode_result { + decode_status status = decode_status::not_attempted; + fc::variant params; // decoded action data (empty on failure) + std::optional return_data; // decoded return_value (empty when no return type) + std::string error_message; // populated only when status == failed + }; + + // At ~500 KB per abi_serializer (for large contracts like sysio.system), + // 256 entries caps cache memory at roughly 130 MB in the worst case. + static constexpr size_t default_cache_capacity = 256; + + explicit abi_data_handler( exception_handler except_handler, abi_lookup_fn lookup_fn = {}, + size_t cache_capacity = default_cache_capacity ) + :_abi_lookup_fn( std::move( lookup_fn ) ) + ,except_handler( std::move( except_handler ) ) + ,_cache_capacity( cache_capacity ) + {} /** - * Add an ABI definition to this data handler - * @param name - the name of the account/contract that this ABI belongs to - * @param abi - the ABI definition of that ABI + * Decode the data + return_value of an action using the ABI that was in + * effect when it executed. Callers inspect decode_result::status to decide + * whether to emit params/return_data or a decode_error field. */ - void add_abi( const chain::name& name, chain::abi_def&& abi ); + decode_result decode(const action_trace_v0& action); /** - * Given an action trace, produce a tuple representing the `data` and `return_value` fields in the trace - * - * @param action - trace of the action including metadata necessary for finding the ABI - * @return tuple where the first element is a variant representing the `data` field of the action interpreted by known ABIs OR an empty variant, and the second element represents the `return_value` field of the trace. + * Tuple-shape wrapper used by the response_formatter::process_block pipeline + * (get_block / get_transaction_trace), whose data_handler_function is keyed + * to the {params, return_data} tuple. Returns empty variants on decode + * failure -- callers that need the decode error surfaced (get_actions / + * get_token_transfers) use decode() directly. */ std::tuple> serialize_to_variant(const std::variant& action); @@ -47,6 +98,10 @@ namespace sysio { :handler(handler) {} + decode_result decode(const action_trace_v0& action) { + return handler->decode(action); + } + std::tuple> serialize_to_variant( const std::variant& action ) { return handler->serialize_to_variant(action); } @@ -55,7 +110,29 @@ namespace sysio { }; private: - std::map> abi_serializer_by_account; + // Look up or construct the abi_serializer in effect for the action. Returns + // nullptr if no ABI is available or construction failed. The cache key is + // (account, effective_abi_global_seq) so multiple actions sharing an ABI + // version all hit the same entry. + std::shared_ptr get_serializer(chain::name account, uint64_t action_global_seq); + + using cache_key = std::pair; + struct cache_key_hash { + size_t operator()(const cache_key& k) const noexcept { + return std::hash{}(k.first) ^ (std::hash{}(k.second) << 1); + } + }; + + abi_lookup_fn _abi_lookup_fn; exception_handler except_handler; + + // LRU cache of (account, effective_abi_global_seq) -> abi_serializer. + // _cache_list: MRU at front, LRU at back. _cache_map: key -> iterator into list. + std::mutex _cache_mtx; + std::list>> _cache_list; + std::unordered_map _cache_map; + const size_t _cache_capacity; }; } } diff --git a/plugins/trace_api_plugin/include/sysio/trace_api/abi_log.hpp b/plugins/trace_api_plugin/include/sysio/trace_api/abi_log.hpp new file mode 100644 index 0000000000..b45cca9558 --- /dev/null +++ b/plugins/trace_api_plugin/include/sysio/trace_api/abi_log.hpp @@ -0,0 +1,129 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace sysio::trace_api { + +// --------------------------------------------------------------------------- +// abi_log: append-only on-disk log of ABI records with an in-memory sorted +// index keyed by (account, global_sequence). +// +// Appends stream records to the end of the file with no rewrite of +// existing records. The lookup index lives in memory and is rebuilt by +// scanning the file at startup. +// +// On-disk format (fc::raw-packed little-endian, x86_64 Linux): +// +// Header (16 bytes): magic "ABIL" (u32), version 1 (u32), reserved (u64) +// Records (repeated until EOF): +// account (u64) +// global_seq (u64) +// blob_size (u64) +// blob_bytes (blob_size bytes) +// crc32 (u32) over (account, global_seq, blob_size, blob_bytes) +// +// Writes are not fsync'd; the on-disk tail may lose the last few records on +// a kernel crash. On startup we scan records, validate CRCs, and truncate +// the file at the first invalid record. Lost records can be rebuilt by +// replaying setabi + lazy-fetch against the chain. +// +// Thread-safety: +// - append() takes _append_mtx for the cfile write, releases it, then +// takes _index_mtx to insert into the lookup index. Two mutexes so +// lookups never wait for file I/O on a concurrent append. +// - lookup() takes _index_mtx briefly to copy (blob_offset, blob_size), +// releases it, then pread()s the blob bytes. pread is atomic w.r.t. +// the fd's shared position and safe to issue concurrently. +// --------------------------------------------------------------------------- + +struct abi_log_header { + // Stored little-endian on disk so a hex dump of the first 4 bytes reads "ABIL". + static constexpr uint32_t magic_value = 0x4C494241; // bytes on disk: 'A','B','I','L' + static constexpr uint32_t current_version = 1; + + uint32_t magic = magic_value; + uint32_t version = current_version; + uint64_t reserved = 0; +}; +static_assert(sizeof(abi_log_header) == 16); + +class abi_log { +public: + explicit abi_log(const std::filesystem::path& path); + + abi_log(const abi_log&) = delete; + abi_log& operator=(const abi_log&) = delete; + + bool valid() const { return _valid; } + + // Append a new ABI record. Thread-safe. Last-write-wins for duplicate + // (account, global_seq) keys. + void append(chain::name account, uint64_t global_seq, std::vector abi_bytes); + + struct lookup_result { + uint64_t effective_global_seq = 0; // global_seq of the ABI record that matched + std::vector abi_bytes; + }; + + // Look up the ABI in effect for account at the largest recorded + // global_seq <= the query. Returns nullopt if no record matches. + // The returned effective_global_seq is the global_seq the ABI was + // recorded at (used as a stable cache key by decoders). + // Thread-safe; may run concurrently with append(). + std::optional lookup(chain::name account, uint64_t global_seq) const; + + // Returns true if at least one record exists for the account at any + // global_sequence. Used by chain extraction to decide whether to lazy- + // fetch an ABI on first encounter. Thread-safe. + bool has_entry(chain::name account) const; + +private: + // Fixed-size record header on disk: (account, global_seq, blob_size). + // Trailer is a u32 crc32 over (header + blob_bytes). + // chain::name is a uint64_t-wrapping struct, so this is 24 bytes with no + // padding and the on-disk wire format is identical to (u64, u64, u64). + struct record_header { + chain::name account; + uint64_t global_seq = 0; + uint64_t blob_size = 0; + }; + static_assert(sizeof(record_header) == 24); + + struct index_entry { + uint64_t blob_file_offset = 0; // file offset of blob_bytes (not the record_header) + uint64_t blob_size = 0; + }; + using index_key = std::pair; + + // Walk the log file from the header onwards. Returns the offset of the + // end of the last valid record. The file is truncated at that offset if + // any trailing bytes were discarded due to CRC failure or truncation. + uint64_t recover_from_disk(const std::filesystem::path& path); + + static uint32_t compute_record_crc(const record_header& hdr, const char* blob, uint64_t blob_size); + + // _append_mtx serializes the cfile write + _end_offset update. + // _index_mtx guards _index. Separate so lookups never block on a + // concurrent append's file I/O. Append acquires _append_mtx first and + // always releases it before acquiring _index_mtx, so no deadlock is + // possible (lookups only take _index_mtx). + std::mutex _append_mtx; + mutable std::mutex _index_mtx; + std::map _index; + fc::cfile _cfile; // held open for appends; reads go through fileno() + pread + uint64_t _end_offset{0}; + bool _valid{false}; +}; + +} // namespace sysio::trace_api + +FC_REFLECT(sysio::trace_api::abi_log_header, (magic)(version)(reserved)) diff --git a/plugins/trace_api_plugin/include/sysio/trace_api/bloom_sidecar.hpp b/plugins/trace_api_plugin/include/sysio/trace_api/bloom_sidecar.hpp new file mode 100644 index 0000000000..315352e121 --- /dev/null +++ b/plugins/trace_api_plugin/include/sysio/trace_api/bloom_sidecar.hpp @@ -0,0 +1,227 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace sysio::trace_api { + +/// Per-slice bloom sidecar: lets get_actions skip a whole slice when the requested receiver (or receiver+action) is +/// not present anywhere in the slice's action traces. Contains two filters in one file: +/// - receivers: boost::bloom::filter over action_trace_v0::receiver +/// - recv_action: boost::bloom::filter over pack_recv_action(receiver, action) +/// A negative bloom probe is authoritative (skip the slice). A positive probe falls through to the normal scan. +/// Missing or corrupt sidecar -> reader is invalid -> caller falls back to full scan. +namespace bloom { + +/// File format constants. Stored little-endian on disk so a hex dump of the first 4 bytes reads "WIRB"; matches the +/// convention in blk_offset_index_header and the rest of the trace_api sidecars. Native-endian, x86_64 Linux only. +inline constexpr uint32_t magic_value = 0x42524957; // bytes on disk: 'W','I','R','B' +inline constexpr uint32_t file_version = 1; +inline constexpr uint32_t k_hashes = 7; ///< Fixed at compile time; reader rejects mismatched files. +inline constexpr double target_fpr = 0.01; ///< 1% false-positive rate. Irrelevant for negatives. +inline constexpr uint32_t min_capacity = 32; ///< Floor on filter sizing to avoid degenerate tiny filters. +/// Defensive upper bound on per-filter bit count in a loaded sidecar - a corrupted or maliciously-crafted file with +/// an absurd capacity value would otherwise trigger a huge std::vector allocation. 128 MiB per filter is ~500x the +/// size of a realistic busy-mainnet slice bloom, so no legitimate file hits it. +inline constexpr uint64_t max_capacity_bits = 128ull * 1024 * 1024 * 8; + +/// Raw on-disk header. Body layout: recv bits (recv_capacity_bits/8 bytes) then recv_action bits, trailing uint32 +/// CRC32 over [header | body]. Fields are ordered so natural alignment produces no padding and the layout is +/// stable across compilers; a `reserved` pad word keeps the uint64 fields 8-byte aligned without #pragma pack. +struct header { + uint32_t magic = magic_value; + uint32_t version = file_version; + uint32_t k_hash_count = k_hashes; + uint32_t n_recv = 0; ///< Distinct receivers inserted (pre-rounding). + uint32_t n_recv_action = 0; ///< Distinct (receiver, action) pairs inserted. + uint32_t reserved = 0; + uint64_t recv_capacity_bits = 0; ///< Filter capacity in bits; reader constructs filter with this. + uint64_t recv_action_capacity_bits = 0; +}; +static_assert(sizeof(header) == 4 * 6 + 8 * 2, "bloom::header layout drift"); + +/// Deterministic packing of (receiver, action) into one 64-bit key for the composite filter. Rotate receiver to +/// separate it from the action in the bit distribution so distinct (r, a) pairs don't collide on the common +/// receiver==action case. Must match between write and read paths. +inline uint64_t pack_recv_action(chain::name r, chain::name a) noexcept { + const uint64_t rv = r.to_uint64_t(); + const uint64_t av = a.to_uint64_t(); + return ((rv << 13) | (rv >> (64 - 13))) ^ av; +} + +using filter_t = boost::bloom::filter; + +} // namespace bloom + +/// Accumulates distinct receivers and (receiver, action) pairs observed while a slice is being written. Finalize +/// sizes and materializes two blooms and writes the sidecar file atomically (temp + rename). Memory cost is two +/// hash sets keyed on uint64_t; at Wire-mainnet scale these stay a few tens of KB per open slice. +class bloom_builder { +public: + void add_action(const action_trace_v0& a) { + _receivers.insert(a.receiver.to_uint64_t()); + _recv_actions.insert(bloom::pack_recv_action(a.receiver, a.action)); + } + + void add_block(const block_trace_v0& bt) { + for (const auto& trx : bt.transactions) { + for (const auto& act : trx.actions) { + add_action(act); + } + } + } + + /// true when no actions have been fed; finalize_and_write still produces a valid file whose probes always miss. + bool empty() const noexcept { return _receivers.empty() && _recv_actions.empty(); } + + std::size_t receiver_count() const noexcept { return _receivers.size(); } + std::size_t recv_action_count() const noexcept { return _recv_actions.size(); } + + /// Writes to `path + ".tmp"` then renames over `path` to keep the sidecar crash-consistent: either the old file + /// remains intact or the new one is fully installed, never a partial file under the canonical name. + void finalize_and_write(const std::filesystem::path& path) const { + const std::size_t n_recv = std::max(_receivers.size(), bloom::min_capacity); + const std::size_t n_recv_action = std::max(_recv_actions.size(), bloom::min_capacity); + + bloom::filter_t recv_f{n_recv, bloom::target_fpr}; + for (uint64_t v : _receivers) recv_f.insert(v); + + bloom::filter_t ra_f{n_recv_action, bloom::target_fpr}; + for (uint64_t v : _recv_actions) ra_f.insert(v); + + bloom::header hdr{}; + hdr.n_recv = static_cast(_receivers.size()); + hdr.n_recv_action = static_cast(_recv_actions.size()); + hdr.recv_capacity_bits = recv_f.capacity(); + hdr.recv_action_capacity_bits = ra_f.capacity(); + + const auto recv_bits = recv_f.array(); + const auto ra_bits = ra_f.array(); + + boost::crc_32_type crc; + crc.process_bytes(&hdr, sizeof(hdr)); + crc.process_bytes(recv_bits.data(), recv_bits.size()); + crc.process_bytes(ra_bits.data(), ra_bits.size()); + const uint32_t crc_v = crc.checksum(); + + const auto tmp = std::filesystem::path(path).concat(".tmp"); + { + // fc::cfile throws std::ios_base::failure on open/write error; the store_provider append path swallows that + // (bloom is advisory) and bare file-system errors surface at the rename. Scope the cfile so it closes (and + // flushes) before we rename, keeping the temp-then-rename atomicity guarantee intact. + fc::cfile out(tmp, fc::cfile::truncate_rw_mode); + out.write(reinterpret_cast(&hdr), sizeof(hdr)); + out.write(reinterpret_cast(recv_bits.data()), recv_bits.size()); + out.write(reinterpret_cast(ra_bits.data()), ra_bits.size()); + out.write(reinterpret_cast(&crc_v), sizeof(crc_v)); + } + std::filesystem::rename(tmp, path); + } + +private: + boost::unordered_flat_set _receivers; + boost::unordered_flat_set _recv_actions; +}; + +/// Load-time view of a sidecar. Constructor is strict: any failure (missing file, bad magic, version mismatch, +/// truncated body, CRC mismatch) leaves the reader in an invalid state and may_contain_* always returns true. +/// Returning true on invalid means "don't skip" - the correct fail-safe since a false negative would silently drop +/// matching actions from the response. +class bloom_reader { +public: + bloom_reader() = default; + + explicit bloom_reader(const std::filesystem::path& path) { + load(path); + } + + bool valid() const noexcept { return _valid; } + + /// Invariant: on invalid reader, returns true so the caller treats the slice as "may contain, scan". + bool may_contain_receiver(chain::name r) const noexcept { + if (!_valid) return true; + return _recv.may_contain(r.to_uint64_t()); + } + + bool may_contain_recv_action(chain::name r, chain::name a) const noexcept { + if (!_valid) return true; + return _recv_action.may_contain(bloom::pack_recv_action(r, a)); + } + +private: + void load(const std::filesystem::path& path) { + // Any failure here leaves _valid == false, which fails safe: the probe methods return true and the caller + // scans the slice. fc::cfile throws std::ios_base::failure on open/read errors; we catch broadly to keep + // that guarantee without propagating to the query path. + try { + std::error_code ec; + const auto file_size = std::filesystem::file_size(path, ec); + if (ec || file_size < sizeof(bloom::header) + sizeof(uint32_t)) return; + + fc::cfile in(path, fc::cfile::update_rw_mode); + + bloom::header hdr; + in.read(reinterpret_cast(&hdr), sizeof(hdr)); + if (hdr.magic != bloom::magic_value) return; + if (hdr.version != bloom::file_version) return; + if (hdr.k_hash_count != bloom::k_hashes) return; + if (hdr.recv_capacity_bits % CHAR_BIT != 0) return; + if (hdr.recv_action_capacity_bits % CHAR_BIT != 0) return; + // Defensive cap: a corrupted or crafted header with an absurd capacity would otherwise trigger a huge + // allocation below. Real bloom sizes are <1 MB even for busy-mainnet slices; 128 MB per filter is a wide + // safety margin. + if (hdr.recv_capacity_bits > bloom::max_capacity_bits) return; + if (hdr.recv_action_capacity_bits > bloom::max_capacity_bits) return; + + const std::size_t recv_bytes = hdr.recv_capacity_bits / CHAR_BIT; + const std::size_t ra_bytes = hdr.recv_action_capacity_bits / CHAR_BIT; + const std::size_t expected_size = sizeof(bloom::header) + recv_bytes + ra_bytes + sizeof(uint32_t); + if (file_size != expected_size) return; + + std::vector recv_buf(recv_bytes); + std::vector ra_buf(ra_bytes); + in.read(reinterpret_cast(recv_buf.data()), recv_bytes); + in.read(reinterpret_cast(ra_buf.data()), ra_bytes); + uint32_t file_crc = 0; + in.read(reinterpret_cast(&file_crc), sizeof(file_crc)); + + boost::crc_32_type crc; + crc.process_bytes(&hdr, sizeof(hdr)); + crc.process_bytes(recv_buf.data(), recv_buf.size()); + crc.process_bytes(ra_buf.data(), ra_buf.size()); + if (crc.checksum() != file_crc) return; + + // boost::bloom guarantees filter{f.capacity()}.capacity() == f.capacity() (see boost/bloom/detail/core.hpp + // :480), so reconstruction with the saved capacity produces an array of the saved byte size. The + // size-equality check below is a belt-and-suspenders guard against a future boost::bloom change. + bloom::filter_t recv_f{hdr.recv_capacity_bits}; + bloom::filter_t ra_f{hdr.recv_action_capacity_bits}; + if (recv_f.array().size() != recv_buf.size()) return; + if (ra_f.array().size() != ra_buf.size()) return; + std::memcpy(recv_f.array().data(), recv_buf.data(), recv_buf.size()); + std::memcpy(ra_f.array().data(), ra_buf.data(), ra_buf.size()); + + _recv = std::move(recv_f); + _recv_action = std::move(ra_f); + _valid = true; + } catch (const std::exception&) { + // File-system or read error: leave _valid == false so may_contain_* returns true -> scan fallback. + _valid = false; + } + } + + bool _valid = false; + bloom::filter_t _recv; + bloom::filter_t _recv_action; +}; + +} // namespace sysio::trace_api diff --git a/plugins/trace_api_plugin/include/sysio/trace_api/chain_extraction.hpp b/plugins/trace_api_plugin/include/sysio/trace_api/chain_extraction.hpp index be25f22136..a51714a601 100644 --- a/plugins/trace_api_plugin/include/sysio/trace_api/chain_extraction.hpp +++ b/plugins/trace_api_plugin/include/sysio/trace_api/chain_extraction.hpp @@ -1,28 +1,49 @@ #pragma once #include -#include #include +#include +#include +#include +#include +#include +#include #include #include #include +#include -namespace sysio { namespace trace_api { +namespace sysio::trace_api { using chain::transaction_id_type; using chain::packed_transaction; +using namespace sysio::chain::literals; + +// Compile-time constants for setabi detection: built from constexpr _n +// literals so we don't pay a chain::name construction cost on every action. +inline constexpr chain::name setabi_action_name = "setabi"_n; template class chain_extraction_impl_type { public: + /** + * Called to fetch the current ABI bytes for an account (lazy init on first encounter). + * Returns nullopt if the account has no ABI. + */ + using abi_fetcher_t = std::function>(chain::name)>; + /** * Chain Extractor for capturing transaction traces, action traces, and block info. - * @param store provider of append & append_lib + * @param store provider of append, append_lib, and append_abi * @param except_handler called on exceptions, logging if any is left to the user + * @param abi_fetcher optional callback to lazily fetch the current ABI for an account; + * called on first encounter of each account; receives global_seq 0 */ - chain_extraction_impl_type( StoreProvider store, exception_handler except_handler ) + chain_extraction_impl_type( StoreProvider store, exception_handler except_handler, + abi_fetcher_t abi_fetcher = {} ) : store(std::move(store)) , except_handler(std::move(except_handler)) + , abi_fetcher(std::move(abi_fetcher)) {} /// connect to chain controller applied_transaction signal @@ -54,6 +75,80 @@ class chain_extraction_impl_type { } else { cached_traces[trace->id] = {trace, t}; } + + // ABI capture: scan all action traces (including inlines) in this transaction. + // + // First pass: collect setabi targets in this trx so the second pass can + // skip the lazy fetch for any account whose ABI is being replaced here. + // on_applied_transaction runs AFTER all actions in the trx have applied, + // so the chain DB already reflects the new ABI; doing a lazy fetch and + // recording it as target@0 would poison lookups for actions on target + // that executed earlier in this same trx (they need the OLD ABI, which + // is no longer reachable from post-apply state). + std::unordered_set setabi_targets_this_trx; + for (const auto& at : trace->action_traces) { + if (!at.receipt) continue; + if (at.act.account == chain::config::system_account_name && + at.act.name == setabi_action_name) { + try { + chain::name target; + chain::bytes abi_bytes; + auto ds = fc::datastream(at.act.data.data(), at.act.data.size()); + fc::raw::unpack(ds, target); + fc::raw::unpack(ds, abi_bytes); + setabi_targets_this_trx.insert(target); + } catch (const std::exception& e) { + fc_wlog(_log, "trace_api: failed to unpack setabi data (collecting targets) at global_seq {}: {}", + at.receipt->global_sequence, e.what()); + } + } + } + + // Second pass: lazy fetch + setabi record, skipping lazy fetch for any + // account whose ABI is being replaced in this trx. + for (const auto& at : trace->action_traces) { + if (!at.receipt) continue; // skip context-free or failed actions + + const chain::name account = at.act.account; + + // Lazy ABI fetch: on first encounter of an account (that isn't having + // its ABI replaced in this same trx), record its current ABI at + // global_seq 0 so pre-plugin-start actions still decode. "First + // encounter" is decided by the abi_log itself: if it has no record + // for this account, we trigger the fetch. Once any record exists + // (lazy or setabi), we never re-fetch. Using the log as + // source-of-truth avoids holding a per-node-lifetime set of all + // accounts ever observed. + if (abi_fetcher + && setabi_targets_this_trx.count(account) == 0 + && !store.has_abi_entry(account)) + { + try { + if (auto abi = abi_fetcher(account)) + store.append_abi(account, 0, std::move(*abi)); + } catch (const std::exception& e) { + fc_dlog(_log, "trace_api: lazy ABI fetch for {} failed: {}", account, e.what()); + } + } + + // setabi: record the new ABI with its exact global_sequence. + if (at.act.account == chain::config::system_account_name && + at.act.name == setabi_action_name) { + try { + chain::name target_account; + chain::bytes abi_bytes; + auto ds = fc::datastream(at.act.data.data(), at.act.data.size()); + fc::raw::unpack(ds, target_account); + fc::raw::unpack(ds, abi_bytes); + store.append_abi(target_account, + at.receipt->global_sequence, + std::vector(abi_bytes.begin(), abi_bytes.end())); + } catch (const std::exception& e) { + fc_wlog(_log, "trace_api: failed to record setabi at global_seq {}: {}", + at.receipt->global_sequence, e.what()); + } + } + } } void on_accepted_block(const chain::signed_block_ptr& block, const chain::block_id_type& id ) { @@ -65,9 +160,53 @@ class chain_extraction_impl_type { } void on_block_start( uint32_t block_num ) { + if (!startup_checked) { + startup_checked = true; + check_continuity(block_num); + } clear_caches(); } + void check_continuity(uint32_t block_num) { + try { + const auto recorded = store.first_and_last_recorded_blocks(); + if (!recorded) { + fc_ilog(_log, "trace_api: no prior trace data found, starting fresh at block {}", block_num); + return; + } + const uint32_t first = recorded->first; + const uint32_t last = recorded->second; + // Overlap or exact continuation: chain head is within or just past existing data. + // Re-applied blocks will overwrite existing slice entries as they are re-recorded. + if (block_num >= first && block_num <= last + 1) + return; + + if (block_num < first) { + throw std::runtime_error(fmt::format( + "trace_api: chain head ({}) is before the first recorded trace block ({}). " + "To recover: load a snapshot whose chain head is within [{}, {}], " + "or copy the trace files covering blocks {}..{} from another node, " + "or delete the trace directory to start fresh (loses historical traces).", + block_num, first, first, last + 1, block_num, first - 1)); + } + // block_num > last + 1: forward gap + throw std::runtime_error(fmt::format( + "trace_api: gap detected in trace data. Last recorded block: {}, current block: {}. " + "To recover: load a snapshot covering block {} (or earlier within the recorded range), " + "or copy the trace files covering blocks {}..{} from another node, " + "or delete the trace directory to start fresh (loses historical traces).", + last, block_num, last + 1, last + 1, block_num - 1)); + } catch (const yield_exception&) { + // Order matters: yield_exception propagates (it's the signal that the + // plugin's own except_handler uses to unwind the controller), while + // other exceptions from store.* calls or the throws above go through + // except_handler so the operator sees a properly formatted message. + throw; + } catch (...) { + except_handler(MAKE_EXCEPTION_WITH_CONTEXT(std::current_exception())); + } + } + void clear_caches() { cached_traces.clear(); onblock_trace.reset(); @@ -116,9 +255,11 @@ class chain_extraction_impl_type { private: StoreProvider store; exception_handler except_handler; + abi_fetcher_t abi_fetcher; std::map cached_traces; std::optional onblock_trace; + bool startup_checked{false}; }; -}} +} // namespace sysio::trace_api diff --git a/plugins/trace_api_plugin/include/sysio/trace_api/data_log.hpp b/plugins/trace_api_plugin/include/sysio/trace_api/data_log.hpp index e75daf50b0..0573a1214f 100644 --- a/plugins/trace_api_plugin/include/sysio/trace_api/data_log.hpp +++ b/plugins/trace_api_plugin/include/sysio/trace_api/data_log.hpp @@ -4,10 +4,10 @@ #include #include -namespace sysio { namespace trace_api { +namespace sysio::trace_api { using data_log_entry = std::variant< block_trace_v0 >; -}} +} // namespace sysio::trace_api diff --git a/plugins/trace_api_plugin/include/sysio/trace_api/extract_util.hpp b/plugins/trace_api_plugin/include/sysio/trace_api/extract_util.hpp index e4c4a2b1aa..d44596296d 100644 --- a/plugins/trace_api_plugin/include/sysio/trace_api/extract_util.hpp +++ b/plugins/trace_api_plugin/include/sysio/trace_api/extract_util.hpp @@ -2,22 +2,34 @@ #include -namespace sysio { namespace trace_api { +namespace sysio::trace_api { inline action_trace_v0 to_action_trace( const chain::action_trace& at ) { action_trace_v0 r; + r.action_ordinal = at.action_ordinal; + r.creator_action_ordinal = at.creator_action_ordinal; + r.closest_unnotified_ancestor_action_ordinal = at.closest_unnotified_ancestor_action_ordinal; r.receiver = at.receiver; r.account = at.act.account; r.action = at.act.name; r.data = at.act.data; r.return_value = at.return_value; + r.cpu_usage_us = at.cpu_usage_us; + r.net_usage = at.net_usage; if( at.receipt ) { r.global_sequence = at.receipt->global_sequence; + r.recv_sequence = at.receipt->recv_sequence; + r.auth_sequence = at.receipt->auth_sequence; + r.code_sequence = at.receipt->code_sequence; + r.abi_sequence = at.receipt->abi_sequence; } r.authorization.reserve( at.act.authorization.size()); for( const auto& auth : at.act.authorization ) { r.authorization.emplace_back( authorization_trace_v0{auth.actor, auth.permission} ); } + for( const auto& delta : at.account_ram_deltas ) { + r.account_ram_deltas.emplace_back( account_delta_v0{delta.account, delta.delta} ); + } return r; } @@ -25,7 +37,6 @@ inline transaction_trace_v0 to_transaction_trace( const cache_trace& t ) { transaction_trace_v0 r; r.id = t.trace->id; if (t.trace->receipt) { - r.status = chain::transaction_receipt_header::status_enum::executed; r.cpu_usage_us = t.trace->total_cpu_usage_us; // Round up net_usage and convert to words r.net_usage_words = (t.trace->net_usage + 7)/8; @@ -38,9 +49,9 @@ inline transaction_trace_v0 to_transaction_trace( const cache_trace& t ) { r.actions.reserve( t.trace->action_traces.size()); for( const auto& at : t.trace->action_traces ) { - if( !at.context_free ) { // not including CFA at this time - r.actions.emplace_back( to_action_trace(at) ); - } + if( at.context_free ) continue; // skip context-free actions + if( at.except ) continue; // skip failed actions + r.actions.emplace_back( to_action_trace(at) ); } return r; } @@ -57,4 +68,4 @@ inline block_trace_v0 create_block_trace( const chain::signed_block_ptr& block, return r; } -} } +} // namespace sysio::trace_api diff --git a/plugins/trace_api_plugin/include/sysio/trace_api/logging.hpp b/plugins/trace_api_plugin/include/sysio/trace_api/logging.hpp new file mode 100644 index 0000000000..4377ce00bf --- /dev/null +++ b/plugins/trace_api_plugin/include/sysio/trace_api/logging.hpp @@ -0,0 +1,16 @@ +#pragma once + +#include +#include + +namespace sysio::trace_api { + +// Shared "trace_api" logger. Configured by the plugin at startup via +// fc::logger::update(logger_name, _log) so operators control verbosity +// through the "trace_api" entry in logging.json. All trace_api_plugin +// log call sites should use this logger (fc_wlog(_log, ...), etc.) so +// that filtering applies uniformly across the plugin. +inline const std::string logger_name{"trace_api"}; +inline fc::logger _log; + +} // namespace sysio::trace_api diff --git a/plugins/trace_api_plugin/include/sysio/trace_api/metadata_log.hpp b/plugins/trace_api_plugin/include/sysio/trace_api/metadata_log.hpp index 21d23d4a54..6930d54bad 100644 --- a/plugins/trace_api_plugin/include/sysio/trace_api/metadata_log.hpp +++ b/plugins/trace_api_plugin/include/sysio/trace_api/metadata_log.hpp @@ -4,15 +4,15 @@ #include #include -namespace sysio { namespace trace_api { +namespace sysio::trace_api { struct block_entry_v0 { chain::block_id_type id; - uint32_t number; - uint64_t offset; + uint32_t number = 0; + uint64_t offset = 0; }; struct lib_entry_v0 { - uint32_t lib; + uint32_t lib = 0; }; using metadata_log_entry = std::variant< @@ -21,7 +21,7 @@ namespace sysio { namespace trace_api { block_trxs_entry >; -}} +} // namespace sysio::trace_api FC_REFLECT(sysio::trace_api::block_entry_v0, (id)(number)(offset)); FC_REFLECT(sysio::trace_api::lib_entry_v0, (lib)); diff --git a/plugins/trace_api_plugin/include/sysio/trace_api/request_handler.hpp b/plugins/trace_api_plugin/include/sysio/trace_api/request_handler.hpp index 4afae4d35a..e7ee2e746f 100644 --- a/plugins/trace_api_plugin/include/sysio/trace_api/request_handler.hpp +++ b/plugins/trace_api_plugin/include/sysio/trace_api/request_handler.hpp @@ -1,6 +1,12 @@ #pragma once +#include +#include +#include #include +#include +#include +#include #include #include #include @@ -15,6 +21,57 @@ namespace sysio::trace_api { }; } + // Serialise an action's authorization vector to {actor, permission} JSON + // objects. Shared by get_block (response_formatter) and the get_actions + // / get_token_transfers handlers below. + inline fc::variants serialize_authorizations(const std::vector& auths) { + fc::variants result; + result.reserve(auths.size()); + for (const auto& a : auths) + result.emplace_back(fc::mutable_variant_object() + ("actor", a.actor.to_string()) + ("permission", a.permission.to_string())); + return result; + } + + // ABI decode payload used by the shared variant builder below. Mirrors + // abi_data_handler::decode_result fields that end up in the HTTP response. + struct decoded_action { + fc::variant params; + std::optional return_data; + std::string error_message; + }; + + enum class variant_shape { + full, // get_actions / get_block: every field on action_trace_v0 + slim, // get_token_transfers: drops ordinals, receipt seqs, ram_deltas, resource usage + }; + + // Shared action->variant builder used by get_actions, get_token_transfers, + // and the legacy process_block path. Lives in request_handler.cpp so the + // formatting logic isn't duplicated in every template instantiation. + fc::mutable_variant_object build_action_variant(const action_trace_v0& a, + const decoded_action& decoded, + variant_shape shape); + + /** + * Filter parameters for the get_actions endpoint. + */ + struct action_query { + std::optional receiver; ///< filter by receiver account (any if unset) + std::optional account; ///< filter by contract/code account (any if unset) + std::optional action; ///< filter by action name (any if unset) + uint32_t block_num_start = 0; + uint32_t block_num_end = std::numeric_limits::max(); + }; + + /** + * Result returned by get_actions. + */ + struct actions_result { + fc::variants actions; + }; + template class request_handler { public: @@ -63,33 +120,194 @@ namespace sysio::trace_api { */ fc::variant get_transaction_trace(chain::transaction_id_type trxid, uint32_t block_height){ _log("get_transaction_trace called" ); - fc::variant result = {}; - // extract the transaction trace from the block trace - auto resp = get_block_trace(block_height); - if (!resp.is_null()) { - auto& b_mvo = resp.get_object(); - if (b_mvo.contains("transactions")) { - auto& transactions = b_mvo["transactions"]; - std::string input_id = trxid.str(); - for (uint32_t i = 0; i < transactions.size(); ++i) { - if (transactions[i].is_null()) continue; - auto& t_mvo = transactions[i].get_object(); - if (t_mvo.contains("id")) { - const auto& t_id = t_mvo["id"].get_string(); - if (t_id == input_id) { - result = transactions[i]; - break; + // Named local with the exact return type so the compiler can NRVO it + // directly into the caller's slot. Scan the raw transaction_trace_v0[] + // for a matching id (cheap sha256 equality) and build the variant for + // ONLY the matching trx. The previous implementation materialised the + // full block variant (ABI-decoding every action) and then string-matched + // through the resulting JSON, which cost O(block) work per lookup. + fc::variant result; + + auto data = logfile_provider.get_block(block_height); + if (!data) { + _log("No block found at block height " + std::to_string(block_height)); + return result; + } + + auto data_handler = [this](const std::variant& action) -> std::tuple> { + return std::visit([&](const auto& a) { + return data_handler_provider.serialize_to_variant(a); + }, action); + }; + const bool irreversible = std::get<1>(*data); + + std::visit([&](const auto& block_trace) { + for (const auto& trx : block_trace.transactions) { + if (trx.id == trxid) { + // Build a single-transaction variant by calling the shared + // formatter on a synthesized block containing only this trx. + // Avoids decoding any other transactions in the block. + auto single = block_trace; // copy + single.transactions = {trx}; + auto block_var = detail::response_formatter::process_block( + data_log_entry{single}, irreversible, data_handler); + auto& txs = block_var.get_object()["transactions"]; + if (!txs.is_null() && txs.size() > 0) + result = txs[size_t{0}]; + return; + } + } + _log("Transaction id " + trxid.str() + " not found in block " + std::to_string(block_height)); + }, std::get<0>(*data)); + + return result; + } + + /** + * Scan a block range for action traces matching the given filter. + * + * Blocks are scanned in ascending order. Within each block, actions are visited in ascending + * global_sequence order. All matching actions in the (caller-clamped) range are returned; + * the caller is responsible for capping block_num_end so that the response stays bounded. + * + * @param query - filter parameters + * @return actions_result containing the matching actions + */ + actions_result get_actions(const action_query& query) { + return get_actions_impl(query, variant_shape::full); + } + + /// Slim response for get_token_transfers: transfer-relevant fields only. + /// Omits execution-tree ordinals, receipt sequences, ram_deltas, and resource usage. + actions_result get_token_transfer_actions(const action_query& query) { + return get_actions_impl(query, variant_shape::slim); + } + + private: + actions_result get_actions_impl(const action_query& query, variant_shape shape) { + actions_result result; + + // Hoist filter state out of the hot loop: avoids re-loading the optional's discriminator and value on every + // action comparison in the inner scan. + const bool has_receiver = query.receiver.has_value(); + const bool has_account = query.account.has_value(); + const bool has_action = query.action.has_value(); + const chain::name receiver_name = has_receiver ? *query.receiver : chain::name{}; + const chain::name account_name = has_account ? *query.account : chain::name{}; + const chain::name action_name = has_action ? *query.action : chain::name{}; + + // Reused across all transactions in all blocks: clear() keeps the vector's capacity so repeated scans of + // trxs with similar action counts avoid per-trx allocations. + std::vector matches; + + // Per-slice bloom skip state. When the caller supplies a receiver (or a non-include_notifications + // request whose receiver is auto-mirrored onto account upstream), probe the slice's receiver bloom and + // advance block_num past the slice on a negative probe. The bloom is opened once per slice (lazy; only + // if skipping is useful for this query) and held for the life of the scan through that slice. If the + // sidecar is missing or CRC-corrupt the bloom_reader is invalid and may_contain_* returns true, which + // preserves the existing scan behaviour. + const uint32_t stride = logfile_provider.slice_stride(); + // Both bloom probes below are gated on has_receiver (the receiver bloom is the only one we can hit with a + // single-filter probe, and the (receiver, action) composite still needs the receiver term). A query with + // only account and/or action set can't benefit from the bloom, so don't even open the sidecar. + const bool skip_eligible = has_receiver; + std::optional current_slice; + bool skip_current_slice = false; + + const uint32_t end = query.block_num_end; + for (uint32_t block_num = query.block_num_start; block_num <= end; ++block_num) { + if (skip_eligible) { + const uint32_t slice = logfile_provider.slice_number(block_num); + if (!current_slice || *current_slice != slice) { + current_slice = slice; + skip_current_slice = false; + bloom_reader r = logfile_provider.get_bloom(slice); + if (r.valid()) { + if (!r.may_contain_receiver(receiver_name)) { + skip_current_slice = true; + } else if (has_action + && !r.may_contain_recv_action(receiver_name, action_name)) { + skip_current_slice = true; } } } - if( result.is_null() ) - _log("Exhausted all " + std::to_string(transactions.size()) + " transactions in block " + b_mvo["number"].as_string() + " without finding trxid " + trxid.str()); + if (skip_current_slice) { + // Jump block_num to the last block of this slice so the for-loop's ++block_num takes us to the + // first block of the next slice. Clamp to the query's end so we don't wrap around if this is + // the last slice in the range. + const uint32_t slice_last = (slice + 1) * stride - 1; + block_num = std::min(slice_last, end); + continue; + } } + + auto data = logfile_provider.get_block(block_num); + if (!data) continue; + + // Block-finality marker mirrors get_block's "status" field. Sourced from the same data log + // tuple so callers can trust trace_api as a single source of truth for "did this action's + // block reach finality." Promotion (pending -> irreversible) happens out-of-band as LIB + // advances; consumers that gate on finality must re-poll, same as get_block today. + const bool irreversible_block = std::get<1>(*data); + const char* const block_status_str = irreversible_block ? "irreversible" : "pending"; + + std::visit([&](const auto& bt) { + for (const auto& trx : bt.transactions) { + // Filter first, sort after. trx.actions is stored in schedule order (how apply_context scheduled + // action slots), which is NOT global_sequence order when a parent action queues both inline actions + // and require_recipient notifications: notifications run before inlines, so the inline's + // global_sequence is higher than later-scheduled notifications'. Sort the matches by + // global_sequence so clients see execution order, matching chain_plugin's push_transaction response. + // global_sequence is unique per action, so sort stability is not required. Sorting only after + // filtering avoids the cost for transactions whose actions are all rejected by the filter - the + // common case when scanning for a specific receiver/account/action across a wide block range. + matches.clear(); + for (const auto& a : trx.actions) { + if (has_receiver && a.receiver != receiver_name) continue; + if (has_account && a.account != account_name) continue; + if (has_action && a.action != action_name) continue; + matches.push_back(&a); + } + if (matches.empty()) continue; + std::ranges::sort(matches, {}, &action_trace_v0::global_sequence); + + // Hoist per-trx variant fields so a multi-match trx doesn't repeat the checksum->hex conversion or + // re-read the same block-level members for each emitted action. trx_cpu_usage_us / + // trx_net_usage_words are full-shape only - they are the parent transaction's resource totals + // (action-level cpu_usage_us / net_usage are per-action and in different units: action net_usage + // is bytes, trx net_usage_words is ceil(net_usage / 8)). Slim (get_token_transfers) omits all + // resource fields, so we don't emit the trx-level totals there either. + const std::string trx_id_str = trx.id.str(); + const uint32_t trx_block = trx.block_num; + const auto& trx_time = trx.block_time; + const auto& trx_pbid = trx.producer_block_id; + const bool full_shape = (shape == variant_shape::full); + + for (const action_trace_v0* ap : matches) { + const auto& a = *ap; + // Decode via the provider; build the variant via the shared helper so get_actions / + // get_token_transfers / get_block all agree on field shapes. + auto dec = data_handler_provider.decode(a); + decoded_action da{std::move(dec.params), std::move(dec.return_data), std::move(dec.error_message)}; + fc::mutable_variant_object av = build_action_variant(a, da, shape); + av("trx_id", trx_id_str) + ("block_num", trx_block) + ("block_time", trx_time) + ("producer_block_id", trx_pbid) + ("block_status", block_status_str); + if (full_shape) { + av("trx_cpu_usage_us", trx.cpu_usage_us) + ("trx_net_usage_words", trx.net_usage_words); + } + result.actions.emplace_back(std::move(av)); + } + } + }, std::get<0>(*data)); } + return result; } - private: LogfileProvider logfile_provider; DataHandlerProvider data_handler_provider; log_handler _log; diff --git a/plugins/trace_api_plugin/include/sysio/trace_api/store_provider.hpp b/plugins/trace_api_plugin/include/sysio/trace_api/store_provider.hpp index 6de1a874c9..d6ebc39d1f 100644 --- a/plugins/trace_api_plugin/include/sysio/trace_api/store_provider.hpp +++ b/plugins/trace_api_plugin/include/sysio/trace_api/store_provider.hpp @@ -1,15 +1,19 @@ #pragma once #include -#include +#include #include +#include #include #include #include +#include +#include #include -#include -#include #include +#include +#include +#include namespace sysio::trace_api { @@ -86,6 +90,30 @@ namespace sysio::trace_api { class store_provider; + // On-disk format: trace_blk_idx_.log + // + // Layout: blk_offset_index_header (16 bytes) followed by a flat array of + // width uint64_t slots. Slot (block_num - slice_base) holds offset+1, + // where offset is the position in trace_.log of that block's trace + // data. Slot value 0 means "not present"; this distinguishes a missing + // block from a block stored at offset 0 (the first block in a slice). + // + // The file is pre-allocated sparse at creation, so any slot write is a + // single 8-byte in-place update. Forks naturally overwrite the slot. + // + // Native-endian, x86_64 Linux only (same convention as other slice files). + struct blk_offset_index_header { + // Stored little-endian on disk so a hex dump of the first 4 bytes reads "BLIX". + static constexpr uint32_t magic_value = 0x58494C42; // bytes on disk: 'B','L','I','X' + static constexpr uint32_t current_version = 1; + + uint32_t magic = magic_value; + uint32_t version = current_version; + uint32_t width = 0; // slice width (block count per slice) + uint32_t reserved = 0; + }; + static_assert(sizeof(blk_offset_index_header) == 16); + /** * Provides access to the slice directory. It is only intended to be used by store_provider * and unit tests. @@ -110,6 +138,18 @@ namespace sysio::trace_api { return block_height / _width; } + /** + * Slice stride (blocks per slice) as configured at construction. + */ + uint32_t width() const noexcept { return _width; } + + /** + * Filesystem path for a slice's receiver bloom sidecar. The file is only read/written via the bloom_builder + * and bloom_reader helpers; no fc::cfile overload is provided because the sidecar is written once at slice + * close (temp + rename) and only mmap-style read by the query path. + */ + std::filesystem::path bloom_slice_path(uint32_t slice_number) const; + /** * Find or create the index file associated with the indicated slice_number * @@ -209,6 +249,60 @@ namespace sysio::trace_api { */ void for_each_trx_id_slice(std::function callback) const; + /** + * Derive the slice number from a trx_id slice file path. + * Parses the block-range start from the filename. Returns nullopt if + * the filename does not parse (callers should fall back to a slower + * lookup path rather than skipping the file silently). + */ + std::optional slice_number_from_path(const std::filesystem::path& trx_id_path) const; + + /** + * Find the trx_id index file for a given slice number (or return nullopt if not present). + */ + std::optional find_trx_id_index_slice(uint32_t slice_number) const; + + /** + * Build the trx_id index for a given slice from its trx_id log file. + * No-op if the index already exists for that slice. + */ + void build_trx_id_index(uint32_t slice_number, const log_handler& log); + + /** + * Build the per-slice receiver bloom sidecar from the slice's trace data log. Called on slices that are fully + * past LIB so the source data is final (no fork can reach back into an already-built sidecar). No-op if the + * sidecar already exists or the slice has no uncompressed trace data. + */ + void build_recv_bloom(uint32_t slice_number, const log_handler& log); + + /** + * Return {first, last} block numbers recorded across all index slice files, or nullopt + * if no data exists. Used at startup to detect gaps between existing trace data and the + * current chain head. Atomic in the sense that both values come from a single directory + * scan, so callers don't need to guard against seeing `first` but not `last`. + */ + std::optional> first_and_last_recorded_blocks() const; + + /** + * Record the offset of a block's trace data in trace_.log, via the block-offset + * sidecar trace_blk_idx_.log. Creates the sidecar on first write to a new slice. + * Writes to an existing slot naturally overwrite it (fork re-writes). + */ + void write_block_offset(uint32_t block_height, uint64_t trace_offset) const; + + /** + * O(1) lookup of the trace-log offset for a block via the block-offset sidecar. + * Returns nullopt when the sidecar is missing, the slot is empty, or the file is invalid. + * Callers should fall back to scanning the metadata log in that case. + */ + std::optional lookup_block_offset(uint32_t block_height) const; + + /** + * Current best-known LIB as reported by append_lib. Thread-safe; used by readers to + * determine whether a given block is irreversible without scanning the metadata log. + */ + uint32_t best_known_lib() const; + /** * set the LIB for maintenance * @param lib @@ -244,6 +338,11 @@ namespace sysio::trace_api { // take an open index slice file and verify its header is valid and prepare the file to be appended to (or read from) void validate_existing_index_slice_file(fc::cfile& index_file, open_state state) const; + // Open the block-offset sidecar for a slice; creates and pre-allocates if missing. + // Validates the header on open. Returns false + leaves blk_idx at the sidecar path + // (unopened) if the existing file has a wrong magic/version/width. + bool open_or_create_blk_offset_slice(uint32_t slice_number, fc::cfile& blk_idx) const; + // helper for methods that process irreversible slice files template void process_irreversible_slice_range(uint32_t lib, uint32_t upper_bound_block, std::optional& lower_bound_slice, F&& f); @@ -254,9 +353,11 @@ namespace sysio::trace_api { std::optional _last_cleaned_up_slice; const std::optional _minimum_uncompressed_irreversible_history_blocks; std::optional _last_compressed_slice; + std::optional _last_indexed_slice; + std::optional _last_bloomed_slice; const size_t _compression_seek_point_stride; - std::mutex _maintenance_mtx; + mutable std::mutex _maintenance_mtx; std::condition_variable _maintenance_condition; std::thread _maintenance_thread; bool _maintenance_shutdown{false}; @@ -278,6 +379,51 @@ namespace sysio::trace_api { void append_lib(uint32_t lib); void append_trx_ids(block_trxs_entry tt); + /** + * Slice stride used for all sidecars. Exposed on the provider so callers (e.g. request_handler's block-range + * scan) can partition queries by slice without having to reach into slice_directory. + */ + uint32_t slice_stride() const noexcept { return _slice_directory.width(); } + + /** + * Slice number containing the given block. + */ + uint32_t slice_number(uint32_t block_height) const noexcept { return _slice_directory.slice_number(block_height); } + + /** + * Open the per-slice bloom sidecar for a given slice number. Returns a bloom_reader whose valid() is false + * when the sidecar is missing, truncated, wrong-version, or CRC-corrupt - in which case the caller MUST fall + * back to a full scan of the slice (an invalid reader returns true from may_contain_*, honoring the fail-safe + * invariant). A positive probe is not authoritative (standard bloom semantics); only a negative probe on a + * valid reader permits skipping. + */ + bloom_reader get_bloom(uint32_t slice_number) const; + + /** + * Record an ABI version for an account at a given global_sequence. + * global_seq == 0 means "captured lazily; exact seq unknown". + * Thread-safe; may be called from the extraction thread. + */ + void append_abi(chain::name account, uint64_t global_seq, std::vector abi_bytes); + + /** + * Return the ABI in effect for account at global_seq (the ABI with the + * largest recorded global_seq <= the query), or nullopt if none is found. + * The returned pair is {effective_global_seq, abi_bytes} where + * effective_global_seq is the recorded setabi's global_seq (0 for the + * lazy-capture sentinel). Decoders use effective_global_seq as a stable + * cache key so actions that share an ABI version all hit the same entry. + * Thread-safe; may be called from the HTTP thread. + */ + std::optional lookup_abi(chain::name account, uint64_t global_seq) const; + + /** + * Return true if any ABI record exists for the account. Used by extraction + * to decide whether to trigger a lazy ABI fetch on first encounter. + * Thread-safe. + */ + bool has_abi_entry(chain::name account) const; + /** * Read the trace for a given block * @param block_height : the height of the data being read @@ -288,6 +434,13 @@ namespace sysio::trace_api { get_block_n get_trx_block_number(const chain::transaction_id_type& trx_id, const yield_function& yield= {}); + /** + * Return {first, last} block numbers recorded across all index slice files, or nullopt + * if the slice directory is empty. Used at startup to verify continuity between existing + * trace data and the current chain head. + */ + std::optional> first_and_last_recorded_blocks() const; + void start_maintenance_thread( log_handler log ) { _slice_directory.start_maintenance_thread( std::move(log) ); } @@ -381,8 +534,14 @@ namespace sysio::trace_api { void validate_existing_index_slice_file(fc::cfile& index, open_state state); slice_directory _slice_directory; + + private: + // ABI sidecar: one global append-only log in the slice directory. + // abi_log serialises its own writes and allows concurrent lookups. + abi_log _abi_log; }; } FC_REFLECT(sysio::trace_api::slice_directory::index_header, (version)) +FC_REFLECT(sysio::trace_api::blk_offset_index_header, (magic)(version)(width)(reserved)) diff --git a/plugins/trace_api_plugin/include/sysio/trace_api/trace.hpp b/plugins/trace_api_plugin/include/sysio/trace_api/trace.hpp index 8c40ddaf9f..2c0e68b576 100644 --- a/plugins/trace_api_plugin/include/sysio/trace_api/trace.hpp +++ b/plugins/trace_api_plugin/include/sysio/trace_api/trace.hpp @@ -3,30 +3,44 @@ #include #include #include +#include #include -namespace sysio { namespace trace_api { +namespace sysio::trace_api { struct authorization_trace_v0 { - chain::name account; + chain::name actor; chain::name permission; }; + struct account_delta_v0 { + chain::name account; + int64_t delta = 0; + }; + struct action_trace_v0 { - uint64_t global_sequence = {}; - chain::name receiver = {}; - chain::name account = {}; - chain::name action = {}; - std::vector authorization = {}; - chain::bytes data = {}; - chain::bytes return_value = {}; + fc::unsigned_int action_ordinal = {}; + fc::unsigned_int creator_action_ordinal = {}; + fc::unsigned_int closest_unnotified_ancestor_action_ordinal = {}; + uint64_t global_sequence = {}; + uint64_t recv_sequence = {}; + boost::container::flat_map auth_sequence = {}; + fc::unsigned_int code_sequence = {}; + fc::unsigned_int abi_sequence = {}; + chain::name receiver = {}; + chain::name account = {}; + chain::name action = {}; + std::vector authorization = {}; + chain::bytes data = {}; + chain::bytes return_value = {}; + std::vector account_ram_deltas = {}; + std::optional cpu_usage_us = {}; + std::optional net_usage = {}; }; struct transaction_trace_v0 { - using status_type = chain::transaction_receipt_header::status_enum; chain::transaction_id_type id = {}; std::vector actions = {}; - fc::enum_type status = {}; uint32_t cpu_usage_us = 0; fc::unsigned_int net_usage_words; std::vector signatures = {}; @@ -57,11 +71,16 @@ namespace sysio { namespace trace_api { uint32_t block_num = 0; }; -} } +} // namespace sysio::trace_api -FC_REFLECT(sysio::trace_api::authorization_trace_v0, (account)(permission)) -FC_REFLECT(sysio::trace_api::action_trace_v0, (global_sequence)(receiver)(account)(action)(authorization)(data)(return_value)) -FC_REFLECT(sysio::trace_api::transaction_trace_v0, (id)(actions)(status)(cpu_usage_us)(net_usage_words)(signatures)(trx_header)(block_num)(block_time)(producer_block_id)) +FC_REFLECT(sysio::trace_api::authorization_trace_v0, (actor)(permission)) +FC_REFLECT(sysio::trace_api::account_delta_v0, (account)(delta)) +FC_REFLECT(sysio::trace_api::action_trace_v0, + (action_ordinal)(creator_action_ordinal)(closest_unnotified_ancestor_action_ordinal) + (global_sequence)(recv_sequence)(auth_sequence)(code_sequence)(abi_sequence) + (receiver)(account)(action)(authorization)(data)(return_value) + (account_ram_deltas)(cpu_usage_us)(net_usage)) +FC_REFLECT(sysio::trace_api::transaction_trace_v0, (id)(actions)(cpu_usage_us)(net_usage_words)(signatures)(trx_header)(block_num)(block_time)(producer_block_id)) FC_REFLECT(sysio::trace_api::block_trace_v0, (id)(number)(previous_id)(timestamp)(producer)(transaction_mroot)(finality_mroot)(transactions)) FC_REFLECT(sysio::trace_api::cache_trace, (trace)(trx)) FC_REFLECT(sysio::trace_api::block_trxs_entry, (ids)(block_num)) diff --git a/plugins/trace_api_plugin/include/sysio/trace_api/trx_id_index.hpp b/plugins/trace_api_plugin/include/sysio/trace_api/trx_id_index.hpp new file mode 100644 index 0000000000..1535d5edcd --- /dev/null +++ b/plugins/trace_api_plugin/include/sysio/trace_api/trx_id_index.hpp @@ -0,0 +1,94 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace sysio::trace_api { + +// --------------------------------------------------------------------------- +// On-disk format: trace_trx_idx_.log +// +// Layout: trx_id_index_header (16 bytes) followed immediately by +// bucket_count trx_id_bucket records (16 bytes each). +// +// Serialized with fc::raw (native-endian, same convention as all other +// trace slice files — x86_64 Linux only). +// +// Open-addressing hash table with linear probing, load factor <= 0.5. +// bucket_count is always a power of two (modulo via bitwise AND). +// Empty slot sentinel: block_num == 0. SYSIO block numbers start at 1. +// --------------------------------------------------------------------------- + +struct trx_id_index_header { + // Stored little-endian on disk so a hex dump of the first 4 bytes reads "TRIX". + static constexpr uint32_t magic_value = 0x58495254; // bytes on disk: 'T','R','I','X' + static constexpr uint32_t current_version = 1; + + uint32_t magic = magic_value; + uint32_t version = current_version; + uint32_t bucket_count = 0; + uint32_t reserved = 0; +}; +static_assert(sizeof(trx_id_index_header) == 16); + +struct trx_id_bucket { + uint64_t prefix64 = 0; // first 8 bytes of trx sha256 interpreted as uint64_t + uint32_t block_num = 0; // 0 = empty; SYSIO block numbers start at 1 + uint32_t reserved = 0; +}; +static_assert(sizeof(trx_id_bucket) == 16); + +// --------------------------------------------------------------------------- +// Writer: accumulate (trx_id, block_num) pairs, write index file when done. +// --------------------------------------------------------------------------- + +class trx_id_index_writer { +public: + void add(const chain::transaction_id_type& trx_id, uint32_t block_num); + void write(const std::filesystem::path& path) const; + size_t entry_count() const { return _entries.size(); } + +private: + static uint64_t prefix_of(const chain::transaction_id_type& id); + + // (prefix64, block_num) pairs in insertion order. write() applies last- + // write-wins per prefix64 when populating the bucket array, so the latest + // add for a given prefix is what ends up in the on-disk hash table. + // std::deque avoids the O(N) reallocation+copy of std::vector growth for + // multi-million-entry slices without needing an up-front reserve hint. + std::deque> _entries; +}; + +// --------------------------------------------------------------------------- +// Reader: load the index from disk and answer prefix lookups. +// Collisions (two trx_ids sharing a 64-bit prefix) are not explicitly +// confirmed — collisions in irreversible sha256 data are negligible. +// --------------------------------------------------------------------------- + +class trx_id_index_reader { +public: + explicit trx_id_index_reader(const std::filesystem::path& path); + + bool valid() const { return _valid; } + + // Returns the block_num for trx_id's prefix, or nullopt if not found. + std::optional lookup(const chain::transaction_id_type& trx_id) const; + +private: + static uint64_t prefix_of(const chain::transaction_id_type& id); + + std::vector _buckets; + bool _valid{false}; +}; + +} // namespace sysio::trace_api + +FC_REFLECT(sysio::trace_api::trx_id_index_header, (magic)(version)(bucket_count)(reserved)) +FC_REFLECT(sysio::trace_api::trx_id_bucket, (prefix64)(block_num)(reserved)) diff --git a/plugins/trace_api_plugin/src/abi_data_handler.cpp b/plugins/trace_api_plugin/src/abi_data_handler.cpp index 338e665aa8..1b03e01afe 100644 --- a/plugins/trace_api_plugin/src/abi_data_handler.cpp +++ b/plugins/trace_api_plugin/src/abi_data_handler.cpp @@ -1,42 +1,122 @@ #include #include +#include namespace sysio::trace_api { - void abi_data_handler::add_abi( const chain::name& name, chain::abi_def&& abi ) { - // currently abis are operator provided so no need to protect against abuse - abi_serializer_by_account.emplace(name, - std::make_shared(std::move(abi), chain::abi_serializer::create_yield_function(fc::microseconds::maximum()))); + std::shared_ptr abi_data_handler::get_serializer(chain::name account, uint64_t action_global_seq) { + if (!_abi_lookup_fn) return nullptr; + + // Resolve the effective ABI version first. Using it as the cache key means + // N actions sharing the same setabi all hit the same entry; keying on the + // action's global_seq instead (as a prior impl did) defeats the cache. + auto lookup = _abi_lookup_fn(account, action_global_seq); + if (!lookup || lookup->abi_bytes.empty()) return nullptr; + + const cache_key key{account, lookup->effective_global_seq}; + + { + std::lock_guard lock(_cache_mtx); + auto it = _cache_map.find(key); + if (it != _cache_map.end()) { + // Move to MRU (front). splice is O(1) on std::list. + _cache_list.splice(_cache_list.begin(), _cache_list, it->second); + return it->second->second; + } + } + + // Miss: build the serializer outside the lock to avoid blocking other + // cache users during a potentially slow unpack. + std::shared_ptr serializer; + try { + chain::abi_def abi; + auto ds = fc::datastream(lookup->abi_bytes.data(), lookup->abi_bytes.size()); + fc::raw::unpack(ds, abi); + serializer = std::make_shared(std::move(abi), + chain::abi_serializer::create_yield_function(fc::microseconds::maximum())); + } catch (...) { + except_handler(MAKE_EXCEPTION_WITH_CONTEXT(std::current_exception())); + return nullptr; + } + + // Insert into cache. Another thread may have raced us -- if so, return that entry. + std::lock_guard lock(_cache_mtx); + auto it = _cache_map.find(key); + if (it != _cache_map.end()) { + _cache_list.splice(_cache_list.begin(), _cache_list, it->second); + return it->second->second; + } + _cache_list.emplace_front(key, serializer); + _cache_map.emplace(key, _cache_list.begin()); + while (_cache_list.size() > _cache_capacity) { + _cache_map.erase(_cache_list.back().first); + _cache_list.pop_back(); + } + return serializer; } - std::tuple> abi_data_handler::serialize_to_variant(const std::variant& action) { - return std::visit([&](const auto& a) -> std::tuple> { - if (abi_serializer_by_account.count(a.account) > 0) { - const auto &serializer_p = abi_serializer_by_account.at(a.account); - auto type_name = serializer_p->get_action_type(a.action); - - if (!type_name.empty()) { - try { - // abi_serializer expects a yield function that takes a recursion depth - // abis are user provided, do not use a deadline - auto abi_yield = [](size_t recursion_depth) { - SYS_ASSERT( recursion_depth < chain::abi_serializer::max_recursion_depth, chain::abi_recursion_depth_exception, - "exceeded max_recursion_depth {} ", chain::abi_serializer::max_recursion_depth ); - }; - std::optional ret_data; - auto params = serializer_p->binary_to_variant(type_name, a.data, abi_yield); - if(a.return_value.size() > 0) { - auto return_type_name = serializer_p->get_action_result_type(a.action); - if (!return_type_name.empty()) { - ret_data = serializer_p->binary_to_variant(return_type_name, a.return_value, abi_yield); - } - } - return {std::move(params), std::move(ret_data)}; - } catch (...) { - except_handler(MAKE_EXCEPTION_WITH_CONTEXT(std::current_exception())); - } + abi_data_handler::decode_result abi_data_handler::decode(const action_trace_v0& a) { + // Named local return so NRVO constructs directly into the caller's slot. + decode_result result; + + auto serializer = get_serializer(a.account, a.global_sequence); + if (!serializer) return result; + + auto type_name = serializer->get_action_type(a.action); + if (type_name.empty()) return result; + + // abis are user provided, do not use a deadline + auto abi_yield = [](size_t recursion_depth) { + SYS_ASSERT( recursion_depth < chain::abi_serializer::max_recursion_depth, + chain::abi_recursion_depth_exception, + "exceeded max_recursion_depth {} ", chain::abi_serializer::max_recursion_depth ); + }; + + // Separate try blocks so that a failure decoding return_value does not + // discard successfully-decoded params (and vice versa). + try { + result.params = serializer->binary_to_variant(type_name, a.data, abi_yield); + result.status = decode_status::ok; + } catch (const std::exception& e) { + result.status = decode_status::failed; + result.error_message = e.what(); + except_handler(MAKE_EXCEPTION_WITH_CONTEXT(std::current_exception())); + return result; + } catch (...) { + result.status = decode_status::failed; + result.error_message = "unknown exception decoding action data"; + except_handler(MAKE_EXCEPTION_WITH_CONTEXT(std::current_exception())); + return result; + } + + if (a.return_value.size() > 0) { + auto return_type_name = serializer->get_action_result_type(a.action); + if (!return_type_name.empty()) { + try { + result.return_data = serializer->binary_to_variant(return_type_name, a.return_value, abi_yield); + } catch (const std::exception& e) { + // Params decoded OK but return_value failed: keep params, flag failure + // and surface the error message. Callers can still emit params. + result.status = decode_status::failed; + result.error_message = std::string("return_value decode failed: ") + e.what(); + except_handler(MAKE_EXCEPTION_WITH_CONTEXT(std::current_exception())); + } catch (...) { + result.status = decode_status::failed; + result.error_message = "unknown exception decoding return_value"; + except_handler(MAKE_EXCEPTION_WITH_CONTEXT(std::current_exception())); } } + } + + return result; + } + + std::tuple> abi_data_handler::serialize_to_variant(const std::variant& action) { + return std::visit([&](const auto& a) -> std::tuple> { + auto r = decode(a); + if (r.status == decode_status::ok) + return {std::move(r.params), std::move(r.return_data)}; + // failed or not_attempted -> legacy empty shape return {}; }, action); } diff --git a/plugins/trace_api_plugin/src/abi_log.cpp b/plugins/trace_api_plugin/src/abi_log.cpp new file mode 100644 index 0000000000..40b870cb4c --- /dev/null +++ b/plugins/trace_api_plugin/src/abi_log.cpp @@ -0,0 +1,274 @@ +#include +#include + +#include + +#include + +#include +#include +#include + +namespace sysio::trace_api { + +namespace { + +// Record layout: header(24) + blob_bytes(blob_size) + crc32(4). +// CRC covers header + blob_bytes (not itself). +constexpr uint64_t record_trailer_size = sizeof(uint32_t); + +// pread loop that tolerates short reads. Returns true iff `n` bytes were +// read into `buf` from `off`. +bool pread_all(int fd, void* buf, size_t n, uint64_t off) { + auto* p = static_cast(buf); + while (n > 0) { + const ssize_t r = ::pread(fd, p, n, static_cast(off)); + if (r > 0) { + p += r; + off += static_cast(r); + n -= static_cast(r); + } else if (r == 0) { + return false; // EOF before all bytes read + } else if (errno == EINTR) { + continue; + } else { + return false; + } + } + return true; +} + +} // namespace + +uint32_t abi_log::compute_record_crc(const record_header& hdr, const char* blob, uint64_t blob_size) { + boost::crc_32_type crc; + crc.process_bytes(&hdr, sizeof(hdr)); + if (blob_size > 0) + crc.process_bytes(blob, blob_size); + return crc.checksum(); +} + +abi_log::abi_log(const std::filesystem::path& path) { + const bool existed = std::filesystem::exists(path); + _cfile.set_file_path(path); + try { + _cfile.open(fc::cfile::create_or_update_rw_mode); + } catch (...) { + fc_wlog(_log, "trace_api: abi_log failed to open {}", path.generic_string()); + // Best-effort clean-up: a failed open on some libc versions leaves a zero-byte + // file behind, which would look like a valid-but-header-less log on the next + // start and fail an unpack inside the existing-file branch. + std::error_code ec; + if (!existed) + std::filesystem::remove(path, ec); + return; + } + + if (!existed) { + // Fresh file: write header, no records yet. "ab+" mode always writes at + // EOF, so the seek below is advisory for the position indicator; the + // write lands at offset 0 by construction (file is freshly opened, EOF = 0). + abi_log_header hdr; + auto data = fc::raw::pack(hdr); + try { + _cfile.write(data.data(), data.size()); + _cfile.flush(); + } catch (...) { + fc_wlog(_log, "trace_api: abi_log failed to write header to {}", path.generic_string()); + return; + } + _end_offset = data.size(); + _valid = true; + return; + } + + // Existing file: validate header, scan records, populate index, truncate any bad tail. + try { + _cfile.seek(0); + auto ds = _cfile.create_datastream(); + abi_log_header hdr; + fc::raw::unpack(ds, hdr); + if (hdr.magic != abi_log_header::magic_value) { + fc_wlog(_log, "trace_api: abi_log {} has wrong magic {:#x}, ignoring", + path.generic_string(), hdr.magic); + return; + } + if (hdr.version != abi_log_header::current_version) { + fc_wlog(_log, "trace_api: abi_log {} has unsupported version {}, ignoring", + path.generic_string(), hdr.version); + return; + } + } catch (...) { + fc_wlog(_log, "trace_api: abi_log {} header read failed", path.generic_string()); + return; + } + + _end_offset = recover_from_disk(path); + _valid = true; +} + +uint64_t abi_log::recover_from_disk(const std::filesystem::path& path) { + const uint64_t header_size = sizeof(abi_log_header); + const uint64_t file_size = std::filesystem::file_size(path); + + // pread correctness: fc::cfile is backed by buffered stdio (FILE*), but + // every append() flushes before returning, and the recover scan only runs + // during the constructor (before any concurrent append). Never introduce + // unflushed buffered writes on the same cfile or pread may see stale data. + uint64_t offset = header_size; + while (offset < file_size) { + const uint64_t record_start = offset; + + // Need at least record_header + crc trailer. + if (file_size - record_start < sizeof(record_header) + record_trailer_size) { + fc_wlog(_log, "trace_api: abi_log {} torn tail at offset {} (less than minimal record), truncating", + path.generic_string(), record_start); + break; + } + + record_header rh{}; + if (!pread_all(_cfile.fileno(), &rh, sizeof(rh), record_start)) { + fc_wlog(_log, "trace_api: abi_log {} failed to read record header at offset {}, truncating", + path.generic_string(), record_start); + break; + } + + const uint64_t blob_file_offset = record_start + sizeof(record_header); + const uint64_t crc_offset = blob_file_offset + rh.blob_size; + const uint64_t record_end = crc_offset + record_trailer_size; + + if (record_end > file_size) { + fc_wlog(_log, "trace_api: abi_log {} record at {} claims blob_size {} but file has only {} bytes remaining, truncating", + path.generic_string(), record_start, rh.blob_size, file_size - blob_file_offset); + break; + } + + std::vector blob; + if (rh.blob_size > 0) { + blob.resize(rh.blob_size); + if (!pread_all(_cfile.fileno(), blob.data(), rh.blob_size, blob_file_offset)) { + fc_wlog(_log, "trace_api: abi_log {} failed to read blob at offset {}, truncating", + path.generic_string(), blob_file_offset); + break; + } + } + + uint32_t stored_crc = 0; + if (!pread_all(_cfile.fileno(), &stored_crc, sizeof(stored_crc), crc_offset)) { + fc_wlog(_log, "trace_api: abi_log {} failed to read crc at offset {}, truncating", + path.generic_string(), crc_offset); + break; + } + + const uint32_t computed_crc = compute_record_crc(rh, blob.data(), rh.blob_size); + if (computed_crc != stored_crc) { + fc_wlog(_log, "trace_api: abi_log {} crc mismatch at record offset {} (stored {:#x} vs computed {:#x}), truncating", + path.generic_string(), record_start, stored_crc, computed_crc); + break; + } + + // std::map::operator[] — duplicate (account, global_seq) silently overwrites (last-write-wins). + _index[{rh.account, rh.global_seq}] = index_entry{blob_file_offset, rh.blob_size}; + offset = record_end; + } + + if (offset < file_size) { + // Drop the bad tail. resize_file works regardless of whether the + // file is currently open; subsequent writes via _cfile will append + // from _end_offset. + std::error_code ec; + std::filesystem::resize_file(path, offset, ec); + if (ec) { + fc_wlog(_log, "trace_api: abi_log {} failed to truncate to {}: {}", + path.generic_string(), offset, ec.message()); + } + } + + return offset; +} + +// NOTE: callers of append()/has_entry() are expected to be single-threaded +// (the chain extraction thread). The append+index-insert sequence is not +// strictly atomic across the two mutexes; a concurrent caller could slip an +// insert for the same key between our write and our index update, producing +// a duplicate record. This is harmless given last-write-wins but not obvious. +void abi_log::append(chain::name account, uint64_t global_seq, std::vector abi_bytes) { + if (!_valid) return; + + record_header rh{ account, global_seq, abi_bytes.size() }; + const uint32_t crc = compute_record_crc(rh, abi_bytes.data(), abi_bytes.size()); + + uint64_t blob_file_offset = 0; + { + std::lock_guard lock(_append_mtx); + try { + // "ab+" mode always writes at EOF; explicit seek would be a no-op here. + _cfile.write(reinterpret_cast(&rh), sizeof(rh)); + if (rh.blob_size > 0) + _cfile.write(abi_bytes.data(), abi_bytes.size()); + _cfile.write(reinterpret_cast(&crc), sizeof(crc)); + _cfile.flush(); + } catch (...) { + fc_wlog(_log, "trace_api: abi_log append failed at offset {}", _end_offset); + return; + } + + blob_file_offset = _end_offset + sizeof(rh); + _end_offset += sizeof(rh) + rh.blob_size + record_trailer_size; + } + + { + std::lock_guard lock(_index_mtx); + // std::map::operator[] — duplicate (account, global_seq) silently overwrites (last-write-wins). + _index[{rh.account, rh.global_seq}] = index_entry{blob_file_offset, rh.blob_size}; + } +} + +bool abi_log::has_entry(chain::name account) const { + if (!_valid) return false; + std::lock_guard lock(_index_mtx); + auto it = _index.lower_bound({account, 0}); + return it != _index.end() && it->first.first == account; +} + +std::optional abi_log::lookup(chain::name account, uint64_t global_seq) const { + // Named local return type so the compiler can NRVO it straight into the + // caller's slot. + std::optional result; + if (!_valid) return result; + + uint64_t blob_file_offset = 0; + uint64_t blob_size = 0; + uint64_t effective_seq = 0; + + { + std::lock_guard lock(_index_mtx); + auto it = _index.upper_bound({account, global_seq}); + if (it == _index.begin()) + return result; + --it; + if (it->first.first != account) + return result; + blob_file_offset = it->second.blob_file_offset; + blob_size = it->second.blob_size; + effective_seq = it->first.second; + } + + if (blob_size == 0) { + result.emplace(lookup_result{effective_seq, {}}); + return result; + } + + // pread correctness: every append() flushes before returning, so the bytes + // we're about to read are visible to the underlying fd. Don't introduce + // unflushed buffered writes on the same cfile. + std::vector out(blob_size); + if (!pread_all(_cfile.fileno(), out.data(), blob_size, blob_file_offset)) { + fc_wlog(_log, "trace_api: abi_log pread of {} bytes at {} failed", blob_size, blob_file_offset); + return result; + } + result.emplace(lookup_result{effective_seq, std::move(out)}); + return result; +} + +} // namespace sysio::trace_api diff --git a/plugins/trace_api_plugin/src/request_handler.cpp b/plugins/trace_api_plugin/src/request_handler.cpp index c67c6b527f..f28afa3595 100644 --- a/plugins/trace_api_plugin/src/request_handler.cpp +++ b/plugins/trace_api_plugin/src/request_handler.cpp @@ -4,55 +4,85 @@ #include -namespace { - using namespace sysio::trace_api; +namespace sysio::trace_api { - std::string to_iso8601_datetime( const fc::time_point& t) { - return t.to_iso_string() + "Z"; - } +fc::mutable_variant_object build_action_variant(const action_trace_v0& a, + const decoded_action& decoded, + variant_shape shape) { + fc::mutable_variant_object v; + // Fields common to all shapes. + v("global_sequence", a.global_sequence) + ("receiver", a.receiver.to_string()) + ("account", a.account.to_string()) + ("name", a.action.to_string()) + ("authorization", serialize_authorizations(a.authorization)) + ("data", fc::to_hex(a.data.data(), a.data.size())) + ("return_value", fc::to_hex(a.return_value.data(), a.return_value.size())); - fc::variants process_authorizations(const std::vector& authorizations) { - fc::variants result; - result.reserve(authorizations.size()); - for ( const auto& a: authorizations) { - result.emplace_back(fc::mutable_variant_object() - ("account", a.account.to_string()) - ("permission", a.permission.to_string()) - ); + if (shape == variant_shape::full) { + v("action_ordinal", a.action_ordinal) + ("creator_action_ordinal", a.creator_action_ordinal) + ("closest_unnotified_ancestor_action_ordinal", a.closest_unnotified_ancestor_action_ordinal) + ("recv_sequence", a.recv_sequence) + ("auth_sequence", a.auth_sequence) + ("code_sequence", a.code_sequence) + ("abi_sequence", a.abi_sequence); + + fc::variants deltas; + deltas.reserve(a.account_ram_deltas.size()); + for (const auto& d : a.account_ram_deltas) { + deltas.emplace_back(fc::mutable_variant_object() + ("account", d.account.to_string()) + ("delta", d.delta)); } + v("account_ram_deltas", std::move(deltas)); - return result; + if (a.cpu_usage_us.has_value()) + v("cpu_usage_us", *a.cpu_usage_us); + if (a.net_usage.has_value()) + v("net_usage", *a.net_usage); + } + if (!decoded.params.is_null()) + v("params", decoded.params); + if (decoded.return_data.has_value()) + v("return_data", *decoded.return_data); + if (!decoded.error_message.empty()) + v("decode_error", decoded.error_message); + + return v; +} + +} // namespace sysio::trace_api + +namespace { + using namespace sysio::trace_api; + + std::string to_iso8601_datetime( const fc::time_point& t) { + return t.to_iso_string() + "Z"; } fc::variants process_actions(const std::vector& actions, const data_handler_function& data_handler) { fc::variants result; result.reserve(actions.size()); - std::vector indices(actions.size()); - std::iota(indices.begin(), indices.end(), 0); - std::sort(indices.begin(), indices.end(), [&actions](const int& lhs, const int& rhs) -> bool { - return actions.at(lhs).global_sequence < actions.at(rhs).global_sequence; + // global_sequence is unique per action (chain invariant), so sort stability is not required. + std::vector sorted; + sorted.reserve(actions.size()); + for (const auto& a : actions) sorted.push_back(&a); + std::sort(sorted.begin(), sorted.end(), [](const auto* l, const auto* r){ + return l->global_sequence < r->global_sequence; }); - for ( int index : indices) { - const auto& a = actions.at(index); - auto action_variant = fc::mutable_variant_object(); - - action_variant("global_sequence", a.global_sequence) - ("receiver", a.receiver.to_string()) - ("account", a.account.to_string()) - ("action", a.action.to_string()) - ("authorization", process_authorizations(a.authorization)) - ("data", fc::to_hex(a.data.data(), a.data.size())); - - action_variant("return_value", fc::to_hex(a.return_value.data(), a.return_value.size())); + + for (const action_trace_v0* ap : sorted) { + const auto& a = *ap; auto [params, return_data] = data_handler(a); - if (!params.is_null()) { - action_variant("params", params); - } - if(return_data.has_value()){ - action_variant("return_data", *return_data); - } + decoded_action decoded{std::move(params), std::move(return_data), {}}; + // legacy process_block path used serialize_to_variant's tuple which doesn't + // convey decode_error, so leave the field empty here; callers that want the + // error path should go through get_actions instead. + fc::mutable_variant_object action_variant = build_action_variant(a, decoded, variant_shape::full); + // block-trace-local fields (populated by the enclosing transaction, not here) result.emplace_back( std::move(action_variant) ); } return result; @@ -69,7 +99,6 @@ namespace { ("block_time", t.block_time) ("producer_block_id", t.producer_block_id) ("actions", process_actions(t.actions, data_handler)) - ("status", t.status) ("cpu_usage_us", t.cpu_usage_us) ("net_usage_words", t.net_usage_words) ("signatures", t.signatures) diff --git a/plugins/trace_api_plugin/src/store_provider.cpp b/plugins/trace_api_plugin/src/store_provider.cpp index 4c7bbccc1c..8b6ad7215e 100644 --- a/plugins/trace_api_plugin/src/store_provider.cpp +++ b/plugins/trace_api_plugin/src/store_provider.cpp @@ -1,5 +1,7 @@ #include +#include +#include #include #include @@ -8,9 +10,28 @@ namespace { static constexpr const char* _trace_prefix = "trace_"; static constexpr const char* _trace_index_prefix = "trace_index_"; static constexpr const char* _trace_trx_id_prefix = "trace_trx_id_"; + static constexpr const char* _trace_trx_id_index_prefix = "trace_trx_idx_"; + static constexpr const char* _trace_blk_idx_prefix = "trace_blk_idx_"; + static constexpr const char* _trace_recv_bloom_prefix = "trace_recv_bloom_"; static constexpr const char* _trace_ext = ".log"; static constexpr const char* _compressed_trace_ext = ".clog"; - static constexpr int _max_filename_size = std::char_traits::length(_trace_index_prefix) + 10 + 1 + 10 + std::char_traits::length(_compressed_trace_ext) + 1; // "trace_index_" + 10-digits + '-' + 10-digits + ".clog" + null-char + // Sized for the longest possible filename across every prefix and + // extension known to this file. Adding a longer prefix above + // automatically grows the buffer; no manual recount needed. + static constexpr size_t _max_prefix_length = std::max({ + std::char_traits::length(_trace_prefix), + std::char_traits::length(_trace_index_prefix), + std::char_traits::length(_trace_trx_id_prefix), + std::char_traits::length(_trace_trx_id_index_prefix), + std::char_traits::length(_trace_blk_idx_prefix), + std::char_traits::length(_trace_recv_bloom_prefix), + }); + static constexpr size_t _max_ext_length = std::max( + std::char_traits::length(_trace_ext), + std::char_traits::length(_compressed_trace_ext) + ); + // prefix + 10-digit start + '-' + 10-digit end + ext + '\0' + static constexpr int _max_filename_size = _max_prefix_length + 10 + 1 + 10 + _max_ext_length + 1; std::string make_filename(const char* slice_prefix, const char* slice_ext, uint32_t slice_number, uint32_t slice_width) { char filename[_max_filename_size] = {}; @@ -32,20 +53,26 @@ namespace { namespace sysio::trace_api { store_provider::store_provider(const std::filesystem::path& slice_dir, uint32_t stride_width, std::optional minimum_irreversible_history_blocks, std::optional minimum_uncompressed_irreversible_history_blocks, size_t compression_seek_point_stride) - : _slice_directory(slice_dir, stride_width, minimum_irreversible_history_blocks, minimum_uncompressed_irreversible_history_blocks, compression_seek_point_stride) { - } + : _slice_directory(slice_dir, stride_width, minimum_irreversible_history_blocks, minimum_uncompressed_irreversible_history_blocks, compression_seek_point_stride) + , _abi_log(slice_dir / "abi_log.log") {} template void store_provider::append(const BlockTrace& bt) { fc::cfile trace; fc::cfile index; const uint32_t slice_number = _slice_directory.slice_number(bt.number); + _slice_directory.find_or_create_slice_pair(slice_number, open_state::write, trace, index); // storing as static_variant to allow adding other data types to the trace file in the future const uint64_t offset = append_store(data_log_entry { bt }, trace); auto be = metadata_log_entry { block_entry_v0 { .id = bt.id, .number = bt.number, .offset = offset }}; append_store(be, index); + + // Record in the block-offset sidecar for O(1) get_block lookups. Fork re-writes + // overwrite the slot; if this step throws the metadata log remains the source of + // truth and get_block falls back to the linear scan. + _slice_directory.write_block_offset(bt.number, offset); } template void store_provider::append(const block_trace_v0& bt); @@ -69,27 +96,38 @@ namespace sysio::trace_api { append_store(entry, trx_id_file); } + bloom_reader store_provider::get_bloom(uint32_t slice_number) const { + const auto path = _slice_directory.bloom_slice_path(slice_number); + std::error_code ec; + if (!std::filesystem::exists(path, ec)) return bloom_reader{}; + return bloom_reader{path}; + } + get_block_t store_provider::get_block(uint32_t block_height, const yield_function& yield) { - std::optional trace_offset; - bool irreversible = false; - scan_metadata_log_from(block_height, 0, [&block_height, &trace_offset, &irreversible](const metadata_log_entry& e) -> bool { - if (std::holds_alternative(e)) { - const auto& block = std::get(e); - if (block.number == block_height) { - trace_offset = block.offset; - } - } else if (std::holds_alternative(e)) { - auto lib = std::get(e).lib; - if (lib >= block_height) { - irreversible = true; - return false; + // Fast path: O(1) random-access lookup of the trace offset via the block-offset sidecar. + std::optional trace_offset = _slice_directory.lookup_block_offset(block_height); + + if (!trace_offset) { + // Fallback: scan the metadata log. Covers slices without a sidecar (e.g. corruption, + // missing sidecar file) or the rare case where a block exists in the metadata log but + // the sidecar write was interrupted. + scan_metadata_log_from(block_height, 0, [&block_height, &trace_offset](const metadata_log_entry& e) -> bool { + if (std::holds_alternative(e)) { + const auto& block = std::get(e); + if (block.number == block_height) { + trace_offset = block.offset; + } } - } - return true; - }, yield); + return true; + }, yield); + } + if (!trace_offset) { return get_block_t{}; } + + const bool irreversible = block_height <= _slice_directory.best_known_lib(); + std::optional entry = read_data_log(block_height, *trace_offset); if (!entry) { return get_block_t{}; @@ -98,11 +136,63 @@ namespace sysio::trace_api { } get_block_n store_provider::get_trx_block_number(const chain::transaction_id_type& trx_id, const yield_function& yield) { - // traversing from last stride to first - // if we find a trx it is either LIB or it is the latest fork, either way we are done + // Fast path: probe the per-slice hash index for each slice newest-to-oldest. + // The index covers only irreversible slices; reversible blocks fall through to + // the linear scan below. std::set trx_block_nums; _slice_directory.for_each_trx_id_slice([&](fc::cfile& trx_id_file) -> bool { + yield(); + + // Derive the slice number from the file path and try the index first. + // If the filename can't be parsed (slice_number_from_path returns nullopt), + // fall through to the linear scan below. + const std::optional slice_number = + _slice_directory.slice_number_from_path(trx_id_file.get_file_path()); + if (slice_number) { + if (auto reader = _slice_directory.find_trx_id_index_slice(*slice_number)) { + if (auto block_num = reader->lookup(trx_id)) { + // Confirm the hit by scanning the candidate block's + // block_trxs_entry for a full trx_id match. A naked 64-bit + // prefix collision (extremely rare with natural sha256 ids, + // but findable in ~2^32 GPU work adversarially) would otherwise + // return the wrong block_num and the downstream get_block + // scan would 404 the real trx. On confirm failure, reset the + // file to offset 0 and fall through to the linear scan below. + bool confirmed = false; + { + metadata_log_entry entry; + auto ds = trx_id_file.create_datastream(); + const uint64_t end = file_size(trx_id_file.get_file_path()); + while (trx_id_file.tellp() < end) { + yield(); + fc::raw::unpack(ds, entry); + if (!std::holds_alternative(entry)) continue; + const auto& te = std::get(entry); + if (te.block_num != *block_num) continue; + for (const auto& id : te.ids) { + if (id == trx_id) { confirmed = true; break; } + } + if (confirmed) break; + } + } + if (confirmed) { + trx_block_nums.insert(*block_num); + return false; // found in an irreversible slice; stop + } + // Prefix collision: reset the file so the linear scan starts + // from offset 0 -- the target trx may still be in a DIFFERENT + // block whose trxs entry we passed over during confirmation. + trx_id_file.seek(0); + // fall through to linear scan + } else { + return true; // not in this indexed slice; continue to next + } + } + } + + // No index for this slice (reversible window or index not yet built): + // fall back to linear scan. metadata_log_entry entry; auto ds = trx_id_file.create_datastream(); const uint64_t end = file_size(trx_id_file.get_file_path()); @@ -129,7 +219,7 @@ namespace sysio::trace_api { return false; // *(--trx_block_nums.end()) is the block with highest block number which is final } } else { - FC_ASSERT( false, "unpacked data should be a block_trxs_entry or a lib_entry_v0" );; + FC_ASSERT( false, "unpacked data should be a block_trxs_entry or a lib_entry_v0" ); } offset = trx_id_file.tellp(); } @@ -218,6 +308,13 @@ namespace sysio::trace_api { return true; } + std::filesystem::path slice_directory::bloom_slice_path(uint32_t slice_number) const { + // Mirrors the filename convention of the other sidecars: -. Callers write through + // bloom_builder::finalize_and_write (temp + rename) and read through bloom_reader, neither of which uses + // fc::cfile, so no open-helper is needed here. + return _slice_dir / make_filename(_trace_recv_bloom_prefix, _trace_ext, slice_number, _width); + } + std::optional slice_directory::find_compressed_trace_slice(uint32_t slice_number, bool open_file ) const { auto filename = make_filename(_trace_prefix, _compressed_trace_ext, slice_number, _width); const auto slice_path = _slice_dir / filename; @@ -259,7 +356,7 @@ namespace sysio::trace_api { if (trace_found != index_found) { const std::string trace_status = trace_found ? "existing" : "new"; const std::string index_status = index_found ? "existing" : "new"; - elog("Trace file is {}, but it's metadata file is {}. This means the files are not consistent.", trace_status, index_status); + fc_elog(_log, "Trace file is {}, but it's metadata file is {}. This means the files are not consistent.", trace_status, index_status); } } @@ -312,6 +409,238 @@ namespace sysio::trace_api { } } + namespace { + // Collect and sort all trace_index_*.log paths; ascending=true gives lowest slice first. + std::vector collect_index_paths(const std::filesystem::path& slice_dir, bool ascending) { + namespace fs = std::filesystem; + std::vector paths; + for (const auto& entry : fs::directory_iterator(slice_dir)) { + if (!entry.is_regular_file()) continue; + const auto name = entry.path().filename().string(); + if (name.rfind(_trace_index_prefix, 0) == 0 && entry.path().extension() == _trace_ext) + paths.push_back(entry.path()); + } + if (ascending) { + std::sort(paths.begin(), paths.end()); + } else { + std::sort(paths.begin(), paths.end(), std::greater<>()); + } + return paths; + } + + // Scan a single index slice file and return the min and max block numbers found. + std::optional> scan_index_slice(const std::filesystem::path& path, uint32_t current_version) { + try { + fc::cfile index; + index.set_file_path(path); + index.open("rb"); + index.seek(0); + const auto header = extract_store(index); + if (header.version != current_version) + return std::nullopt; + const uint64_t end = file_size(path); + std::optional lo, hi; + while (index.tellp() < end) { + const auto e = extract_store(index); + if (std::holds_alternative(e)) { + const auto num = std::get(e).number; + if (!lo || num < *lo) lo = num; + if (!hi || num > *hi) hi = num; + } + } + if (lo && hi) + return std::make_pair(*lo, *hi); + } catch (...) { + // malformed or partially written slice + fc_wlog(_log, "trace_api: cannot scan index slice '{}'", path.string()); + } + return std::nullopt; + } + } // anonymous namespace + + std::optional> slice_directory::first_and_last_recorded_blocks() const { + // Named local with the exact return type so the compiler can NRVO it directly + // into the caller's slot. Single directory scan: collect ascending paths, + // walk from both ends. Returning both bounds from one call guarantees callers + // see a consistent view -- either both values are present or neither is. + std::optional> result; + + const auto paths = collect_index_paths(_slice_dir, /*ascending=*/true); + std::optional first_block; + for (const auto& path : paths) { + if (const auto r = scan_index_slice(path, _current_version)) { + first_block = r->first; + break; + } + } + if (!first_block) + return result; + + // first_block was found, so at least one slice has block entries; the reverse + // pass is guaranteed to find at least one (worst case, the same slice). + uint32_t last_block = *first_block; + for (auto it = paths.rbegin(); it != paths.rend(); ++it) { + if (const auto r = scan_index_slice(*it, _current_version)) { + last_block = r->second; + break; + } + } + + result.emplace(*first_block, last_block); + return result; + } + + std::optional> store_provider::first_and_last_recorded_blocks() const { + return _slice_directory.first_and_last_recorded_blocks(); + } + + void store_provider::append_abi(chain::name account, uint64_t global_seq, std::vector abi_bytes) { + _abi_log.append(account, global_seq, std::move(abi_bytes)); + } + + std::optional store_provider::lookup_abi(chain::name account, uint64_t global_seq) const { + return _abi_log.lookup(account, global_seq); + } + + bool store_provider::has_abi_entry(chain::name account) const { + return _abi_log.has_entry(account); + } + + std::optional slice_directory::slice_number_from_path(const std::filesystem::path& trx_id_path) const { + // Filename format: trace_trx_id_XXXXXXXXXX-YYYYYYYYYY.log + // Parse the start block number (XXXXXXXXXX) and divide by _width. + // Named local matching the return type so the compiler can NRVO the + // optional directly into the caller's slot. + const auto name = trx_id_path.filename().string(); + const auto prefix_len = std::char_traits::length(_trace_trx_id_prefix); + std::optional result; + try { + const uint32_t start_block = static_cast(std::stoul(name.substr(prefix_len, 10))); + result = start_block / _width; + } catch (...) { + fc_wlog(_log, "trace_api: cannot parse slice start-block from filename '{}'", name); + } + return result; + } + + std::optional slice_directory::find_trx_id_index_slice(uint32_t slice_number) const { + auto filename = make_filename(_trace_trx_id_index_prefix, _trace_ext, slice_number, _width); + const auto path = _slice_dir / filename; + if (!std::filesystem::exists(path)) + return std::nullopt; + trx_id_index_reader reader(path); + if (!reader.valid()) + return std::nullopt; + return reader; + } + + void slice_directory::build_trx_id_index(uint32_t slice_number, const log_handler& log) { + auto idx_filename = make_filename(_trace_trx_id_index_prefix, _trace_ext, slice_number, _width); + const auto idx_path = _slice_dir / idx_filename; + if (std::filesystem::exists(idx_path)) + return; // already built + + fc::cfile trx_id_file; + if (!find_trx_id_slice(slice_number, open_state::read, trx_id_file)) + return; // no source data + + log(std::string("Building trx_id index for slice: ") + std::to_string(slice_number)); + + // Dedup pass: the trx_id log can hold MULTIPLE block_trxs_entry records + // with the same block_num when the chain forks at that height (each + // accepted block, including forked-out ones, writes one entry). The + // last entry for each block_num reflects the canonical post-fork- + // resolution state, matching the semantics of the linear-scan path in + // get_trx_block_number which uses trx_block_nums.erase to drop forked- + // out entries. Build a canonical map first, then feed it to the + // writer (which itself does last-write-wins per prefix). + std::map> canonical; + const uint64_t end = file_size(trx_id_file.get_file_path()); + while (trx_id_file.tellp() < end) { + metadata_log_entry entry; + auto ds = trx_id_file.create_datastream(); + fc::raw::unpack(ds, entry); + if (std::holds_alternative(entry)) { + const auto& te = std::get(entry); + canonical[te.block_num] = te.ids; // last entry per block_num wins + } + } + + trx_id_index_writer writer; + for (const auto& [block_num, ids] : canonical) { + for (const auto& id : ids) + writer.add(id, block_num); + } + + // Write to a temp path and atomically rename so concurrent readers never + // see a partially written index file. + const auto tmp_path = idx_path.parent_path() / (idx_path.filename().string() + ".tmp"); + writer.write(tmp_path); + std::filesystem::rename(tmp_path, idx_path); + log(std::string("Built trx_id index for slice: ") + std::to_string(slice_number) + + " (" + std::to_string(writer.entry_count()) + " entries)"); + } + + void slice_directory::build_recv_bloom(uint32_t slice_number, const log_handler& log) { + const auto bloom_path = bloom_slice_path(slice_number); + if (std::filesystem::exists(bloom_path)) + return; // already built + + // Locate the slice's trace data log (trace_.log). run_maintenance_tasks orders bloom building before + // compression so a freshly-irreversible slice still has its uncompressed .log. If only a compressed .clog + // exists (e.g. upgrading a node that predates the bloom) or the file is missing, skip; the query path treats + // a missing sidecar as "scan this slice". Don't decompress-then-scan - compressed slices are aged and rarely + // queried. Look up the path without opening so we can check size before committing to an open. + fc::cfile trace; + const bool dont_open_file = false; + if (!find_trace_slice(slice_number, open_state::read, trace, dont_open_file)) { + log(std::string("trace_api: skipping receiver bloom for slice ") + std::to_string(slice_number) + + " (no uncompressed trace data; already compressed or never written)"); + return; + } + // Empty trace file => no actions to bloom. Production slices always have on-block traces so this only fires + // in tests that pre-create slice files; keeping it guards the maintenance path from writing a zero-entry + // sidecar that'd just clutter the directory. + const auto trace_path = trace.get_file_path(); + std::error_code ec; + const uint64_t trace_size = std::filesystem::file_size(trace_path, ec); + if (ec || trace_size == 0) return; + + log(std::string("Building receiver bloom for slice: ") + std::to_string(slice_number)); + + trace.open(fc::cfile::update_rw_mode); + + bloom_builder builder; + bool processed_any_block = false; + try { + // Stream through the data log record-by-record. Fork re-writes leave stale block_trace_v0 records in the + // file (the blk_offset sidecar only points to the canonical one), so the bloom will contain a superset of + // the canonical receivers. That's fine: bloom allows false positives; a receiver present only in a forked- + // out copy just probes as present and the query scan finds no canonical match for it. + while (trace.tellp() < trace_size) { + data_log_entry entry; + auto ds = trace.create_datastream(); + fc::raw::unpack(ds, entry); + std::visit([&builder](const auto& bt) { builder.add_block(bt); }, entry); + processed_any_block = true; + } + } FC_LOG_AND_DROP(); + + if (!processed_any_block) { + // No parseable records (corrupted or malformed data log). Don't write a default-sized sidecar - let the + // query path fall back to scanning this slice, which is the correct behavior for unreadable input. + return; + } + + try { + builder.finalize_and_write(bloom_path); + } FC_LOG_AND_DROP(); + + log(std::string("Built receiver bloom for slice: ") + std::to_string(slice_number) + + " (" + std::to_string(builder.receiver_count()) + " receivers, " + + std::to_string(builder.recv_action_count()) + " (receiver, action) pairs)"); + } + void slice_directory::set_lib(uint32_t lib) { { std::scoped_lock lock(_maintenance_mtx); @@ -320,6 +649,96 @@ namespace sysio::trace_api { _maintenance_condition.notify_one(); } + uint32_t slice_directory::best_known_lib() const { + std::scoped_lock lock(_maintenance_mtx); + return _best_known_lib; + } + + bool slice_directory::open_or_create_blk_offset_slice(uint32_t slice_number, fc::cfile& blk_idx) const { + const bool found = find_slice(_trace_blk_idx_prefix, slice_number, blk_idx, /*open_file=*/false); + if (found) { + // Existing file: open for random-access read/write ("rb+"). + blk_idx.open(fc::cfile::update_rw_mode); + blk_offset_index_header header{}; + try { + blk_idx.seek(0); + blk_idx.read(reinterpret_cast(&header), sizeof(header)); + } catch (...) { + blk_idx.close(); + return false; + } + if (header.magic != blk_offset_index_header::magic_value || + header.version != blk_offset_index_header::current_version || + header.width != _width) { + blk_idx.close(); + return false; + } + return true; + } + + // New file: create with truncate+read/write ("wb+") so subsequent seek()+write() + // behave as random-access (append mode "ab+" would ignore the seek on writes). + blk_idx.open(fc::cfile::truncate_rw_mode); + blk_offset_index_header header{}; + header.width = _width; + blk_idx.seek(0); + blk_idx.write(reinterpret_cast(&header), sizeof(header)); + // Pre-allocate by writing the final byte. Linux fills the gap sparsely. + const uint64_t final_byte = sizeof(blk_offset_index_header) + uint64_t{_width} * sizeof(uint64_t) - 1; + const char zero = 0; + blk_idx.seek(final_byte); + blk_idx.write(&zero, 1); + blk_idx.flush(); + return true; + } + + void slice_directory::write_block_offset(uint32_t block_height, uint64_t trace_offset) const { + const uint32_t slice_number = this->slice_number(block_height); + fc::cfile blk_idx; + if (!open_or_create_blk_offset_slice(slice_number, blk_idx)) { + // Existing sidecar is unusable (wrong magic/version/width). Remove and recreate; + // readers fall back to the metadata-log scan in the meantime. + std::filesystem::remove(blk_idx.get_file_path()); + if (!open_or_create_blk_offset_slice(slice_number, blk_idx)) { + return; + } + } + const uint32_t slot = block_height % _width; + const uint64_t slot_pos = sizeof(blk_offset_index_header) + uint64_t{slot} * sizeof(uint64_t); + const uint64_t encoded = trace_offset + 1; // 0 reserved as "empty" + blk_idx.seek(slot_pos); + blk_idx.write(reinterpret_cast(&encoded), sizeof(encoded)); + blk_idx.flush(); + } + + std::optional slice_directory::lookup_block_offset(uint32_t block_height) const { + const uint32_t slice_number = this->slice_number(block_height); + fc::cfile blk_idx; + const bool found = find_slice(_trace_blk_idx_prefix, slice_number, blk_idx, /*open_file=*/false); + if (!found) return std::nullopt; + + try { + blk_idx.open(fc::cfile::update_rw_mode); + blk_offset_index_header header{}; + blk_idx.seek(0); + blk_idx.read(reinterpret_cast(&header), sizeof(header)); + if (header.magic != blk_offset_index_header::magic_value || + header.version != blk_offset_index_header::current_version || + header.width != _width) { + return std::nullopt; + } + const uint32_t slot = block_height % _width; + const uint64_t slot_pos = sizeof(blk_offset_index_header) + uint64_t{slot} * sizeof(uint64_t); + uint64_t encoded = 0; + blk_idx.seek(slot_pos); + blk_idx.read(reinterpret_cast(&encoded), sizeof(encoded)); + if (encoded == 0) return std::nullopt; + return encoded - 1; + } catch (...) { + return std::nullopt; + } + } + void slice_directory::start_maintenance_thread(log_handler log) { _maintenance_thread = std::thread([this, log=std::move(log)](){ fc::set_thread_name( "trace-mx" ); @@ -377,6 +796,23 @@ namespace sysio::trace_api { } void slice_directory::run_maintenance_tasks(uint32_t lib, const log_handler& log) { + // Build trx_id indexes for all newly irreversible slices (min_irreversible=0: + // index as soon as a slice's block range is fully below LIB). + process_irreversible_slice_range(lib, 0, _last_indexed_slice, [this, &log](uint32_t slice_to_index){ + try { + build_trx_id_index(slice_to_index, log); + } FC_LOG_AND_DROP(); + }); + + // Build receiver bloom sidecars on the same schedule as trx_id indexes - any slice fully past LIB has its data + // final, so forks can't corrupt the sidecar after it's written. Ordering before compression keeps the source + // .log available for the stream-scan. + process_irreversible_slice_range(lib, 0, _last_bloomed_slice, [this, &log](uint32_t slice_to_bloom){ + try { + build_recv_bloom(slice_to_bloom, log); + } FC_LOG_AND_DROP(); + }); + if (_minimum_irreversible_history_blocks) { process_irreversible_slice_range(lib, *_minimum_irreversible_history_blocks, _last_cleaned_up_slice, [this, &log](uint32_t slice_to_clean){ fc::cfile trace; @@ -402,6 +838,25 @@ namespace sysio::trace_api { log(std::string("Removing: ") + trx_id.get_file_path().generic_string()); std::filesystem::remove(trx_id.get_file_path()); } + auto idx_filename = make_filename(_trace_trx_id_index_prefix, _trace_ext, slice_to_clean, _width); + const auto idx_path = _slice_dir / idx_filename; + if (std::filesystem::exists(idx_path)) { + log(std::string("Removing: ") + idx_path.generic_string()); + std::filesystem::remove(idx_path); + } + + auto blk_idx_filename = make_filename(_trace_blk_idx_prefix, _trace_ext, slice_to_clean, _width); + const auto blk_idx_path = _slice_dir / blk_idx_filename; + if (std::filesystem::exists(blk_idx_path)) { + log(std::string("Removing: ") + blk_idx_path.generic_string()); + std::filesystem::remove(blk_idx_path); + } + + const auto bloom_path = bloom_slice_path(slice_to_clean); + if (std::filesystem::exists(bloom_path)) { + log(std::string("Removing: ") + bloom_path.generic_string()); + std::filesystem::remove(bloom_path); + } auto ctrace = find_compressed_trace_slice(slice_to_clean, dont_open_file); if (ctrace) { diff --git a/plugins/trace_api_plugin/src/trace_api_plugin.cpp b/plugins/trace_api_plugin/src/trace_api_plugin.cpp index 1457919659..88046b8db0 100644 --- a/plugins/trace_api_plugin/src/trace_api_plugin.cpp +++ b/plugins/trace_api_plugin/src/trace_api_plugin.cpp @@ -1,8 +1,9 @@ #include #include -#include #include +#include +#include #include #include @@ -13,13 +14,11 @@ using namespace sysio::trace_api; using namespace sysio::trace_api::configuration_utils; +using namespace sysio::chain::literals; using boost::signals2::scoped_connection; namespace { - const std::string logger_name("trace_api"); - fc::logger _log; - std::string to_detail_string(const std::exception_ptr& e) { try { std::rethrow_exception(e); @@ -91,6 +90,34 @@ namespace { store->append_trx_ids(std::move(tt)); } + std::optional> first_and_last_recorded_blocks() const { + return store->first_and_last_recorded_blocks(); + } + + void append_abi(chain::name account, uint64_t global_seq, std::vector abi_bytes) { + store->append_abi(account, global_seq, std::move(abi_bytes)); + } + + std::optional lookup_abi(chain::name account, uint64_t global_seq) const { + return store->lookup_abi(account, global_seq); + } + + bool has_abi_entry(chain::name account) const { + return store->has_abi_entry(account); + } + + uint32_t slice_stride() const noexcept { + return store->slice_stride(); + } + + uint32_t slice_number(uint32_t block_height) const noexcept { + return store->slice_number(block_height); + } + + bloom_reader get_bloom(uint32_t slice_number) const { + return store->get_bloom(slice_number); + } + std::shared_ptr store; }; } @@ -106,13 +133,20 @@ struct trace_api_common_impl { cfg_options("trace-dir", bpo::value()->default_value("traces"), "the location of the trace directory (absolute path or relative to application data dir)"); cfg_options("trace-slice-stride", bpo::value()->default_value(10'000), - "the number of blocks each \"slice\" of trace data will contain on the filesystem"); + "Number of blocks each \"slice\" of trace data will contain on the filesystem.\n" + "Must be in the range [1, 1000000]. Larger values reduce file count but bloat the\n" + "block-offset sidecar pre-allocation and stress the per-slice trx_id hash index."); cfg_options("trace-minimum-irreversible-history-blocks", boost::program_options::value()->default_value(-1), "Number of blocks to ensure are kept past LIB for retrieval before \"slice\" files can be automatically removed.\n" "A value of -1 indicates that automatic removal of \"slice\" files will be turned off."); cfg_options("trace-minimum-uncompressed-irreversible-history-blocks", boost::program_options::value()->default_value(-1), "Number of blocks to ensure are uncompressed past LIB. Compressed \"slice\" files are still accessible but may carry a performance loss on retrieval\n" "A value of -1 indicates that automatic compression of \"slice\" files will be turned off."); + cfg_options("trace-max-block-range", bpo::value()->default_value(1000), + "Maximum number of blocks scanned by a single get_actions or get_token_transfers request.\n" + "Must be in [1, 10000]. block_num_end is silently clamped to block_num_start + this - 1.\n" + "Clients paginate by advancing block_num_start by this amount each call. The response\n" + "envelope reports the actual range scanned."); } void plugin_initialize(const appbase::variables_map& options) { @@ -125,6 +159,8 @@ struct trace_api_common_impl { resmon_plugin->monitor_directory(trace_dir); slice_stride = options.at("trace-slice-stride").as(); + SYS_ASSERT(slice_stride >= 1 && slice_stride <= 1'000'000, chain::plugin_config_exception, + "\"trace-slice-stride\" must be in [1, 1000000]; got {}", slice_stride); const int32_t blocks = options.at("trace-minimum-irreversible-history-blocks").as(); SYS_ASSERT(blocks >= -1, chain::plugin_config_exception, @@ -141,6 +177,11 @@ struct trace_api_common_impl { minimum_uncompressed_irreversible_history_blocks = uncompressed_blocks; } + const uint32_t block_range = options.at("trace-max-block-range").as(); + SYS_ASSERT(block_range >= 1 && block_range <= 10'000, chain::plugin_config_exception, + "\"trace-max-block-range\" must be in [1, 10000]; got {}", block_range); + max_block_range = block_range; + store = std::make_shared( trace_dir, slice_stride, @@ -170,6 +211,7 @@ struct trace_api_common_impl { static constexpr int32_t manual_slice_file_value = -1; static constexpr uint32_t compression_seek_point_stride = 6 * 1024 * 1024; // 6 MiB strides for clog seek points + uint32_t max_block_range = 100; std::shared_ptr store; }; @@ -181,51 +223,24 @@ struct trace_api_rpc_plugin_impl : public std::enable_shared_from_this& common ) :common(common) {} - static void set_program_options(appbase::options_description& cli, appbase::options_description& cfg) { - auto cfg_options = cfg.add_options(); - cfg_options("trace-rpc-abi", bpo::value>()->composing(), - "ABIs used when decoding trace RPC responses.\n" - "There must be at least one ABI specified OR the flag trace-no-abis must be used.\n" - "ABIs are specified as \"Key=Value\" pairs in the form =\n" - "Where can be:\n" - " an absolute path to a file containing a valid JSON-encoded ABI\n" - " a relative path from `data-dir` to a file containing a valid JSON-encoded ABI\n" - ); - cfg_options("trace-no-abis", - "Use to indicate that the RPC responses will not use ABIs.\n" - "Failure to specify this option when there are no trace-rpc-abi configuations will result in an Error.\n" - "This option is mutually exclusive with trace-rpc-api" - ); - } - - void plugin_initialize(const appbase::variables_map& options) { - ilog("initializing trace api rpc plugin"); - std::shared_ptr data_handler = std::make_shared([](const exception_with_context& e){ - log_exception(e, fc::log_level::debug); - if (std::get<0>(e)) { // rethrow so caller is notified of error - std::rethrow_exception(std::get<0>(e)); - } - }); - - if( options.count("trace-rpc-abi") ) { - SYS_ASSERT(options.count("trace-no-abis") == 0, chain::plugin_config_exception, - "Trace API is configured with ABIs however trace-no-abis is set"); - const std::vector key_value_pairs = options["trace-rpc-abi"].as>(); - for (const auto& entry : key_value_pairs) { - try { - auto kv = parse_kv_pairs(entry); - auto account = chain::name(kv.first); - auto abi = abi_def_from_file(kv.second, app().data_dir()); - data_handler->add_abi(account, std::move(abi)); - } catch (...) { - elog("Malformed trace-rpc-abi provider: \"{}\"", entry); - throw; + void plugin_initialize(const appbase::variables_map&) { + fc_ilog(_log, "trace_api: initializing trace api rpc plugin"); + max_block_range = common->max_block_range; + auto store = common->store; + auto data_handler = std::make_shared( + [](const exception_with_context& e) { + // Log at debug and fall back to raw hex -- do not rethrow, since + // ABI capture is automatic and decoding failures should be soft. + log_exception(e, fc::log_level::debug); + }, + [store](chain::name account, uint64_t global_seq) -> std::optional { + std::optional out; + if (auto r = store->lookup_abi(account, global_seq)) { + out.emplace(abi_data_handler::lookup_entry{r->effective_global_seq, std::move(r->abi_bytes)}); } + return out; } - } else { - SYS_ASSERT(options.count("trace-no-abis") != 0, chain::plugin_config_exception, - "Trace API is not configured with ABIs and trace-no-abis is not set"); - } + ); req_handler = std::make_shared( shared_store_provider(common->store), @@ -326,12 +341,132 @@ struct trace_api_rpc_plugin_impl : public std::enable_shared_from_this(); + if (obj.contains("block_num_end")) + query.block_num_end = obj["block_num_end"].as(); + if (obj.contains("include_notifications")) + include_notifications = obj["include_notifications"].as_bool(); + } catch (...) { + error_results results{400, "Bad request body"}; + cb( 400, fc::variant( results )); + return; + } + } + + // Default = canonical actions only (receiver == account). When the caller + // specifies exactly one of receiver/account and does not opt in to + // notifications, mirror the specified value onto the missing one. + if (!include_notifications) { + if (query.account && !query.receiver) + query.receiver = query.account; + else if (query.receiver && !query.account) + query.account = query.receiver; + } + + if (query.block_num_start > query.block_num_end) { + error_results results{400, "block_num_start must be <= block_num_end"}; + cb( 400, fc::variant( results )); + return; + } + + clamp_block_end(query); + + try { + auto result = req_handler->get_actions(query); + cb( 200, fc::mutable_variant_object() + ("block_num_start", query.block_num_start) + ("block_num_end", query.block_num_end) + ("actions", result.actions) ); + } catch (...) { + http_plugin::handle_exception("trace_api", "get_actions", body, cb); + } + }}); + + http.add_async_handler({"/v1/trace_api/get_token_transfers", + api_category::trace_api, + [this](std::string, std::string body, url_response_callback cb) + { + // Convenience wrapper: receiver==account==token_contract, action="transfer" + // gives exactly one result per transfer (inline notifications excluded). + static constexpr chain::name default_token_contract = "sysio.token"_n; + static constexpr chain::name transfer_action_name = "transfer"_n; + action_query query; + query.action = transfer_action_name; + query.receiver = default_token_contract; + query.account = default_token_contract; + + if (!body.empty()) { + try { + auto input = fc::json::from_string(body); + const auto& obj = input.get_object(); + if (obj.contains("token_contract")) { + chain::name tc = chain::name(obj["token_contract"].as_string()); + query.receiver = tc; + query.account = tc; + } + if (obj.contains("block_num_start")) + query.block_num_start = obj["block_num_start"].as(); + if (obj.contains("block_num_end")) + query.block_num_end = obj["block_num_end"].as(); + } catch (...) { + error_results results{400, "Bad request body"}; + cb( 400, fc::variant( results )); + return; + } + } + + if (query.block_num_start > query.block_num_end) { + error_results results{400, "block_num_start must be <= block_num_end"}; + cb( 400, fc::variant( results )); + return; + } + + clamp_block_end(query); + + try { + auto result = req_handler->get_token_transfer_actions(query); + cb( 200, fc::mutable_variant_object() + ("block_num_start", query.block_num_start) + ("block_num_end", query.block_num_end) + ("transfers", result.actions) ); + } catch (...) { + http_plugin::handle_exception("trace_api", "get_token_transfers", body, cb); + } + }}); } void plugin_shutdown() { } + // Silently clamp block_num_end so the scan spans at most max_block_range blocks. + // No 400 returned -- wide-range requests are a normal pagination pattern. + // The response envelope reports the actual range so clients can detect the clamp. + void clamp_block_end(action_query& query) const { + const uint64_t max_end = uint64_t{query.block_num_start} + max_block_range - 1; + if (max_end < query.block_num_end) + query.block_num_end = static_cast(max_end); + } + std::shared_ptr common; + uint32_t max_block_range = 100; using request_handler_t = request_handler, abi_data_handler::shared_provider>; std::shared_ptr req_handler; @@ -342,16 +477,36 @@ struct trace_api_plugin_impl { :common(common) {} void plugin_initialize(const appbase::variables_map& options) { - ilog("initializing trace api plugin"); + fc_ilog(_log, "trace_api: initializing trace api plugin"); auto log_exceptions_and_shutdown = [](const exception_with_context& e) { log_exception(e, fc::log_level::error); app().quit(); throw yield_exception("shutting down"); }; - extraction = std::make_shared(shared_store_provider(common->store), log_exceptions_and_shutdown); - auto& chain = app().find_plugin()->chain(); + // Lazy ABI fetcher: called from applied_transaction (chain write thread) on first + // encounter of each account. Captures current ABI from the chain DB at that point. + chain_extraction_t::abi_fetcher_t abi_fetcher = [&chain](chain::name account) + -> std::optional> { + std::optional> result; + try { + const auto* meta = chain.find_account_metadata(account); + if (meta && meta->abi.size() > 0) + result.emplace(meta->abi.data(), meta->abi.data() + meta->abi.size()); + } catch (const std::exception& e) { + fc_dlog(_log, "trace_api: lazy ABI fetch for {} failed: {}", account, e.what()); + } catch (...) { + fc_dlog(_log, "trace_api: lazy ABI fetch for {} failed: unknown", account); + } + return result; + }; + + extraction = std::make_shared( + shared_store_provider(common->store), + log_exceptions_and_shutdown, + std::move(abi_fetcher)); + applied_transaction_connection.emplace( chain.applied_transaction().connect([this](std::tuple t) { emit_killer([&](){ @@ -409,7 +564,6 @@ trace_api_plugin::~trace_api_plugin() = default; void trace_api_plugin::set_program_options(appbase::options_description& cli, appbase::options_description& cfg) { trace_api_common_impl::set_program_options(cli, cfg); - trace_api_rpc_plugin_impl::set_program_options(cli, cfg); } void trace_api_plugin::plugin_initialize(const appbase::variables_map& options) { @@ -446,7 +600,6 @@ trace_api_rpc_plugin::~trace_api_rpc_plugin() = default; void trace_api_rpc_plugin::set_program_options(appbase::options_description& cli, appbase::options_description& cfg) { trace_api_common_impl::set_program_options(cli, cfg); - trace_api_rpc_plugin_impl::set_program_options(cli, cfg); } void trace_api_rpc_plugin::plugin_initialize(const appbase::variables_map& options) { diff --git a/plugins/trace_api_plugin/src/trx_id_index.cpp b/plugins/trace_api_plugin/src/trx_id_index.cpp new file mode 100644 index 0000000000..ff28a909d7 --- /dev/null +++ b/plugins/trace_api_plugin/src/trx_id_index.cpp @@ -0,0 +1,169 @@ +#include +#include +#include + +#include + +#include + +#include +#include +#include + +namespace sysio::trace_api { + +uint64_t trx_id_index_writer::prefix_of(const chain::transaction_id_type& id) { + // transaction_id_type is fc::sha256 — use the first 8 bytes as the hash key. + // fc::sha256 is a plain struct with no padding before its data. + uint64_t p; + static_assert(sizeof(chain::transaction_id_type) >= sizeof(p)); + std::memcpy(&p, &id, sizeof(p)); + return p; +} + +void trx_id_index_writer::add(const chain::transaction_id_type& trx_id, uint32_t block_num) { + _entries.emplace_back(prefix_of(trx_id), block_num); +} + +void trx_id_index_writer::write(const std::filesystem::path& path) const { + // Target load factor <= 0.5; bucket_count is a power of two for fast modulo. + // Guard the uint32_t cast: in practice a slice holds at most ~10M trxs, + // but a future regression could exceed UINT32_MAX and silently truncate. + SYS_ASSERT(_entries.size() <= std::numeric_limits::max() / 2 - 1, + chain::plugin_exception, + "trx_id_index entry count {} exceeds uint32 range", _entries.size()); + const uint32_t n = static_cast(_entries.size()); + const uint32_t bucket_count = std::max(4u, std::bit_ceil(n * 2 + 1)); + const uint32_t mask = bucket_count - 1; + + // Value-initialize all buckets to zero. block_num == 0 is the empty-slot + // sentinel that terminates the probe loop below; do NOT replace this with + // reserve()+emplace_back() or any path that leaves block_num uninitialized. + std::vector buckets(bucket_count); + + // Last-write-wins per prefix: probe forward until either an empty bucket + // (fresh insert) OR a bucket already holding this prefix (overwrite). + // Combined with the index-builder's per-block_num dedup pass, this means a + // trx that's been re-recorded under a different block_num after a fork + // resolves to the latest entry, matching the linear-scan get_trx_block_number + // path which returns *(--trx_block_nums.end()) (highest/most recent). + for (const auto& [prefix, block_num] : _entries) { + uint32_t idx = static_cast(prefix) & mask; + while (buckets[idx].block_num != 0 && buckets[idx].prefix64 != prefix) { + idx = (idx + 1) & mask; + } + buckets[idx].prefix64 = prefix; + buckets[idx].block_num = block_num; + } + + fc::cfile f; + f.set_file_path(path); + f.open(fc::cfile::create_or_update_rw_mode); + + trx_id_index_header header; + header.bucket_count = bucket_count; + auto hdr_data = fc::raw::pack(header); + f.write(hdr_data.data(), hdr_data.size()); + + // Bulk write: see the bulk-read note in trx_id_index_reader's constructor + // for the layout-equivalence rationale. + f.write(reinterpret_cast(buckets.data()), + buckets.size() * sizeof(trx_id_bucket)); + f.flush(); +} + +// --------------------------------------------------------------------------- + +uint64_t trx_id_index_reader::prefix_of(const chain::transaction_id_type& id) { + uint64_t p; + std::memcpy(&p, &id, sizeof(p)); + return p; +} + +// Hard cap on bucket_count read from disk. At 16 bytes per bucket, 2^28 +// buckets = 4 GB. A realistic worst-case slice (default 10K blocks at +// ~1K trxs/block = 10M trxs at load factor 0.5 = 2^25 buckets) fits 8x +// inside this cap. Anything larger is treated as a corrupt/malicious file. +static constexpr uint32_t max_bucket_count = 1u << 28; + +trx_id_index_reader::trx_id_index_reader(const std::filesystem::path& path) { + try { + fc::cfile f; + f.set_file_path(path); + f.open("rb"); + f.seek(0); + + const auto header = extract_store(f); + if (header.magic != trx_id_index_header::magic_value) { + fc_wlog(_log,"trace_api: trx_id index {} has wrong magic, ignoring", path.generic_string()); + return; + } + if (header.version != trx_id_index_header::current_version) { + fc_wlog(_log,"trace_api: trx_id index {} has unsupported version {}, ignoring", + path.generic_string(), header.version); + return; + } + if (header.bucket_count == 0) { + _valid = true; + return; + } + + // Open-addressing math (mask = bucket_count - 1) requires a power of two. + if (!std::has_single_bit(header.bucket_count)) { + fc_wlog(_log,"trace_api: trx_id index {} bucket_count {} is not a power of two, ignoring", + path.generic_string(), header.bucket_count); + return; + } + // Cap allocation against malicious / corrupt headers. + if (header.bucket_count > max_bucket_count) { + fc_wlog(_log,"trace_api: trx_id index {} bucket_count {} exceeds cap {}, ignoring", + path.generic_string(), header.bucket_count, max_bucket_count); + return; + } + // File length must equal header + bucket_count * sizeof(bucket). + const uint64_t expected_size = sizeof(trx_id_index_header) + + uint64_t{header.bucket_count} * sizeof(trx_id_bucket); + const uint64_t actual_size = std::filesystem::file_size(path); + if (actual_size != expected_size) { + fc_wlog(_log,"trace_api: trx_id index {} size {} != expected {}, ignoring", + path.generic_string(), actual_size, expected_size); + return; + } + + _buckets.resize(header.bucket_count); + // Bulk read: trx_id_bucket is {u64, u32, u32} with no padding and + // static_assert(sizeof(trx_id_bucket) == 16) in the header. fc::raw::pack + // writes those fields little-endian, identical to the in-memory layout on + // x86_64 LE -- so a single read into the contiguous vector is equivalent + // to the field-by-field unpack and avoids the per-bucket call overhead. + f.read(reinterpret_cast(_buckets.data()), + static_cast(header.bucket_count) * sizeof(trx_id_bucket)); + _valid = true; + } catch (...) { + fc_wlog(_log,"trace_api: failed to load trx_id index from {}", path.generic_string()); + } +} + +std::optional trx_id_index_reader::lookup(const chain::transaction_id_type& trx_id) const { + if (!_valid || _buckets.empty()) + return std::nullopt; + + const uint64_t prefix = prefix_of(trx_id); + const uint32_t bucket_count = static_cast(_buckets.size()); + const uint32_t mask = bucket_count - 1; + uint32_t idx = static_cast(prefix) & mask; + + // Bounded probe loop: a well-formed index has load factor <= 0.5 and an + // empty bucket terminates the chain. The bound guards against a corrupt + // file with no empty buckets at all (would otherwise loop forever). + for (uint32_t probes = 0; probes < bucket_count; ++probes) { + if (_buckets[idx].block_num == 0) + return std::nullopt; + if (_buckets[idx].prefix64 == prefix) + return _buckets[idx].block_num; + idx = (idx + 1) & mask; + } + return std::nullopt; +} + +} // namespace sysio::trace_api diff --git a/plugins/trace_api_plugin/test/CMakeLists.txt b/plugins/trace_api_plugin/test/CMakeLists.txt index aab0919ff5..60b34afef8 100644 --- a/plugins/trace_api_plugin/test/CMakeLists.txt +++ b/plugins/trace_api_plugin/test/CMakeLists.txt @@ -1,10 +1,15 @@ add_executable( test_trace_api_plugin + test_continuity.cpp test_extraction.cpp test_responses.cpp test_trace_file.cpp test_data_handlers.cpp test_configuration_utils.cpp test_compressed_file.cpp + test_trx_id_index.cpp + test_abi_log.cpp + test_get_actions.cpp + test_bloom_sidecar.cpp main.cpp ) target_link_libraries( test_trace_api_plugin trace_api_plugin ) diff --git a/plugins/trace_api_plugin/test/include/sysio/trace_api/test_common.hpp b/plugins/trace_api_plugin/test/include/sysio/trace_api/test_common.hpp index a602429acf..a572f3ab58 100644 --- a/plugins/trace_api_plugin/test/include/sysio/trace_api/test_common.hpp +++ b/plugins/trace_api_plugin/test/include/sysio/trace_api/test_common.hpp @@ -115,26 +115,39 @@ namespace sysio::trace_api { inline bool operator==(const authorization_trace_v0& lhs, const authorization_trace_v0& rhs) { return - lhs.account == rhs.account && + lhs.actor == rhs.actor && lhs.permission == rhs.permission; } + inline bool operator==(const account_delta_v0& lhs, const account_delta_v0& rhs) { + return lhs.account == rhs.account && lhs.delta == rhs.delta; + } + inline bool operator==(const action_trace_v0& lhs, const action_trace_v0& rhs) { return + lhs.action_ordinal == rhs.action_ordinal && + lhs.creator_action_ordinal == rhs.creator_action_ordinal && + lhs.closest_unnotified_ancestor_action_ordinal == rhs.closest_unnotified_ancestor_action_ordinal && lhs.global_sequence == rhs.global_sequence && + lhs.recv_sequence == rhs.recv_sequence && + lhs.auth_sequence == rhs.auth_sequence && + lhs.code_sequence == rhs.code_sequence && + lhs.abi_sequence == rhs.abi_sequence && lhs.receiver == rhs.receiver && lhs.account == rhs.account && lhs.action == rhs.action && lhs.authorization == rhs.authorization && lhs.data == rhs.data && - lhs.return_value == rhs.return_value; + lhs.return_value == rhs.return_value && + lhs.account_ram_deltas == rhs.account_ram_deltas && + lhs.cpu_usage_us == rhs.cpu_usage_us && + lhs.net_usage == rhs.net_usage; } inline bool operator==(const transaction_trace_v0& lhs, const transaction_trace_v0& rhs) { return lhs.id == rhs.id && lhs.actions == rhs.actions && - lhs.status == rhs.status && lhs.cpu_usage_us == rhs.cpu_usage_us && lhs.net_usage_words == rhs.net_usage_words && lhs.signatures == rhs.signatures && diff --git a/plugins/trace_api_plugin/test/test_abi_log.cpp b/plugins/trace_api_plugin/test/test_abi_log.cpp new file mode 100644 index 0000000000..663c3826b1 --- /dev/null +++ b/plugins/trace_api_plugin/test/test_abi_log.cpp @@ -0,0 +1,313 @@ +#include +#include +#include + +#include +#include + +#include +#include + +using namespace sysio; +using namespace sysio::trace_api; +using namespace sysio::trace_api::test_common; + +namespace { + +struct abi_log_fixture { + fc::temp_directory tempdir; + + std::filesystem::path log_path() const { + return tempdir.path() / "abi_log.log"; + } + + static std::vector make_abi(const std::string& tag) { + return std::vector(tag.begin(), tag.end()); + } + + // Open a log, run callable with it, then let it destruct (closes the file). + template + void with_log(F&& f) { + abi_log log(log_path()); + BOOST_REQUIRE(log.valid()); + f(log); + } +}; + +} // namespace + +BOOST_AUTO_TEST_SUITE(abi_log_tests) + +// --------------------------------------------------------------------------- +// Round-trip: writer/reader in the same process +// --------------------------------------------------------------------------- + +BOOST_FIXTURE_TEST_CASE(empty_log_is_valid, abi_log_fixture) { + abi_log log(log_path()); + BOOST_CHECK(log.valid()); + BOOST_CHECK(!log.lookup("sysio.token"_n, 1)); + BOOST_CHECK(!log.lookup("sysio.token"_n, 0)); +} + +BOOST_FIXTURE_TEST_CASE(single_entry_round_trip, abi_log_fixture) { + abi_log log(log_path()); + auto blob = make_abi("abi-v1"); + log.append("sysio.token"_n, 100, blob); + + auto result = log.lookup("sysio.token"_n, 100); + BOOST_REQUIRE(result.has_value()); + BOOST_CHECK_EQUAL_COLLECTIONS(result->abi_bytes.begin(), result->abi_bytes.end(), blob.begin(), blob.end()); +} + +BOOST_FIXTURE_TEST_CASE(multiple_accounts_round_trip, abi_log_fixture) { + abi_log log(log_path()); + auto blob1 = make_abi("token-abi-v1"); + auto blob2 = make_abi("eosio-abi-v1"); + log.append("sysio.token"_n, 50, blob1); + log.append("sysio"_n, 200, blob2); + + auto r1 = log.lookup("sysio.token"_n, 50); + BOOST_REQUIRE(r1.has_value()); + BOOST_CHECK_EQUAL_COLLECTIONS(r1->abi_bytes.begin(), r1->abi_bytes.end(), blob1.begin(), blob1.end()); + + auto r2 = log.lookup("sysio"_n, 200); + BOOST_REQUIRE(r2.has_value()); + BOOST_CHECK_EQUAL_COLLECTIONS(r2->abi_bytes.begin(), r2->abi_bytes.end(), blob2.begin(), blob2.end()); +} + +// --------------------------------------------------------------------------- +// Version selection via upper_bound +// --------------------------------------------------------------------------- + +BOOST_FIXTURE_TEST_CASE(returns_abi_in_effect_at_global_seq, abi_log_fixture) { + abi_log log(log_path()); + auto v1 = make_abi("abi-v1"); + auto v2 = make_abi("abi-v2"); + auto v3 = make_abi("abi-v3"); + + log.append("sysio.token"_n, 100, v1); + log.append("sysio.token"_n, 200, v2); + log.append("sysio.token"_n, 300, v3); + + // Before any version + BOOST_CHECK(!log.lookup("sysio.token"_n, 99)); + + // Exactly at v1 + auto at100 = log.lookup("sysio.token"_n, 100); + BOOST_REQUIRE(at100.has_value()); + BOOST_CHECK_EQUAL_COLLECTIONS(at100->abi_bytes.begin(), at100->abi_bytes.end(), v1.begin(), v1.end()); + + // Between v1 and v2 -> v1 + auto at150 = log.lookup("sysio.token"_n, 150); + BOOST_REQUIRE(at150.has_value()); + BOOST_CHECK_EQUAL_COLLECTIONS(at150->abi_bytes.begin(), at150->abi_bytes.end(), v1.begin(), v1.end()); + + // Exactly at v2 + auto at200 = log.lookup("sysio.token"_n, 200); + BOOST_REQUIRE(at200.has_value()); + BOOST_CHECK_EQUAL_COLLECTIONS(at200->abi_bytes.begin(), at200->abi_bytes.end(), v2.begin(), v2.end()); + + // After v3 + auto at500 = log.lookup("sysio.token"_n, 500); + BOOST_REQUIRE(at500.has_value()); + BOOST_CHECK_EQUAL_COLLECTIONS(at500->abi_bytes.begin(), at500->abi_bytes.end(), v3.begin(), v3.end()); +} + +BOOST_FIXTURE_TEST_CASE(lookup_wrong_account_returns_nullopt, abi_log_fixture) { + abi_log log(log_path()); + log.append("sysio.token"_n, 100, make_abi("token-abi")); + BOOST_CHECK(!log.lookup("sysio.msig"_n, 100)); +} + +BOOST_FIXTURE_TEST_CASE(accounts_do_not_bleed_into_each_other, abi_log_fixture) { + abi_log log(log_path()); + log.append("sysio.token"_n, 100, make_abi("token-100")); + log.append("sysio"_n, 200, make_abi("sysio-200")); + + auto token_result = log.lookup("sysio.token"_n, 250); + BOOST_REQUIRE(token_result.has_value()); + BOOST_CHECK(token_result->abi_bytes == make_abi("token-100")); + + BOOST_CHECK(!log.lookup("sysio.token"_n, 50)); +} + +BOOST_FIXTURE_TEST_CASE(empty_blob_round_trip, abi_log_fixture) { + abi_log log(log_path()); + log.append("clearme"_n, 999, {}); // empty ABI + + auto result = log.lookup("clearme"_n, 999); + BOOST_REQUIRE(result.has_value()); + BOOST_CHECK(result->abi_bytes.empty()); +} + +BOOST_FIXTURE_TEST_CASE(last_write_wins_for_duplicate_key, abi_log_fixture) { + abi_log log(log_path()); + log.append("acct"_n, 100, make_abi("first")); + log.append("acct"_n, 100, make_abi("second")); // overwrites in index + + auto result = log.lookup("acct"_n, 100); + BOOST_REQUIRE(result.has_value()); + BOOST_CHECK(result->abi_bytes == make_abi("second")); +} + +// --------------------------------------------------------------------------- +// Restart: entries persist across open/close cycles +// --------------------------------------------------------------------------- + +BOOST_FIXTURE_TEST_CASE(append_after_restart, abi_log_fixture) { + // Session 1: write two records, close. + { + abi_log log(log_path()); + log.append("sysio.token"_n, 100, make_abi("token-100")); + log.append("sysio"_n, 200, make_abi("sysio-200")); + } + + // Session 2: reopen, verify old records, append a new one, verify all three. + abi_log log(log_path()); + BOOST_REQUIRE(log.valid()); + + auto r1 = log.lookup("sysio.token"_n, 100); + BOOST_REQUIRE(r1.has_value()); + BOOST_CHECK(r1->abi_bytes == make_abi("token-100")); + + auto r2 = log.lookup("sysio"_n, 200); + BOOST_REQUIRE(r2.has_value()); + BOOST_CHECK(r2->abi_bytes == make_abi("sysio-200")); + + log.append("newacct"_n, 300, make_abi("new-300")); + + auto r3 = log.lookup("newacct"_n, 300); + BOOST_REQUIRE(r3.has_value()); + BOOST_CHECK(r3->abi_bytes == make_abi("new-300")); +} + +// --------------------------------------------------------------------------- +// Error paths: bad header +// --------------------------------------------------------------------------- + +BOOST_FIXTURE_TEST_CASE(bad_magic_is_invalid, abi_log_fixture) { + { + std::ofstream f(log_path(), std::ios::binary); + abi_log_header hdr; + hdr.magic = 0xDEADBEEF; + f.write(reinterpret_cast(&hdr), sizeof(hdr)); + } + abi_log log(log_path()); + BOOST_CHECK(!log.valid()); +} + +BOOST_FIXTURE_TEST_CASE(bad_version_is_invalid, abi_log_fixture) { + { + std::ofstream f(log_path(), std::ios::binary); + abi_log_header hdr; + hdr.version = 99; + f.write(reinterpret_cast(&hdr), sizeof(hdr)); + } + abi_log log(log_path()); + BOOST_CHECK(!log.valid()); +} + +// --------------------------------------------------------------------------- +// Corruption recovery: torn tail +// --------------------------------------------------------------------------- + +// Chop bytes off the end of the file. The final record's trailing CRC is +// truncated, so recovery drops it at reopen. +BOOST_FIXTURE_TEST_CASE(truncated_tail_is_recovered, abi_log_fixture) { + // Session 1: write three records. + { + abi_log log(log_path()); + log.append("a"_n, 100, make_abi("a-100")); + log.append("b"_n, 200, make_abi("b-200")); + log.append("c"_n, 300, make_abi("c-300")); + } + + // Lop off the last 3 bytes (inside record-c's crc). + { + const auto size = std::filesystem::file_size(log_path()); + std::filesystem::resize_file(log_path(), size - 3); + } + + // Session 2: recover. a and b survive, c is gone. + abi_log log(log_path()); + BOOST_REQUIRE(log.valid()); + BOOST_CHECK(log.lookup("a"_n, 100).has_value()); + BOOST_CHECK(log.lookup("b"_n, 200).has_value()); + BOOST_CHECK(!log.lookup("c"_n, 300)); + + // File should have been truncated at end of record-b, so a new append + // lands at that position and is recoverable. + log.append("d"_n, 400, make_abi("d-400")); + auto r = log.lookup("d"_n, 400); + BOOST_REQUIRE(r.has_value()); + BOOST_CHECK(r->abi_bytes == make_abi("d-400")); +} + +// Flip a byte inside record-b's blob so its CRC fails. Recovery truncates +// everything from the start of record-b onwards (including record-c). +BOOST_FIXTURE_TEST_CASE(crc_mismatch_drops_record_and_tail, abi_log_fixture) { + { + abi_log log(log_path()); + log.append("a"_n, 100, make_abi("a-blob")); + log.append("b"_n, 200, make_abi("b-blob")); + log.append("c"_n, 300, make_abi("c-blob")); + } + + // Layout per record: header(24) + blob + crc(4). + // Record a: offset 16 (header), header end 40, blob 40..46, crc 46..50 + // Record b: offset 50 (header), header end 74, blob 74..80, crc 80..84 + // Flip byte at offset 75 (middle of b's blob). + { + std::fstream f(log_path(), std::ios::binary | std::ios::in | std::ios::out); + f.seekp(75); + char c = 0; + f.read(&c, 1); + c ^= 0xff; + f.seekp(75); + f.write(&c, 1); + } + + abi_log log(log_path()); + BOOST_REQUIRE(log.valid()); + BOOST_CHECK(log.lookup("a"_n, 100).has_value()); + BOOST_CHECK(!log.lookup("b"_n, 200)); + BOOST_CHECK(!log.lookup("c"_n, 300)); + + // File truncated at end of record-a (offset 50). New append works. + log.append("d"_n, 400, make_abi("d-blob")); + auto r = log.lookup("d"_n, 400); + BOOST_REQUIRE(r.has_value()); + BOOST_CHECK(r->abi_bytes == make_abi("d-blob")); +} + +// --------------------------------------------------------------------------- +// Many records +// --------------------------------------------------------------------------- + +BOOST_FIXTURE_TEST_CASE(many_accounts_many_versions, abi_log_fixture) { + const int NUM_ACCOUNTS = 10; + const int VERSIONS_PER_ACCOUNT = 5; + + abi_log log(log_path()); + + // Add in reverse order to verify the in-memory index sort is correct. + for (int a = NUM_ACCOUNTS - 1; a >= 0; --a) { + for (int v = VERSIONS_PER_ACCOUNT - 1; v >= 0; --v) { + auto acct = chain::name(static_cast(a + 1) * 0x10000000000000ULL); + uint64_t seq = static_cast(a * 100 + v * 10 + 1); + log.append(acct, seq, make_abi("a" + std::to_string(a) + "v" + std::to_string(v))); + } + } + + for (int a = 0; a < NUM_ACCOUNTS; ++a) { + auto acct = chain::name(static_cast(a + 1) * 0x10000000000000ULL); + int v = VERSIONS_PER_ACCOUNT - 1; + uint64_t seq = static_cast(a * 100 + v * 10 + 1); + auto result = log.lookup(acct, seq); + BOOST_REQUIRE_MESSAGE(result.has_value(), "missing entry for account " << a << " version " << v); + auto expected = make_abi("a" + std::to_string(a) + "v" + std::to_string(v)); + BOOST_CHECK_EQUAL_COLLECTIONS(result->abi_bytes.begin(), result->abi_bytes.end(), expected.begin(), expected.end()); + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/plugins/trace_api_plugin/test/test_bloom_sidecar.cpp b/plugins/trace_api_plugin/test/test_bloom_sidecar.cpp new file mode 100644 index 0000000000..4730c4d452 --- /dev/null +++ b/plugins/trace_api_plugin/test/test_bloom_sidecar.cpp @@ -0,0 +1,266 @@ +#include +#include + +#include +#include + +#include +#include +#include + +using namespace sysio; +using namespace sysio::trace_api; +using sysio::chain::name; +using sysio::chain::operator""_n; + +namespace { + +/// Build a minimal action_trace_v0 with only the fields the bloom consumes. +action_trace_v0 act(name receiver, name account, name action) { + action_trace_v0 a{}; + a.receiver = receiver; + a.account = account; + a.action = action; + return a; +} + +/// Pack three actions into a single-transaction block_trace so tests can exercise add_block without pulling in the +/// full extraction machinery. Field defaults are fine - bloom_builder only reads receiver/account/action. +block_trace_v0 block_with(std::vector actions) { + transaction_trace_v0 t{}; + t.actions = std::move(actions); + block_trace_v0 bt{}; + bt.transactions.push_back(std::move(t)); + return bt; +} + +} // namespace + +BOOST_AUTO_TEST_SUITE(bloom_sidecar_tests) + +/// A slice with a known set of receivers yields hits for every inserted receiver and (receiver, action) pair, and +/// misses for names not inserted. The miss rate for well-separated unknowns should be at or below the target FPR; +/// with 32 inserted items at p=0.01 we expect roughly zero false positives on the 8 probe names we test. +BOOST_AUTO_TEST_CASE(roundtrip_hits_and_misses) { + fc::temp_directory tempdir; + const auto path = tempdir.path() / "bloom_roundtrip.log"; + + const std::vector present = { "alice"_n, "bob"_n, "charlie"_n, "sysio.token"_n }; + const std::vector absent = { "dave"_n, "eve"_n, "unknown.acc"_n, "never.seen"_n }; + const name transfer = "transfer"_n; + const name setabi = "setabi"_n; + + bloom_builder b; + for (auto r : present) { + b.add_action(act(r, r, transfer)); + } + BOOST_REQUIRE_EQUAL(b.receiver_count(), present.size()); + BOOST_REQUIRE_EQUAL(b.recv_action_count(), present.size()); + b.finalize_and_write(path); + + bloom_reader r(path); + BOOST_REQUIRE(r.valid()); + + for (auto rcv : present) { + BOOST_TEST_INFO("present receiver " << rcv.to_string()); + BOOST_CHECK(r.may_contain_receiver(rcv)); + BOOST_CHECK(r.may_contain_recv_action(rcv, transfer)); + } + + std::size_t false_positives = 0; + for (auto rcv : absent) { + if (r.may_contain_receiver(rcv)) ++false_positives; + } + BOOST_CHECK_LE(false_positives, 1u); + + // A (receiver, action) probe with a present receiver but unseen action should almost always miss, since the + // composite key has more entropy than receiver alone. + std::size_t composite_fp = 0; + for (auto rcv : present) { + if (r.may_contain_recv_action(rcv, setabi)) ++composite_fp; + } + BOOST_CHECK_LE(composite_fp, 1u); +} + +/// Building from an empty slice still produces a valid file; probes always miss. This is the "no transactions in +/// any block of the slice" case. +BOOST_AUTO_TEST_CASE(empty_builder_produces_valid_file_all_miss) { + fc::temp_directory tempdir; + const auto path = tempdir.path() / "bloom_empty.log"; + + bloom_builder b; + BOOST_REQUIRE(b.empty()); + b.finalize_and_write(path); + + bloom_reader r(path); + BOOST_REQUIRE(r.valid()); + + for (auto n : { "alice"_n, "bob"_n, "sysio.token"_n, "anything"_n }) { + BOOST_CHECK(!r.may_contain_receiver(n)); + BOOST_CHECK(!r.may_contain_recv_action(n, "transfer"_n)); + } +} + +/// add_block feeds every action_trace in every transaction into both filters. +BOOST_AUTO_TEST_CASE(add_block_walks_all_transactions) { + fc::temp_directory tempdir; + const auto path = tempdir.path() / "bloom_block.log"; + + bloom_builder b; + const auto bt = block_with({ + act("alice"_n, "sysio.token"_n, "transfer"_n), + act("sysio.token"_n, "sysio.token"_n, "transfer"_n), + act("bob"_n, "sysio.token"_n, "transfer"_n), + }); + b.add_block(bt); + b.finalize_and_write(path); + + bloom_reader r(path); + BOOST_REQUIRE(r.valid()); + BOOST_CHECK(r.may_contain_receiver("alice"_n)); + BOOST_CHECK(r.may_contain_receiver("bob"_n)); + BOOST_CHECK(r.may_contain_receiver("sysio.token"_n)); + BOOST_CHECK(r.may_contain_recv_action("alice"_n, "transfer"_n)); + BOOST_CHECK(r.may_contain_recv_action("bob"_n, "transfer"_n)); +} + +/// Missing file: reader is invalid and probes default to true so the caller falls back to scanning the slice +/// instead of silently dropping matches. +BOOST_AUTO_TEST_CASE(missing_file_is_invalid_fail_safe_true) { + fc::temp_directory tempdir; + const auto path = tempdir.path() / "does_not_exist.log"; + + bloom_reader r(path); + BOOST_CHECK(!r.valid()); + BOOST_CHECK(r.may_contain_receiver("alice"_n)); + BOOST_CHECK(r.may_contain_recv_action("alice"_n, "transfer"_n)); +} + +/// Bad magic is rejected. Overwrites the first byte of a freshly built file. +BOOST_AUTO_TEST_CASE(bad_magic_is_invalid) { + fc::temp_directory tempdir; + const auto path = tempdir.path() / "bloom_badmagic.log"; + + bloom_builder b; + b.add_action(act("alice"_n, "sysio.token"_n, "transfer"_n)); + b.finalize_and_write(path); + + { + std::fstream f(path.string(), std::ios::binary | std::ios::in | std::ios::out); + f.seekp(0); + char bad = 'X'; + f.write(&bad, 1); + } + + bloom_reader r(path); + BOOST_CHECK(!r.valid()); +} + +/// A single flipped bit in the body invalidates the CRC, and the reader rejects the file rather than trusting +/// potentially corrupted filter state. +BOOST_AUTO_TEST_CASE(corrupted_body_rejected_by_crc) { + fc::temp_directory tempdir; + const auto path = tempdir.path() / "bloom_crc.log"; + + bloom_builder b; + b.add_action(act("alice"_n, "sysio.token"_n, "transfer"_n)); + b.finalize_and_write(path); + + // Flip a bit inside the first receiver-bloom byte (well after the header, well before the trailing CRC). + const auto hdr_size = sizeof(bloom::header); + { + std::fstream f(path.string(), std::ios::binary | std::ios::in | std::ios::out); + f.seekg(hdr_size); + char b0 = 0; + f.read(&b0, 1); + b0 ^= 0x01; + f.seekp(hdr_size); + f.write(&b0, 1); + } + + bloom_reader r(path); + BOOST_CHECK(!r.valid()); +} + +/// Truncated file: drop the last few bytes. Reader detects the size mismatch and rejects the file. +BOOST_AUTO_TEST_CASE(truncated_file_rejected) { + fc::temp_directory tempdir; + const auto path = tempdir.path() / "bloom_trunc.log"; + + bloom_builder b; + b.add_action(act("alice"_n, "sysio.token"_n, "transfer"_n)); + b.finalize_and_write(path); + + const auto full_size = std::filesystem::file_size(path); + BOOST_REQUIRE(full_size > 8); + std::filesystem::resize_file(path, full_size - 8); + + bloom_reader r(path); + BOOST_CHECK(!r.valid()); +} + +/// File written with a different bloom version number is rejected. Prevents silently mis-probing a future-format +/// file. +BOOST_AUTO_TEST_CASE(version_mismatch_rejected) { + fc::temp_directory tempdir; + const auto path = tempdir.path() / "bloom_ver.log"; + + bloom_builder b; + b.add_action(act("alice"_n, "sysio.token"_n, "transfer"_n)); + b.finalize_and_write(path); + + // Rewrite the version field to a future value. The reader compares against bloom::file_version and the CRC + // recomputation will not rescue the file either, but version gate fires first. + { + std::fstream f(path.string(), std::ios::binary | std::ios::in | std::ios::out); + f.seekp(offsetof(bloom::header, version)); + uint32_t bumped = bloom::file_version + 1; + f.write(reinterpret_cast(&bumped), sizeof(bumped)); + } + + bloom_reader r(path); + BOOST_CHECK(!r.valid()); +} + +/// Regression pin against boost::bloom capacity round-trip. boost::bloom's detail/core.hpp:480 documents the +/// invariant filter{f.capacity()}.capacity() == f.capacity(), which we rely on to reconstruct the filter from the +/// saved bit count. If a future boost upgrade quietly breaks this, bloom_reader::load would start rejecting every +/// sidecar on the array-size guard and every query would silently scan instead of skip. This test freezes the +/// invariant by inserting a known set, writing, reading back, and probing every inserted item for a hit. +BOOST_AUTO_TEST_CASE(filter_capacity_roundtrip_invariant) { + fc::temp_directory tempdir; + const auto path = tempdir.path() / "bloom_capacity.log"; + + // Range of item counts spanning the min_capacity floor (32) up into busy-slice territory (1000), so a rounding + // regression that only shows up at certain sizes has a chance to trigger. + for (std::size_t n : { std::size_t{1}, std::size_t{10}, std::size_t{50}, std::size_t{500}, std::size_t{1000} }) { + BOOST_TEST_INFO("n=" << n); + + bloom_builder b; + for (std::size_t i = 0; i < n; ++i) { + // Synthesize distinct names; name stores as uint64 so any distinct 64-bit values work. + chain::name receiver(0x1000'0000'0000'0000ull | static_cast(i)); + chain::name action (0x2000'0000'0000'0000ull | static_cast(i)); + action_trace_v0 a{}; + a.receiver = receiver; + a.account = receiver; + a.action = action; + b.add_action(a); + } + b.finalize_and_write(path); + + bloom_reader r(path); + BOOST_REQUIRE(r.valid()); + + for (std::size_t i = 0; i < n; ++i) { + chain::name receiver(0x1000'0000'0000'0000ull | static_cast(i)); + chain::name action (0x2000'0000'0000'0000ull | static_cast(i)); + BOOST_REQUIRE_MESSAGE(r.may_contain_receiver(receiver), + "receiver " << receiver.to_string() << " should probe as present"); + BOOST_REQUIRE_MESSAGE(r.may_contain_recv_action(receiver, action), + "(receiver, action) should probe as present"); + } + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/plugins/trace_api_plugin/test/test_continuity.cpp b/plugins/trace_api_plugin/test/test_continuity.cpp new file mode 100644 index 0000000000..06d946d8dc --- /dev/null +++ b/plugins/trace_api_plugin/test/test_continuity.cpp @@ -0,0 +1,168 @@ +#include + +#include +#include + +using namespace sysio; +using namespace sysio::trace_api; +using namespace sysio::trace_api::test_common; + +namespace { + +struct continuity_mock_store { + // first == nullopt means no data at all + continuity_mock_store(std::optional first_block, std::optional last_block) + : _first_block(first_block), _last_block(last_block) {} + + template + void append(const BlockTrace&) {} + void append_lib(uint32_t) {} + void append_trx_ids(block_trxs_entry) {} + + std::optional> first_and_last_recorded_blocks() const { + if (!_first_block) return std::nullopt; + return std::make_pair(*_first_block, _last_block.value_or(*_first_block)); + } + + std::optional _first_block; + std::optional _last_block; +}; + +struct continuity_fixture { + // Convenience: data [first, last] exists, or nullopt for empty store. + // Each try_block_start() constructs a fresh extractor instance -- models + // an independent node startup. For tests that need to assert the check + // fires only once across multiple block_start signals on the SAME + // extractor, do not use this fixture; build the extractor inline (see + // check_only_on_first_block_start below). + continuity_fixture(std::optional first_block, std::optional last_block) + : store_first(first_block), store_last(last_block) + {} + + bool try_block_start(uint32_t block_num) { + bool threw = false; + auto except = exception_handler{[&threw](const exception_with_context&) { + threw = true; + throw yield_exception("continuity error"); + }}; + chain_extraction_impl_type impl( + continuity_mock_store{store_first, store_last}, std::move(except)); + try { + impl.signal_block_start(block_num); + } catch (const yield_exception&) { + // expected on continuity error + } + return !threw; + } + + std::optional store_first; + std::optional store_last; +}; + +} // namespace + +BOOST_AUTO_TEST_SUITE(continuity_tests) + + // Empty slice dir: any starting block is a fresh start, always succeeds + BOOST_AUTO_TEST_CASE(fresh_start_block_1) { + continuity_fixture f(std::nullopt, std::nullopt); + BOOST_CHECK(f.try_block_start(1)); + } + + BOOST_AUTO_TEST_CASE(fresh_start_high_block) { + continuity_fixture f(std::nullopt, std::nullopt); + BOOST_CHECK(f.try_block_start(50'000'000)); + } + + // Exact continuation: chain head == last_recorded + 1 + BOOST_AUTO_TEST_CASE(exact_continuation_from_block_1) { + continuity_fixture f(1, 1); + BOOST_CHECK(f.try_block_start(2)); + } + + BOOST_AUTO_TEST_CASE(exact_continuation_mid_chain) { + continuity_fixture f(1, 50'000'000); + BOOST_CHECK(f.try_block_start(50'000'001)); + } + + // Overlap: chain head is within existing data range (snapshot older than trace end) + // Should succeed — re-applied blocks will overwrite. + BOOST_AUTO_TEST_CASE(overlap_at_first_recorded) { + continuity_fixture f(500, 600); + BOOST_CHECK(f.try_block_start(500)); // chain head == first recorded + } + + BOOST_AUTO_TEST_CASE(overlap_mid_range) { + continuity_fixture f(500, 600); + BOOST_CHECK(f.try_block_start(550)); // chain head in middle of existing data + } + + BOOST_AUTO_TEST_CASE(overlap_at_last_recorded) { + continuity_fixture f(500, 600); + BOOST_CHECK(f.try_block_start(600)); // chain head == last recorded (last block replay) + } + + // Gap forward: chain head > last_recorded + 1, must error + BOOST_AUTO_TEST_CASE(gap_forward_small) { + continuity_fixture f(1, 100); + BOOST_CHECK(!f.try_block_start(105)); + } + + BOOST_AUTO_TEST_CASE(gap_forward_large) { + continuity_fixture f(1, 50'000'000); + BOOST_CHECK(!f.try_block_start(60'000'000)); + } + + // Snapshot before trace data begins: chain head < first_recorded, must error + BOOST_AUTO_TEST_CASE(snapshot_before_data_start) { + continuity_fixture f(500, 600); + BOOST_CHECK(!f.try_block_start(400)); // chain head < first recorded + } + + BOOST_AUTO_TEST_CASE(snapshot_well_before_data_start) { + continuity_fixture f(50'000'000, 60'000'000); + BOOST_CHECK(!f.try_block_start(1)); + } + + // check_continuity called only once: subsequent block_start calls do not re-check + BOOST_AUTO_TEST_CASE(check_only_on_first_block_start) { + bool threw = false; + auto except = exception_handler{[&threw](const exception_with_context&) { + threw = true; + throw yield_exception("continuity error"); + }}; + + // fresh start, no prior data + chain_extraction_impl_type impl( + continuity_mock_store{std::nullopt, std::nullopt}, std::move(except)); + + impl.signal_block_start(100); // first call: fresh start, succeeds + BOOST_CHECK(!threw); + + impl.signal_block_start(200); // second call: no re-check, also succeeds + BOOST_CHECK(!threw); + } + + // The continuity check flips its "already checked" flag BEFORE running, so + // subsequent block_start signals skip the check regardless of what the + // except_handler did on the first one. Verify via a non-throwing handler + // (which would otherwise re-enter the check if the flag weren't set early). + BOOST_AUTO_TEST_CASE(check_not_rerun_after_non_throwing_except_handler) { + int handler_calls = 0; + auto except = exception_handler{[&handler_calls](const exception_with_context&) { + ++handler_calls; + // deliberately do NOT throw -- simulates a handler that just logs + }}; + + // Prior data ending at 100; first block_start at 200 is a forward gap. + chain_extraction_impl_type impl( + continuity_mock_store{1, 100}, std::move(except)); + + impl.signal_block_start(200); // gap detected; handler called once + BOOST_CHECK_EQUAL(handler_calls, 1); + + impl.signal_block_start(300); // second call must NOT re-invoke the check + BOOST_CHECK_EQUAL(handler_calls, 1); + } + +BOOST_AUTO_TEST_SUITE_END() diff --git a/plugins/trace_api_plugin/test/test_data_handlers.cpp b/plugins/trace_api_plugin/test/test_data_handlers.cpp index 32aa14a172..7d1ea665e3 100644 --- a/plugins/trace_api_plugin/test/test_data_handlers.cpp +++ b/plugins/trace_api_plugin/test/test_data_handlers.cpp @@ -3,17 +3,37 @@ #include #include +#include using namespace sysio; using namespace sysio::trace_api; using namespace sysio::trace_api::test_common; +namespace { + // Pack an abi_def into raw bytes that abi_data_handler can unpack + std::vector pack_abi(const chain::abi_def& abi) { + return fc::raw::pack(abi); + } + + // Build a lookup_fn that returns packed ABI bytes for a given account. + // effective_global_seq is fixed at 0 for test purposes -- the handler's + // cache key becomes (account, 0) regardless of the action's global_seq. + abi_data_handler::abi_lookup_fn make_lookup(chain::name account, std::vector abi_bytes) { + return [account, bytes = std::move(abi_bytes)](chain::name a, uint64_t) -> std::optional { + if (a == account) return abi_data_handler::lookup_entry{0, bytes}; + return std::nullopt; + }; + } +} + BOOST_AUTO_TEST_SUITE(abi_data_handler_tests) BOOST_AUTO_TEST_CASE(empty_data) { - auto action = action_trace_v0 { - 0, "alice"_n, "alice"_n, "foo"_n, {}, {}, {} - }; + action_trace_v0 action; + action.global_sequence = 0; + action.receiver = "alice"_n; + action.account = "alice"_n; + action.action = "foo"_n; std::variant action_trace_t = action; abi_data_handler handler(exception_handler{}); @@ -28,9 +48,12 @@ BOOST_AUTO_TEST_SUITE(abi_data_handler_tests) { // Without return_value { - auto action = action_trace_v0 { - 0, "alice"_n, "alice"_n, "foo"_n, {}, {0x00, 0x01, 0x02, 0x03}, {} - }; + action_trace_v0 action; + action.global_sequence = 0; + action.receiver = "alice"_n; + action.account = "alice"_n; + action.action = "foo"_n; + action.data = {0x00, 0x01, 0x02, 0x03}; std::variant action_trace_t = action; abi_data_handler handler(exception_handler{}); @@ -43,9 +66,13 @@ BOOST_AUTO_TEST_SUITE(abi_data_handler_tests) // With return_value { - auto action = action_trace_v0 { - 0, "alice"_n, "alice"_n, "foo"_n, {}, {0x00, 0x01, 0x02, 0x03}, {0x04, 0x05, 0x06, 0x07} - }; + action_trace_v0 action; + action.global_sequence = 0; + action.receiver = "alice"_n; + action.account = "alice"_n; + action.action = "foo"_n; + action.data = {0x00, 0x01, 0x02, 0x03}; + action.return_value = {0x04, 0x05, 0x06, 0x07}; std::variant action_trace_t = action; abi_data_handler handler(exception_handler{}); @@ -59,9 +86,12 @@ BOOST_AUTO_TEST_SUITE(abi_data_handler_tests) BOOST_AUTO_TEST_CASE(basic_abi) { - auto action = action_trace_v0 { - 0, "alice"_n, "alice"_n, "foo"_n, {}, {0x00, 0x01, 0x02, 0x03}, {} - }; + action_trace_v0 action; + action.global_sequence = 0; + action.receiver = "alice"_n; + action.account = "alice"_n; + action.action = "foo"_n; + action.data = {0x00, 0x01, 0x02, 0x03}; std::variant action_trace_t = action; @@ -76,8 +106,7 @@ BOOST_AUTO_TEST_SUITE(abi_data_handler_tests) ); abi.version = "sysio::abi/1."; - abi_data_handler handler(exception_handler{}); - handler.add_abi("alice"_n, std::move(abi)); + abi_data_handler handler(exception_handler{}, make_lookup("alice"_n, pack_abi(abi))); fc::variant expected = fc::mutable_variant_object() ("a", 0) @@ -93,9 +122,13 @@ BOOST_AUTO_TEST_SUITE(abi_data_handler_tests) BOOST_AUTO_TEST_CASE(basic_abi_with_return_value) { - auto action = action_trace_v0 { - 0, "alice"_n, "alice"_n, "foo"_n, {}, {0x00, 0x01, 0x02, 0x03}, {0x04, 0x05, 0x06} - }; + action_trace_v0 action; + action.global_sequence = 0; + action.receiver = "alice"_n; + action.account = "alice"_n; + action.action = "foo"_n; + action.data = {0x00, 0x01, 0x02, 0x03}; + action.return_value = {0x04, 0x05, 0x06}; std::variant action_trace_t = action; @@ -112,8 +145,7 @@ BOOST_AUTO_TEST_SUITE(abi_data_handler_tests) abi.version = "sysio::abi/1."; abi.action_results = { std::vector{ chain::action_result_def{ "foo"_n, "foor"} } }; - abi_data_handler handler(exception_handler{}); - handler.add_abi("alice"_n, std::move(abi)); + abi_data_handler handler(exception_handler{}, make_lookup("alice"_n, pack_abi(abi))); fc::variant expected = fc::mutable_variant_object() ("a", 0) @@ -134,9 +166,13 @@ BOOST_AUTO_TEST_SUITE(abi_data_handler_tests) BOOST_AUTO_TEST_CASE(basic_abi_wrong_type) { - auto action = action_trace_v0 { - 0, "alice"_n, "alice"_n, "foo"_n, {}, {0x00, 0x01, 0x02, 0x03}, {0x04, 0x05, 0x06, 0x07} - }; + action_trace_v0 action; + action.global_sequence = 0; + action.receiver = "alice"_n; + action.account = "alice"_n; + action.action = "foo"_n; + action.data = {0x00, 0x01, 0x02, 0x03}; + action.return_value = {0x04, 0x05, 0x06, 0x07}; std::variant action_trace_t = action; @@ -151,8 +187,7 @@ BOOST_AUTO_TEST_SUITE(abi_data_handler_tests) ); abi.version = "sysio::abi/1."; - abi_data_handler handler(exception_handler{}); - handler.add_abi("alice"_n, std::move(abi)); + abi_data_handler handler(exception_handler{}, make_lookup("alice"_n, pack_abi(abi))); auto expected = fc::variant(); @@ -164,9 +199,12 @@ BOOST_AUTO_TEST_SUITE(abi_data_handler_tests) BOOST_AUTO_TEST_CASE(basic_abi_insufficient_data) { - auto action = action_trace_v0 { - 0, "alice"_n, "alice"_n, "foo"_n, {}, {0x00, 0x01, 0x02}, {} - }; + action_trace_v0 action; + action.global_sequence = 0; + action.receiver = "alice"_n; + action.account = "alice"_n; + action.action = "foo"_n; + action.data = {0x00, 0x01, 0x02}; std::variant action_trace_t = action; @@ -182,8 +220,10 @@ BOOST_AUTO_TEST_SUITE(abi_data_handler_tests) abi.version = "sysio::abi/1."; bool log_called = false; - abi_data_handler handler([&log_called](const exception_with_context& ){log_called = true;}); - handler.add_abi("alice"_n, std::move(abi)); + abi_data_handler handler( + [&log_called](const exception_with_context& ){log_called = true;}, + make_lookup("alice"_n, pack_abi(abi)) + ); auto expected = fc::variant(); @@ -197,9 +237,13 @@ BOOST_AUTO_TEST_SUITE(abi_data_handler_tests) // If no ABI provided for return type then do not attempt to decode it BOOST_AUTO_TEST_CASE(basic_abi_no_return_abi_when_return_value_provided) { - auto action = action_trace_v0 { - 0, "alice"_n, "alice"_n, "foo"_n, {}, {0x00, 0x01, 0x02, 0x03}, {0x04, 0x05, 0x06} - }; + action_trace_v0 action; + action.global_sequence = 0; + action.receiver = "alice"_n; + action.account = "alice"_n; + action.action = "foo"_n; + action.data = {0x00, 0x01, 0x02, 0x03}; + action.return_value = {0x04, 0x05, 0x06}; std::variant action_trace_t = action; @@ -214,8 +258,7 @@ BOOST_AUTO_TEST_SUITE(abi_data_handler_tests) ); abi.version = "sysio::abi/1."; - abi_data_handler handler(exception_handler{}); - handler.add_abi("alice"_n, std::move(abi)); + abi_data_handler handler(exception_handler{}, make_lookup("alice"_n, pack_abi(abi))); fc::variant expected = fc::mutable_variant_object() ("a", 0) diff --git a/plugins/trace_api_plugin/test/test_extraction.cpp b/plugins/trace_api_plugin/test/test_extraction.cpp index e0610dec76..9a264c0e6c 100644 --- a/plugins/trace_api_plugin/test/test_extraction.cpp +++ b/plugins/trace_api_plugin/test/test_extraction.cpp @@ -42,6 +42,28 @@ namespace { "sysio.token"_n, "transfer"_n, make_transfer_data( from, to, quantity, std::move(memo) ) ); } + // sysio::setabi action data: (name account, vector abi). The extraction path + // unpacks it via fc::raw in chain_extraction.hpp. + std::vector make_setabi_data( chain::name target, const std::vector& abi_bytes ) { + fc::datastream ps; + fc::raw::pack(ps, target, abi_bytes); + std::vector data( ps.tellp() ); + if (!data.empty()) { + fc::datastream ds(data.data(), data.size()); + fc::raw::pack(ds, target, abi_bytes); + } + return data; + } + + auto make_setabi_action( chain::name target, const std::vector& abi_bytes ) { + return chain::action( {}, chain::config::system_account_name, "setabi"_n, + make_setabi_data( target, abi_bytes ) ); + } + + auto make_simple_action( chain::name account, chain::name act_name ) { + return chain::action( {}, account, act_name, std::vector{} ); + } + auto make_packed_trx( std::vector actions ) { chain::signed_transaction trx; trx.actions = std::move( actions ); @@ -61,7 +83,6 @@ namespace { chain::action_trace make_action_trace( uint64_t global_sequence, chain::action act, chain::name receiver ) { chain::action_trace result; - // don't think we need any information other than receiver and global sequence result.receipt.emplace(chain::action_receipt{ receiver, digest_type::hash(act), @@ -73,6 +94,11 @@ namespace { }); result.receiver = receiver; result.act = std::move(act); + // chain::action_trace::cpu_usage_us / net_usage are populated for input actions; + // the block_extraction fixtures expect these to round-trip as fc::unsigned_int{0} + // on every action, so set them here rather than at every call site. + result.cpu_usage_us = fc::unsigned_int{0}; + result.net_usage = fc::unsigned_int{0}; return result; } @@ -105,14 +131,38 @@ struct extraction_test_fixture { fixture.id_log[tt.block_num] = tt.ids; } + std::optional> first_and_last_recorded_blocks() const { + return std::nullopt; // no prior data in unit tests + } + + void append_abi(chain::name account, uint64_t global_seq, std::vector abi_bytes) { + fixture.abi_calls.push_back({account, global_seq, std::move(abi_bytes)}); + } + + bool has_abi_entry(chain::name account) const { + for (const auto& c : fixture.abi_calls) + if (c.account == account) return true; + return false; + } + extraction_test_fixture& fixture; }; + struct abi_call { + chain::name account; + uint64_t global_seq = 0; + std::vector abi_bytes; + }; + extraction_test_fixture() : extraction_impl(mock_logfile_provider_type(*this), exception_handler{} ) { } + void signal_block_start( uint32_t block_num ) { + extraction_impl.signal_block_start(block_num); + } + void signal_applied_transaction( const chain::transaction_trace_ptr& trace, const chain::packed_transaction_ptr& ptrx ) { extraction_impl.signal_applied_transaction(trace, ptrx); } @@ -125,6 +175,7 @@ struct extraction_test_fixture { uint32_t max_lib = 0; std::vector data_log = {}; std::unordered_map> id_log; + std::vector abi_calls; chain_extraction_impl_type extraction_impl; }; @@ -153,34 +204,41 @@ BOOST_AUTO_TEST_SUITE(block_extraction) { chain::packed_transaction(ptrx1) } ); signal_accepted_block( bp1 ); - const std::vector expected_action_traces { - { - 0, - "sysio.token"_n, "sysio.token"_n, "transfer"_n, - {{"alice"_n, "active"_n}}, - make_transfer_data("alice"_n, "bob"_n, "0.0001 SYS"_t, "Memo!"), - {} - }, - { - 1, - "alice"_n, "sysio.token"_n, "transfer"_n, - {{"alice"_n, "active"_n}}, - make_transfer_data("alice"_n, "bob"_n, "0.0001 SYS"_t, "Memo!"), - {} - }, - { - 2, - "bob"_n, "sysio.token"_n, "transfer"_n, - {{"alice"_n, "active"_n}}, - make_transfer_data("alice"_n, "bob"_n, "0.0001 SYS"_t, "Memo!"), - {} - } - }; + action_trace_v0 eat1{}; + eat1.global_sequence = 0; + eat1.receiver = "sysio.token"_n; + eat1.account = "sysio.token"_n; + eat1.action = "transfer"_n; + eat1.authorization = {{"alice"_n, "active"_n}}; + eat1.data = make_transfer_data("alice"_n, "bob"_n, "0.0001 SYS"_t, "Memo!"); + eat1.cpu_usage_us = fc::unsigned_int{0}; + eat1.net_usage = fc::unsigned_int{0}; + + action_trace_v0 eat2{}; + eat2.global_sequence = 1; + eat2.receiver = "alice"_n; + eat2.account = "sysio.token"_n; + eat2.action = "transfer"_n; + eat2.authorization = {{"alice"_n, "active"_n}}; + eat2.data = make_transfer_data("alice"_n, "bob"_n, "0.0001 SYS"_t, "Memo!"); + eat2.cpu_usage_us = fc::unsigned_int{0}; + eat2.net_usage = fc::unsigned_int{0}; + + action_trace_v0 eat3{}; + eat3.global_sequence = 2; + eat3.receiver = "bob"_n; + eat3.account = "sysio.token"_n; + eat3.action = "transfer"_n; + eat3.authorization = {{"alice"_n, "active"_n}}; + eat3.data = make_transfer_data("alice"_n, "bob"_n, "0.0001 SYS"_t, "Memo!"); + eat3.cpu_usage_us = fc::unsigned_int{0}; + eat3.net_usage = fc::unsigned_int{0}; + + const std::vector expected_action_traces { eat1, eat2, eat3 }; const transaction_trace_v0 expected_transaction_trace { ptrx1.id(), expected_action_traces, - fc::enum_type{0}, 0, 0, ptrx1.get_signatures(), @@ -239,41 +297,44 @@ BOOST_AUTO_TEST_SUITE(block_extraction) { chain::packed_transaction(ptrx1), chain::packed_transaction(ptrx2), chain::packed_transaction(ptrx3) } ); signal_accepted_block( bp1 ); - const std::vector expected_action_trace1 { - { - 0, - "sysio.token"_n, "sysio.token"_n, "transfer"_n, - {{"alice"_n, "active"_n}}, - make_transfer_data("alice"_n, "bob"_n, "0.0001 SYS"_t, "Memo!"), - {} - } - }; - - const std::vector expected_action_trace2 { - { - 1, - "bob"_n, "sysio.token"_n, "transfer"_n, - {{ "bob"_n, "active"_n }}, - make_transfer_data( "bob"_n, "alice"_n, "0.0001 SYS"_t, "Memo!" ), - {} - } - }; - - const std::vector expected_action_trace3 { - { - 2, - "fred"_n, "sysio.token"_n, "transfer"_n, - {{ "fred"_n, "active"_n }}, - make_transfer_data( "fred"_n, "bob"_n, "0.0001 SYS"_t, "Memo!" ), - {} - } - }; + action_trace_v0 eat1{}; + eat1.global_sequence = 0; + eat1.receiver = "sysio.token"_n; + eat1.account = "sysio.token"_n; + eat1.action = "transfer"_n; + eat1.authorization = {{"alice"_n, "active"_n}}; + eat1.data = make_transfer_data("alice"_n, "bob"_n, "0.0001 SYS"_t, "Memo!"); + eat1.cpu_usage_us = fc::unsigned_int{0}; + eat1.net_usage = fc::unsigned_int{0}; + + action_trace_v0 eat2{}; + eat2.global_sequence = 1; + eat2.receiver = "bob"_n; + eat2.account = "sysio.token"_n; + eat2.action = "transfer"_n; + eat2.authorization = {{ "bob"_n, "active"_n }}; + eat2.data = make_transfer_data( "bob"_n, "alice"_n, "0.0001 SYS"_t, "Memo!" ); + eat2.cpu_usage_us = fc::unsigned_int{0}; + eat2.net_usage = fc::unsigned_int{0}; + + action_trace_v0 eat3{}; + eat3.global_sequence = 2; + eat3.receiver = "fred"_n; + eat3.account = "sysio.token"_n; + eat3.action = "transfer"_n; + eat3.authorization = {{ "fred"_n, "active"_n }}; + eat3.data = make_transfer_data( "fred"_n, "bob"_n, "0.0001 SYS"_t, "Memo!" ); + eat3.cpu_usage_us = fc::unsigned_int{0}; + eat3.net_usage = fc::unsigned_int{0}; + + const std::vector expected_action_trace1 { eat1 }; + const std::vector expected_action_trace2 { eat2 }; + const std::vector expected_action_trace3 { eat3 }; const std::vector expected_transaction_traces { { ptrx1.id(), expected_action_trace1, - fc::enum_type{chain::transaction_receipt_header::status_enum::executed}, 0, 0, ptrx1.get_signatures(), @@ -285,7 +346,6 @@ BOOST_AUTO_TEST_SUITE(block_extraction) { ptrx2.id(), expected_action_trace2, - fc::enum_type{chain::transaction_receipt_header::status_enum::executed}, 0, 0, ptrx2.get_signatures(), @@ -297,7 +357,6 @@ BOOST_AUTO_TEST_SUITE(block_extraction) { ptrx3.id(), expected_action_trace3, - fc::enum_type{chain::transaction_receipt_header::status_enum::executed}, 0, 0, ptrx3.get_signatures(), @@ -326,3 +385,192 @@ BOOST_AUTO_TEST_SUITE(block_extraction) } BOOST_AUTO_TEST_SUITE_END() + + +// --------------------------------------------------------------------------- +// ABI capture: lazy fetch + setabi interaction +// --------------------------------------------------------------------------- + +struct abi_capture_fixture { + struct mock_store { + explicit mock_store(abi_capture_fixture& f) : fixture(f) {} + + template void append(const BT&) {} + void append_lib(uint32_t) {} + void append_trx_ids(const block_trxs_entry&) {} + std::optional> first_and_last_recorded_blocks() const { return std::nullopt; } + + void append_abi(chain::name account, uint64_t global_seq, std::vector abi_bytes) { + fixture.abi_calls.push_back({account, global_seq, std::move(abi_bytes)}); + } + + bool has_abi_entry(chain::name account) const { + for (const auto& c : fixture.abi_calls) + if (c.account == account) return true; + return false; + } + + abi_capture_fixture& fixture; + }; + + struct abi_call { + chain::name account; + uint64_t global_seq = 0; + std::vector abi_bytes; + }; + + using extraction_t = chain_extraction_impl_type; + + // Fetcher returns whatever is keyed in fetcher_state (mimics reading the + // chain DB's account_metadata_object post-apply). + std::map> fetcher_state; + + extraction_t::abi_fetcher_t make_fetcher() { + return [this](chain::name account) -> std::optional> { + auto it = fetcher_state.find(account); + if (it == fetcher_state.end()) return std::nullopt; + return it->second; + }; + } + + std::vector abi_calls; + + // Feed a pre-built transaction through the extraction impl. Returns the extraction + // so additional trxs can be fed through the same instance (mimicking multi-trx flows). + std::unique_ptr extraction = std::make_unique( + mock_store{*this}, exception_handler{}, make_fetcher()); + + void signal(const chain::transaction_trace_ptr& trace, const chain::packed_transaction_ptr& ptrx) { + extraction->signal_applied_transaction(trace, ptrx); + } +}; + + +BOOST_AUTO_TEST_SUITE(abi_capture_tests) + +// Trx = [X.foo, sysio.setabi(X, newAbi), X.bar]. X has NEVER been observed before. +// Without the fix: lazy fetch on iter 1 would read post-apply state (newAbi) and +// record X@0=newAbi, which would be served for X.foo's pre-setabi global_seq and +// decode with the wrong schema. Fix: scan for setabi targets first and skip +// lazy fetch for them. X.foo remains undecodable (there's no pre-setabi ABI +// anywhere reachable), but returning raw hex is strictly safer than wrong data. +BOOST_FIXTURE_TEST_CASE(lazy_fetch_skipped_for_same_trx_setabi_target, abi_capture_fixture) +{ + auto new_abi = std::vector{'n', 'e', 'w'}; + fetcher_state["x"_n] = new_abi; // what a post-apply lazy fetch would return + + auto x_foo_action = make_simple_action("x"_n, "foo"_n); + auto setabi_action = make_setabi_action("x"_n, new_abi); + auto x_bar_action = make_simple_action("x"_n, "bar"_n); + + auto x_foo = make_action_trace(100, x_foo_action, "x"_n); + auto setabi = make_action_trace(101, setabi_action, chain::config::system_account_name); + auto x_bar = make_action_trace(102, x_bar_action, "x"_n); + + auto ptrx = make_packed_trx({ x_foo_action, setabi_action, x_bar_action }); + auto trace = make_transaction_trace( + ptrx.id(), 1, 1, chain::transaction_receipt_header::executed, + { x_foo, setabi, x_bar }); + + signal(trace, std::make_shared(ptrx)); + + // Exactly one append: the setabi at its own global_sequence. + // No X@0 (the poisoning case), and NO X@100/102 (those are not setabis). + BOOST_REQUIRE_EQUAL(abi_calls.size(), 1u); + BOOST_TEST(abi_calls[0].account == "x"_n); + BOOST_TEST(abi_calls[0].global_seq == 101u); + BOOST_TEST(abi_calls[0].abi_bytes == new_abi); +} + +// Common case: X already has a prior setabi record (from an earlier trx), so +// _seen_accounts contains X. A later trx does [X.foo, setabi(X, newAbi), X.bar]. +// The lazy fetch is a no-op (X already seen); the setabi record is appended. +// After the trx, the in-memory log should contain BOTH records, so a lookup +// for X.foo's global_sequence (< setabi_seq) resolves to the old ABI via +// upper_bound step-back in abi_log. This is the fix's key property: it does +// not clobber the prior entry that lets pre-setabi actions decode correctly. +BOOST_FIXTURE_TEST_CASE(prior_setabi_survives_later_setabi_in_same_trx, abi_capture_fixture) +{ + auto old_abi = std::vector{'o', 'l', 'd'}; + auto new_abi = std::vector{'n', 'e', 'w'}; + + // Trx 1: the original setabi that registered X@50=old_abi. + { + auto setabi_old = make_setabi_action("x"_n, old_abi); + auto setabi_old_trace = make_action_trace(50, setabi_old, chain::config::system_account_name); + auto ptrx = make_packed_trx({ setabi_old }); + auto trace = make_transaction_trace( + ptrx.id(), 1, 1, chain::transaction_receipt_header::executed, + { setabi_old_trace }); + signal(trace, std::make_shared(ptrx)); + } + + BOOST_REQUIRE_EQUAL(abi_calls.size(), 1u); + BOOST_TEST(abi_calls[0].account == "x"_n); + BOOST_TEST(abi_calls[0].global_seq == 50u); + BOOST_TEST(abi_calls[0].abi_bytes == old_abi); + + // Fetcher now returns newAbi since in reality the chain DB has been updated. + fetcher_state["x"_n] = new_abi; + + // Trx 2: X.foo (ran under oldAbi), setabi(X, newAbi), X.bar (ran under newAbi). + { + auto x_foo_action = make_simple_action("x"_n, "foo"_n); + auto setabi_action = make_setabi_action("x"_n, new_abi); + auto x_bar_action = make_simple_action("x"_n, "bar"_n); + + auto x_foo = make_action_trace(200, x_foo_action, "x"_n); + auto setabi = make_action_trace(201, setabi_action, chain::config::system_account_name); + auto x_bar = make_action_trace(202, x_bar_action, "x"_n); + + auto ptrx = make_packed_trx({ x_foo_action, setabi_action, x_bar_action }); + auto trace = make_transaction_trace( + ptrx.id(), 1, 1, chain::transaction_receipt_header::executed, + { x_foo, setabi, x_bar }); + signal(trace, std::make_shared(ptrx)); + } + + // Expect exactly two appends total: the prior setabi plus the new one. + // No spurious X@0 lazy-fetch that would poison X.foo's lookup. + BOOST_REQUIRE_EQUAL(abi_calls.size(), 2u); + BOOST_TEST(abi_calls[1].account == "x"_n); + BOOST_TEST(abi_calls[1].global_seq == 201u); + BOOST_TEST(abi_calls[1].abi_bytes == new_abi); + + // Hand-check of the lookup contract (which abi_log tests cover end-to-end): + // lookup(X, 200) -> upper_bound finds X@201, step back to X@50 = OLD ABI. Correct. + // lookup(X, 201) -> upper_bound finds > X@201 (none), step back to X@201 = NEW. Correct. + // lookup(X, 202) -> same as above, NEW. Correct. +} + +// An unrelated account in the same trx as a setabi should still get a lazy +// fetch — the skip is narrowly scoped to the setabi target. +BOOST_FIXTURE_TEST_CASE(lazy_fetch_fires_for_non_setabi_target_in_same_trx, abi_capture_fixture) +{ + auto y_abi = std::vector{'y'}; + auto x_abi = std::vector{'x'}; + fetcher_state["y"_n] = y_abi; + + auto y_foo_action = make_simple_action("y"_n, "foo"_n); + auto setabi_action = make_setabi_action("x"_n, x_abi); + + auto y_foo = make_action_trace(300, y_foo_action, "y"_n); + auto setabi = make_action_trace(301, setabi_action, chain::config::system_account_name); + + auto ptrx = make_packed_trx({ y_foo_action, setabi_action }); + auto trace = make_transaction_trace( + ptrx.id(), 1, 1, chain::transaction_receipt_header::executed, + { y_foo, setabi }); + signal(trace, std::make_shared(ptrx)); + + // Two appends: Y@0 lazy fetch (Y isn't a setabi target) and X@301 setabi. + BOOST_REQUIRE_EQUAL(abi_calls.size(), 2u); + BOOST_TEST(abi_calls[0].account == "y"_n); + BOOST_TEST(abi_calls[0].global_seq == 0u); + BOOST_TEST(abi_calls[0].abi_bytes == y_abi); + BOOST_TEST(abi_calls[1].account == "x"_n); + BOOST_TEST(abi_calls[1].global_seq == 301u); + BOOST_TEST(abi_calls[1].abi_bytes == x_abi); +} + +BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/plugins/trace_api_plugin/test/test_get_actions.cpp b/plugins/trace_api_plugin/test/test_get_actions.cpp new file mode 100644 index 0000000000..f656831a49 --- /dev/null +++ b/plugins/trace_api_plugin/test/test_get_actions.cpp @@ -0,0 +1,636 @@ +#include + +#include + +#include + +#include +#include +#include +#include + +using namespace sysio; +using namespace sysio::trace_api; +using namespace sysio::trace_api::test_common; + +namespace { + +// --------------------------------------------------------------------------- +// Fixture +// --------------------------------------------------------------------------- + +struct get_actions_fixture { + struct mock_logfile_provider { + mock_logfile_provider(get_actions_fixture& f) : fixture(f) {} + + get_block_t get_block(uint32_t height) { + auto it = fixture.blocks.find(height); + if (it == fixture.blocks.end()) return {}; + // pending_blocks is a per-block override for tests that need to exercise + // the "pending" branch of block_status emission. Default is irreversible. + const bool irreversible = fixture.pending_blocks.find(height) == fixture.pending_blocks.end(); + return std::make_tuple(data_log_entry{it->second}, irreversible); + } + + // Stride/slice mapping is a fixture knob so tests can exercise the per-slice bloom skip path with a small + // stride rather than the production default of 10,000 blocks. + uint32_t slice_stride() const noexcept { return fixture.mock_slice_stride; } + uint32_t slice_number(uint32_t block_num) const noexcept { return block_num / fixture.mock_slice_stride; } + + // Default: no sidecar -> invalid bloom_reader -> may_contain_* returns true -> caller scans as before. Tests + // that want to exercise skipping install a function that returns a valid reader for specific slices. + bloom_reader get_bloom(uint32_t slice_number) const { + return fixture.mock_get_bloom(slice_number); + } + + get_actions_fixture& fixture; + }; + + struct mock_data_handler_provider { + mock_data_handler_provider(get_actions_fixture& f) : fixture(f) {} + + std::tuple> serialize_to_variant(const action_trace_v0& a) { + return fixture.mock_data_handler(a); + } + + // Production shared_provider exposes decode(); the request_handler's + // get_actions path calls decode() directly so a decode_error field can + // be surfaced on failure. The mock builds a decode_result from the + // tuple returned by mock_data_handler -- decode never fails in tests. + abi_data_handler::decode_result decode(const action_trace_v0& a) { + auto [params, return_data] = fixture.mock_data_handler(a); + abi_data_handler::decode_result r; + r.params = std::move(params); + r.return_data = std::move(return_data); + r.status = r.params.is_null() + ? abi_data_handler::decode_status::not_attempted + : abi_data_handler::decode_status::ok; + return r; + } + + get_actions_fixture& fixture; + }; + + using impl_type = request_handler; + + get_actions_fixture() + : impl(mock_logfile_provider(*this), mock_data_handler_provider(*this), + [](const std::string& msg){ fc_dlog(fc::logger::default_logger(), "{}", msg); }) + {} + + actions_result get_actions(const action_query& query) { + return impl.get_actions(query); + } + + actions_result get_token_transfer_actions(const action_query& query) { + return impl.get_token_transfer_actions(query); + } + + // Default: no ABI decoding — params/return_data absent from result + std::function>(const action_trace_v0&)> + mock_data_handler = [](const action_trace_v0&) -> std::tuple> { + return {}; + }; + + std::map blocks; + std::set pending_blocks; // blocks that should report "pending" instead of the default "irreversible" + uint32_t mock_slice_stride = 10; + std::function mock_get_bloom = [](uint32_t) { return bloom_reader{}; }; + impl_type impl; +}; + +// --------------------------------------------------------------------------- +// Builders +// --------------------------------------------------------------------------- + +action_trace_v0 make_action(uint64_t seq, chain::name receiver, chain::name account, + chain::name act, chain::bytes data = {}) { + action_trace_v0 a{}; + a.global_sequence = seq; + a.receiver = receiver; + a.account = account; + a.action = act; + a.data = std::move(data); + return a; +} + +transaction_trace_v0 make_trx(chain::transaction_id_type id, uint32_t block_num, + std::vector actions) { + transaction_trace_v0 trx; + trx.id = id; + trx.actions = std::move(actions); + trx.block_num = block_num; + trx.block_time = chain::block_timestamp_type(0); + return trx; +} + +block_trace_v0 make_block(uint32_t num, std::vector trxs) { + block_trace_v0 blk; + blk.number = num; + blk.producer = "bp.one"_n; + blk.transactions = std::move(trxs); + return blk; +} + +// Convenience: one action per block, sequential global_sequences +static const chain::transaction_id_type TRX1 = + "0000000000000000000000000000000000000000000000000000000000000001"_h; +static const chain::transaction_id_type TRX2 = + "0000000000000000000000000000000000000000000000000000000000000002"_h; +static const chain::transaction_id_type TRX3 = + "0000000000000000000000000000000000000000000000000000000000000003"_h; + +} // anonymous namespace + +// --------------------------------------------------------------------------- +// Test suite +// --------------------------------------------------------------------------- + +BOOST_AUTO_TEST_SUITE(get_actions_tests) + +// No blocks in the queried range -> empty result +BOOST_FIXTURE_TEST_CASE(empty_range, get_actions_fixture) +{ + action_query q; + q.block_num_start = 1; + q.block_num_end = 10; + + auto r = get_actions(q); + + BOOST_TEST(r.actions.empty()); +} + +// Filter that matches nothing returns empty result +BOOST_FIXTURE_TEST_CASE(no_matching_filter, get_actions_fixture) +{ + blocks[1] = make_block(1, { + make_trx(TRX1, 1, { make_action(1, "sysio.token"_n, "sysio.token"_n, "transfer"_n) }) + }); + + action_query q; + q.block_num_start = 1; + q.block_num_end = 1; + q.receiver = "alice"_n; // alice is not a receiver in this block + + auto r = get_actions(q); + + BOOST_TEST(r.actions.empty()); +} + +// receiver filter: keeps only actions where receiver matches +BOOST_FIXTURE_TEST_CASE(filter_by_receiver, get_actions_fixture) +{ + // Two actions: original transfer (receiver=sysio.token) + inline notification (receiver=bob) + blocks[1] = make_block(1, { + make_trx(TRX1, 1, { + make_action(1, "sysio.token"_n, "sysio.token"_n, "transfer"_n), + make_action(2, "bob"_n, "sysio.token"_n, "transfer"_n) + }) + }); + + action_query q; + q.block_num_start = 1; + q.block_num_end = 1; + q.receiver = "sysio.token"_n; + + auto r = get_actions(q); + + BOOST_REQUIRE_EQUAL(r.actions.size(), 1u); + BOOST_TEST(r.actions[0].get_object()["receiver"].as_string() == "sysio.token"); + BOOST_TEST(r.actions[0].get_object()["global_sequence"].as_uint64() == 1u); +} + +// account filter: keeps only actions where account (code) matches +BOOST_FIXTURE_TEST_CASE(filter_by_account, get_actions_fixture) +{ + blocks[1] = make_block(1, { + make_trx(TRX1, 1, { + make_action(1, "alice"_n, "sysio.token"_n, "transfer"_n), + make_action(2, "alice"_n, "mycontract"_n, "foo"_n) + }) + }); + + action_query q; + q.block_num_start = 1; + q.block_num_end = 1; + q.account = "sysio.token"_n; + + auto r = get_actions(q); + + BOOST_REQUIRE_EQUAL(r.actions.size(), 1u); + BOOST_TEST(r.actions[0].get_object()["account"].as_string() == "sysio.token"); + BOOST_TEST(r.actions[0].get_object()["global_sequence"].as_uint64() == 1u); +} + +// action name filter: keeps only the named action +BOOST_FIXTURE_TEST_CASE(filter_by_action_name, get_actions_fixture) +{ + blocks[1] = make_block(1, { + make_trx(TRX1, 1, { + make_action(1, "sysio.token"_n, "sysio.token"_n, "transfer"_n), + make_action(2, "sysio"_n, "sysio"_n, "newaccount"_n) + }) + }); + + action_query q; + q.block_num_start = 1; + q.block_num_end = 1; + q.action = "transfer"_n; + + auto r = get_actions(q); + + BOOST_REQUIRE_EQUAL(r.actions.size(), 1u); + BOOST_TEST(r.actions[0].get_object()["name"].as_string() == "transfer"); +} + +// Actions are returned across multiple blocks; missing blocks in the log are skipped +BOOST_FIXTURE_TEST_CASE(multi_block_scan, get_actions_fixture) +{ + blocks[1] = make_block(1, { make_trx(TRX1, 1, { make_action(1, "a"_n, "tok"_n, "transfer"_n) }) }); + blocks[2] = make_block(2, { make_trx(TRX2, 2, { make_action(2, "a"_n, "tok"_n, "transfer"_n) }) }); + // block 3 is missing (gap in trace log) + blocks[4] = make_block(4, { make_trx(TRX3, 4, { make_action(3, "a"_n, "tok"_n, "transfer"_n) }) }); + + action_query q; + q.block_num_start = 1; + q.block_num_end = 5; + + auto r = get_actions(q); + + BOOST_REQUIRE_EQUAL(r.actions.size(), 3u); + BOOST_TEST(r.actions[0].get_object()["block_num"].as() == 1u); + BOOST_TEST(r.actions[1].get_object()["block_num"].as() == 2u); + BOOST_TEST(r.actions[2].get_object()["block_num"].as() == 4u); +} + +// Per-trx cpu / net totals are emitted on every action variant in the full +// (get_actions) shape, alongside trx_id / block_num / block_time / +// producer_block_id, so callers can attribute resource usage to the parent +// transaction without a separate lookup. Deliberately distinct from +// action-level cpu_usage_us / net_usage which are per-action and in different +// units (action net_usage is bytes; trx net_usage_words is ceil/8). +BOOST_FIXTURE_TEST_CASE(emits_trx_resource_totals_in_full_shape, get_actions_fixture) +{ + transaction_trace_v0 trx = make_trx(TRX1, 1, { + make_action(1, "sysio.token"_n, "sysio.token"_n, "transfer"_n), + make_action(2, "bob"_n, "sysio.token"_n, "transfer"_n) + }); + trx.cpu_usage_us = 1234; + trx.net_usage_words = fc::unsigned_int{56}; + blocks[1] = make_block(1, { std::move(trx) }); + + action_query q; + q.block_num_start = 1; + q.block_num_end = 1; + + auto r = get_actions(q); + + BOOST_REQUIRE_EQUAL(r.actions.size(), 2u); + for (const auto& a : r.actions) { + const auto& obj = a.get_object(); + BOOST_TEST(obj["trx_cpu_usage_us"].as() == 1234u); + BOOST_TEST(obj["trx_net_usage_words"].as() == 56u); + } +} + +// Slim (get_token_transfers) omits all resource fields - both action-level +// cpu_usage_us / net_usage and the trx-level totals. Per-trx context +// (trx_id, block_num, etc.) still appears so transfers can be located in the +// chain. +BOOST_FIXTURE_TEST_CASE(slim_shape_omits_trx_resource_totals, get_actions_fixture) +{ + transaction_trace_v0 trx = make_trx(TRX1, 1, { + make_action(1, "sysio.token"_n, "sysio.token"_n, "transfer"_n) + }); + trx.cpu_usage_us = 1234; + trx.net_usage_words = fc::unsigned_int{56}; + blocks[1] = make_block(1, { std::move(trx) }); + + action_query q; + q.block_num_start = 1; + q.block_num_end = 1; + q.receiver = "sysio.token"_n; + q.account = "sysio.token"_n; + q.action = "transfer"_n; + + auto r = get_token_transfer_actions(q); + + BOOST_REQUIRE_EQUAL(r.actions.size(), 1u); + const auto& obj = r.actions[0].get_object(); + BOOST_TEST(obj.contains("trx_id")); + BOOST_TEST(!obj.contains("trx_cpu_usage_us")); + BOOST_TEST(!obj.contains("trx_net_usage_words")); + BOOST_TEST(!obj.contains("cpu_usage_us")); + BOOST_TEST(!obj.contains("net_usage")); +} + +// Every action carries a block_status mirroring get_block's "status" field, sourced from the same +// data log tuple so trace_api remains the single source of truth for "is this action's block final." +// Both shapes (full / slim) emit it: an exchange consuming get_token_transfers needs finality just +// as much as a general consumer of get_actions. Operators that want only-irreversible responses can +// run nodeop in irreversible mode -- every block returned will then carry "irreversible". +BOOST_FIXTURE_TEST_CASE(emits_block_status_per_action, get_actions_fixture) +{ + blocks[1] = make_block(1, { make_trx(TRX1, 1, { make_action(1, "a"_n, "tok"_n, "transfer"_n) }) }); + blocks[2] = make_block(2, { make_trx(TRX2, 2, { make_action(2, "a"_n, "tok"_n, "transfer"_n) }) }); + pending_blocks.insert(2); // block 1 irreversible, block 2 still pending + + action_query q; + q.block_num_start = 1; + q.block_num_end = 2; + + auto r_full = get_actions(q); + BOOST_REQUIRE_EQUAL(r_full.actions.size(), 2u); + BOOST_TEST(r_full.actions[0].get_object()["block_status"].as_string() == "irreversible"); + BOOST_TEST(r_full.actions[1].get_object()["block_status"].as_string() == "pending"); + + q.receiver = "a"_n; + q.account = "tok"_n; + q.action = "transfer"_n; + auto r_slim = get_token_transfer_actions(q); + BOOST_REQUIRE_EQUAL(r_slim.actions.size(), 2u); + BOOST_TEST(r_slim.actions[0].get_object()["block_status"].as_string() == "irreversible"); + BOOST_TEST(r_slim.actions[1].get_object()["block_status"].as_string() == "pending"); +} + +// ABI-decoded params are included in the result when the data handler returns them +BOOST_FIXTURE_TEST_CASE(abi_decoded_params_included, get_actions_fixture) +{ + blocks[1] = make_block(1, { + make_trx(TRX1, 1, { + make_action(1, "sysio.token"_n, "sysio.token"_n, "transfer"_n, {0x01, 0x02}) + }) + }); + + mock_data_handler = [](const action_trace_v0&) -> std::tuple> { + return { fc::mutable_variant_object()("amount", 100), std::nullopt }; + }; + + action_query q; + q.block_num_start = 1; + q.block_num_end = 1; + + auto r = get_actions(q); + + BOOST_REQUIRE_EQUAL(r.actions.size(), 1u); + const auto& obj = r.actions[0].get_object(); + BOOST_REQUIRE(obj.contains("params")); + BOOST_TEST(obj["params"].get_object()["amount"].as() == 100); + BOOST_TEST(!obj.contains("return_data")); +} + +// When the data handler returns null, no params field is emitted +BOOST_FIXTURE_TEST_CASE(no_params_when_handler_returns_null, get_actions_fixture) +{ + blocks[1] = make_block(1, { + make_trx(TRX1, 1, { make_action(1, "alice"_n, "contract"_n, "foo"_n, {static_cast(0xAB)}) }) + }); + + action_query q; + q.block_num_start = 1; + q.block_num_end = 1; + + auto r = get_actions(q); + + BOOST_REQUIRE_EQUAL(r.actions.size(), 1u); + const auto& obj = r.actions[0].get_object(); + BOOST_TEST(!obj.contains("params")); + BOOST_TEST(obj["data"].as_string() == "ab"); // raw hex always present +} + +// Verifies the receiver+account+action filter used by get_token_transfers: +// exactly one result per transfer (the original; notification copy is excluded) +BOOST_FIXTURE_TEST_CASE(token_transfer_filter_excludes_notifications, get_actions_fixture) +{ + blocks[1] = make_block(1, { + make_trx(TRX1, 1, { + make_action(1, "sysio.token"_n, "sysio.token"_n, "transfer"_n), // original + make_action(2, "bob"_n, "sysio.token"_n, "transfer"_n) // inline notification + }) + }); + + // This is the filter preset used by POST /v1/trace_api/get_token_transfers + action_query q; + q.block_num_start = 1; + q.block_num_end = 1; + q.receiver = "sysio.token"_n; + q.account = "sysio.token"_n; + q.action = "transfer"_n; + + auto r = get_actions(q); + + BOOST_REQUIRE_EQUAL(r.actions.size(), 1u); + BOOST_TEST(r.actions[0].get_object()["global_sequence"].as_uint64() == 1u); + BOOST_TEST(r.actions[0].get_object()["receiver"].as_string() == "sysio.token"); +} + +// Actions within a transaction are returned sorted by global_sequence (execution order), +// not the schedule order in which the chain stored them. The divergence matters when +// an action queues both an inline AND a require_recipient notification — notifications +// execute before inlines, so the inline's global_sequence is higher than later-scheduled +// notifications'. Sorting gives clients a consistent execution view matching what +// chain_plugin's push_transaction does and what the legacy get_block response returned. +BOOST_FIXTURE_TEST_CASE(actions_sorted_by_global_sequence, get_actions_fixture) +{ + blocks[1] = make_block(1, { + make_trx(TRX1, 1, { + make_action(5, "a"_n, "tok"_n, "transfer"_n), // schedule order: 5 + make_action(1, "a"_n, "tok"_n, "transfer"_n), // schedule order: 1 + make_action(3, "a"_n, "tok"_n, "transfer"_n) // schedule order: 3 + }) + }); + + action_query q; + q.block_num_start = 1; + q.block_num_end = 1; + + auto r = get_actions(q); + + BOOST_REQUIRE_EQUAL(r.actions.size(), 3u); + BOOST_TEST(r.actions[0].get_object()["global_sequence"].as_uint64() == 1u); + BOOST_TEST(r.actions[1].get_object()["global_sequence"].as_uint64() == 3u); + BOOST_TEST(r.actions[2].get_object()["global_sequence"].as_uint64() == 5u); +} + +// Realistic scenario exercising top-level actions, notifications (receiver != account), +// inline actions triggered by a notification handler, and the inline's own notifications. +// +// Scenario: alice transfers 1 SYS to bob.contract via sysio.token::transfer. +// bob.contract has a notif handler that fires an inline logger::log action. +// logger::log has an audit account notification handler. +// +// Expected global_sequence order (chain-assigned, monotonic): +// 100: sysio.token::transfer (receiver=sysio.token, account=sysio.token) <- original +// 101: sysio.token::transfer (receiver=alice, account=sysio.token) <- notif to sender +// 102: sysio.token::transfer (receiver=bob.contract,account=sysio.token) <- notif to recipient +// 103: logger::log (receiver=logger, account=logger) <- inline from 102 +// 104: logger::log (receiver=audit, account=logger) <- notif from 103 +BOOST_FIXTURE_TEST_CASE(complex_inline_and_notification_ordering, get_actions_fixture) +{ + blocks[1] = make_block(1, { + make_trx(TRX1, 1, { + make_action(100, "sysio.token"_n, "sysio.token"_n, "transfer"_n), + make_action(101, "alice"_n, "sysio.token"_n, "transfer"_n), + make_action(102, "bob.contract"_n, "sysio.token"_n, "transfer"_n), + make_action(103, "logger"_n, "logger"_n, "log"_n), + make_action(104, "audit"_n, "logger"_n, "log"_n) + }) + }); + + // No filter: all 5 actions in global_sequence order. + { + action_query q; + q.block_num_start = 1; + q.block_num_end = 1; + + auto r = get_actions(q); + + BOOST_REQUIRE_EQUAL(r.actions.size(), 5u); + BOOST_TEST(r.actions[0].get_object()["global_sequence"].as_uint64() == 100u); + BOOST_TEST(r.actions[1].get_object()["global_sequence"].as_uint64() == 101u); + BOOST_TEST(r.actions[2].get_object()["global_sequence"].as_uint64() == 102u); + BOOST_TEST(r.actions[3].get_object()["global_sequence"].as_uint64() == 103u); + BOOST_TEST(r.actions[4].get_object()["global_sequence"].as_uint64() == 104u); + } + + // receiver=sysio.token: only the original execution; notifications to alice/bob excluded. + { + action_query q; + q.block_num_start = 1; + q.block_num_end = 1; + q.receiver = "sysio.token"_n; + + auto r = get_actions(q); + + BOOST_REQUIRE_EQUAL(r.actions.size(), 1u); + BOOST_TEST(r.actions[0].get_object()["global_sequence"].as_uint64() == 100u); + BOOST_TEST(r.actions[0].get_object()["receiver"].as_string() == "sysio.token"); + } + + // account=sysio.token: original + both notifications (3 rows), ordered by global_seq. + { + action_query q; + q.block_num_start = 1; + q.block_num_end = 1; + q.account = "sysio.token"_n; + + auto r = get_actions(q); + + BOOST_REQUIRE_EQUAL(r.actions.size(), 3u); + BOOST_TEST(r.actions[0].get_object()["global_sequence"].as_uint64() == 100u); + BOOST_TEST(r.actions[1].get_object()["global_sequence"].as_uint64() == 101u); + BOOST_TEST(r.actions[2].get_object()["global_sequence"].as_uint64() == 102u); + BOOST_TEST(r.actions[0].get_object()["receiver"].as_string() == "sysio.token"); + BOOST_TEST(r.actions[1].get_object()["receiver"].as_string() == "alice"); + BOOST_TEST(r.actions[2].get_object()["receiver"].as_string() == "bob.contract"); + } + + // account=logger: the inline from bob.contract's notif handler + its notification to audit. + { + action_query q; + q.block_num_start = 1; + q.block_num_end = 1; + q.account = "logger"_n; + + auto r = get_actions(q); + + BOOST_REQUIRE_EQUAL(r.actions.size(), 2u); + BOOST_TEST(r.actions[0].get_object()["global_sequence"].as_uint64() == 103u); + BOOST_TEST(r.actions[1].get_object()["global_sequence"].as_uint64() == 104u); + BOOST_TEST(r.actions[0].get_object()["receiver"].as_string() == "logger"); + BOOST_TEST(r.actions[1].get_object()["receiver"].as_string() == "audit"); + } + + // receiver=bob.contract + action=transfer: exactly the one notification to the recipient. + { + action_query q; + q.block_num_start = 1; + q.block_num_end = 1; + q.receiver = "bob.contract"_n; + q.action = "transfer"_n; + + auto r = get_actions(q); + + BOOST_REQUIRE_EQUAL(r.actions.size(), 1u); + BOOST_TEST(r.actions[0].get_object()["global_sequence"].as_uint64() == 102u); + } +} + +// Per-slice bloom skip: a valid bloom that does not contain the queried receiver causes get_actions_impl to advance +// past the entire slice without scanning any of its blocks. The fixture observes "no scan" by having get_block +// return a single well-known action in every block; if the scan ran, the result would include that action. +BOOST_FIXTURE_TEST_CASE(bloom_skips_entire_slice_when_receiver_absent, get_actions_fixture) { + fc::temp_directory tempdir; + + // Three slices of 10 blocks each; populate every block so a non-skipped scan would always find the single action. + mock_slice_stride = 10; + for (uint32_t n = 1; n < 30; ++n) { + blocks[n] = make_block(n, { make_trx(TRX1, n, { make_action(n, "alice"_n, "alice"_n, "transfer"_n) }) }); + } + + // Build bloom sidecars for slices 0, 1, 2. Slice 1 is the only one that contains alice; slices 0 and 2 have no + // receivers at all (empty blooms -> every probe misses). + auto bloom_for = [&tempdir](std::size_t idx, bool with_alice) { + bloom_builder b; + if (with_alice) { + action_trace_v0 a{}; + a.receiver = "alice"_n; + a.account = "alice"_n; + a.action = "transfer"_n; + b.add_action(a); + } + const auto path = tempdir.path() / ("bloom_slice_" + std::to_string(idx) + ".log"); + b.finalize_and_write(path); + return path; + }; + const auto slice0_path = bloom_for(0, /*with_alice=*/false); + const auto slice1_path = bloom_for(1, /*with_alice=*/true); + const auto slice2_path = bloom_for(2, /*with_alice=*/false); + + mock_get_bloom = [slice0_path, slice1_path, slice2_path](uint32_t slice) -> bloom_reader { + switch (slice) { + case 0: return bloom_reader{slice0_path}; + case 1: return bloom_reader{slice1_path}; + case 2: return bloom_reader{slice2_path}; + default: return bloom_reader{}; + } + }; + + action_query q; + q.block_num_start = 1; + q.block_num_end = 29; + q.receiver = "alice"_n; + + auto r = get_actions(q); + + // All hits come from slice 1 (blocks 10..19). Slices 0 and 2 were bloom-skipped without any get_block call. + BOOST_REQUIRE_EQUAL(r.actions.size(), 10u); + for (const auto& a : r.actions) { + const auto block_num = a.get_object()["block_num"].as_uint64(); + BOOST_TEST(block_num >= 10u); + BOOST_TEST(block_num <= 19u); + } +} + +// Sanity check that a query with no filter cannot bloom-skip: even if the mock would return an empty bloom for +// every slice, we still scan because there's nothing to probe against. Without this behaviour callers would see +// empty results on unfiltered queries once sidecars exist. +BOOST_FIXTURE_TEST_CASE(bloom_not_consulted_when_no_filter, get_actions_fixture) { + mock_slice_stride = 10; + for (uint32_t n = 1; n < 15; ++n) { + blocks[n] = make_block(n, { make_trx(TRX1, n, { make_action(n, "alice"_n, "alice"_n, "transfer"_n) }) }); + } + // If the handler ever calls get_bloom under this configuration, fail loudly. + mock_get_bloom = [](uint32_t) -> bloom_reader { + BOOST_FAIL("get_bloom should not be called when no filter is set"); + return bloom_reader{}; + }; + + action_query q; + q.block_num_start = 1; + q.block_num_end = 14; + auto r = get_actions(q); + BOOST_CHECK_EQUAL(r.actions.size(), 14u); +} + +BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/plugins/trace_api_plugin/test/test_responses.cpp b/plugins/trace_api_plugin/test/test_responses.cpp index 6232659470..20a37f51e1 100644 --- a/plugins/trace_api_plugin/test/test_responses.cpp +++ b/plugins/trace_api_plugin/test/test_responses.cpp @@ -110,20 +110,20 @@ BOOST_AUTO_TEST_SUITE(trace_responses) BOOST_FIXTURE_TEST_CASE(basic_block_response, response_test_fixture) { - auto action_trace = action_trace_v0 { - 0, - "receiver"_n, "contract"_n, "action"_n, - {{ "alice"_n, "active"_n }}, - { 0x00, 0x01, 0x02, 0x03 }, - { 0x04, 0x05, 0x06, 0x07 } - }; + action_trace_v0 action_trace{}; + action_trace.global_sequence = 0; + action_trace.receiver = "receiver"_n; + action_trace.account = "contract"_n; + action_trace.action = "action"_n; + action_trace.authorization = {{ "alice"_n, "active"_n }}; + action_trace.data = { 0x00, 0x01, 0x02, 0x03 }; + action_trace.return_value = { 0x04, 0x05, 0x06, 0x07 }; auto transaction_trace = transaction_trace_v0 { "0000000000000000000000000000000000000000000000000000000000000001"_h, std::vector { action_trace }, - fc::enum_type{chain::transaction_receipt_header::status_enum::executed}, 10, 5, std::vector{ chain::signature_type() }, @@ -163,13 +163,19 @@ BOOST_AUTO_TEST_SUITE(trace_responses) ("producer_block_id", fc::variant()) ("actions", fc::variants({ fc::mutable_variant_object() + ("action_ordinal", 0) + ("creator_action_ordinal", 0) + ("closest_unnotified_ancestor_action_ordinal", 0) ("global_sequence", 0) + ("recv_sequence", 0) + ("code_sequence", 0) + ("abi_sequence", 0) ("receiver", "receiver") ("account", "contract") - ("action", "action") + ("name", "action") ("authorization", fc::variants({ fc::mutable_variant_object() - ("account", "alice") + ("actor", "alice") ("permission", "active") })) ("data", "00010203") @@ -179,7 +185,6 @@ BOOST_AUTO_TEST_SUITE(trace_responses) ("return_data", fc::mutable_variant_object() ("hex", "04050607")) })) - ("status", "executed") ("cpu_usage_us", 10) ("net_usage_words", 5) ("signatures", fc::variants({"SIG_K1_111111111111111111111111111111111111111111111111111111111111111116uk5ne"})) @@ -206,6 +211,15 @@ BOOST_AUTO_TEST_SUITE(trace_responses) BOOST_FIXTURE_TEST_CASE(basic_block_response_no_params, response_test_fixture) { + action_trace_v0 inner_action{}; + inner_action.global_sequence = 0; + inner_action.receiver = "receiver"_n; + inner_action.account = "contract"_n; + inner_action.action = "action"_n; + inner_action.authorization = {{ "alice"_n, "active"_n }}; + inner_action.data = { 0x00, 0x01, 0x02, 0x03 }; + inner_action.return_value = { 0x04, 0x05, 0x06, 0x07 }; + auto block_trace = block_trace_v0 { "b000000000000000000000000000000000000000000000000000000000000001"_h, 1, @@ -218,15 +232,8 @@ BOOST_AUTO_TEST_SUITE(trace_responses) { "0000000000000000000000000000000000000000000000000000000000000001"_h, std::vector { - { - 0, - "receiver"_n, "contract"_n, "action"_n, - {{ "alice"_n, "active"_n }}, - { 0x00, 0x01, 0x02, 0x03 }, - { 0x04, 0x05, 0x06, 0x07 } - } + inner_action }, - fc::enum_type{chain::transaction_receipt_header::status_enum::executed}, 10, 5, std::vector{ chain::signature_type() }, @@ -255,19 +262,24 @@ BOOST_AUTO_TEST_SUITE(trace_responses) ("producer_block_id", fc::variant()) ("actions", fc::variants({ fc::mutable_variant_object() + ("action_ordinal", 0) + ("creator_action_ordinal", 0) + ("closest_unnotified_ancestor_action_ordinal", 0) ("global_sequence", 0) + ("recv_sequence", 0) + ("code_sequence", 0) + ("abi_sequence", 0) ("receiver", "receiver") ("account", "contract") - ("action", "action") + ("name", "action") ("authorization", fc::variants({ fc::mutable_variant_object() - ("account", "alice") + ("actor", "alice") ("permission", "active") })) ("data", "00010203") ("return_value", "04050607") })) - ("status", "executed") ("cpu_usage_us", 10) ("net_usage_words", 5) ("signatures", fc::variants({"SIG_K1_111111111111111111111111111111111111111111111111111111111111111116uk5ne"})) @@ -299,34 +311,38 @@ BOOST_AUTO_TEST_SUITE(trace_responses) BOOST_FIXTURE_TEST_CASE(basic_block_response_unsorted, response_test_fixture) { - std::vector actions = { - { - 1, - "receiver"_n, "contract"_n, "action"_n, - {{ "alice"_n, "active"_n }}, - { 0x01, 0x01, 0x01, 0x01 }, - { 0x05, 0x05, 0x05, 0x05 } - }, - { - 0, - "receiver"_n, "contract"_n, "action"_n, - {{ "alice"_n, "active"_n }}, - { 0x00, 0x00, 0x00, 0x00 }, - { 0x04, 0x04, 0x04, 0x04 } - }, - { - 2, - "receiver"_n, "contract"_n, "action"_n, - {{ "alice"_n, "active"_n }}, - { 0x02, 0x02, 0x02, 0x02 }, - { 0x06, 0x06, 0x06, 0x06 } - } - }; + action_trace_v0 at1{}; + at1.global_sequence = 1; + at1.receiver = "receiver"_n; + at1.account = "contract"_n; + at1.action = "action"_n; + at1.authorization = {{ "alice"_n, "active"_n }}; + at1.data = { 0x01, 0x01, 0x01, 0x01 }; + at1.return_value = { 0x05, 0x05, 0x05, 0x05 }; + + action_trace_v0 at0{}; + at0.global_sequence = 0; + at0.receiver = "receiver"_n; + at0.account = "contract"_n; + at0.action = "action"_n; + at0.authorization = {{ "alice"_n, "active"_n }}; + at0.data = { 0x00, 0x00, 0x00, 0x00 }; + at0.return_value = { 0x04, 0x04, 0x04, 0x04 }; + + action_trace_v0 at2{}; + at2.global_sequence = 2; + at2.receiver = "receiver"_n; + at2.account = "contract"_n; + at2.action = "action"_n; + at2.authorization = {{ "alice"_n, "active"_n }}; + at2.data = { 0x02, 0x02, 0x02, 0x02 }; + at2.return_value = { 0x06, 0x06, 0x06, 0x06 }; + + std::vector actions = { at1, at0, at2 }; auto transaction_trace = transaction_trace_v0 { "0000000000000000000000000000000000000000000000000000000000000001"_h, actions, - fc::enum_type{chain::transaction_receipt_header::status_enum::executed}, 10, 5, { chain::signature_type() }, @@ -366,45 +382,62 @@ BOOST_AUTO_TEST_SUITE(trace_responses) ("producer_block_id", fc::variant()) ("actions", fc::variants({ fc::mutable_variant_object() + ("action_ordinal", 0) + ("creator_action_ordinal", 0) + ("closest_unnotified_ancestor_action_ordinal", 0) ("global_sequence", 0) + ("recv_sequence", 0) + ("code_sequence", 0) + ("abi_sequence", 0) ("receiver", "receiver") ("account", "contract") - ("action", "action") + ("name", "action") ("authorization", fc::variants({ fc::mutable_variant_object() - ("account", "alice") + ("actor", "alice") ("permission", "active") })) ("data", "00000000") ("return_value", "04040404") , fc::mutable_variant_object() + ("action_ordinal", 0) + ("creator_action_ordinal", 0) + ("closest_unnotified_ancestor_action_ordinal", 0) ("global_sequence", 1) + ("recv_sequence", 0) + ("code_sequence", 0) + ("abi_sequence", 0) ("receiver", "receiver") ("account", "contract") - ("action", "action") + ("name", "action") ("authorization", fc::variants({ fc::mutable_variant_object() - ("account", "alice") + ("actor", "alice") ("permission", "active") })) ("data", "01010101") ("return_value", "05050505") , fc::mutable_variant_object() + ("action_ordinal", 0) + ("creator_action_ordinal", 0) + ("closest_unnotified_ancestor_action_ordinal", 0) ("global_sequence", 2) + ("recv_sequence", 0) + ("code_sequence", 0) + ("abi_sequence", 0) ("receiver", "receiver") ("account", "contract") - ("action", "action") + ("name", "action") ("authorization", fc::variants({ fc::mutable_variant_object() - ("account", "alice") + ("actor", "alice") ("permission", "active") })) ("data", "02020202") ("return_value", "06060606") })) - ("status", "executed") ("cpu_usage_us", 10) ("net_usage_words", 5) ("signatures", fc::variants({"SIG_K1_111111111111111111111111111111111111111111111111111111111111111116uk5ne"})) @@ -491,4 +524,4 @@ BOOST_AUTO_TEST_SUITE(trace_responses) BOOST_TEST(null_response.is_null()); } -BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/plugins/trace_api_plugin/test/test_trace_file.cpp b/plugins/trace_api_plugin/test/test_trace_file.cpp index 9235af1fa5..bd47726b39 100644 --- a/plugins/trace_api_plugin/test/test_trace_file.cpp +++ b/plugins/trace_api_plugin/test/test_trace_file.cpp @@ -12,34 +12,39 @@ using open_state = slice_directory::open_state; namespace { struct test_fixture { - std::vector actions = { + std::vector actions { { - 1, - "receiver"_n, "contract"_n, "action"_n, - {{ "alice"_n, "active"_n }}, - { 0x01, 0x01, 0x01, 0x01 }, - { 0x05, 0x05, 0x05, 0x05 } + .global_sequence = 1, + .receiver = "receiver"_n, + .account = "contract"_n, + .action = "action"_n, + .authorization = {{ "alice"_n, "active"_n }}, + .data = { 0x01, 0x01, 0x01, 0x01 }, + .return_value = { 0x05, 0x05, 0x05, 0x05 } }, { - 0, - "receiver"_n, "contract"_n, "action"_n, - {{ "alice"_n, "active"_n }}, - { 0x00, 0x00, 0x00, 0x00 }, - { 0x04, 0x04, 0x04, 0x04} + .global_sequence = 0, + .receiver = "receiver"_n, + .account = "contract"_n, + .action = "action"_n, + .authorization = {{ "alice"_n, "active"_n }}, + .data = { 0x00, 0x00, 0x00, 0x00 }, + .return_value = { 0x04, 0x04, 0x04, 0x04 } }, { - 2, - "receiver"_n, "contract"_n, "action"_n, - {{ "alice"_n, "active"_n }}, - { 0x02, 0x02, 0x02, 0x02 }, - { 0x06, 0x06, 0x06, 0x06 } + .global_sequence = 2, + .receiver = "receiver"_n, + .account = "contract"_n, + .action = "action"_n, + .authorization = {{ "alice"_n, "active"_n }}, + .data = { 0x02, 0x02, 0x02, 0x02 }, + .return_value = { 0x06, 0x06, 0x06, 0x06 } } }; transaction_trace_v0 transaction_trace { "0000000000000000000000000000000000000000000000000000000000000001"_h, actions, - fc::enum_type{chain::transaction_receipt_header::status_enum::executed}, 10, 5, { chain::signature_type() }, @@ -73,44 +78,46 @@ namespace { }; const block_trace_v0 bt1 { - "0000000000000000000000000000000000000000000000000000000000000001"_h, - 1, - "0000000000000000000000000000000000000000000000000000000000000003"_h, - chain::block_timestamp_type(1), - "bp.one"_n, - "0000000000000000000000000000000000000000000000000000000000000000"_h, - "0000000000000000000000000000000000000000000000000000000000000000"_h, - { - { - "0000000000000000000000000000000000000000000000000000000000000001"_h, - { + .id = "0000000000000000000000000000000000000000000000000000000000000001"_h, + .number = 1, + .previous_id = "0000000000000000000000000000000000000000000000000000000000000003"_h, + .timestamp = chain::block_timestamp_type(1), + .producer = "bp.one"_n, + .transaction_mroot = "0000000000000000000000000000000000000000000000000000000000000000"_h, + .finality_mroot = "0000000000000000000000000000000000000000000000000000000000000000"_h, + .transactions = { + transaction_trace_v0 { + .id = "0000000000000000000000000000000000000000000000000000000000000001"_h, + .actions = { { - 0, - "sysio.token"_n, "sysio.token"_n, "transfer"_n, - {{ "alice"_n, "active"_n }}, - make_transfer_data( "alice"_n, "bob"_n, "0.0001 SYS"_t, "Memo!" ), - {} + .global_sequence = 0, + .receiver = "sysio.token"_n, + .account = "sysio.token"_n, + .action = "transfer"_n, + .authorization = {{ "alice"_n, "active"_n }}, + .data = make_transfer_data( "alice"_n, "bob"_n, "0.0001 SYS"_t, "Memo!" ) }, { - 1, - "alice"_n, "sysio.token"_n, "transfer"_n, - {{ "alice"_n, "active"_n }}, - make_transfer_data( "alice"_n, "bob"_n, "0.0001 SYS"_t, "Memo!" ), - {} + .global_sequence = 1, + .receiver = "alice"_n, + .account = "sysio.token"_n, + .action = "transfer"_n, + .authorization = {{ "alice"_n, "active"_n }}, + .data = make_transfer_data( "alice"_n, "bob"_n, "0.0001 SYS"_t, "Memo!" ) }, { - 2, - "bob"_n, "sysio.token"_n, "transfer"_n, - {{ "alice"_n, "active"_n }}, - make_transfer_data( "alice"_n, "bob"_n, "0.0001 SYS"_t, "Memo!" ), - {} + .global_sequence = 2, + .receiver = "bob"_n, + .account = "sysio.token"_n, + .action = "transfer"_n, + .authorization = {{ "alice"_n, "active"_n }}, + .data = make_transfer_data( "alice"_n, "bob"_n, "0.0001 SYS"_t, "Memo!" ) } }, - fc::enum_type{chain::transaction_receipt_header::status_enum::executed}, - 10, - 5, - std::vector{chain::signature_type()}, - chain::transaction_header{chain::time_point_sec(), 1, 0, 100, 50, 0} + .cpu_usage_us = 10, + .net_usage_words = 5, + .signatures = {chain::signature_type()}, + .trx_header = chain::transaction_header{chain::time_point_sec(), 1, 0, 100, 50, 0} } } }; @@ -127,7 +134,6 @@ namespace { { "f000000000000000000000000000000000000000000000000000000000000004"_h, {}, - fc::enum_type{chain::transaction_receipt_header::status_enum::executed}, 10, 5, std::vector{chain::signature_type()}, @@ -162,6 +168,7 @@ namespace { } using store_provider::scan_metadata_log_from; using store_provider::read_data_log; + using store_provider::_slice_directory; }; class vslice_datastream; @@ -830,46 +837,66 @@ BOOST_AUTO_TEST_SUITE(slice_tests) sp.append(block_trace1); sp.append_lib(1); sp.append(block_trace2); - int count = 0; - get_block_t block1 = sp.get_block(1, [&count]() { - if (++count >= 3) { - throw yield_exception(""); - } - }); + + // get_block uses the trace_blk_idx_.log sidecar for O(1) lookup and does + // not iterate, so the yield callback is not invoked on the fast path. + get_block_t block1 = sp.get_block(1, [](){ BOOST_FAIL("yield must not be called on sidecar fast path"); }); BOOST_REQUIRE(block1); BOOST_REQUIRE(std::get<1>(*block1)); const auto block1_bt = std::get<0>(*block1); BOOST_REQUIRE_EQUAL(std::get(block1_bt), block_trace1); - count = 0; - get_block_t block2 = sp.get_block(5, [&count]() { - if (++count >= 4) { - throw yield_exception(""); - } - }); + get_block_t block2 = sp.get_block(5, [](){ BOOST_FAIL("yield must not be called on sidecar fast path"); }); BOOST_REQUIRE(block2); BOOST_REQUIRE(!std::get<1>(*block2)); const auto block2_bt = std::get<0>(*block2); BOOST_REQUIRE_EQUAL(std::get(block2_bt), block_trace2); - count = 0; - try { - sp.get_block(5,[&count]() { - if (++count >= 3) { - throw yield_exception(""); - } - }); - BOOST_FAIL("Should not have completed scan"); - } catch (const yield_exception& ex) { - } + // Missing block: sidecar slot is empty, we fall back to scanning the metadata log. + get_block_t block_missing = sp.get_block(2); + BOOST_REQUIRE(!block_missing); + } - count = 0; - block2 = sp.get_block(2,[&count]() { - if (++count >= 4) { - throw yield_exception(""); - } - }); - BOOST_REQUIRE(!block2); + // Sidecar must be removed when its slice is cleaned up. + BOOST_FIXTURE_TEST_CASE(test_blk_offset_sidecar_cleanup, test_fixture) + { + fc::temp_directory tempdir; + const uint32_t width = 100; + slice_directory sd(tempdir.path(), width, /*min_irr=*/0u, std::optional(), 0); + + // Create sidecar + matching trace/index for slice 0 so cleanup has something to do. + sd.write_block_offset(1, 0); + fc::cfile f; + sd.find_or_create_trace_slice(0, open_state::read, f); + sd.find_or_create_index_slice(0, open_state::read, f); + + const auto sidecar = tempdir.path() / "trace_blk_idx_0000000000-0000000100.log"; + BOOST_REQUIRE(std::filesystem::exists(sidecar)); + + // Advance LIB several slices past 0 so slice 0 is rotated out. + sd.run_maintenance_tasks(width * 5, [](auto&&){}); + + BOOST_REQUIRE(!std::filesystem::exists(sidecar)); + } + + // Fork re-writes the block to a new offset. The sidecar slot must be overwritten so + // get_block returns the latest (fork-resolved) copy, matching the scan-based "last wins". + BOOST_FIXTURE_TEST_CASE(test_blk_offset_fork_rewrite, test_fixture) + { + fc::temp_directory tempdir; + store_provider sp(tempdir.path(), 100, std::optional(), std::optional(), 0); + + // Different block bodies but same block number. Simulates a fork re-applying block 1. + block_trace_v0 forked = block_trace1; + forked.producer = "bp.two"_n; + + sp.append(block_trace1); + sp.append(forked); + + auto result = sp.get_block(1); + BOOST_REQUIRE(result); + const auto bt = std::get(std::get<0>(*result)); + BOOST_REQUIRE_EQUAL(bt, forked); } // Verify basics of get_trx_block_number() @@ -883,7 +910,6 @@ BOOST_AUTO_TEST_SUITE(slice_tests) transaction_trace_v0 trx_trace1 { trx_id1, actions, - fc::enum_type{chain::transaction_receipt_header::status_enum::executed}, 10, 5, { chain::signature_type() }, @@ -893,7 +919,6 @@ BOOST_AUTO_TEST_SUITE(slice_tests) transaction_trace_v0 trx_trace2 { trx_id2, actions, - fc::enum_type{chain::transaction_receipt_header::status_enum::executed}, 10, 5, { chain::signature_type() }, @@ -996,7 +1021,6 @@ BOOST_AUTO_TEST_SUITE(slice_tests) transaction_trace_v0 trx_trace1 { target_trx_id, actions, - fc::enum_type{chain::transaction_receipt_header::status_enum::executed}, 10, 5, { chain::signature_type() }, @@ -1006,7 +1030,6 @@ BOOST_AUTO_TEST_SUITE(slice_tests) transaction_trace_v0 trx_trace2 { "0000000000000000000000000000000000000000000000000000000000000002"_h, actions, - fc::enum_type{chain::transaction_receipt_header::status_enum::executed}, 10, 5, { chain::signature_type() }, @@ -1124,8 +1147,6 @@ BOOST_AUTO_TEST_SUITE(slice_tests) transaction_trace_v0 trx_trace1 { target_trx_id, actions, - fc::enum_type{ - chain::transaction_receipt_header::status_enum::executed}, 10, 5, {chain::signature_type()}, @@ -1178,4 +1199,274 @@ BOOST_AUTO_TEST_SUITE(slice_tests) BOOST_REQUIRE_EQUAL(*block_num, trx_block_num); // target trx is in final block } + // build_trx_id_index must apply the same fork-resolution logic as the linear + // scan in get_trx_block_number: when the trx_id log holds multiple + // block_trxs_entry records for the same block_num (one per accepted block at + // that height, including forked-out ones), only the LAST entry per block_num + // reflects the canonical post-fork state. Trxs that appeared in an earlier + // entry for that block_num but not the last one were forked out and must NOT + // appear in the index. Trxs that moved to a different block in the canonical + // fork must resolve to the new block_num. + BOOST_FIXTURE_TEST_CASE(test_build_trx_id_index_dedups_forked_trxs, test_fixture) + { + // Distinct first-8-byte prefixes so each trx maps to a unique bucket + // (the writer's prefix64 = first 8 bytes of the trx_id). + const chain::transaction_id_type trx_a = + "a1a2a3a4a5a6a7a8000000000000000000000000000000000000000000000001"_h; + const chain::transaction_id_type trx_b = + "b1b2b3b4b5b6b7b8000000000000000000000000000000000000000000000002"_h; + const chain::transaction_id_type trx_c = + "c1c2c3c4c5c6c7c8000000000000000000000000000000000000000000000003"_h; + + fc::temp_directory tempdir; + const uint32_t width = 100; + test_store_provider sp(tempdir.path(), width); + + // First accepted block at height 1: contains trx_a + trx_b. + sp.append_trx_ids(block_trxs_entry{ .ids = {trx_a, trx_b}, .block_num = 1 }); + // Block 1 forks out and is replaced by a different block at height 1: + // canonical block 1 contains only trx_c (trx_a moved, trx_b removed). + sp.append_trx_ids(block_trxs_entry{ .ids = {trx_c}, .block_num = 1 }); + // trx_a re-appears at block 2 in the canonical chain. + sp.append_trx_ids(block_trxs_entry{ .ids = {trx_a}, .block_num = 2 }); + + // Build the index for slice 0 directly (bypass the maintenance thread). + sp._slice_directory.build_trx_id_index(0, [](const std::string&){}); + + // Open the resulting on-disk index and verify lookups. + auto reader = sp._slice_directory.find_trx_id_index_slice(0); + BOOST_REQUIRE(reader.has_value()); + BOOST_REQUIRE(reader->valid()); + + // trx_a moved to block 2 in the canonical chain -> lookup returns 2, + // NOT 1 (the forked-out occurrence). + auto a = reader->lookup(trx_a); + BOOST_REQUIRE(a.has_value()); + BOOST_CHECK_EQUAL(*a, 2u); + + // trx_b was removed entirely (only present in the forked-out block 1). + // Its bucket must be empty. + auto b = reader->lookup(trx_b); + BOOST_CHECK(!b.has_value()); + + // trx_c is in canonical block 1. + auto c = reader->lookup(trx_c); + BOOST_REQUIRE(c.has_value()); + BOOST_CHECK_EQUAL(*c, 1u); + } + + // Receiver bloom sidecar is built by run_maintenance_tasks at slice irreversibility, not during append. Before + // LIB crosses the slice, the sidecar must be absent (queries fall back to scan); once LIB advances past the slice, + // a maintenance pass produces a valid sidecar whose probes hit every receiver actually present in the slice and + // miss for receivers that were never appended. This exercises the full on-LIB build path including the data log + // stream-scan and the atomic sidecar write. + BOOST_FIXTURE_TEST_CASE(slice_dir_recv_bloom_build_on_lib, test_fixture) + { + fc::temp_directory tempdir; + const uint32_t width = 10; + // No compression, no deletion - keep the bloom build path focused. + test_store_provider sp(tempdir.path(), width); + + // Build two block_trace_v0s in slice 0 (block numbers 1 and 2), each with one transaction whose actions touch + // a distinct, known set of receivers. + auto make_bt = [](uint32_t num, chain::checksum256_type id, std::vector receivers) { + block_trace_v0 bt; + bt.id = id; + bt.number = num; + transaction_trace_v0 trx; + trx.id = id; + trx.block_num = num; + uint64_t seq = uint64_t{num} * 100; + for (auto r : receivers) { + action_trace_v0 a{}; + a.global_sequence = seq++; + a.receiver = r; + a.account = "sysio.token"_n; + a.action = "transfer"_n; + trx.actions.push_back(std::move(a)); + } + bt.transactions.push_back(std::move(trx)); + return bt; + }; + + auto id1 = "b000000000000000000000000000000000000000000000000000000000000001"_h; + auto id2 = "b000000000000000000000000000000000000000000000000000000000000002"_h; + sp.append(make_bt(1, id1, { "alice"_n, "bob"_n })); + sp.append(make_bt(2, id2, { "charlie"_n })); + + const auto bloom_path = sp._slice_directory.bloom_slice_path(0); + + // Before LIB, no sidecar should exist - the append path must not have built anything on the fly. + BOOST_CHECK(!std::filesystem::exists(bloom_path)); + + // Advance LIB so slice 0 (blocks 0..9) is past LIB. run_maintenance_tasks processes irreversible slices with + // min_irreversible=0, so a LIB inside slice 1 (block >= 10) makes slice 0 eligible. + sp._slice_directory.run_maintenance_tasks(/*lib=*/15, [](const std::string&){}); + + BOOST_REQUIRE(std::filesystem::exists(bloom_path)); + + bloom_reader r(bloom_path); + BOOST_REQUIRE(r.valid()); + BOOST_CHECK(r.may_contain_receiver("alice"_n)); + BOOST_CHECK(r.may_contain_receiver("bob"_n)); + BOOST_CHECK(r.may_contain_receiver("charlie"_n)); + BOOST_CHECK(r.may_contain_recv_action("alice"_n, "transfer"_n)); + BOOST_CHECK(r.may_contain_recv_action("charlie"_n, "transfer"_n)); + // A receiver that was never appended should probe as absent (allowing for the 1% FPR on the small-capacity + // filter - try several unrelated names and tolerate at most one spurious hit). + std::size_t false_positives = 0; + for (auto n : { "never1"_n, "never2"_n, "never3"_n, "never4"_n, "never5"_n }) { + if (r.may_contain_receiver(n)) ++false_positives; + } + BOOST_CHECK_LE(false_positives, 1u); + + // Re-running maintenance is idempotent: the bloom path still exists and the file wasn't clobbered. + sp._slice_directory.run_maintenance_tasks(/*lib=*/15, [](const std::string&){}); + BOOST_REQUIRE(std::filesystem::exists(bloom_path)); + } + + // Fork behavior inside a single slice. The extraction path re-applies forked blocks by calling append() again + // with a new block_trace_v0 at the same block number. The data log ends up with BOTH the forked-out trace and + // the canonical trace: the blk_offset sidecar points only to the canonical offset, but the pre-fork record is + // still physically present in the file. Because the bloom is built by streaming the entire data log (not by + // walking blk_offset), it naturally includes receivers from forked-out blocks too. That is safe: a bloom may + // have false positives (a probe "hits" for a receiver that isn't in the canonical chain) but must never have + // false negatives (a probe "misses" for a receiver that IS in the canonical chain). The test asserts both halves + // of the invariant - canonical receivers probe as present, AND a forked-out receiver also probes as present + // (harmless false positive), AND a never-appended receiver does not. + BOOST_FIXTURE_TEST_CASE(slice_dir_recv_bloom_fork_in_slice, test_fixture) + { + fc::temp_directory tempdir; + const uint32_t width = 10; + test_store_provider sp(tempdir.path(), width); + + auto make_bt = [](uint32_t num, chain::checksum256_type id, chain::name receiver) { + block_trace_v0 bt; + bt.id = id; + bt.number = num; + transaction_trace_v0 trx; + trx.id = id; + trx.block_num = num; + action_trace_v0 a{}; + a.global_sequence = uint64_t{num} * 100; + a.receiver = receiver; + a.account = "sysio.token"_n; + a.action = "transfer"_n; + trx.actions.push_back(std::move(a)); + bt.transactions.push_back(std::move(trx)); + return bt; + }; + + // Initial chain: block 1 with alice, block 2 with bob. + sp.append(make_bt(1, "b000000000000000000000000000000000000000000000000000000000000001"_h, "alice"_n)); + sp.append(make_bt(2, "b000000000000000000000000000000000000000000000000000000000000002"_h, "bob"_n)); + + // Fork: chain switches to a different branch. Block 2 gets replayed with a different trace containing eve. + // Controller fires accepted_block again with the new block_trace; store_provider::append writes the new trace + // to the data log (appending, not overwriting in place) and updates the blk_offset sidecar to point at the new + // offset. The stale "bob" record still occupies its original position in the trace file. + sp.append(make_bt(2, "b0000000000000000000000000000000000000000000000000000000000000b2"_h, "eve"_n)); + + // Advance LIB past slice 0 so maintenance builds the bloom. + sp._slice_directory.run_maintenance_tasks(/*lib=*/15, [](const std::string&){}); + const auto bloom_path = sp._slice_directory.bloom_slice_path(0); + BOOST_REQUIRE(std::filesystem::exists(bloom_path)); + bloom_reader r(bloom_path); + BOOST_REQUIRE(r.valid()); + + // Canonical receivers must probe as present - this is the correctness invariant. + BOOST_CHECK(r.may_contain_receiver("alice"_n)); + BOOST_CHECK(r.may_contain_receiver("eve"_n)); + // Forked-out receiver also probes as present because the stream-scan includes its stale record. This is a + // benign false positive; the query scan will then visit that slice and find no canonical match for "bob". + BOOST_CHECK(r.may_contain_receiver("bob"_n)); + // Sanity: a receiver that was never in any branch at any time should still miss (modulo FPR). + std::size_t false_positives = 0; + for (auto n : { "never1"_n, "never2"_n, "never3"_n, "never4"_n, "never5"_n }) { + if (r.may_contain_receiver(n)) ++false_positives; + } + BOOST_CHECK_LE(false_positives, 1u); + } + + // Fork that crosses a slice boundary. The scenario that motivated moving the bloom write to LIB (rather than + // doing it at slice roll-over during append): the tail of slice K is replayed after the head of slice K+1 is + // already in flight. Under the earlier roll-over-based design the back-and-forth would have overwritten slice + // K's bloom with an incomplete one built only from the replayed blocks. Under the LIB-based design the sidecar + // isn't written until the slice is fully irreversible, so forks can't reach back into an already-written sidecar. + BOOST_FIXTURE_TEST_CASE(slice_dir_recv_bloom_cross_slice_fork, test_fixture) + { + fc::temp_directory tempdir; + const uint32_t width = 10; + test_store_provider sp(tempdir.path(), width); + + auto make_bt = [](uint32_t num, chain::checksum256_type id, chain::name receiver) { + block_trace_v0 bt; + bt.id = id; + bt.number = num; + transaction_trace_v0 trx; + trx.id = id; + trx.block_num = num; + action_trace_v0 a{}; + a.global_sequence = uint64_t{num} * 100; + a.receiver = receiver; + a.account = "sysio.token"_n; + a.action = "transfer"_n; + trx.actions.push_back(std::move(a)); + bt.transactions.push_back(std::move(trx)); + return bt; + }; + + // Normal forward progress through slice 0: blocks 1..9 each with a distinct receiver. These will all end up + // in slice 0's bloom if LIB crosses cleanly. + for (uint32_t n = 1; n <= 9; ++n) { + chain::name r(0x4000'0000'0000'0000ull | n); // synthesize distinct names + chain::checksum256_type id; + std::memcpy(id.data(), &n, sizeof(n)); + sp.append(make_bt(n, id, r)); + } + // Block 10 lands in slice 1. + sp.append(make_bt(10, "b00000000000000000000000000000000000000000000000000000000000000a"_h, "frank"_n)); + + // Simulate a fork that replays the last block of slice 0 with a different trace, then replays slice 1's first + // block. This is exactly the cross-slice rollback pattern that broke the earlier design. + sp.append(make_bt(9, "b0000000000000000000000000000000000000000000000000000000000000f9"_h, "grace"_n)); + sp.append(make_bt(10, "b00000000000000000000000000000000000000000000000000000000000000b"_h, "harry"_n)); + + // Neither slice 0 nor slice 1 has been built yet: no LIB has crossed them. + BOOST_CHECK(!std::filesystem::exists(sp._slice_directory.bloom_slice_path(0))); + BOOST_CHECK(!std::filesystem::exists(sp._slice_directory.bloom_slice_path(1))); + + // Advance LIB past slice 0 but still within slice 1. Slice 0 should now be bloomed; slice 1 should still be + // absent (it's still in flight, potentially subject to further forks). + sp._slice_directory.run_maintenance_tasks(/*lib=*/12, [](const std::string&){}); + BOOST_REQUIRE(std::filesystem::exists(sp._slice_directory.bloom_slice_path(0))); + BOOST_CHECK (!std::filesystem::exists(sp._slice_directory.bloom_slice_path(1))); + + // Slice 0's bloom must contain every receiver that was ever recorded in it - canonical and forked-out alike. + // The key invariant: a query for "grace" (canonical tail of slice 0) MUST hit the bloom. Under the pre-fix + // design this was the receiver that could get lost. + bloom_reader r0(sp._slice_directory.bloom_slice_path(0)); + BOOST_REQUIRE(r0.valid()); + for (uint32_t n = 1; n <= 8; ++n) { + chain::name expected(0x4000'0000'0000'0000ull | n); + BOOST_TEST_INFO("canonical slice-0 receiver " << expected.to_string()); + BOOST_CHECK(r0.may_contain_receiver(expected)); + } + BOOST_CHECK(r0.may_contain_receiver("grace"_n)); // canonical post-fork tail of slice 0 + // Forked-out block 9 receiver (the 0x4000... name for n=9): also present because stream-scan includes it. + { + chain::name pre_fork_9(0x4000'0000'0000'0000ull | 9); + BOOST_CHECK(r0.may_contain_receiver(pre_fork_9)); + } + + // Advance LIB past slice 1 and rebuild. Slice 1 must now be bloomed and must contain harry (canonical) - bob + // frank was the forked-out block 10, which the stream-scan still finds. + sp._slice_directory.run_maintenance_tasks(/*lib=*/25, [](const std::string&){}); + BOOST_REQUIRE(std::filesystem::exists(sp._slice_directory.bloom_slice_path(1))); + bloom_reader r1(sp._slice_directory.bloom_slice_path(1)); + BOOST_REQUIRE(r1.valid()); + BOOST_CHECK(r1.may_contain_receiver("harry"_n)); // canonical slice-1 + BOOST_CHECK(r1.may_contain_receiver("frank"_n)); // forked-out slice-1 (harmless false positive) + } + BOOST_AUTO_TEST_SUITE_END() diff --git a/plugins/trace_api_plugin/test/test_trx_id_index.cpp b/plugins/trace_api_plugin/test/test_trx_id_index.cpp new file mode 100644 index 0000000000..92ea9eca4a --- /dev/null +++ b/plugins/trace_api_plugin/test/test_trx_id_index.cpp @@ -0,0 +1,339 @@ +#include +#include +#include +#include + +#include +#include + +using namespace sysio; +using namespace sysio::trace_api; +using namespace sysio::trace_api::test_common; + +namespace { + +// Build a sha256 whose first 8 bytes (the prefix64) equal the given value. +// On little-endian x86 this is the little-endian encoding written into bytes 0-7. +chain::transaction_id_type make_trx_id(uint64_t prefix64, uint8_t disambiguator = 0) { + chain::transaction_id_type id; + std::memcpy(id.data(), &prefix64, sizeof(prefix64)); + // vary the last byte so ids with the same prefix64 are still distinct objects + id.data()[id.data_size() - 1] = disambiguator; + return id; +} + +struct trx_id_index_fixture { + fc::temp_directory tempdir; + + std::filesystem::path index_path() const { + return tempdir.path() / "test_trx_idx.log"; + } +}; + +} // namespace + +BOOST_AUTO_TEST_SUITE(trx_id_index_tests) + +// --------------------------------------------------------------------------- +// Writer / Reader round-trip +// --------------------------------------------------------------------------- + +BOOST_FIXTURE_TEST_CASE(empty_writer_is_valid, trx_id_index_fixture) { + trx_id_index_writer w; + BOOST_REQUIRE_EQUAL(w.entry_count(), 0u); + w.write(index_path()); + + trx_id_index_reader r(index_path()); + BOOST_CHECK(r.valid()); + // empty index: any lookup should return nullopt + BOOST_CHECK(!r.lookup(make_trx_id(0))); + BOOST_CHECK(!r.lookup(make_trx_id(42))); +} + +BOOST_FIXTURE_TEST_CASE(single_entry_round_trip, trx_id_index_fixture) { + auto id = make_trx_id(0xDEADBEEFCAFEBABEULL); + const uint32_t block = 12345; + + trx_id_index_writer w; + w.add(id, block); + w.write(index_path()); + + trx_id_index_reader r(index_path()); + BOOST_REQUIRE(r.valid()); + auto result = r.lookup(id); + BOOST_REQUIRE(result.has_value()); + BOOST_CHECK_EQUAL(*result, block); +} + +BOOST_FIXTURE_TEST_CASE(multiple_entries_round_trip, trx_id_index_fixture) { + // 20 distinct entries with distinct prefix64 values, varied block numbers + const int N = 20; + std::vector> entries; + trx_id_index_writer w; + for (int i = 0; i < N; ++i) { + auto id = make_trx_id(static_cast(i) * 0x0101010101010101ULL + i, static_cast(i)); + uint32_t block = 1000 + i; + w.add(id, block); + entries.emplace_back(id, block); + } + BOOST_REQUIRE_EQUAL(w.entry_count(), static_cast(N)); + w.write(index_path()); + + trx_id_index_reader r(index_path()); + BOOST_REQUIRE(r.valid()); + for (const auto& [id, expected_block] : entries) { + auto result = r.lookup(id); + BOOST_REQUIRE_MESSAGE(result.has_value(), "entry not found for block " << expected_block); + BOOST_CHECK_EQUAL(*result, expected_block); + } +} + +BOOST_FIXTURE_TEST_CASE(not_found_returns_nullopt, trx_id_index_fixture) { + trx_id_index_writer w; + w.add(make_trx_id(0xAAAAAAAAAAAAAAAAULL), 100); + w.write(index_path()); + + trx_id_index_reader r(index_path()); + BOOST_REQUIRE(r.valid()); + // id not in the index + BOOST_CHECK(!r.lookup(make_trx_id(0xBBBBBBBBBBBBBBBBULL))); +} + +// --------------------------------------------------------------------------- +// Linear probing: two entries that hash to the same initial slot +// --------------------------------------------------------------------------- + +BOOST_FIXTURE_TEST_CASE(linear_probing_on_prefix_collision, trx_id_index_fixture) { + // With 2 entries, bucket_count = bit_ceil(2*2+1) = bit_ceil(5) = 8, mask = 7. + // Both ids below have uint32_t(prefix64) & 7 == 0, so they start at slot 0. + // The second must be linearly probed to slot 1. + const uint64_t prefix_A = 0x0000000000000000ULL; // slot = 0 & 7 = 0 + const uint64_t prefix_B = 0x0000000000000008ULL; // slot = 8 & 7 = 0 + auto id_A = make_trx_id(prefix_A, 1); + auto id_B = make_trx_id(prefix_B, 2); + + trx_id_index_writer w; + w.add(id_A, 101); + w.add(id_B, 202); + w.write(index_path()); + + trx_id_index_reader r(index_path()); + BOOST_REQUIRE(r.valid()); + + auto result_A = r.lookup(id_A); + BOOST_REQUIRE(result_A.has_value()); + BOOST_CHECK_EQUAL(*result_A, 101u); + + auto result_B = r.lookup(id_B); + BOOST_REQUIRE(result_B.has_value()); + BOOST_CHECK_EQUAL(*result_B, 202u); +} + +// --------------------------------------------------------------------------- +// Last-write-wins for duplicate prefix64 +// --------------------------------------------------------------------------- + +BOOST_FIXTURE_TEST_CASE(duplicate_prefix64_last_write_wins, trx_id_index_fixture) { + // Two adds with the same prefix64 should result in the second's block_num + // overwriting the first. Combined with build_trx_id_index's per-block_num + // dedup, this gives lookups the same "latest fork wins" semantic as the + // linear scan in get_trx_block_number. + const uint64_t shared_prefix = 0xCAFEBABEDEADBEEFULL; + auto id1 = make_trx_id(shared_prefix, 1); + auto id2 = make_trx_id(shared_prefix, 2); // same prefix64, different tail + + trx_id_index_writer w; + w.add(id1, 10); + w.add(id2, 20); + w.write(index_path()); + + trx_id_index_reader r(index_path()); + BOOST_REQUIRE(r.valid()); + + // Both id1 and id2 share the prefix, so a lookup of either resolves the + // single bucket — and that bucket holds the LATEST value (20). + auto result1 = r.lookup(id1); + BOOST_REQUIRE(result1.has_value()); + BOOST_CHECK_EQUAL(*result1, 20u); + + auto result2 = r.lookup(id2); + BOOST_REQUIRE(result2.has_value()); + BOOST_CHECK_EQUAL(*result2, 20u); +} + +// --------------------------------------------------------------------------- +// Error paths +// --------------------------------------------------------------------------- + +BOOST_FIXTURE_TEST_CASE(missing_file_is_invalid, trx_id_index_fixture) { + trx_id_index_reader r(tempdir.path() / "nonexistent.log"); + BOOST_CHECK(!r.valid()); +} + +BOOST_FIXTURE_TEST_CASE(bad_magic_is_invalid, trx_id_index_fixture) { + // Write a file with a bad magic value + fc::cfile f; + f.set_file_path(index_path()); + f.open(fc::cfile::create_or_update_rw_mode); + trx_id_index_header hdr; + hdr.magic = 0xDEADBEEF; // wrong magic + hdr.version = trx_id_index_header::current_version; + hdr.bucket_count = 0; + auto data = fc::raw::pack(hdr); + f.write(data.data(), data.size()); + f.flush(); + f.close(); + + trx_id_index_reader r(index_path()); + BOOST_CHECK(!r.valid()); +} + +BOOST_FIXTURE_TEST_CASE(bad_version_is_invalid, trx_id_index_fixture) { + fc::cfile f; + f.set_file_path(index_path()); + f.open(fc::cfile::create_or_update_rw_mode); + trx_id_index_header hdr; + hdr.magic = trx_id_index_header::magic_value; + hdr.version = 99; // unsupported version + hdr.bucket_count = 0; + auto data = fc::raw::pack(hdr); + f.write(data.data(), data.size()); + f.flush(); + f.close(); + + trx_id_index_reader r(index_path()); + BOOST_CHECK(!r.valid()); +} + +// --------------------------------------------------------------------------- +// Load factor / large table +// --------------------------------------------------------------------------- + +BOOST_FIXTURE_TEST_CASE(large_entry_set_all_found, trx_id_index_fixture) { + // Write 200 entries. bucket_count = bit_ceil(201*2) = bit_ceil(401) = 512. + // Load factor = 200/512 ~ 0.39 — well under 0.5. + const int N = 200; + std::vector> entries; + trx_id_index_writer w; + for (int i = 0; i < N; ++i) { + // Spread prefix64 values across the full range + uint64_t prefix = static_cast(i) * 0x9E3779B97F4A7C15ULL; // Fibonacci hashing + auto id = make_trx_id(prefix, static_cast(i & 0xFF)); + uint32_t block = 100000 + static_cast(i); + w.add(id, block); + entries.emplace_back(id, block); + } + BOOST_REQUIRE_EQUAL(w.entry_count(), static_cast(N)); + w.write(index_path()); + + trx_id_index_reader r(index_path()); + BOOST_REQUIRE(r.valid()); + for (const auto& [id, expected_block] : entries) { + auto result = r.lookup(id); + BOOST_REQUIRE_MESSAGE(result.has_value(), "entry not found for block " << expected_block); + BOOST_CHECK_EQUAL(*result, expected_block); + } +} + +// --------------------------------------------------------------------------- +// Defensive validation of corrupt / hostile index files +// --------------------------------------------------------------------------- + +BOOST_FIXTURE_TEST_CASE(bucket_count_not_power_of_two_is_invalid, trx_id_index_fixture) { + fc::cfile f; + f.set_file_path(index_path()); + f.open(fc::cfile::create_or_update_rw_mode); + trx_id_index_header hdr; + hdr.bucket_count = 5; // not a power of two + auto data = fc::raw::pack(hdr); + f.write(data.data(), data.size()); + // Pad to expected size so the file-size check would otherwise pass. + trx_id_bucket empty{}; + for (int i = 0; i < 5; ++i) { + auto bdata = fc::raw::pack(empty); + f.write(bdata.data(), bdata.size()); + } + f.flush(); + f.close(); + + trx_id_index_reader r(index_path()); + BOOST_CHECK(!r.valid()); +} + +BOOST_FIXTURE_TEST_CASE(bucket_count_above_cap_is_invalid, trx_id_index_fixture) { + // Just write the header claiming an absurd bucket_count. We intentionally + // do NOT write the buckets payload (that would be ~4GB); the reader must + // reject the header before attempting to allocate. + fc::cfile f; + f.set_file_path(index_path()); + f.open(fc::cfile::create_or_update_rw_mode); + trx_id_index_header hdr; + hdr.bucket_count = (1u << 29); // beyond the cap of 1<<28 + auto data = fc::raw::pack(hdr); + f.write(data.data(), data.size()); + f.flush(); + f.close(); + + trx_id_index_reader r(index_path()); + BOOST_CHECK(!r.valid()); +} + +BOOST_FIXTURE_TEST_CASE(file_size_mismatch_is_invalid, trx_id_index_fixture) { + // Header claims 8 buckets but file only contains 2. + fc::cfile f; + f.set_file_path(index_path()); + f.open(fc::cfile::create_or_update_rw_mode); + trx_id_index_header hdr; + hdr.bucket_count = 8; + auto data = fc::raw::pack(hdr); + f.write(data.data(), data.size()); + trx_id_bucket empty{}; + for (int i = 0; i < 2; ++i) { + auto bdata = fc::raw::pack(empty); + f.write(bdata.data(), bdata.size()); + } + f.flush(); + f.close(); + + trx_id_index_reader r(index_path()); + BOOST_CHECK(!r.valid()); +} + +// Hand-craft a fully populated bucket array (load factor 1.0). A naive probe +// loop would spin forever looking for an empty slot. The bounded probe must +// terminate and return nullopt for a missing prefix. +BOOST_FIXTURE_TEST_CASE(lookup_terminates_on_full_table, trx_id_index_fixture) { + constexpr uint32_t bucket_count = 8; // power of two, small for the test + fc::cfile f; + f.set_file_path(index_path()); + f.open(fc::cfile::create_or_update_rw_mode); + trx_id_index_header hdr; + hdr.bucket_count = bucket_count; + auto hdr_data = fc::raw::pack(hdr); + f.write(hdr_data.data(), hdr_data.size()); + + // Fill EVERY bucket with a non-zero block_num and a unique non-matching + // prefix. Choose prefixes whose initial slot is bucket 0 so the probe + // walks the whole table looking for prefix 0xCAFEBABE... + for (uint32_t i = 0; i < bucket_count; ++i) { + trx_id_bucket b{}; + // (bucket_count is a power of two, so any prefix64 value has its slot + // determined by the low log2(bucket_count) bits — set those to 0.) + b.prefix64 = (uint64_t{i} << 8); // all start at slot 0, distinct values + b.block_num = 1000 + i; // non-zero + auto bdata = fc::raw::pack(b); + f.write(bdata.data(), bdata.size()); + } + f.flush(); + f.close(); + + trx_id_index_reader r(index_path()); + BOOST_REQUIRE(r.valid()); + + // Lookup a prefix that isn't stored. Must terminate (not hang) and return nullopt. + auto missing = make_trx_id(0xDEADBEEFCAFEBABEULL); + auto result = r.lookup(missing); + BOOST_CHECK(!result.has_value()); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/plugins/trace_api_plugin/trace_api_plugin.md b/plugins/trace_api_plugin/trace_api_plugin.md new file mode 100644 index 0000000000..cd5327be91 --- /dev/null +++ b/plugins/trace_api_plugin/trace_api_plugin.md @@ -0,0 +1,957 @@ +# trace_api_plugin + +Full-history action trace plugin for Wire nodeop. Captures every action +trace as blocks are applied, persists them in a structured on-disk store, +and exposes HTTP endpoints for querying traces, transactions, actions, and +token transfers. + +--- + +## Table of contents + +1. [Overview](#overview) +2. [Quick start](#quick-start) +3. [Configuration options](#configuration-options) +4. [HTTP API reference](#http-api-reference) + - [get_block](#get_block) + - [get_transaction_trace](#get_transaction_trace) + - [get_actions](#get_actions) + - [get_token_transfers](#get_token_transfers) +5. [Pagination guide](#pagination-guide) +6. [Exchange / indexer integration guide](#exchange--indexer-integration-guide) +7. [ABI decoding](#abi-decoding) +8. [Operations](#operations) + - [Maintenance and retention](#maintenance-and-retention) + - [Startup continuity check](#startup-continuity-check) + - [Plugin variants](#plugin-variants) +9. [Implementation details](#implementation-details) + - [On-disk layout](#on-disk-layout) + - [ABI capture mechanics](#abi-capture-mechanics) + +--- + +## Overview + +The trace_api_plugin writes a complete record of every action executed on +chain (including inline actions), alongside the ABI in effect at the time +each contract was called. That data is kept on disk indefinitely (or for a +configurable retention window) and is served through a set of HTTP endpoints +without touching the chainbase database. + +Key design points: + +- **No chainbase dependency at query time** — responses are built entirely + from the trace files on disk. +- **Inline actions included** — every entry in `chain::transaction_trace:: + action_traces` is stored, so inline notifications are captured alongside + the originating action. +- **Versioned ABI decoding** — the ABI in effect at the moment each + `setabi` transaction was applied is captured in `abi_log.log`. Queries + decode `data` and `return_value` fields using the historically correct + ABI, not the current on-chain ABI. +- **O(1) transaction lookup** — a per-slice hash index maps `trx_id` to + `block_num` so `get_transaction_trace` does not scan the chain. + +--- + +## Quick start + +Add to `config.ini` or pass on the command line: + +```ini +plugin = sysio::trace_api_plugin +trace-dir = traces +``` + +Or via CLI: + +```bash +nodeop --plugin sysio::trace_api_plugin --trace-dir /var/lib/nodeop/traces +``` + +The plugin also requires `chain_plugin` and `http_plugin` (both loaded by +default). + +--- + +## Configuration options + +| Option | Default | Description | +|--------|---------|-------------| +| `trace-dir` | `traces` | Directory for trace files. Relative paths are resolved from the node's data directory. | +| `trace-slice-stride` | `10000` | Number of blocks per slice file. Must be in `[1, 1000000]`. Larger values reduce file count but bloat the block-offset sidecar's per-slice pre-allocation (`stride * 8` bytes, sparse) and stress the per-slice trx_id hash index (rejected if it would need more than 2^28 buckets). Also bounds the worst-case scan cost of `get_actions` on a positive bloom probe - smaller strides mean less work per hit-slice at the cost of more sidecar files (see [Slice stride vs. query latency](#slice-stride-vs-query-latency)). Setting takes effect on nodeop restart; existing slices retain their old naming. | +| `trace-minimum-irreversible-history-blocks` | `-1` | Blocks past LIB to retain before old slices can be auto-deleted. `-1` disables automatic deletion (keep forever). | +| `trace-minimum-uncompressed-irreversible-history-blocks` | `-1` | Blocks past LIB to keep uncompressed. Slices older than this threshold are transparently compressed. `-1` disables automatic compression. | +| `trace-max-block-range` | `1000` | Maximum number of blocks scanned by a single `get_actions` or `get_token_transfers` request. Must be in `[1, 10000]`. `block_num_end` is silently clamped to `block_num_start + trace-max-block-range - 1` when a request asks for more. The response envelope always reports the actual range scanned. | + +### Recommended production settings + +```ini +plugin = sysio::trace_api_plugin +trace-dir = /var/lib/nodeop/traces +trace-slice-stride = 10000 +# Keep 2 weeks of uncompressed data (~2M blocks/day = ~28M blocks) +trace-minimum-uncompressed-irreversible-history-blocks = 28000000 +# Keep 1 year total (compressed) +trace-minimum-irreversible-history-blocks = 365000000 +# Widen per-request scan window for private/trusted nodes +trace-max-block-range = 5000 +``` + +For a full-history archive node omit or set both retention options to `-1`. + +--- + +## HTTP API reference + +All endpoints accept `POST` with a JSON body. The base URL is +`/v1/trace_api/`. + +--- + +### get_block + +Retrieve the full action trace for a single block. + +**Endpoint:** `POST /v1/trace_api/get_block` + +**Request:** + +```json +{ "block_num": 1000 } +``` + +**Response (200):** + +```json +{ + "id": "000003e8...", + "number": 1000, + "previous_id": "000003e7...", + "status": "irreversible", + "timestamp": "2025-01-01T00:05:00.000Z", + "producer": "bp.one", + "transaction_mroot": "0000...0000", + "finality_mroot": "0000...0000", + "transactions": [ + { + "id": "abcd1234...", + "block_num": 1000, + "block_time": "2025-01-01T00:05:00.000Z", + "producer_block_id": "000003e8...", + "actions": [ + { + "action_ordinal": 1, + "creator_action_ordinal": 0, + "closest_unnotified_ancestor_action_ordinal": 0, + "global_sequence": 12345, + "recv_sequence": 55, + "auth_sequence": [["alice", 42]], + "code_sequence": 3, + "abi_sequence": 3, + "receiver": "sysio.token", + "account": "sysio.token", + "name": "transfer", + "authorization": [{ "actor": "alice", "permission": "active" }], + "data": "0000000000855c34...", + "return_value": "", + "account_ram_deltas": [], + "cpu_usage_us": 100, + "net_usage": 16, + "params": { + "from": "alice", + "to": "bob", + "quantity": "1.0000 SYS", + "memo": "payment" + } + } + ], + "cpu_usage_us": 200, + "net_usage_words": 16, + "signatures": ["SIG_K1_..."], + "transaction_header": { + "expiration": "2025-01-01T00:05:30", + "ref_block_num": 999, + "ref_block_prefix": 12345678, + "max_net_usage_words": 0, + "max_cpu_usage_ms": 0, + "delay_sec": 0 + } + } + ] +} +``` + +**Error responses:** + +| Code | Condition | +|------|-----------| +| 400 | `block_num` missing or not a number | +| 404 | Block not found in trace store | +| 500 | Internal error reading the trace store | + +--- + +### get_transaction_trace + +Retrieve the trace for a single transaction by its ID. + +**Endpoint:** `POST /v1/trace_api/get_transaction_trace` + +**Request:** + +```json +{ "id": "abcd1234ef567890abcd1234ef567890abcd1234ef567890abcd1234ef567890" } +``` + +The block number is resolved via the per-slice `trx_id` index (O(1)); no +block scanning is performed. + +**Response (200):** A single transaction object in the same shape as one +element of `transactions` in the `get_block` response (`id`, `block_num`, +`block_time`, `producer_block_id`, `actions`, `cpu_usage_us`, +`net_usage_words`, `signatures`, `transaction_header`). + +**Error responses:** + +| Code | Condition | +|------|-----------| +| 400 | `id` missing or malformed | +| 404 | Transaction not found in index, or block trace not found | +| 500 | Internal error reading the trace store | + +--- + +### get_actions + +Paginated search over action traces in a block range, with optional filters +on receiver, account (contract code), and action name. + +**Endpoint:** `POST /v1/trace_api/get_actions` + +**Request fields:** + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `block_num_start` | uint32 | `0` | First block to scan (inclusive). | +| `block_num_end` | uint32 | `UINT32_MAX` | Last block to scan (inclusive). Silently clamped server-side to `block_num_start + trace-max-block-range - 1`. The response reports the actual range scanned. | +| `receiver` | string | *(any)* | Filter: match `act.receiver`. | +| `account` | string | *(any)* | Filter: match `act.account` (the contract whose code ran). | +| `action` | string | *(any)* | Filter: match action name. | +| `include_notifications` | bool | `false` | Notification handling (see below). | + +**Notifications (`include_notifications`):** + +- `false` (default): when exactly one of `receiver` / `account` is + specified, the other is implicitly constrained to the same value — you + get the canonical execution only (`act.account == act.receiver == filter_value`). +- `true`: only the explicitly-specified filters apply. Notifications where + `act.account != act.receiver` are included. + +When both `receiver` and `account` are explicitly specified, the flag has +no effect — both filters apply literally. + +Returned actions within a transaction are sorted by `global_sequence` +(execution order), matching the behavior of `get_block` and chain_plugin's +`push_transaction` response. See the [Pagination guide](#pagination-guide) +for the cursor pattern. + +**Request example:** + +```json +{ + "block_num_start": 1, + "block_num_end": 10000, + "account": "sysio.token", + "action": "transfer" +} +``` + +**Response (200):** + +```json +{ + "block_num_start": 1, + "block_num_end": 1000, + "actions": [ + { + "action_ordinal": 1, + "creator_action_ordinal": 0, + "closest_unnotified_ancestor_action_ordinal": 0, + "global_sequence": 101, + "recv_sequence": 55, + "auth_sequence": [["alice", 42]], + "code_sequence": 3, + "abi_sequence": 3, + "receiver": "sysio.token", + "account": "sysio.token", + "name": "transfer", + "authorization": [{"actor": "alice", "permission": "active"}], + "data": "0000000000855c34...", + "return_value": "", + "account_ram_deltas": [], + "cpu_usage_us": 100, + "net_usage": 16, + "params": { + "from": "alice", + "to": "bob", + "quantity": "1.0000 SYS", + "memo": "payment" + }, + "trx_id": "abcd1234...", + "block_num": 1000, + "block_time": "2025-01-01T00:05:00.000Z", + "producer_block_id": "000003e8...", + "block_status": "irreversible", + "trx_cpu_usage_us": 200, + "trx_net_usage_words": 16 + } + ] +} +``` + +`block_num_start` and `block_num_end` on the response reflect the actual +range scanned (after clamping), so a client can detect a clamp and +resume pagination from `block_num_end + 1`. + +**Response fields:** + +| Field | Description | +|-------|-------------| +| `block_num_start` | First block number actually scanned. | +| `block_num_end` | Last block number actually scanned (after clamping). | +| `actions` | Array of matching action objects, ordered by `(block_num, global_sequence)`. | + +**Action object fields:** + +| Field | Description | +|-------|-------------| +| `action_ordinal` | Position of this action in the transaction's execution tree (1-based). | +| `creator_action_ordinal` | Ordinal of the action that created this one (0 for top-level actions). | +| `closest_unnotified_ancestor_action_ordinal` | Ordinal of the nearest ancestor whose receiver has not already been notified. | +| `global_sequence` | Monotonically increasing sequence number across the entire chain. | +| `recv_sequence` | Per-receiver sequence number. | +| `auth_sequence` | Array of `[actor, sequence]` pairs, one per authorizing account. | +| `code_sequence` | Number of times the contract's code has been updated up to this action. | +| `abi_sequence` | Number of times the contract's ABI has been updated up to this action. | +| `receiver` | The account that received (and may have processed) the action. | +| `account` | The contract account whose code was executed. | +| `name` | The action name. | +| `authorization` | Array of `{actor, permission}` objects. | +| `data` | Raw action payload as hex. | +| `return_value` | Raw return value as hex (empty string when none). | +| `account_ram_deltas` | Array of `{account, delta}` objects capturing RAM allocation changes. | +| `cpu_usage_us` | Producer-set CPU in microseconds for this action (present only for input/top-level actions). | +| `net_usage` | Producer-set NET usage in bytes for this action (present only for input/top-level actions). | +| `params` | ABI-decoded action payload (omitted when ABI unavailable or decode failed). | +| `return_data` | ABI-decoded return value (omitted when ABI unavailable or no return type defined). | +| `decode_error` | Error message; present only when ABI decoding failed and the response falls back to raw hex. | +| `trx_id` | ID of the transaction that contains this action. | +| `block_num` | Block number. | +| `block_time` | Block timestamp (ISO-8601). | +| `producer_block_id` | Block ID as reported by the producer (null for pending blocks). | +| `block_status` | Finality of this action's block: `"irreversible"` once the block is at or before LIB, `"pending"` otherwise. Mirrors the `status` field on `get_block`. A pending block can later be promoted to irreversible as LIB advances; consumers that gate on finality must re-poll. Operators that only want to serve already-final data can run nodeop with `read-mode = irreversible`, which causes every block returned by trace_api to carry `"irreversible"`. | +| `trx_cpu_usage_us` | Parent transaction's total CPU in microseconds. | +| `trx_net_usage_words` | Parent transaction's total NET usage in words (`ceil(net_usage / 8)`). | + +**Error responses:** + +| Code | Condition | +|------|-----------| +| 400 | Malformed request body, or `block_num_start > block_num_end`. | +| 500 | Internal error reading the trace store. | + +#### Receiver vs account + +Every SYSIO action has two account fields: + +- **`account`** — the contract whose code is executed (always the contract + that defines the action). +- **`receiver`** — the account receiving the notification. For the + originating action `receiver == account`. For inline notifications sent + to other accounts, `receiver != account`. + +A `sysio.token::transfer` from alice to bob produces three action traces +in the store: + +| global_seq | account | receiver | role | +|-----------|---------|----------|------| +| N | `sysio.token` | `sysio.token` | Canonical execution | +| N+1 | `sysio.token` | `alice` | Notification to sender | +| N+2 | `sysio.token` | `bob` | Notification to recipient | + +The default query (no `include_notifications`) implicitly constrains +`receiver == account` when you specify one of them, returning only the +canonical row. To see notifications, set `include_notifications: true`. + +--- + +### get_token_transfers + +Convenience wrapper around `get_actions` preset to return only token +`transfer` actions for a given contract. Uses +`receiver = account = token_contract, action = "transfer"` so exactly one +entry per transfer is returned (the canonical execution; inline +notifications are excluded). + +**Endpoint:** `POST /v1/trace_api/get_token_transfers` + +**Request fields:** + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `token_contract` | string | `sysio.token` | Contract account to filter on. | +| `block_num_start` | uint32 | `0` | First block to scan (inclusive). | +| `block_num_end` | uint32 | `UINT32_MAX` | Last block to scan (inclusive). Silently clamped to `block_num_start + trace-max-block-range - 1`. | + +**Request example:** + +```json +{ + "token_contract": "sysio.token", + "block_num_start": 1, + "block_num_end": 50000 +} +``` + +**Response (200):** + +```json +{ + "block_num_start": 1, + "block_num_end": 1000, + "transfers": [ + { + "global_sequence": 101, + "receiver": "sysio.token", + "account": "sysio.token", + "name": "transfer", + "authorization": [{"actor": "alice", "permission": "active"}], + "data": "...", + "return_value": "", + "params": { + "from": "alice", + "to": "bob", + "quantity": "1.0000 SYS", + "memo": "payment" + }, + "trx_id": "abcd1234...", + "block_num": 1000, + "block_time": "2025-01-01T00:05:00.000Z", + "producer_block_id": "000003e8...", + "block_status": "irreversible" + } + ] +} +``` + +The response uses `"transfers"` as the array key instead of `"actions"`. +`get_token_transfers` returns a **slim subset** of the fields that +`get_actions` returns — it omits execution-tree ordinals +(`action_ordinal`, `creator_action_ordinal`, +`closest_unnotified_ancestor_action_ordinal`), per-receipt sequence +numbers (`recv_sequence`, `auth_sequence`, `code_sequence`, +`abi_sequence`), `account_ram_deltas`, and the resource usage fields +(action-level `cpu_usage_us` / `net_usage` and trx-level +`trx_cpu_usage_us` / `trx_net_usage_words`). These are rarely useful for token-transfer +exchange/indexer workflows. If you need them, call `get_actions` with +`receiver = account = , action = "transfer"` instead. + +`block_status` IS retained -- exchanges crediting transfers need finality +just as much as general action consumers. See the `get_actions` field +table above for its semantics, including the irreversible-mode operator +note. + +**Error responses:** + +| Code | Condition | +|------|-----------| +| 400 | Malformed request body, or `block_num_start > block_num_end`. | +| 500 | Internal error reading the trace store. | + +--- + +## Pagination guide + +`get_actions` and `get_token_transfers` cap the per-request scan window at +`trace-max-block-range` blocks (default 1000). `block_num_end` is silently +clamped to `block_num_start + trace-max-block-range - 1` if the client asks +for more. The response always includes the actual `block_num_start` and +`block_num_end` scanned so the client can page reliably. + +Within that window, ALL matching actions are returned — there is no +per-result limit and no in-window cursor. + +To page across a wide range, advance `block_num_start` by the response's +`block_num_end + 1` each call: + +``` +# Page 1: blocks 1..1000 (assuming trace-max-block-range = 1000) +POST /v1/trace_api/get_actions +{ "account": "sysio.token", "action": "transfer", + "block_num_start": 1, "block_num_end": 1000000 } + +# Response: { "block_num_start": 1, "block_num_end": 1000, "actions": [...] } + +# Page 2: blocks 1001..2000 +POST /v1/trace_api/get_actions +{ "account": "sysio.token", "action": "transfer", + "block_num_start": 1001, "block_num_end": 1000000 } +``` + +Continue until `block_num_end` returned by the server equals the +requested `block_num_end` (no clamp happened), or until you catch up to +the chain head. Use `get_block` or out-of-band head-block knowledge to +know when to stop. + +Notes: +- Within each transaction, actions are sorted by `global_sequence` + (execution order, not schedule order). See "Receiver vs account" for + why this matters when an action queues both inlines and notifications. +- The maximum supported `trace-max-block-range` is 10,000. Raise it via + `config.ini` on private/trusted nodes; public nodes should typically + leave it at the default. + +--- + +## Exchange / indexer integration guide + +### Widening the per-request scan window + +Exchanges running their own private nodeop can raise +`trace-max-block-range` in `config.ini` to reduce the number of round +trips per backfill: + +```ini +# config.ini — safe on a private/trusted node +trace-max-block-range = 5000 +``` + +Values up to 10,000 are accepted. Larger windows produce larger responses +(which hit `http-max-response-time-ms` and `http-max-body-size` limits) +and tie up an HTTP thread for longer on busy contracts. Pick the largest +window that still returns in a reasonable time for your typical contract +activity. + +### Detecting deposits + +To find all incoming transfers to your account (`exchange1111`) on +`sysio.token`: + +```bash +curl -s -X POST http://127.0.0.1:8888/v1/trace_api/get_token_transfers \ + -H 'Content-Type: application/json' \ + -d '{ + "block_num_start": 100000, + "block_num_end": 100999 + }' | jq '.transfers[] | select(.params.to == "exchange1111")' +``` + +`get_token_transfers` with no additional filter returns one entry per +transfer across all accounts. Filter `params.to` client-side, or scan +with `get_actions` and post-filter as needed. + +The `receiver = account` preset guarantees that each on-chain transfer +appears exactly once regardless of how many accounts were notified +inline. + +### Non-system token contracts + +```bash +curl -s -X POST http://127.0.0.1:8888/v1/trace_api/get_token_transfers \ + -H 'Content-Type: application/json' \ + -d '{ "token_contract": "mytoken1111", "block_num_start": 1, "block_num_end": 9999 }' +``` + +### Watching for smart-contract activity + +```bash +# All canonical actions executed by a DEX contract in blocks 5000–6000 +curl -s -X POST http://127.0.0.1:8888/v1/trace_api/get_actions \ + -H 'Content-Type: application/json' \ + -d '{ "account": "my.dex", "block_num_start": 5000, "block_num_end": 6000 }' + +# Same, but including inline notifications the DEX sent to other accounts +curl -s -X POST http://127.0.0.1:8888/v1/trace_api/get_actions \ + -H 'Content-Type: application/json' \ + -d '{ "account": "my.dex", "block_num_start": 5000, "block_num_end": 6000, "include_notifications": true }' +``` + +### Inline actions + +Inline actions (e.g. `sysio.token::transfer` called from inside another +contract) appear as separate entries in `get_actions` results with their +own `global_sequence` values. The parent and child share the same +`trx_id`. Use `get_block` or `get_transaction_trace` if you need the full +causal tree. + +--- + +## ABI decoding + +When serving any trace endpoint the plugin attempts to decode the raw +`data` and `return_value` bytes of each action using the ABI captured in +`abi_log.log` at the point that action executed. + +- The ABI in effect when an action ran is used — not the current on-chain + ABI. This matters when a contract calls `setabi` mid-history: older + actions decode against the older schema. +- If the ABI is unavailable (contract never captured, ABI log missing, or + the action predates ABI capture on this node), `data` and + `return_value` are returned as raw hex. +- If ABI decoding throws (e.g., a schema/data mismatch in one contract), + the response includes a `decode_error` field and falls back to raw hex + for that action only. Unrelated actions in the same block are + unaffected. + +When decoded, `params` and `return_data` appear alongside the raw `data` +and `return_value` hex fields. + +See [ABI capture mechanics](#abi-capture-mechanics) for how the ABI log +is populated and the edge cases around same-transaction `setabi`. + +--- + +## Operations + +### Maintenance and retention + +#### Automatic retention + +Set `trace-minimum-irreversible-history-blocks` to the number of blocks +you want to retain past LIB. Slices that fall entirely before +`LIB - retention_blocks` are eligible for deletion. + +Set `trace-minimum-uncompressed-irreversible-history-blocks` similarly to +control the compression boundary. Slices are still accessible when +compressed but random-access reads may be slightly slower. + +#### Manual deletion + +Stop nodeop before deleting trace files manually. Delete the full set of +slice files for a given range (`trace_*`, `trace_index_*`, +`trace_blk_idx_*`, `trace_trx_idx_*`). Partial deletion (e.g. removing +only the trace file but not the index) will cause `bad_data_exception` +errors on the next startup. + +#### Snapshot restores + +After restoring from a snapshot, the trace store's recorded range may +not match the chain's new head. If chain head is within the recorded +range, replay overlaps existing slices silently. If there's a gap, the +startup continuity check aborts (see below). + +#### abi_log.log + +`abi_log.log` is append-only. If it is lost or corrupted, delete it and +restart nodeop. The plugin will rebuild it as new `setabi` transactions +are applied or previously-unseen contracts are touched (lazy current-ABI +fetch). Historical ABI lookup for actions before the loss will fall back +to raw hex. + +--- + +### Startup continuity check + +On the first `block_start` signal after plugin startup, the trace +store's recorded block range is compared against the chain's current +head, and the plugin chooses one of four outcomes: + +| Situation | Behavior | +|-----------|----------| +| No prior trace data (empty slice dir) | `info` log, fresh start; tracing begins at the current head. | +| Chain head is within `[first_recorded, last_recorded + 1]` (exact continuation OR overlap from a snapshot replay) | Silent — re-applied blocks naturally overwrite existing slice entries. | +| Chain head is **before** the first recorded block | Plugin throws; `error` log, **node shuts down**. | +| Chain head is **after** `last_recorded + 1` (forward gap) | Plugin throws; `error` log, **node shuts down**. | + +The shutdown is intentional: a gap means the trace store is no longer a +faithful continuous record of chain history, and silently accepting it +would let `get_block` / `get_transaction_trace` return inconsistent data +for blocks on either side of the gap. + +To recover, pick one: + +- **Load a snapshot whose chain head is within the existing recorded + range (or one block past it).** Replay covers the existing slice + entries; tracing continues with no gap. +- **Copy the missing slice files from another node** that has the + missing range, then restart. +- **Delete the trace directory and start fresh.** Tracing resumes from + the current chain head; old block traces are lost. + +The check fires only on the first `block_start` after the plugin loads, +so recovery actions take effect on the next startup. + +--- + +### Plugin variants + +Two plugin classes are registered: + +| Class | Purpose | +|-------|---------| +| `trace_api_plugin` | Full plugin: captures trace data AND exposes HTTP endpoints. Use this in production. | +| `trace_api_rpc_plugin` | HTTP-only: exposes endpoints against a trace directory written by another node. Use when separating the writer node from the query node. | + +Both accept the same configuration options. + +--- + +## Implementation details + +### On-disk layout + +All files live inside `trace-dir`. The directory is monitored by +`resource_monitor_plugin` when that plugin is loaded. + +#### Slice files + +Blocks are grouped into contiguous slices of `trace-slice-stride` blocks +each. Each slice is represented by five files that share a common range +suffix `-` (zero-padded to 10 digits): + +| File | Description | +|------|-------------| +| `trace_-.log` | Serialized `block_trace_v0` records (action data). | +| `trace_index_-.log` | Append-only metadata log of `block_entry_v0` and `lib_entry_v0` records. Source of truth; used as a fallback for `get_block` and to track LIB advancement within the slice. | +| `trace_blk_idx_-.log` | Block-offset sidecar. Enables O(1) `get_block` lookups regardless of the block's position within the slice. | +| `trace_trx_idx_-.log` | Transaction-id hash index. | +| `trace_recv_bloom_-.log` | Per-slice bloom filter over action receivers and (receiver, action) pairs. `get_actions` consults it to skip slices that cannot contain the requested filter value. | + +When a slice is compressed the trace file is replaced by: + +| File | Description | +|------|-------------| +| `trace_-.clog` | zlib-compressed trace data with embedded seek points for random access. | + +The metadata, block-offset, trx-id, and receiver-bloom sidecars are not compressed - they are already compact and need random access. + +**Example** (10 000-block stride, blocks 0–29 999): + +``` +traces/ + trace_0000000000-0000010000.log + trace_index_0000000000-0000010000.log + trace_blk_idx_0000000000-0000010000.log + trace_trx_idx_0000000000-0000010000.log + trace_recv_bloom_0000000000-0000010000.log + trace_0000010000-0000020000.log + trace_index_0000010000-0000020000.log + trace_blk_idx_0000010000-0000020000.log + trace_trx_idx_0000010000-0000020000.log + trace_recv_bloom_0000010000-0000020000.log + trace_0000020000-0000030000.clog <- compressed + trace_index_0000020000-0000030000.log + trace_blk_idx_0000020000-0000030000.log + trace_trx_idx_0000020000-0000030000.log + trace_recv_bloom_0000020000-0000030000.log + abi_log.log +``` + +#### Block-offset index + +`trace_blk_idx_-.log` is a flat fixed-size array of 64-bit +trace-log offsets, one entry per block in the slice, used by +`/v1/trace_api/get_block` for O(1) block lookups. + +- **Header** (16 bytes): magic `BLIX`, version 1, slice width, reserved. +- **Slots** (8 bytes each): `offset + 1` into `trace_-.log`, + or 0 when the slot is empty. The `+1` encoding reserves 0 as an empty + sentinel since a block's trace data can legitimately live at offset 0. + +The sidecar is written synchronously alongside the metadata log as each +block is persisted. Forks that re-apply the same block number overwrite +the slot naturally. If the sidecar is missing or reports an empty slot, +`get_block` falls back to scanning the metadata log. + +#### Transaction-id index + +`trace_trx_idx_-.log` is a compact open-addressing hash +table (load factor ≤ 0.5, linear probing) that maps a 64-bit prefix of +a transaction SHA-256 to the block number containing that transaction. + +- **Header** (16 bytes): magic `TRIX`, version 1, bucket_count, reserved. +- **Buckets** (16 bytes each): `prefix64 (u64)` + `block_num (u32)` + + `reserved (u32)`. Empty slots have `block_num == 0`. + +On hash hit, the candidate block is scanned for a full trx_id match to +defeat 64-bit prefix collisions (a miss on the full compare re-probes +the hash table). The index is built once per slice when the slice's +last block becomes irreversible. Queries against +`/v1/trace_api/get_transaction_trace` use this index for O(1) +`trx_id → block_num` resolution. + +#### Receiver bloom sidecar + +`trace_recv_bloom_-.log` is a per-slice pair of bloom +filters used by `/v1/trace_api/get_actions` to skip slices that cannot +contain the requested filter value. Without it, a request for a +rarely-active receiver across a wide block range would have to fetch +and scan every block in every slice; with it, slices that do not +contain the receiver are dismissed by a single O(1) probe and the +scanner advances `block_num` to the next slice boundary. + +Contents: + +- **Receiver filter** - `boost::bloom::filter` over + `action_trace_v0::receiver` (name stored as its 64-bit value). +- **Composite filter** - `boost::bloom::filter` over a + deterministic pack of `(receiver, action)` pairs. Probed when the + caller supplies both a `receiver` and an `action` filter, giving an + extra selectivity win for `get_token_transfers`-style lookups. + +Format: + +``` +Header (40 bytes): + magic (u32) = 0x42524957 ("WIRB" on little-endian) + version (u32) = 1 + k_hashes (u32) = 7 + n_recv (u32) - distinct receivers inserted + n_recv_action (u32) - distinct (receiver, action) pairs inserted + reserved (u32) + recv_capacity_bits (u64) - filter bit count + recv_action_capacity_bits (u64) +Body: + receiver bloom bits (recv_capacity_bits / 8 bytes) + (receiver, action) bits (recv_action_capacity_bits / 8 bytes) + crc32 (u32) over header + body +``` + +Filters are sized for a 1% false-positive rate with a minimum floor of +32 items to avoid degenerate tiny bit arrays on sparse slices. Total +sidecar size is dominated by the number of distinct receivers seen in +the slice (the composite filter scales with (receiver, action) pairs, +typically 2-3x the receiver count). A busy mainnet slice sits around +10 KB; an empty slice produces a minimal always-miss file. + +Build model: the bloom is built by `slice_directory::build_recv_bloom` +on the same schedule as the trx_id index - when the slice becomes +fully irreversible (its last block is below LIB), the maintenance pass +opens the slice's uncompressed data log, streams through each +`block_trace_v0` record in order, and inserts every action's receiver +(and `(receiver, action)` pair) into two +`boost::unordered_flat_set` accumulators. The filters are +then sized, populated, and written (temp + rename). Deferring the +write to irreversibility means forks cannot corrupt an already-written +sidecar: a fork cannot reach back across LIB, so the slice's data log +is final by the time the bloom is built. Fork re-writes leave stale +`block_trace_v0` records in the data log (the blk_offset sidecar +points only to the canonical offset); the stream-scan visits those +stale records too, so the bloom ends up as a superset of the canonical +receivers. That is safe - bloom allows false positives, and a +forked-out receiver probing as present just means the query scan +visits that slice and finds no canonical match. + +Query model: `get_actions` probes the bloom once per distinct slice in +the queried block range. A negative probe is authoritative and the +entire slice is skipped with no `get_block` call. A positive probe +(or a missing/corrupt sidecar) falls through to the existing scan. +Unfiltered queries (no `receiver`, no `account`, no `action`) do not +consult the bloom. + +Retention: `slice_directory::run_maintenance_tasks` removes the bloom +sidecar alongside the slice's other files when the slice ages out of +`minimum_irreversible_history_blocks`. + +##### Slice stride vs. query latency + +The bloom is per-slice, so on a **positive** probe the scanner still +reads every block in the slice before returning. That cost is bounded +by `trace-slice-stride`: larger strides mean more work per hit-slice, +smaller strides mean finer skip granularity at the cost of more +sidecar files. + +Rough per-slice scan cost on SSD with slices in the OS page cache +(dominated by deserializing `block_trace_v0` records): + +| `trace-slice-stride` | Scan after bloom hit (warm) | Scan (compressed, cold) | Files per slice | +|----------------------|-----------------------------|--------------------------|-----------------| +| 10000 (default) | ~50-100 ms | ~200-500 ms | 5 (+ .clog) | +| 2500 | ~15-25 ms | ~50-125 ms | 5 (+ .clog) | +| 1000 | ~5-10 ms | ~20-50 ms | 5 (+ .clog) | + +Smaller strides approximate a finer-grained bloom skip (bigger +stride-shrink == more precise "miss" resolution) at the cost of +linearly more files on disk. At stride 1000 a year of busy-chain +history lands around 380 000 slice files total, which modern +filesystems handle but makes directory listings slow. + +Queries that cluster near head (the common case) visit only a handful +of slices regardless of stride, so stride mostly affects deep-history +lookups on sparse accounts. For nodes that primarily serve recent +queries the default is fine; nodes that expect frequent deep scans can +benefit from dropping to 2500 or lower. + +`trace-slice-stride` takes effect on nodeop restart. Existing slices +keep their old `-` naming; new slices written after the +restart use the new stride. Nothing is migrated - query paths read +whatever sidecars exist for the slice covering a given block. + +#### ABI log + +`abi_log.log` is an append-only file that persists the ABI published by +each contract account across all `setabi` transactions observed since +the node started (or since the file was first written). + +Format: + +``` +Header (16 bytes): magic "ABIL" (u32), version 1 (u32), reserved (u64) +Records (repeated until EOF): + account (u64) + global_seq (u64) + blob_size (u64) + blob_bytes (blob_size bytes) + crc32 (u32) over (account, global_seq, blob_size, blob_bytes) +``` + +An in-memory index keyed by `(account, global_sequence)` is built at +startup by walking the file record-by-record and validating each CRC. +Runtime lookups go through the index; the matching blob is then read +from the file via `pread()`. Appends stream new records to the end of +the file under a mutex, with no rewrite of existing records. + +Writes are not fsync'd; the on-disk tail may lose the last few records +on a kernel crash. On startup the recovery scan detects torn or +CRC-mismatched records and truncates the file at the first bad one — +any lost records are rebuilt the next time their contract is touched +(via an observed `setabi` or the lazy current-ABI fetch). + +--- + +### ABI capture mechanics + +ABI records enter `abi_log.log` through two paths: + +1. **`setabi` observation** — every time a transaction contains a + `sysio::setabi` action, the plugin decodes the action's payload and + records `(target_account, setabi.global_sequence, abi_bytes)`. This + gives exact ABI-version boundaries at the granularity of + global_sequence. + +2. **Lazy current-ABI fetch** — on the first action observed for an + account that has no prior `abi_log` entry, the plugin reads the + account's current ABI from the chain DB and records it at + `global_sequence = 0`. The `0` is a sentinel meaning "ABI as of + first observation; exact recorded sequence unknown." Lookups step + back from the query `global_sequence` to the largest recorded entry + ≤ query, so a 0 sentinel matches any action of that account that + predates the first recorded real setabi. + +#### Same-transaction `setabi` caveat + +If the plugin observes a contract for the *first* time via a transaction +that *also* contains a `setabi` for that same contract, actions on that +contract which executed *before* the setabi within that transaction +cannot be decoded and are returned as raw hex. The pre-setabi ABI is no +longer reachable from the post-apply chain state (by the time the +applied_transaction signal fires, the chain DB already reflects the new +ABI), so the plugin deliberately does not record the post-apply (new) +ABI as the contract's pre-observation baseline — doing so would decode +pre-setabi actions with the wrong schema. + +Once the contract has been observed at least once (via any earlier +transaction, or via a setabi-free transaction), later same-trx setabis +do not have this limitation: pre-setabi actions decode correctly with +the previously-recorded ABI. diff --git a/tests/PerformanceHarness/performance_test_basic.py b/tests/PerformanceHarness/performance_test_basic.py index f56a1833f5..3d647da69b 100755 --- a/tests/PerformanceHarness/performance_test_basic.py +++ b/tests/PerformanceHarness/performance_test_basic.py @@ -300,7 +300,7 @@ def fileOpenMode(self, filePath) -> str: return append_write def isOnBlockTransaction(self, transaction): - if transaction['actions'][0]['account'] != 'sysio' or transaction['actions'][0]['action'] != 'onblock': + if transaction['actions'][0]['account'] != 'sysio' or transaction['actions'][0]['name'] != 'onblock': return False return True diff --git a/tests/TestHarness/Cluster.py b/tests/TestHarness/Cluster.py index 591f99cc4c..f924becf3e 100644 --- a/tests/TestHarness/Cluster.py +++ b/tests/TestHarness/Cluster.py @@ -283,11 +283,9 @@ def launch(self, pnodes=1, unstartedNodes=0, totalNodes=1, prodCount=21, topo="m if Utils.Debug and "--contracts-console" not in extraNodeopArgs: nodeopArgs += " --contracts-console" if PFSetupPolicy.hasPreactivateFeature(pfSetupPolicy): - nodeopArgs += " --plugin sysio::producer_api_plugin" + nodeopArgs += " --plugin sysio::producer_api_plugin " if prodsEnableTraceApi: nodeopArgs += " --plugin sysio::trace_api_plugin " - if extraNodeopArgs.find("--trace-rpc-abi") == -1: - nodeopArgs += " --trace-no-abis " httpMaxResponseTimeSet = False if specificExtraNodeopArgs is not None: assert(isinstance(specificExtraNodeopArgs, dict)) diff --git a/tests/TestHarness/launcher.py b/tests/TestHarness/launcher.py index 821abdab9c..67bb9e1ee3 100644 --- a/tests/TestHarness/launcher.py +++ b/tests/TestHarness/launcher.py @@ -565,8 +565,7 @@ def construct_command_line(self, instance: nodeDefinition): '--p2p-peer-address', '--p2p-auto-bp-peer', '--peer-key', '--peer-private-key', # producer_plugin '--producer-name', '--signature-provider', '--greylist-account', '--disable-subjective-account-billing', - # trace_api_plugin - '--trace-rpc-abi'] + ] for arg in specificList: if '-' in arg and arg not in repeatable: if arg in sysdcmd: diff --git a/tests/cli_test.py b/tests/cli_test.py index bee578999b..e40ef07dd0 100755 --- a/tests/cli_test.py +++ b/tests/cli_test.py @@ -353,7 +353,7 @@ def abi_file_with_nodeop_test(): os.makedirs(data_dir, exist_ok=True) walletMgr = WalletMgr(True) walletMgr.launch() - cmd = "./programs/nodeop/nodeop -e -p sysio --signature-provider wire-1,wire,wire,SYS6MRyAjQq8ud7hVNYcfnVPJqcVpscN5So8BhtHuGYqET5GDW5CV,KEY:5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3 --plugin sysio::trace_api_plugin --trace-no-abis --plugin sysio::producer_plugin --plugin sysio::producer_api_plugin --plugin sysio::chain_api_plugin --plugin sysio::chain_plugin --plugin sysio::http_plugin --access-control-allow-origin=* --http-validate-host=false --max-transaction-time=-1 --resource-monitor-not-shutdown-on-threshold-exceeded " + "--data-dir " + data_dir + " --config-dir " + data_dir + cmd = "./programs/nodeop/nodeop -e -p sysio --signature-provider wire-1,wire,wire,SYS6MRyAjQq8ud7hVNYcfnVPJqcVpscN5So8BhtHuGYqET5GDW5CV,KEY:5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3 --plugin sysio::trace_api_plugin --plugin sysio::producer_plugin --plugin sysio::producer_api_plugin --plugin sysio::chain_api_plugin --plugin sysio::chain_plugin --plugin sysio::http_plugin --access-control-allow-origin=* --http-validate-host=false --max-transaction-time=-1 --resource-monitor-not-shutdown-on-threshold-exceeded " + "--data-dir " + data_dir + " --config-dir " + data_dir node = Node('localhost', 8888, nodeId, data_dir=Path(data_dir), config_dir=Path(data_dir), cmd=shlex.split(cmd), launch_time=datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S'), walletMgr=walletMgr) if not node or not Utils.waitForBool(node.checkPulse, timeout=15): Utils.Print("ERROR: node doesn't appear to be running...") diff --git a/tests/nodeop_lib_test.py b/tests/nodeop_lib_test.py index c35d075bba..ec17111c7b 100755 --- a/tests/nodeop_lib_test.py +++ b/tests/nodeop_lib_test.py @@ -62,8 +62,7 @@ if localTest and not dontLaunch: Print("Stand up cluster") - abs_path = os.path.abspath(os.getcwd() + '/contracts/sysio.token/sysio.token.abi') - traceNodeopArgs=" --http-max-response-time-ms 990000 --trace-rpc-abi sysio.token=" + abs_path + traceNodeopArgs=" --http-max-response-time-ms 990000" extraNodeopArgs=traceNodeopArgs + " --plugin sysio::prometheus_plugin --database-map-mode mapped_private " specificNodeopInstances={0: "bin/nodeop"} if cluster.launch(totalNodes=total_nodes, pnodes=pnodes, topo=topo, prodCount=prodCount, activateIF=activateIF, onlyBios=onlyBios, dontBootstrap=dontBootstrap, extraNodeopArgs=extraNodeopArgs, specificNodeopInstances=specificNodeopInstances) is False: @@ -290,8 +289,8 @@ amountVal=None key="" try: - key = "[actions][0][action]" - typeVal = transaction["actions"][0]["action"] + key = "[actions][0][name]" + typeVal = transaction["actions"][0]["name"] key = "[actions][0][params][quantity]" amountVal = transaction["actions"][0]["params"]["quantity"] amountVal = int(decimal.Decimal(amountVal.split()[0]) * 10000) diff --git a/tests/nodeop_run_test.py b/tests/nodeop_run_test.py index 6bbe871592..641d8926e8 100755 --- a/tests/nodeop_run_test.py +++ b/tests/nodeop_run_test.py @@ -60,8 +60,7 @@ if localTest and not dontLaunch: Print("Stand up cluster") - abs_path = os.path.abspath(os.getcwd() + '/contracts/sysio.token/sysio.token.abi') - traceNodeopArgs=" --http-max-response-time-ms 990000 --trace-rpc-abi sysio.token=" + abs_path + traceNodeopArgs=" --http-max-response-time-ms 990000" extraNodeopArgs=traceNodeopArgs + " --plugin sysio::prometheus_plugin --database-map-mode mapped_private " specificNodeopInstances={0: "bin/nodeop"} if cluster.launch(totalNodes=2, prodCount=prodCount, activateIF=activateIF, onlyBios=onlyBios, dontBootstrap=dontBootstrap, extraNodeopArgs=extraNodeopArgs, specificNodeopInstances=specificNodeopInstances) is False: @@ -288,8 +287,8 @@ amountVal=None key="" try: - key = "[actions][0][action]" - typeVal = transaction["actions"][0]["action"] + key = "[actions][0][name]" + typeVal = transaction["actions"][0]["name"] key = "[actions][0][params][quantity]" amountVal = transaction["actions"][0]["params"]["quantity"] amountVal = int(decimal.Decimal(amountVal.split()[0]) * 10000) @@ -300,6 +299,45 @@ if typeVal != "transfer" or amountVal != 975311: errorExit("FAILURE - get transaction trans_id failed: %s %s %s" % (transId, typeVal, amountVal), raw=True) + # ------------------------------------------------------------------------- + # Verify trace_api get_actions and get_token_transfers endpoints + # ------------------------------------------------------------------------- + blockNum = transaction["block_num"] + + Print("Verify trace_api get_actions returns transfer with decoded params") + actResult = node.processUrllibRequest("trace_api", "get_actions", { + "block_num_start": blockNum, + "block_num_end": blockNum, + "account": "sysio.token", + "action": "transfer" + }, silentErrors=False, exitOnError=True) + assert actResult["code"] == 200, f"get_actions returned HTTP {actResult['code']}" + actionsPayload = actResult["payload"] + matchingActions = [a for a in actionsPayload["actions"] if a["trx_id"] == transId] + assert len(matchingActions) >= 1, \ + f"get_actions: expected at least one action for trxid {transId}, got: {actionsPayload}" + assert "params" in matchingActions[0], \ + f"get_actions: expected decoded 'params' field, got: {matchingActions[0]}" + assert "quantity" in matchingActions[0]["params"], \ + f"get_actions: expected 'quantity' in params, got: {matchingActions[0]['params']}" + + Print("Verify trace_api get_token_transfers returns exactly one entry per transfer") + xferResult = node.processUrllibRequest("trace_api", "get_token_transfers", { + "block_num_start": blockNum, + "block_num_end": blockNum + }, silentErrors=False, exitOnError=True) + assert xferResult["code"] == 200, f"get_token_transfers returned HTTP {xferResult['code']}" + xfersPayload = xferResult["payload"] + matchingXfers = [t for t in xfersPayload["transfers"] if t["trx_id"] == transId] + assert len(matchingXfers) == 1, \ + f"get_token_transfers: expected exactly 1 entry for trxid {transId} (receiver filter should exclude inline notifications), got {len(matchingXfers)}: {matchingXfers}" + assert matchingXfers[0]["receiver"] == "sysio.token", \ + f"get_token_transfers: expected receiver 'sysio.token', got: {matchingXfers[0]['receiver']}" + assert "params" in matchingXfers[0], \ + f"get_token_transfers: expected decoded 'params', got: {matchingXfers[0]}" + assert "quantity" in matchingXfers[0]["params"], \ + f"get_token_transfers: expected 'quantity' in params, got: {matchingXfers[0]['params']}" + Print("Currency Contract Tests") Print("verify no contract in place") Print("Get code hash for account %s" % (currencyAccount.name)) diff --git a/tests/plugin_http_api_test.py b/tests/plugin_http_api_test.py index 094a55d47c..3882e15d29 100755 --- a/tests/plugin_http_api_test.py +++ b/tests/plugin_http_api_test.py @@ -92,7 +92,7 @@ def startEnv(self) : "net_api_plugin", "producer_plugin", "producer_api_plugin", "chain_api_plugin", "http_plugin", "db_size_api_plugin", "prometheus_plugin"] nodeop_plugins = "--plugin sysio::" + " --plugin sysio::".join(plugin_names) - nodeop_flags = (" --data-dir=%s --config-dir=%s --trace-dir=%s --trace-no-abis --access-control-allow-origin=%s " + nodeop_flags = (" --data-dir=%s --config-dir=%s --trace-dir=%s --access-control-allow-origin=%s " "--signature-provider wire-1,wire,wire,SYS6MRyAjQq8ud7hVNYcfnVPJqcVpscN5So8BhtHuGYqET5GDW5CV,KEY:5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3 " "--contracts-console --http-validate-host=%s --verbose-http-errors --max-transaction-time -1 --abi-serializer-max-time-ms 30000 --http-max-response-time-ms 30000 " "--p2p-peer-address localhost:9011 --resource-monitor-not-shutdown-on-threshold-exceeded ") % (self.data_dir, self.config_dir, self.data_dir, "\'*\'", "false") diff --git a/tests/resource_monitor_plugin_test.py b/tests/resource_monitor_plugin_test.py index ff641b3a0c..97e70b7719 100755 --- a/tests/resource_monitor_plugin_test.py +++ b/tests/resource_monitor_plugin_test.py @@ -178,9 +178,9 @@ def testAll(): testCommon("Resmon not enabled: no arguments", "", ["interval set to 2", "threshold set to 90", "Shutdown flag when threshold exceeded set to true", "snapshots's file system to be monitored", "blocks's file system to be monitored", "state's file system to be monitored"]) # default arguments with registered directories - testCommon("Resmon not enabled: Producer, Chain, State History and Trace Api", "--plugin sysio::state_history_plugin --state-history-dir=/tmp/state-history --disable-replay-opts --plugin sysio::trace_api_plugin --trace-dir=/tmp/trace --trace-no-abis", ["interval set to 2", "threshold set to 90", "Shutdown flag when threshold exceeded set to true", "snapshots's file system to be monitored", "blocks's file system to be monitored", "state's file system to be monitored", "state-history's file system to be monitored", "trace's file system to be monitored"]) + testCommon("Resmon not enabled: Producer, Chain, State History and Trace Api", "--plugin sysio::state_history_plugin --state-history-dir=/tmp/state-history --disable-replay-opts --plugin sysio::trace_api_plugin --trace-dir=/tmp/trace", ["interval set to 2", "threshold set to 90", "Shutdown flag when threshold exceeded set to true", "snapshots's file system to be monitored", "blocks's file system to be monitored", "state's file system to be monitored", "state-history's file system to be monitored", "trace's file system to be monitored"]) - testCommon("Resmon enabled: Producer, Chain, State History and Trace Api", "--plugin sysio::resource_monitor_plugin --plugin sysio::state_history_plugin --state-history-dir=/tmp/state-history --disable-replay-opts --plugin sysio::trace_api_plugin --trace-dir=/tmp/trace --trace-no-abis --resource-monitor-space-threshold=80 --resource-monitor-interval-seconds=3", ["snapshots's file system to be monitored", "blocks's file system to be monitored", "state's file system to be monitored", "state-history's file system to be monitored", "trace's file system to be monitored", "threshold set to 80", "interval set to 3", "Shutdown flag when threshold exceeded set to true"]) + testCommon("Resmon enabled: Producer, Chain, State History and Trace Api", "--plugin sysio::resource_monitor_plugin --plugin sysio::state_history_plugin --state-history-dir=/tmp/state-history --disable-replay-opts --plugin sysio::trace_api_plugin --trace-dir=/tmp/trace --resource-monitor-space-threshold=80 --resource-monitor-interval-seconds=3", ["snapshots's file system to be monitored", "blocks's file system to be monitored", "state's file system to be monitored", "state-history's file system to be monitored", "trace's file system to be monitored", "threshold set to 80", "interval set to 3", "Shutdown flag when threshold exceeded set to true"]) # Only test minimum warning threshold (i.e. 6) to trigger warning as much as possible testInterval("Resmon enabled: set warning interval", diff --git a/tests/trace_plugin_test.py b/tests/trace_plugin_test.py index 63ad70854a..d9ec626bbd 100755 --- a/tests/trace_plugin_test.py +++ b/tests/trace_plugin_test.py @@ -20,8 +20,7 @@ class TraceApiPluginTest(unittest.TestCase): # start kiod and nodeop def startEnv(self) : account_names = ["alice", "bob", "charlie"] - abs_path = os.path.abspath(os.getcwd() + '/contracts/sysio.token/sysio.token.abi') - extraNodeopArgs = " --verbose-http-errors --trace-slice-stride 10000 --trace-rpc-abi sysio.token=" + abs_path + extraNodeopArgs = " --verbose-http-errors --trace-slice-stride 10000" extraNodeopArgs+=" --production-pause-vote-timeout-ms 0" self.cluster.launch(totalNodes=2, activateIF=True, extraNodeopArgs=extraNodeopArgs) self.walletMgr.launch() diff --git a/tools/cluster_manager.py b/tools/cluster_manager.py index b453e69e47..6ccef013af 100755 --- a/tools/cluster_manager.py +++ b/tools/cluster_manager.py @@ -254,7 +254,6 @@ def _create_cluster( " --contracts-console" " --plugin sysio::producer_api_plugin" " --plugin sysio::trace_api_plugin" - " --trace-no-abis" " --http-max-response-time-ms 990000" ) args_arr.extend(["--nodeop", nodeop_args]) diff --git a/tutorials/bios-boot-tutorial/bios-boot-tutorial.py b/tutorials/bios-boot-tutorial/bios-boot-tutorial.py index 996eb1e32a..95128b4c85 100755 --- a/tutorials/bios-boot-tutorial/bios-boot-tutorial.py +++ b/tutorials/bios-boot-tutorial/bios-boot-tutorial.py @@ -94,7 +94,7 @@ def startNode(nodeIndex, account): run('mkdir -p ' + dir) otherOpts = ''.join(list(map(lambda i: ' --p2p-peer-address localhost:' + str(9000 + i), range(nodeIndex)))) if not nodeIndex: otherOpts += ( - ' --plugin sysio::trace_api_plugin --trace-no-abis' + ' --plugin sysio::trace_api_plugin' ) # if SVN blsFinKeys cmd = (