diff --git a/tests/PerformanceHarness/log_reader.py b/tests/PerformanceHarness/log_reader.py index be11a9f472..de39d200ac 100644 --- a/tests/PerformanceHarness/log_reader.py +++ b/tests/PerformanceHarness/log_reader.py @@ -144,11 +144,16 @@ def calcdTimeEpoch(self): @timestamp.setter def timestamp(self, time: str): - self._timestamp = time[:-1] + # Sources differ on the trailing 'Z': trace_api/get_block emits ISO8601 with a 'Z' + # suffix, while chain/get_block_info serializes block_timestamp_type via fc's + # to_iso_string() with no zone marker. Strip 'Z' only when it is present so we keep + # full sub-second precision (e.g. '.500') regardless of which endpoint produced the + # value, and so parsing tolerates either shape. # When we no longer support Python 3.6, would be great to update to use this - # self._calcdTimeEpoch = datetime.fromisoformat(time[:-1]).timestamp() - #Note block timestamp formatted like: '2022-09-30T16:48:13.500Z', but 'Z' is not part of python's recognized iso format, so strip it off the end - self._calcdTimeEpoch = datetime.strptime(time[:-1], "%Y-%m-%dT%H:%M:%S.%f").timestamp() + # self._calcdTimeEpoch = datetime.fromisoformat(stripped).timestamp() + stripped = time[:-1] if time.endswith('Z') else time + self._timestamp = stripped + self._calcdTimeEpoch = datetime.strptime(stripped, "%Y-%m-%dT%H:%M:%S.%f").timestamp() @timestamp.deleter def timestamp(self): diff --git a/tests/PerformanceHarness/performance_test_basic.py b/tests/PerformanceHarness/performance_test_basic.py index 3d647da69b..0d27791008 100755 --- a/tests/PerformanceHarness/performance_test_basic.py +++ b/tests/PerformanceHarness/performance_test_basic.py @@ -305,24 +305,71 @@ def isOnBlockTransaction(self, transaction): return True def queryBlockTrxData(self, node, blockDataPath, blockTrxDataPath, startBlockNum, endBlockNum): + # Use trace_api/get_actions for trx-level data and chain/get_block_info for block metadata. + # The old trace_api/get_block serialized the entire block (including base58-encoded + # signatures - OpenSSL BN_div alloc storm) even though the harness only needs trx id, + # block num/time, cpu/net usage, and the block header. + # Filter by the configured action name (transfer/cpu/ram/net/newaccount/doit/...) so + # onblock and other unrelated trxs are skipped server-side without base58 work. + # Per-trx cpu/net come from trx_cpu_usage_us / trx_net_usage_words on each action variant + # (the parent transaction's totals); the action-level cpu_usage_us / net_usage would + # undercount multi-action trxs and use different units (action net_usage is bytes; trx + # net_usage_words is ceil(bytes / 8)). + # Block finality (irreversible/pending) comes from block_status on each action -- trace_api + # is the single source of truth so we never mix in a chain/get_info LIB read that could + # disagree with what the trace data reflects. For blocks where the action filter dropped + # everything (typically the leading/trailing ramp window), fall back to trace_api/get_block + # for that one block: empty blocks carry no harness-action trxs so the payload is small. + actionFilter = None + if getattr(self, 'userTrxDataDict', None): + cfgActions = self.userTrxDataDict.get('actions') or [] + if cfgActions: + actionFilter = cfgActions[0].get('actionName') for blockNum in range(startBlockNum, endBlockNum + 1): blockCpuTotal, blockNetTotal, blockTransactionTotal = 0, 0, 0 - block = node.processUrllibRequest("trace_api", "get_block", {"block_num":blockNum}, silentErrors=False, exitOnError=True) + blockStatus = None + blockInfo = node.processUrllibRequest("chain", "get_block_info", {"block_num":blockNum}, silentErrors=False, exitOnError=True) + actionsQuery = {"block_num_start": blockNum, "block_num_end": blockNum} + if actionFilter: + actionsQuery["action"] = actionFilter + actionsResp = node.processUrllibRequest("trace_api", "get_actions", actionsQuery, silentErrors=False, exitOnError=True) btdf_append_write = self.fileOpenMode(blockTrxDataPath) with open(blockTrxDataPath, btdf_append_write) as trxDataFile: - for trx in block['payload']['transactions']: - if not self.isOnBlockTransaction(trx): - trx_data = trxData(blockNum=trx["block_num"], cpuUsageUs=trx["cpu_usage_us"], - netUsageUs=trx["net_usage_words"], blockTime=trx["block_time"]) - self.data.trxDict.update(dict([(trx["id"], trx_data)])) - [ trxDataFile.write(f"{trx['id']},{trx['block_num']},{trx['block_time']},{trx['cpu_usage_us']},{trx['net_usage_words']},{trx['actions']}\n") ] - blockCpuTotal += trx["cpu_usage_us"] - blockNetTotal += trx["net_usage_words"] - blockTransactionTotal += 1 - block_data = blockData(blockId=block["payload"]["id"], blockNum=block['payload']['number'], + seen = set() # one trxData entry per trx even if multiple actions match the filter + for action in actionsResp['payload']['actions']: + # If no action filter configured, still skip onblock explicitly. + if not actionFilter and action.get('account') == 'sysio' and action.get('name') == 'onblock': + continue + trxId = action['trx_id'] + if trxId in seen: + continue + seen.add(trxId) + # All actions in a block share the same block_status; capture once. + if blockStatus is None: + blockStatus = action.get('block_status') + cpu = action.get('trx_cpu_usage_us', 0) or 0 + net = action.get('trx_net_usage_words', 0) or 0 + trx_data = trxData(blockNum=action['block_num'], cpuUsageUs=cpu, + netUsageUs=net, blockTime=action['block_time']) + self.data.trxDict.update({trxId: trx_data}) + trxDataFile.write(f"{trxId},{action['block_num']},{action['block_time']},{cpu},{net},\n") + blockCpuTotal += cpu + blockNetTotal += net + blockTransactionTotal += 1 + if blockStatus is None: + # No harness-relevant action in this block. Fall back to trace_api/get_block so + # the row's status still comes from trace_api's data log rather than mixing in a + # different source. Cost is bounded -- empty/onblock-only blocks have no trx + # signatures to base58-encode. + blockResp = node.processUrllibRequest("trace_api", "get_block", {"block_num": blockNum}, + silentErrors=False, exitOnError=True) + blockStatus = blockResp['payload'].get('status', 'unknown') + block_data = blockData(blockId=blockInfo["payload"]["id"], + blockNum=blockInfo['payload']['block_num'], transactions=blockTransactionTotal, net=blockNetTotal, cpu=blockCpuTotal, - producer=block["payload"]["producer"], status=block["payload"]["status"], - _timestamp=block["payload"]["timestamp"]) + producer=blockInfo["payload"]["producer"], + status=blockStatus, + _timestamp=blockInfo["payload"]["timestamp"]) self.data.blockList.append(block_data) self.data.blockDict[str(blockNum)] = block_data bdf_append_write = self.fileOpenMode(blockDataPath)