diff --git a/.vscode/cspell.json b/.vscode/cspell.json index 4f298a27d7..042ece778f 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -226,6 +226,7 @@ "PCERT", "PBYTE", "pdbs", + "perfstress", "phoebusm", "Piotrowski", "pkcs", diff --git a/sdk/core/perf/CMakeLists.txt b/sdk/core/perf/CMakeLists.txt index 881774cb98..a138164474 100644 --- a/sdk/core/perf/CMakeLists.txt +++ b/sdk/core/perf/CMakeLists.txt @@ -18,9 +18,11 @@ set( inc/azure/perf/argagg.hpp inc/azure/perf/base_test.hpp inc/azure/perf/dynamic_test_options.hpp + inc/azure/perf/latency_stats.hpp inc/azure/perf/options.hpp inc/azure/perf/program.hpp inc/azure/perf/random_stream.hpp + inc/azure/perf/result_output.hpp inc/azure/perf/test.hpp inc/azure/perf/test_metadata.hpp inc/azure/perf/test_options.hpp @@ -30,9 +32,11 @@ set( AZURE_PERFORMANCE_SOURCE src/arg_parser.cpp src/base_test.cpp + src/latency_stats.cpp src/options.cpp src/program.cpp src/random_stream.cpp + src/result_output.cpp ) add_library(azure-perf ${AZURE_PERFORMANCE_HEADER} ${AZURE_PERFORMANCE_SOURCE}) diff --git a/sdk/core/perf/inc/azure/perf.hpp b/sdk/core/perf/inc/azure/perf.hpp index 0c9d154292..038a1eedfd 100644 --- a/sdk/core/perf/inc/azure/perf.hpp +++ b/sdk/core/perf/inc/azure/perf.hpp @@ -12,8 +12,11 @@ #include "azure/perf/argagg.hpp" #include "azure/perf/base_test.hpp" #include "azure/perf/dynamic_test_options.hpp" +#include "azure/perf/latency_stats.hpp" #include "azure/perf/options.hpp" #include "azure/perf/program.hpp" +#include "azure/perf/random_stream.hpp" +#include "azure/perf/result_output.hpp" #include "azure/perf/test.hpp" #include "azure/perf/test_metadata.hpp" #include "azure/perf/test_options.hpp" diff --git a/sdk/core/perf/inc/azure/perf/latency_stats.hpp b/sdk/core/perf/inc/azure/perf/latency_stats.hpp new file mode 100644 index 0000000000..d3ce093c4f --- /dev/null +++ b/sdk/core/perf/inc/azure/perf/latency_stats.hpp @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * @file + * @brief Per-operation latency collector and percentile summary. + * + */ + +#pragma once + +#include +#include +#include +#include +#include + +namespace Azure { namespace Perf { + + /** + * @brief Thread-safe collector of per-operation latency samples. + * + * @remark Records nanosecond-resolution durations from many worker threads and computes + * percentile summaries (p50/p90/p95/p99/max) on demand. Designed to match the latency + * reporting added in the Go perf framework so cross-language results are comparable. + * + */ + class LatencyCollector { + public: + /** + * @brief A single latency sample, optionally tagged by call type. + * + */ + struct Sample + { + std::chrono::nanoseconds Duration{0}; + std::string CallType; + }; + + /** + * @brief Latency summary expressed in milliseconds, matching the .NET + * `Azure.Test.Perf` percentile distribution: 50, 75, 90, 99, 99.9, 99.99, 99.999, 100. + * + */ + struct Summary + { + uint64_t Count = 0; + double P50Ms = 0; + double P75Ms = 0; + double P90Ms = 0; + double P99Ms = 0; + double P999Ms = 0; + double P9999Ms = 0; + double P99999Ms = 0; + double P100Ms = 0; + double MeanMs = 0; + }; + + /** + * @brief Record a single latency sample with no call-type tag. + * + * @param duration The latency to record. + */ + void Record(std::chrono::nanoseconds duration); + + /** + * @brief Record a single latency sample tagged with a call type. + * + * @param callType A short label for the operation (e.g. "Upload"). + * @param duration The latency to record. + */ + void Record(std::string const& callType, std::chrono::nanoseconds duration); + + /** + * @brief Clear all recorded samples. + * + */ + void Reset(); + + /** + * @brief Compute the summary over all recorded samples. + * + * @return The percentile summary. + */ + Summary Summarize() const; + + /** + * @brief Compute summaries grouped by call type. + * + * @return A vector of (callType, summary) pairs, sorted by callType. + */ + std::vector> SummarizeByCallType() const; + + /** + * @brief Snapshot all recorded samples (copy). + * + */ + std::vector Samples() const; + + private: + mutable std::mutex m_mutex; + std::vector m_samples; + }; + +}} // namespace Azure::Perf diff --git a/sdk/core/perf/inc/azure/perf/options.hpp b/sdk/core/perf/inc/azure/perf/options.hpp index ab9d87016b..e635b79998 100644 --- a/sdk/core/perf/inc/azure/perf/options.hpp +++ b/sdk/core/perf/inc/azure/perf/options.hpp @@ -108,6 +108,22 @@ namespace Azure { namespace Perf { */ std::vector TestProxies; + /** + * @brief Interval in seconds between live status lines printed during a run. + * + */ + int StatusInterval = 1; + + /** + * @brief When set, write a per-operation results file (JSON) containing the per-op + * latency (ms) and the per-op size (bytes) for every measured operation, matching the + * .NET `OperationResult { Time, Size }` schema. + * + * @remark Only populated when #Latency is also enabled. + * + */ + std::string ResultsFile; + /** * @brief Create an array of the performance framework options. * diff --git a/sdk/core/perf/inc/azure/perf/result_output.hpp b/sdk/core/perf/inc/azure/perf/result_output.hpp new file mode 100644 index 0000000000..7c78bffb89 --- /dev/null +++ b/sdk/core/perf/inc/azure/perf/result_output.hpp @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * @file + * @brief Run-summary helpers: `--results-file` writer and `#StartJobStatistics` printer. + * + */ + +#pragma once + +#include "azure/perf/latency_stats.hpp" + +#include +#include +#include + +namespace Azure { namespace Perf { + + /** + * @brief A consolidated run summary used by the framework. Fields mirror the data + * already printed in the .NET reference framework's results block. + * + */ + struct RunSummary + { + std::string TestName; + int Parallel = 1; + int DurationSeconds = 0; + int Warmup = 0; + int Iterations = 1; + uint64_t TotalOperations = 0; + double WeightedAverageSeconds = 0; + double OperationsPerSecond = 0; + double SecondsPerOperation = 0; + LatencyCollector::Summary Latency; + std::vector> LatencyByCallType; + }; + + /** + * @brief A single per-operation result, matching the .NET + * `Azure.Test.Perf.OperationResult { Time, Size }` schema. + * + * `Time` is the operation latency in milliseconds; `Size` is the operation size in + * bytes (or -1 if the test does not have a meaningful size). + * + */ + struct OperationResult + { + double Time = 0; + int64_t Size = -1; + }; + + /** + * @brief Write per-operation results to `path` as a JSON array of + * `OperationResult { Time, Size }` objects, matching the .NET `--results-file` output + * shape. + * + * @param path Destination file. + * @param results The per-operation samples to write. + */ + void WriteResultsFile(std::string const& path, std::vector const& results); + + /** + * @brief Print the `#StartJobStatistics`/`#EndJobStatistics` JSON block consumed by the + * perf-automation tool. + * + * @details The payload matches the .NET reference framework's `BenchmarkOutput` + * envelope: + * ``` + * { "Metadata": [ + * {"Source","Name","ShortDescription","LongDescription","Format"} + * ], + * "Measurements": [ + * {"Timestamp","Name","Value"} + * ] + * } + * ``` + * + * @param summary The run summary to serialize. + */ + void PrintJobStatistics(RunSummary const& summary); + +}} // namespace Azure::Perf diff --git a/sdk/core/perf/src/arg_parser.cpp b/sdk/core/perf/src/arg_parser.cpp index ecb4cc461e..36ff216981 100644 --- a/sdk/core/perf/src/arg_parser.cpp +++ b/sdk/core/perf/src/arg_parser.cpp @@ -78,6 +78,11 @@ Azure::Perf::GlobalTestOptions Azure::Perf::Program::ArgParser::Parse( { options.NoCleanup = parsedArgs["NoCleanup"].as(); } + // .NET-compatible bare-switch alias --no-cleanup; presence implies true. + if (parsedArgs["NoCleanupSwitch"]) + { + options.NoCleanup = true; + } if (parsedArgs["Parallel"]) { options.Parallel = parsedArgs["Parallel"]; @@ -103,6 +108,14 @@ Azure::Perf::GlobalTestOptions Azure::Perf::Program::ArgParser::Parse( options.TestProxies.push_back(proxy); } } + if (parsedArgs["StatusInterval"]) + { + options.StatusInterval = parsedArgs["StatusInterval"]; + } + if (parsedArgs["ResultsFile"]) + { + options.ResultsFile = parsedArgs["ResultsFile"].as(); + } return options; } diff --git a/sdk/core/perf/src/latency_stats.cpp b/sdk/core/perf/src/latency_stats.cpp new file mode 100644 index 0000000000..015ce192a5 --- /dev/null +++ b/sdk/core/perf/src/latency_stats.cpp @@ -0,0 +1,126 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "azure/perf/latency_stats.hpp" + +#include +#include + +namespace { + +double NanosToMs(std::chrono::nanoseconds ns) +{ + return std::chrono::duration(ns).count(); +} + +// Compute the value at percentile `p` (0..100) using nearest-rank. +// `sortedMs` must be sorted ascending. Returns 0 for an empty input. +double Percentile(std::vector const& sortedMs, double p) +{ + if (sortedMs.empty()) + { + return 0.0; + } + if (sortedMs.size() == 1) + { + return sortedMs.front(); + } + double rank = (p / 100.0) * static_cast(sortedMs.size() - 1); + size_t lo = static_cast(rank); + size_t hi = (lo + 1 < sortedMs.size()) ? lo + 1 : lo; + double frac = rank - static_cast(lo); + return sortedMs[lo] * (1.0 - frac) + sortedMs[hi] * frac; +} + +Azure::Perf::LatencyCollector::Summary SummaryFromMs(std::vector& msValues) +{ + Azure::Perf::LatencyCollector::Summary s; + s.Count = msValues.size(); + if (msValues.empty()) + { + return s; + } + std::sort(msValues.begin(), msValues.end()); + double sum = 0; + for (auto v : msValues) + { + sum += v; + } + s.MeanMs = sum / static_cast(msValues.size()); + s.P50Ms = Percentile(msValues, 50); + s.P75Ms = Percentile(msValues, 75); + s.P90Ms = Percentile(msValues, 90); + s.P99Ms = Percentile(msValues, 99); + s.P999Ms = Percentile(msValues, 99.9); + s.P9999Ms = Percentile(msValues, 99.99); + s.P99999Ms = Percentile(msValues, 99.999); + s.P100Ms = msValues.back(); + return s; +} + +} // namespace + +namespace Azure { namespace Perf { + + void LatencyCollector::Record(std::chrono::nanoseconds duration) + { + std::lock_guard lock(m_mutex); + m_samples.push_back(Sample{duration, std::string{}}); + } + + void LatencyCollector::Record(std::string const& callType, std::chrono::nanoseconds duration) + { + std::lock_guard lock(m_mutex); + m_samples.push_back(Sample{duration, callType}); + } + + void LatencyCollector::Reset() + { + std::lock_guard lock(m_mutex); + m_samples.clear(); + } + + LatencyCollector::Summary LatencyCollector::Summarize() const + { + std::vector snapshot; + { + std::lock_guard lock(m_mutex); + snapshot = m_samples; + } + std::vector msValues; + msValues.reserve(snapshot.size()); + for (auto const& s : snapshot) + { + msValues.push_back(NanosToMs(s.Duration)); + } + return SummaryFromMs(msValues); + } + + std::vector> + LatencyCollector::SummarizeByCallType() const + { + std::vector snapshot; + { + std::lock_guard lock(m_mutex); + snapshot = m_samples; + } + std::map> buckets; + for (auto const& s : snapshot) + { + buckets[s.CallType].push_back(NanosToMs(s.Duration)); + } + std::vector> result; + for (auto& kv : buckets) + { + result.emplace_back(kv.first, SummaryFromMs(kv.second)); + } + return result; + } + + std::vector LatencyCollector::Samples() const + { + std::lock_guard lock(m_mutex); + return m_samples; + } + +}} // namespace Azure::Perf diff --git a/sdk/core/perf/src/options.cpp b/sdk/core/perf/src/options.cpp index 44921c84dc..c80ee0c27e 100644 --- a/sdk/core/perf/src/options.cpp +++ b/sdk/core/perf/src/options.cpp @@ -16,7 +16,9 @@ void Azure::Perf::to_json(Azure::Core::Json::_internal::json& j, const GlobalTes {"Latency", p.Latency}, {"NoCleanup", p.NoCleanup}, {"Parallel", p.Parallel}, - {"Warmup", p.Warmup}}; + {"Warmup", p.Warmup}, + {"StatusInterval", p.StatusInterval}, + {"ResultsFile", p.ResultsFile.empty() ? "N/A" : p.ResultsFile}}; if (p.Port) { j["Port"] = p.Port.Value(); @@ -56,7 +58,6 @@ std::vector Azure::Perf::GlobalTestOptions::GetOptionMe [Option('p', "parallel", Default = 1, HelpText = "Number of operations to execute in parallel")] [Option("port", HelpText = "Port to redirect HTTP requests")] [Option('r', "rate", HelpText = "Target throughput (ops/sec)")] - [Option("sync", HelpText = "Runs sync version of test")] -- Not supported [Option('w', "warmup", Default = 5, HelpText = "Duration of warmup in seconds")] [Option('x', "proxy", Default = "", HelpText = "Proxy server")] */ @@ -78,6 +79,11 @@ std::vector Azure::Perf::GlobalTestOptions::GetOptionMe "Track and print per-operation latency statistics. Default to false.", 1}, {"NoCleanup", {"--noclean"}, "Disables test clean up. Default to false.", 1}, + // .NET-compatible bare-switch alias for --noclean. + {"NoCleanupSwitch", + {"--no-cleanup"}, + "Disables test clean up (bare switch, matches .NET --no-cleanup).", + 0}, {"Parallel", {"-p", "--parallel"}, "Number of operations to execute in parallel. Default to 1.", @@ -85,8 +91,19 @@ std::vector Azure::Perf::GlobalTestOptions::GetOptionMe {"Port", {"--port"}, "Port to redirect HTTP requests. Default to no redirection.", 1}, {"Rate", {"-r", "--rate"}, "Target throughput (ops/sec). Default to no throughput.", 1}, - {"Sync", {"-y", "--sync"}, "Runs sync version of test, not implemented", 0}, + // Accepted for cross-language CLI compatibility (perf-automation appends --sync for + // sync-only languages). C++ is sync-only and has no async variant, so the flag is + // parsed and intentionally ignored. + {"Sync", {"-y", "--sync"}, "Accepted for compatibility; ignored (C++ is sync-only).", 0}, {"TestProxies", {"-x", "--test-proxies"}, "URIs of TestProxy Servers (separated by ';')", 1}, {"Warmup", {"-w", "--warmup"}, "Duration of warmup in seconds. Default to 5 seconds.", 1}, + {"StatusInterval", + {"--status-interval"}, + "Interval in seconds between live status lines. Default to 1.", + 1}, + {"ResultsFile", + {"--results-file"}, + "Write per-operation results ({Time, Size}) as JSON to this file. Requires --latency.", + 1}, }; } diff --git a/sdk/core/perf/src/program.cpp b/sdk/core/perf/src/program.cpp index fac6dec3f7..06f0132ca1 100644 --- a/sdk/core/perf/src/program.cpp +++ b/sdk/core/perf/src/program.cpp @@ -4,15 +4,20 @@ #include "azure/perf/program.hpp" #include "azure/perf/argagg.hpp" +#include "azure/perf/latency_stats.hpp" +#include "azure/perf/result_output.hpp" #include #include #include #include +#include #include #include +#include #include +#include #include namespace { @@ -151,13 +156,23 @@ inline void RunLoop( uint64_t& completedOperations, std::chrono::nanoseconds& lastCompletionTimes, bool latency, + Azure::Perf::LatencyCollector* latencyCollector, bool& isCancelled) { - (void)latency; auto start = std::chrono::system_clock::now(); while (!isCancelled) { - test.Run(context); + if (latency && latencyCollector != nullptr) + { + auto opStart = std::chrono::steady_clock::now(); + test.Run(context); + auto opEnd = std::chrono::steady_clock::now(); + latencyCollector->Record(opEnd - opStart); + } + else + { + test.Run(context); + } completedOperations += 1; lastCompletionTimes = std::chrono::system_clock::now() - start; } @@ -224,61 +239,79 @@ inline void RunTests( std::vector> const& tests, Azure::Perf::GlobalTestOptions const& options, std::string const& title, + Azure::Perf::LatencyCollector* latencyCollector, + Azure::Perf::RunSummary* outSummary, bool warmup = false) { - (void)title; auto parallelTestsCount = options.Parallel; auto durationInSeconds = warmup ? options.Warmup : options.Duration; - // auto jobStatistics = warmup ? false : options.JobStatistics; - // auto latency = warmup ? false : options.Latency; + auto recordLatency = warmup ? false : options.Latency; std::vector completedOperations(parallelTestsCount); std::vector lastCompletionTimes(parallelTestsCount); + // Per-iteration reset: clear the latency collector so each iteration produces an + // independent summary, matching the Go perf-framework lifecycle. + if (recordLatency && latencyCollector != nullptr) + { + latencyCollector->Reset(); + } + /********************* Progress Reporter ******************************/ Azure::Core::Context progressToken; uint64_t lastCompleted = 0; - auto progressThread = std::thread( - [&title, &completedOperations, &lastCompletionTimes, &lastCompleted, &progressToken]() { - std::cout << std::endl - << "=== " << title << " ===" << std::endl - << "Current\t\tTotal\t\tAverage" << std::endl; - while (!progressToken.IsCancelled()) - { - using namespace std::chrono_literals; - std::this_thread::sleep_for(1000ms); - auto total = Sum(completedOperations); - auto current = total - lastCompleted; - auto avg = Sum(ZipAvg(completedOperations, lastCompletionTimes)); - lastCompleted = total; - std::cout << current << "\t\t" << total << "\t\t" << avg << std::endl; - } - }); + int statusInterval = (options.StatusInterval > 0) ? options.StatusInterval : 1; + auto progressThread = std::thread([&title, + &completedOperations, + &lastCompletionTimes, + &lastCompleted, + &progressToken, + statusInterval]() { + std::cout << std::endl + << "=== " << title << " ===" << std::endl + << "Current\t\tTotal\t\tAverage" << std::endl; + while (!progressToken.IsCancelled()) + { + std::this_thread::sleep_for(std::chrono::seconds(statusInterval)); + auto total = Sum(completedOperations); + auto current = total - lastCompleted; + auto avg = Sum(ZipAvg(completedOperations, lastCompletionTimes)); + lastCompleted = total; + std::cout << current << "\t\t" << total << "\t\t" << avg << std::endl; + } + }); /********************* parallel test creation ******************************/ std::vector tasks(tests.size()); auto deadLineSeconds = std::chrono::seconds(durationInSeconds); for (size_t index = 0; index != tests.size(); index++) { - tasks[index] = std::thread( - [index, &tests, &completedOperations, &lastCompletionTimes, &deadLineSeconds, &context]() { - bool isCancelled = false; - // Azure::Context is not good performer for checking cancellation inside the test loop - auto manualCancellation = std::thread([&deadLineSeconds, &isCancelled] { - std::this_thread::sleep_for(deadLineSeconds); - isCancelled = true; - }); - - RunLoop( - context, - *tests[index], - completedOperations[index], - lastCompletionTimes[index], - false, - isCancelled); - - manualCancellation.join(); - }); + tasks[index] = std::thread([index, + &tests, + &completedOperations, + &lastCompletionTimes, + &deadLineSeconds, + &context, + latencyCollector, + recordLatency]() { + bool isCancelled = false; + // Azure::Context is not good performer for checking cancellation inside the test loop + auto manualCancellation = std::thread([&deadLineSeconds, &isCancelled] { + std::this_thread::sleep_for(deadLineSeconds); + isCancelled = true; + }); + + RunLoop( + context, + *tests[index], + completedOperations[index], + lastCompletionTimes[index], + recordLatency, + latencyCollector, + isCancelled); + + manualCancellation.join(); + }); } // Wait for all tests to complete setUp for (auto& t : tasks) @@ -297,6 +330,8 @@ inline void RunTests( auto secondsPerOperation = 1 / operationsPerSecond; auto weightedAverageSeconds = totalOperations / operationsPerSecond; + // Match the established `Completed N operations in a weighted-average of Ts (X ops/s, + // Y s/op)` line format that downstream tools (Cpp.cs's ops/s regex) key off. std::cout << std::endl << "Completed " << FormatNumber(totalOperations, false) << " operations in a weighted-average of " @@ -304,6 +339,53 @@ inline void RunTests( << FormatNumber(operationsPerSecond) << " ops/s, " << secondsPerOperation << " s/op)" << std::endl << std::endl; + + if (!warmup && outSummary != nullptr) + { + outSummary->TotalOperations = totalOperations; + outSummary->OperationsPerSecond = operationsPerSecond; + outSummary->SecondsPerOperation = secondsPerOperation; + outSummary->WeightedAverageSeconds = weightedAverageSeconds; + if (recordLatency && latencyCollector != nullptr) + { + outSummary->Latency = latencyCollector->Summarize(); + outSummary->LatencyByCallType = latencyCollector->SummarizeByCallType(); + + auto const& s = outSummary->Latency; + if (s.Count > 0) + { + // Match the .NET Azure.Test.Perf latency distribution exactly: + // format string is `{percentile,7:N3}% {ms,8:N2}ms` -- i.e., 7-char-wide + // percentile with 3 decimals, then "% ", then 8-char-wide latency with + // 2 decimals, then "ms". Reproduce the format here for byte-near parity. + struct Row + { + double Pct; + double Ms; + }; + Row rows[] = { + {50.0, s.P50Ms}, + {75.0, s.P75Ms}, + {90.0, s.P90Ms}, + {99.0, s.P99Ms}, + {99.9, s.P999Ms}, + {99.99, s.P9999Ms}, + {99.999, s.P99999Ms}, + {100.0, s.P100Ms}, + }; + std::cout << "=== Latency Distribution ===" << std::endl; + for (auto const& row : rows) + { + std::ostringstream pctSs; + pctSs << std::fixed << std::setprecision(3) << row.Pct; + std::ostringstream msSs; + msSs << std::fixed << std::setprecision(2) << row.Ms; + std::cout << std::right << std::setw(7) << pctSs.str() << "% " << std::right + << std::setw(8) << msSs.str() << "ms" << std::endl; + } + } + } + } } } // namespace @@ -327,7 +409,6 @@ void Azure::Perf::Program::Run( // Parse args only to get the test name first auto testMetadata = GetTestMetadata(tests, argc, argv); - auto const& testGenerator = testMetadata->Factory; if (testMetadata == nullptr) { // Wrong input. Print what are the options. @@ -335,6 +416,7 @@ void Azure::Perf::Program::Run( return; } + auto const& testGenerator = testMetadata->Factory; // Initial test to get it's options, we can use a dummy parser results argagg::parser_results argResults; @@ -375,6 +457,9 @@ void Azure::Perf::Program::Run( } } + /******************** Per-run latency collector (when --latency) ****************/ + Azure::Perf::LatencyCollector latencyCollector; + /******************** Global Set up ******************************/ std::cout << std::endl << "=== Global Setup ===" << std::endl; test->GlobalSetup(); @@ -419,18 +504,67 @@ void Azure::Perf::Program::Run( /******************** WarmUp ******************************/ if (options.Warmup) { - RunTests(context, parallelTest, options, "Warmup", true); + RunTests(context, parallelTest, options, "Warmup", nullptr, nullptr, true); } /******************** Tests ******************************/ std::string iterationInfo; + Azure::Perf::RunSummary finalSummary; + finalSummary.TestName = testMetadata->Name; + finalSummary.Parallel = options.Parallel; + finalSummary.DurationSeconds = options.Duration; + finalSummary.Warmup = options.Warmup; + finalSummary.Iterations = options.Iterations; for (int iteration = 0; iteration < options.Iterations; iteration++) { if (iteration > 0) { iterationInfo.append(FormatNumber(iteration)); } - RunTests(context, parallelTest, options, "Test" + iterationInfo); + RunTests( + context, + parallelTest, + options, + "Test" + iterationInfo, + options.Latency ? &latencyCollector : nullptr, + &finalSummary); + } + + /******************** End-of-run artifacts ************************/ + if (options.Latency && !options.ResultsFile.empty()) + { + // Match the .NET `--results-file` shape: an array of OperationResult { Time, Size }. + // Time is per-op latency in ms; Size is taken from the test's --size option when + // present, otherwise -1 (mirroring .NET's `(options as SizeOptions)?.Size ?? -1`). + int64_t opSize = -1; + try + { + if (argResults["Size"]) + { + opSize = argResults["Size"].as(); + } + } + catch (std::exception const&) + { + opSize = -1; + } + + std::vector ops; + auto samples = latencyCollector.Samples(); + ops.reserve(samples.size()); + for (auto const& s : samples) + { + Azure::Perf::OperationResult r; + r.Time = std::chrono::duration(s.Duration).count(); + r.Size = opSize; + ops.push_back(std::move(r)); + } + Azure::Perf::WriteResultsFile(options.ResultsFile, ops); + } + + if (options.JobStatistics) + { + Azure::Perf::PrintJobStatistics(finalSummary); } std::cout << std::endl << "=== Pre-Cleanup ===" << std::endl; diff --git a/sdk/core/perf/src/result_output.cpp b/sdk/core/perf/src/result_output.cpp new file mode 100644 index 0000000000..9e21f6aee7 --- /dev/null +++ b/sdk/core/perf/src/result_output.cpp @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "azure/perf/result_output.hpp" + +#include + +#include +#include +#include +#include +#include +#include + +namespace { + +using Azure::Core::Json::_internal::json; +using Azure::Perf::OperationResult; +using Azure::Perf::RunSummary; + +bool TryOpen(std::ofstream& f, std::string const& path) +{ + f.open(path, std::ios::out | std::ios::trunc); + if (!f.is_open()) + { + std::cerr << "warning: failed to open output file " << path << std::endl; + return false; + } + return true; +} + +// ISO-8601 UTC timestamp matching .NET's DateTime.ToString("O") JSON serialization, +// which emits fractional seconds at 100-nanosecond (7-digit) resolution. +std::string IsoUtcNow() +{ + using namespace std::chrono; + auto now = system_clock::now(); + auto secs = time_point_cast(now); + // 100-nanosecond ticks within the current second, matching .NET DateTime "fffffff". + // system_clock typically has microsecond resolution on Windows; pad with trailing zeros. + auto ticks = duration_cast>>(now - secs).count(); + std::time_t tt = system_clock::to_time_t(secs); + std::tm tm{}; +#if defined(_WIN32) + gmtime_s(&tm, &tt); +#else + gmtime_r(&tt, &tm); +#endif + std::ostringstream os; + os << std::put_time(&tm, "%Y-%m-%dT%H:%M:%S") << "." << std::setw(7) << std::setfill('0') << ticks + << "Z"; + return os.str(); +} + +} // namespace + +namespace Azure { namespace Perf { + + void WriteResultsFile(std::string const& path, std::vector const& results) + { + if (path.empty()) + { + return; + } + std::ofstream f; + if (!TryOpen(f, path)) + { + return; + } + // Match the .NET Azure.Test.Perf OperationResult JSON shape exactly: + // [ { "Time": , "Size": }, ... ] + json arr = json::array(); + for (auto const& r : results) + { + arr.push_back(json{{"Time", r.Time}, {"Size", r.Size}}); + } + f << arr.dump(2) << std::endl; + } + + void PrintJobStatistics(RunSummary const& summary) + { + // Match the .NET BenchmarkOutput shape AND key order exactly so perf-automation's + // downstream parser sees the same fields as for .NET runs: + // { "Metadata": [ { Source, Name, ShortDescription, LongDescription, Format } ], + // "Measurements": [ { Timestamp, Name, Value } ] } + // We serialize manually because nlohmann::json sorts object keys alphabetically by + // default; .NET emits keys in declaration order. + std::ostringstream os; + os << "{\"Metadata\":[{" + << "\"Source\":\"PerfStress\"," + << "\"Name\":\"perfstress/throughput\"," + << "\"ShortDescription\":\"Throughput (ops/sec)\"," + << "\"LongDescription\":\"Throughput (ops/sec)\"," + << "\"Format\":\"n2\"" + << "}],\"Measurements\":[{" + << "\"Timestamp\":\"" << IsoUtcNow() << "\"," + << "\"Name\":\"perfstress/throughput\"," + << "\"Value\":" << json(summary.OperationsPerSecond).dump() << "}]}"; + std::cout << "#StartJobStatistics" << std::endl; + std::cout << os.str() << std::endl; + std::cout << "#EndJobStatistics" << std::endl; + } + +}} // namespace Azure::Perf diff --git a/sdk/core/perf/test/CMakeLists.txt b/sdk/core/perf/test/CMakeLists.txt index bf213d5ab2..d764cfefc2 100644 --- a/sdk/core/perf/test/CMakeLists.txt +++ b/sdk/core/perf/test/CMakeLists.txt @@ -19,7 +19,9 @@ include(GoogleTest) add_executable ( azure-perf-unit-test + src/latency_stats_test.cpp src/random_stream_test.cpp + src/result_output_test.cpp ) if (MSVC) diff --git a/sdk/core/perf/test/src/latency_stats_test.cpp b/sdk/core/perf/test/src/latency_stats_test.cpp new file mode 100644 index 0000000000..402b385d9d --- /dev/null +++ b/sdk/core/perf/test/src/latency_stats_test.cpp @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include + +#include +#include +#include + +#include + +using namespace std::chrono; +using Azure::Perf::LatencyCollector; + +TEST(latency_stats, percentiles) +{ + LatencyCollector c; + // Insert 1..100 ms; percentiles should land near the respective ranks. + for (int i = 1; i <= 100; ++i) + { + c.Record(milliseconds(i)); + } + auto s = c.Summarize(); + EXPECT_EQ(s.Count, 100u); + EXPECT_NEAR(s.P50Ms, 50.0, 1.5); + EXPECT_NEAR(s.P75Ms, 75.0, 1.5); + EXPECT_NEAR(s.P90Ms, 90.0, 1.5); + EXPECT_NEAR(s.P99Ms, 99.0, 1.5); + // p99.9/p99.99/p99.999 on a 100-sample input all collapse near the top of the range. + EXPECT_GE(s.P999Ms, 99.0); + EXPECT_GE(s.P9999Ms, 99.0); + EXPECT_GE(s.P99999Ms, 99.0); + EXPECT_LE(s.P999Ms, 100.0); + EXPECT_LE(s.P9999Ms, 100.0); + EXPECT_LE(s.P99999Ms, 100.0); + EXPECT_NEAR(s.P100Ms, 100.0, 0.001); + EXPECT_NEAR(s.MeanMs, 50.5, 0.001); +} + +TEST(latency_stats, by_call_type) +{ + LatencyCollector c; + c.Record("A", milliseconds(10)); + c.Record("A", milliseconds(30)); + c.Record("B", milliseconds(50)); + auto byType = c.SummarizeByCallType(); + ASSERT_EQ(byType.size(), 2u); + EXPECT_EQ(byType[0].first, "A"); + EXPECT_EQ(byType[0].second.Count, 2u); + EXPECT_EQ(byType[1].first, "B"); + EXPECT_EQ(byType[1].second.Count, 1u); +} + +TEST(latency_stats, reset) +{ + LatencyCollector c; + c.Record(milliseconds(5)); + c.Reset(); + EXPECT_EQ(c.Summarize().Count, 0u); +} + +TEST(latency_stats, concurrent_record) +{ + LatencyCollector c; + constexpr int N = 8; + constexpr int Per = 1000; + std::vector threads; + for (int t = 0; t < N; ++t) + { + threads.emplace_back([&c]() { + for (int i = 0; i < Per; ++i) + { + c.Record(microseconds(100)); + } + }); + } + for (auto& th : threads) + { + th.join(); + } + EXPECT_EQ(c.Summarize().Count, static_cast(N * Per)); +} diff --git a/sdk/core/perf/test/src/result_output_test.cpp b/sdk/core/perf/test/src/result_output_test.cpp new file mode 100644 index 0000000000..95bac92b9f --- /dev/null +++ b/sdk/core/perf/test/src/result_output_test.cpp @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include + +#include +#include +#include +#include + +#include + +using Azure::Perf::OperationResult; +using Azure::Perf::RunSummary; + +namespace { +std::string TempPath(std::string const& tag) +{ + auto base = ::testing::TempDir(); + if (!base.empty() && base.back() != '/' && base.back() != '\\') + { +#ifdef _WIN32 + base += '\\'; +#else + base += '/'; +#endif + } + return base + "azure_perf_test_" + tag + ".json"; +} + +bool FileExists(std::string const& path) +{ + std::ifstream f(path.c_str()); + return f.good(); +} + +std::string Slurp(std::string const& path) +{ + std::ifstream f(path.c_str(), std::ios::binary); + std::ostringstream ss; + ss << f.rdbuf(); + return ss.str(); +} +} // namespace + +TEST(result_output, write_results_file_matches_dotnet_schema) +{ + auto path = TempPath("results"); + std::vector results; + results.push_back({12.5, 1024}); + results.push_back({8.0, 1024}); + Azure::Perf::WriteResultsFile(path, results); + EXPECT_TRUE(FileExists(path)); + auto contents = Slurp(path); + // .NET OperationResult JSON shape: { "Time": ..., "Size": ... }. + // Field names are PascalCase and must match the .NET reference framework exactly. + EXPECT_NE(contents.find("\"Time\""), std::string::npos); + EXPECT_NE(contents.find("\"Size\""), std::string::npos); + EXPECT_NE(contents.find("12.5"), std::string::npos); + EXPECT_NE(contents.find("1024"), std::string::npos); + // Schema must NOT contain the legacy / Go-style field names. + EXPECT_EQ(contents.find("\"operation\""), std::string::npos); + EXPECT_EQ(contents.find("\"latencyMs\""), std::string::npos); + EXPECT_EQ(contents.find("\"sizeBytes\""), std::string::npos); + std::remove(path.c_str()); +} + +TEST(result_output, print_job_statistics_matches_dotnet_envelope) +{ + RunSummary s; + s.OperationsPerSecond = 1234.5; + + // Capture std::cout. + std::stringstream buffer; + auto* oldBuf = std::cout.rdbuf(buffer.rdbuf()); + Azure::Perf::PrintJobStatistics(s); + std::cout.rdbuf(oldBuf); + + auto out = buffer.str(); + EXPECT_NE(out.find("#StartJobStatistics"), std::string::npos); + EXPECT_NE(out.find("#EndJobStatistics"), std::string::npos); + // Match the .NET BenchmarkOutput envelope AND key order: + // { "Metadata": [...], "Measurements": [...] } -- Metadata must appear before Measurements. + std::size_t metaPos = out.find("\"Metadata\""); + std::size_t measPos = out.find("\"Measurements\""); + EXPECT_NE(metaPos, std::string::npos); + EXPECT_NE(measPos, std::string::npos); + EXPECT_LT(metaPos, measPos); + EXPECT_NE(out.find("\"Source\""), std::string::npos); + EXPECT_NE(out.find("PerfStress"), std::string::npos); + EXPECT_NE(out.find("perfstress/throughput"), std::string::npos); + EXPECT_NE(out.find("1234.5"), std::string::npos); +} diff --git a/sdk/storage/azure-storage-blobs/test/perf/inc/azure/storage/blobs/test/download_blob_test.hpp b/sdk/storage/azure-storage-blobs/test/perf/inc/azure/storage/blobs/test/download_blob_test.hpp index e1871135b7..9686c5f4ee 100644 --- a/sdk/storage/azure-storage-blobs/test/perf/inc/azure/storage/blobs/test/download_blob_test.hpp +++ b/sdk/storage/azure-storage-blobs/test/perf/inc/azure/storage/blobs/test/download_blob_test.hpp @@ -13,6 +13,7 @@ #include #include +#include #include #include @@ -23,10 +24,24 @@ namespace Azure { namespace Storage { namespace Blobs { namespace Test { /** * @brief A test to measure downloading a blob. * + * @details `--download-method` chooses between: + * - `buffer` (default, preserves existing behavior): allocate a contiguous buffer and + * call `DownloadTo(buffer, size)`. + * - `stream`: stream the response with `Download()` and drain its body stream without + * materializing the payload in RAM. Use for multi-GiB sizes. + * + * `--block-size` and `--concurrency` are forwarded to `DownloadBlobToOptions` for the + * `buffer` method. */ class DownloadBlob : public Azure::Storage::Blobs::Test::BlobsTest { private: std::unique_ptr> m_downloadBuffer; + long m_size = 0; + std::string m_downloadMethod = "buffer"; + long m_blockSize = 0; + int m_concurrency = 0; + + static constexpr size_t StreamDrainBufferSize = 1024 * 1024; public: /** @@ -45,22 +60,64 @@ namespace Azure { namespace Storage { namespace Blobs { namespace Test { // Call base to create blob client BlobsTest::Setup(); - long size = m_options.GetMandatoryOption("Size"); - - m_downloadBuffer = std::make_unique>(size); - - auto rawData = std::make_unique>(size); - auto content = Azure::Core::IO::MemoryBodyStream(*rawData); - m_blobClient->Upload(content); + m_size = m_options.GetMandatoryOption("Size"); + m_downloadMethod = m_options.GetOptionOrDefault("DownloadMethod", "buffer"); + m_blockSize = m_options.GetOptionOrDefault("BlockSize", 0); + m_concurrency = m_options.GetOptionOrDefault("Concurrency", 0); + + if (m_downloadMethod == "buffer") + { + m_downloadBuffer = std::make_unique>(m_size); + } + else if (m_downloadMethod != "stream") + { + throw std::runtime_error( + "Invalid --download-method '" + m_downloadMethod + + "'. Expected one of: buffer, stream."); + } + + // Stage the blob with random data. Use the streaming RandomStream so very large + // sizes do not materialize a contiguous staging buffer. + auto staging = Azure::Perf::RandomStream::Create(m_size); + m_blobClient->Upload(*staging); } /** * @brief Define the test * */ - void Run(Azure::Core::Context const&) override + void Run(Azure::Core::Context const& context) override { - m_blobClient->DownloadTo(m_downloadBuffer->data(), m_downloadBuffer->size()); + if (m_downloadMethod == "stream") + { + auto response = m_blobClient->Download({}, context); + auto& bodyStream = response.Value.BodyStream; + if (bodyStream) + { + // Drain into a thread-local heap buffer; a stack buffer this large would + // overflow the default Windows thread stack (1 MiB) under high --parallel. + static thread_local std::vector buffer(StreamDrainBufferSize); + while (true) + { + auto read = bodyStream->Read(buffer.data(), buffer.size(), context); + if (read == 0) + { + break; + } + } + } + return; + } + Azure::Storage::Blobs::DownloadBlobToOptions opts; + if (m_blockSize > 0) + { + opts.TransferOptions.ChunkSize = m_blockSize; + } + if (m_concurrency > 0) + { + opts.TransferOptions.Concurrency = m_concurrency; + } + m_blobClient->DownloadTo(m_downloadBuffer->data(), m_downloadBuffer->size(), opts); } /** @@ -77,7 +134,20 @@ namespace Azure { namespace Storage { namespace Blobs { namespace Test { {"--token-credential"}, "Use a token credential to run the test. By default, a connection string is used.", 0}, - {"Size", {"--size"}, "Size of payload (in bytes)", 1, true}}; + {"Size", {"--size"}, "Size of payload (in bytes)", 1, true}, + {"DownloadMethod", + {"--download-method"}, + "Download method: 'buffer' (default, contiguous buffer via DownloadTo) or " + "'stream' (drain the response BodyStream, no contiguous buffer).", + 1}, + {"BlockSize", + {"--block-size"}, + "Chunk size (bytes) for buffer-mode DownloadTo. Default: client default.", + 1}, + {"Concurrency", + {"--concurrency"}, + "Per-operation concurrency for buffer-mode DownloadTo. Default: client default.", + 1}}; } /** diff --git a/sdk/storage/azure-storage-blobs/test/perf/inc/azure/storage/blobs/test/list_blob_test.hpp b/sdk/storage/azure-storage-blobs/test/perf/inc/azure/storage/blobs/test/list_blob_test.hpp index 99172b79a9..e20d6ac68a 100644 --- a/sdk/storage/azure-storage-blobs/test/perf/inc/azure/storage/blobs/test/list_blob_test.hpp +++ b/sdk/storage/azure-storage-blobs/test/perf/inc/azure/storage/blobs/test/list_blob_test.hpp @@ -26,6 +26,9 @@ namespace Azure { namespace Storage { namespace Blobs { namespace Test { * */ class ListBlob : public Azure::Storage::Blobs::Test::BlobsTest { + private: + int m_pageSize = 0; + public: /** * @brief Construct a new ListBlob test. @@ -42,7 +45,18 @@ namespace Azure { namespace Storage { namespace Blobs { namespace Test { { // Call base to create blob client BlobsTest::Setup(); - long count = m_options.GetMandatoryOption("Count"); + // --num-blobs is the canonical name (matches the Go perf harness); --count is kept + // for backward compatibility with existing test definitions. + long count = 0; + if (m_options.HasOption("NumBlobs")) + { + count = m_options.GetMandatoryOption("NumBlobs"); + } + else + { + count = m_options.GetMandatoryOption("Count"); + } + m_pageSize = m_options.GetOptionOrDefault("PageSize", 0); auto rawData = std::make_unique>(1); auto content = Azure::Core::IO::MemoryBodyStream(*rawData); @@ -62,8 +76,13 @@ namespace Azure { namespace Storage { namespace Blobs { namespace Test { */ void Run(Azure::Core::Context const& context) override { + Azure::Storage::Blobs::ListBlobsOptions opts; + if (m_pageSize > 0) + { + opts.PageSizeHint = m_pageSize; + } // Loop each page - auto page = m_containerClient->ListBlobs({}, context); + auto page = m_containerClient->ListBlobs(opts, context); for (; page.HasPage(); page.MoveToNextPage(context)) { // loop each blob @@ -87,7 +106,12 @@ namespace Azure { namespace Storage { namespace Blobs { namespace Test { {"--token-credential"}, "Use a token credential to run the test. By default, a connection string is used.", 0}, - {"Count", {"--count"}, "Number of blobs to list", 1, true}}; + {"Count", {"--count"}, "Number of blobs to list (legacy alias of --num-blobs).", 1}, + {"NumBlobs", {"--num-blobs"}, "Number of blobs to list.", 1}, + {"PageSize", + {"--page-size"}, + "Server page size hint for ListBlobs. Default: server default.", + 1}}; } /** diff --git a/sdk/storage/azure-storage-blobs/test/perf/inc/azure/storage/blobs/test/upload_blob_test.hpp b/sdk/storage/azure-storage-blobs/test/perf/inc/azure/storage/blobs/test/upload_blob_test.hpp index b313ff69d9..0e27f7cea6 100644 --- a/sdk/storage/azure-storage-blobs/test/perf/inc/azure/storage/blobs/test/upload_blob_test.hpp +++ b/sdk/storage/azure-storage-blobs/test/perf/inc/azure/storage/blobs/test/upload_blob_test.hpp @@ -24,11 +24,27 @@ namespace Azure { namespace Storage { namespace Blobs { namespace Test { /** * @brief A test to measure uploading a blob. * + * @details Supports three upload methods selected via `--upload-method`: + * - `buffer` (default, preserves existing behavior): build a contiguous in-memory + * payload and call `BlockBlobClient::UploadFrom(buffer, size)`. Guarded by a + * `size * parallel` memory-budget check to avoid OOM kills. + * - `stream`: do not materialize the payload; stream a circular `RandomStream` into + * `BlockBlobClient::Upload(BodyStream)`. Use for multi-GiB sizes. + * - `single`: same as `buffer` but uses the single-shot `Upload(BodyStream)` for the + * in-memory buffer (no chunked staging). Useful to compare buffered vs. chunked + * upload paths. + * + * `--block-size` and `--concurrency` are forwarded to `UploadBlockBlobFromOptions` for + * the `buffer` method. */ class UploadBlob : public Azure::Storage::Blobs::Test::BlobsTest { private: // C++ can upload and download from contiguous memory or file only std::vector m_uploadBuffer; + long m_size = 0; + std::string m_uploadMethod = "buffer"; + long m_blockSize = 0; + int m_concurrency = 0; public: /** @@ -47,8 +63,22 @@ namespace Azure { namespace Storage { namespace Blobs { namespace Test { // Call base to create blob client BlobsTest::Setup(); - long size = m_options.GetMandatoryOption("Size"); - m_uploadBuffer = Azure::Perf::RandomStream::Create(size)->ReadToEnd(Azure::Core::Context{}); + m_size = m_options.GetMandatoryOption("Size"); + m_uploadMethod = m_options.GetOptionOrDefault("UploadMethod", "buffer"); + m_blockSize = m_options.GetOptionOrDefault("BlockSize", 0); + m_concurrency = m_options.GetOptionOrDefault("Concurrency", 0); + + if (m_uploadMethod == "buffer" || m_uploadMethod == "single") + { + m_uploadBuffer + = Azure::Perf::RandomStream::Create(m_size)->ReadToEnd(Azure::Core::Context{}); + } + else if (m_uploadMethod != "stream") + { + throw std::runtime_error( + "Invalid --upload-method '" + m_uploadMethod + + "'. Expected one of: buffer, stream, single."); + } } /** @@ -57,7 +87,29 @@ namespace Azure { namespace Storage { namespace Blobs { namespace Test { */ void Run(Azure::Core::Context const&) override { - m_blobClient->UploadFrom(m_uploadBuffer.data(), m_uploadBuffer.size()); + if (m_uploadMethod == "stream") + { + auto stream = Azure::Perf::RandomStream::Create(m_size); + m_blobClient->Upload(*stream); + return; + } + if (m_uploadMethod == "single") + { + auto stream = Azure::Core::IO::MemoryBodyStream(m_uploadBuffer); + m_blobClient->Upload(stream); + return; + } + // Default: buffer (chunked via UploadFrom). + Azure::Storage::Blobs::UploadBlockBlobFromOptions opts; + if (m_blockSize > 0) + { + opts.TransferOptions.ChunkSize = m_blockSize; + } + if (m_concurrency > 0) + { + opts.TransferOptions.Concurrency = m_concurrency; + } + m_blobClient->UploadFrom(m_uploadBuffer.data(), m_uploadBuffer.size(), opts); } /** @@ -73,7 +125,21 @@ namespace Azure { namespace Storage { namespace Blobs { namespace Test { {"--token-credential"}, "Use a token credential to run the test. By default, a connection string is used.", 0}, - {"Size", {"--size", "-s"}, "Size of payload (in bytes)", 1, true}}; + {"Size", {"--size", "-s"}, "Size of payload (in bytes)", 1, true}, + {"UploadMethod", + {"--upload-method"}, + "Upload method: 'buffer' (default, chunked UploadFrom), 'stream' (Upload " + "BodyStream from a circular RandomStream, no contiguous buffer), or 'single' " + "(single-shot Upload of an in-memory buffer).", + 1}, + {"BlockSize", + {"--block-size"}, + "Chunk size (bytes) for buffer-mode UploadFrom. Default: client default.", + 1}, + {"Concurrency", + {"--concurrency"}, + "Per-operation concurrency for buffer-mode UploadFrom. Default: client default.", + 1}}; } /**