Skip to content

Commit

Permalink
LiveMetric: realtime statistics for debug UI.
Browse files Browse the repository at this point in the history 
Statistics include latest/mean/variance/min/max.
  • Loading branch information
@hysw
hysw committed Jan 16, 2024
1 parent a1392e5 commit 557479c
Show file tree
Hide file tree
Showing 7 changed files with 466 additions and 65 deletions.
167 changes: 135 additions & 32 deletions benchmarks/graphics_pipeline/GraphicsBenchmarkApp.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
#include "GraphicsBenchmarkApp.h"
#include "SphereMesh.h"

#include "ppx/application.h"
#include "ppx/graphics_util.h"
#include "ppx/grfx/grfx_format.h"
#include "ppx/timer.h"
Expand Down Expand Up @@ -300,30 +301,51 @@ void GraphicsBenchmarkApp::Setup()
void GraphicsBenchmarkApp::SetupMetrics()
{
Application::SetupMetrics();
if (HasActiveMetricsRun()) {
ppx::metrics::MetricMetadata metadata = {ppx::metrics::MetricType::GAUGE, "CPU Submission Time", "ms", ppx::metrics::MetricInterpretation::LOWER_IS_BETTER, {0.f, 10000.f}};
mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime] = AddMetric(metadata);
PPX_ASSERT_MSG(mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime] != ppx::metrics::kInvalidMetricID, "Failed to add CPU Submission Time metric");

metadata = {ppx::metrics::MetricType::GAUGE, "Bandwidth", "GB/s", ppx::metrics::MetricInterpretation::HIGHER_IS_BETTER, {0.f, 10000.f}};
mMetricsData.metrics[MetricsData::kTypeBandwidth] = AddMetric(metadata);
PPX_ASSERT_MSG(mMetricsData.metrics[MetricsData::kTypeBandwidth] != ppx::metrics::kInvalidMetricID, "Failed to add Bandwidth metric");
}
mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime] = AllocateMetricID();
mMetricsData.metrics[MetricsData::kTypeBandwidth] = AllocateMetricID();
mMetricsData.metrics[MetricsData::kTypeGPUWorkDuration] = AllocateMetricID();

AddLiveMetric(mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime]);
AddLiveMetric(mMetricsData.metrics[MetricsData::kTypeBandwidth]);
AddLiveMetric(mMetricsData.metrics[MetricsData::kTypeGPUWorkDuration]);
}

void GraphicsBenchmarkApp::UpdateMetrics()
void GraphicsBenchmarkApp::SetupMetricsRun()
{
if (!HasActiveMetricsRun()) {
return;
Application::SetupMetricsRun();
{
ppx::metrics::MetricMetadata metadata = {
ppx::metrics::MetricType::GAUGE,
"CPU Submission Time",
"ms",
ppx::metrics::MetricInterpretation::LOWER_IS_BETTER,
{0.f, 10000.f}};
ppx::metrics::MetricID res = AddMetric(metadata, mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime]);
PPX_ASSERT_MSG(res != ppx::metrics::kInvalidMetricID, "Failed to add CPU Submission Time metric");
}
{
ppx::metrics::MetricMetadata metadata = {
ppx::metrics::MetricType::GAUGE,
"Bandwidth",
"GB/s",
ppx::metrics::MetricInterpretation::HIGHER_IS_BETTER,
{0.f, 10000.f}};
ppx::metrics::MetricID res = AddMetric(metadata, mMetricsData.metrics[MetricsData::kTypeBandwidth]);
PPX_ASSERT_MSG(res != ppx::metrics::kInvalidMetricID, "Failed to add Bandwidth metric");
}
}

void GraphicsBenchmarkApp::UpdateMetrics()
{
uint64_t frequency = 0;
PPX_CHECKED_CALL(GetGraphicsQueue()->GetTimestampFrequency(&frequency));

ppx::metrics::MetricData data = {ppx::metrics::MetricType::GAUGE};
data.gauge.seconds = GetElapsedSeconds();
data.gauge.value = mCPUSubmissionTime;
RecordMetricData(mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime], data);
{
ppx::metrics::MetricData data = {ppx::metrics::MetricType::GAUGE};
data.gauge.seconds = GetElapsedSeconds();
data.gauge.value = mCPUSubmissionTime;
RecordLiveMetricData(mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime], data);
}

const float gpuWorkDurationInSec = static_cast<float>(mGpuWorkDuration / static_cast<double>(frequency));
const grfx::Format swapchainColorFormat = GetSwapchain()->GetColorFormat();
Expand All @@ -335,13 +357,14 @@ void GraphicsBenchmarkApp::UpdateMetrics()
const uint32_t height = isOffscreen ? mOffscreenFrame.back().height : swapchainHeight;
const uint32_t quadCount = pFullscreenQuadsCount->GetValue();

if (quadCount) {
// Skip the first kSkipFrameCount frames after the knob of quad count being changed to avoid noise
constexpr uint32_t kSkipFrameCount = 2;
if (pFullscreenQuadsCount->DigestUpdate()) {
mSkipRecordBandwidthMetricFrameCounter = kSkipFrameCount;
}
{
ppx::metrics::MetricData data = {ppx::metrics::MetricType::GAUGE};
data.gauge.seconds = GetElapsedSeconds();
data.gauge.value = gpuWorkDurationInSec * 1000.0f;
RecordLiveMetricData(mMetricsData.metrics[MetricsData::kTypeGPUWorkDuration], data);
}

if (quadCount) {
if (mSkipRecordBandwidthMetricFrameCounter == 0) {
const auto texelSize = static_cast<float>(grfx::GetFormatDescription(colorFormat)->bytesPerTexel);
const float dataWriteInGb = (static_cast<float>(width) * static_cast<float>(height) * texelSize * quadCount) / (1024.f * 1024.f * 1024.f);
Expand All @@ -350,7 +373,7 @@ void GraphicsBenchmarkApp::UpdateMetrics()
ppx::metrics::MetricData data = {ppx::metrics::MetricType::GAUGE};
data.gauge.seconds = GetElapsedSeconds();
data.gauge.value = bandwidth;
RecordMetricData(mMetricsData.metrics[MetricsData::kTypeBandwidth], data);
RecordLiveMetricData(mMetricsData.metrics[MetricsData::kTypeBandwidth], data);
}
else {
--mSkipRecordBandwidthMetricFrameCounter;
Expand Down Expand Up @@ -893,6 +916,22 @@ void GraphicsBenchmarkApp::ProcessInput()
}

void GraphicsBenchmarkApp::ProcessKnobs()
{
// Skip the first kSkipFrameCount frames after the knob of quad count being changed to avoid noise
constexpr uint32_t kSkipFrameCount = 2;

const bool sphereChanged = ProcessSphereKnobs();
const bool quadChanged = ProcessQuadsKnobs();
const bool framebufferChanged = ProcessOffscreenRenderKnobs();
const bool anyUpdate = sphereChanged || quadChanged || framebufferChanged;

if (anyUpdate) {
ClearLiveMetricsHistory();
mSkipRecordBandwidthMetricFrameCounter = kSkipFrameCount;
}
}

bool GraphicsBenchmarkApp::ProcessSphereKnobs()
{
// Detect if any knob value has been changed
// Note: DigestUpdate should be called only once per frame (DigestUpdate unflags the internal knob variable)!
Expand All @@ -901,6 +940,14 @@ void GraphicsBenchmarkApp::ProcessKnobs()
const bool depthTestWriteKnobChanged = pDepthTestWrite->DigestUpdate();
const bool enableSpheresKnobChanged = pEnableSpheres->DigestUpdate();
const bool sphereInstanceCountKnobChanged = pSphereInstanceCount->DigestUpdate();
const bool debugViewChanged = pDebugViews->DigestUpdate();
const bool anyUpdate =
(allTexturesTo1x1KnobChanged ||
alphaBlendKnobChanged ||
depthTestWriteKnobChanged ||
enableSpheresKnobChanged ||
sphereInstanceCountKnobChanged ||
debugViewChanged);

// TODO: Ideally, the `maxValue` of the drawcall-count slider knob should be changed at runtime.
// Currently, the value of the drawcall-count is adjusted to the sphere-count in case the
Expand Down Expand Up @@ -948,16 +995,20 @@ void GraphicsBenchmarkApp::ProcessKnobs()
}
}
}
return anyUpdate;
}

ProcessQuadsKnobs();
bool GraphicsBenchmarkApp::ProcessOffscreenRenderKnobs()
{
const bool offscreenChanged = pRenderOffscreen->DigestUpdate();
const bool framebufferChanged = pResolution->DigestUpdate() || pFramebufferFormat->DigestUpdate();
const bool anyUpdate = offscreenChanged || framebufferChanged;

bool offscreenChanged = pRenderOffscreen->DigestUpdate();
if (offscreenChanged) {
pBlitOffscreen->SetVisible(pRenderOffscreen->GetValue());
pFramebufferFormat->SetVisible(pRenderOffscreen->GetValue());
pResolution->SetVisible(pRenderOffscreen->GetValue());
}
bool framebufferChanged = pResolution->DigestUpdate() || pFramebufferFormat->DigestUpdate();

if ((offscreenChanged && pRenderOffscreen->GetValue()) || framebufferChanged) {
std::pair<int, int> resolution = pResolution->GetValue();
Expand All @@ -966,10 +1017,17 @@ void GraphicsBenchmarkApp::ProcessKnobs()
int fbHeight = (resolution.second > 0 ? resolution.second : GetSwapchain()->GetHeight());
UpdateOffscreenBuffer(RenderFormat(), fbWidth, fbHeight);
}
return anyUpdate;
}

void GraphicsBenchmarkApp::ProcessQuadsKnobs()
bool GraphicsBenchmarkApp::ProcessQuadsKnobs()
{
const bool countUpdated = pFullscreenQuadsCount->DigestUpdate();
const bool typeUpdated = pFullscreenQuadsType->DigestUpdate();
const bool renderpassUpdated = pFullscreenQuadsSingleRenderpass->DigestUpdate();
const bool colorUpdated = pFullscreenQuadsColor->DigestUpdate();
const bool anyUpdate = countUpdated || typeUpdated || renderpassUpdated || colorUpdated;

// Set Visibilities
if (pFullscreenQuadsCount->GetValue() > 0) {
pFullscreenQuadsType->SetVisible(true);
Expand All @@ -986,6 +1044,7 @@ void GraphicsBenchmarkApp::ProcessQuadsKnobs()
pFullscreenQuadsSingleRenderpass->SetVisible(false);
pFullscreenQuadsColor->SetVisible(false);
}
return anyUpdate;
}

#if defined(PPX_BUILD_XR)
Expand Down Expand Up @@ -1169,7 +1228,23 @@ void GraphicsBenchmarkApp::UpdateGUI()

void GraphicsBenchmarkApp::DrawExtraInfo()
{
constexpr const char* kUTF8PlusMinus = "\xC2\xB1"; // \u00B1

ImGui::Columns(2);

const auto cpuSubmissionTimeLive = GetLiveStatistics(mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime]);
ImGui::Text("CPU Submission Time");
ImGui::NextColumn();
ImGui::Text(
"%.4f ms min=%.4f max=%.4f\n%.4f%s%.4f ms",
cpuSubmissionTimeLive.Latest(),
cpuSubmissionTimeLive.Min(),
cpuSubmissionTimeLive.Max(),
cpuSubmissionTimeLive.Mean(),
kUTF8PlusMinus,
cpuSubmissionTimeLive.StandardDeviation());
ImGui::NextColumn();

if (HasActiveMetricsRun()) {
const auto cpuSubmissionTime = GetGaugeBasicStatistics(mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime]);

Expand Down Expand Up @@ -1198,17 +1273,32 @@ void GraphicsBenchmarkApp::DrawExtraInfo()

uint64_t frequency = 0;
PPX_CHECKED_CALL(GetGraphicsQueue()->GetTimestampFrequency(&frequency));
const float gpuWorkDurationInSec = static_cast<float>(mGpuWorkDuration / static_cast<double>(frequency));
const float gpuWorkDurationInMs = gpuWorkDurationInSec * 1000.0f;
const float sPerTick = 1.0f / static_cast<float>(frequency);
const float msPerTick = 1000.0f / static_cast<float>(frequency);

const auto gpuWork = GetLiveStatistics(mMetricsData.metrics[MetricsData::kTypeGPUWorkDuration]);
const double gpuWorkStd = gpuWork.StandardDeviation();
ImGui::Text("GPU Work Duration");
ImGui::NextColumn();
ImGui::Text("%.2f ms ", gpuWorkDurationInMs);
ImGui::Text(
"%.4f ms min=%.4f max=%.4f\n%.4f%s%.4f ms",
gpuWork.Latest(),
gpuWork.Min(),
gpuWork.Max(),
gpuWork.Mean(),
kUTF8PlusMinus,
gpuWork.StandardDeviation());
ImGui::NextColumn();

const float gpuFPS = static_cast<float>(frequency / static_cast<double>(mGpuWorkDuration));
const double gpuFPS = 1000.0 / gpuWork.Latest();
const double gpuAvgFPS = 1000.0 / gpuWork.Mean();
// The standard deviation of fps is a bit ill-defined, use a reasonable substitute.
const double gpuStdFPS = std::max(
1000.0 / std::max(gpuWork.Mean() - gpuWorkStd, gpuWork.Min()) - gpuAvgFPS,
gpuAvgFPS - 1000.0 / std::min(gpuWork.Mean() + gpuWorkStd, gpuWork.Max()));
ImGui::Text("GPU FPS");
ImGui::NextColumn();
ImGui::Text("%.2f fps ", gpuFPS);
ImGui::Text("%.2f fps\n%.2f%s%.2f fps", gpuFPS, gpuAvgFPS, kUTF8PlusMinus, gpuStdFPS);
ImGui::NextColumn();

const grfx::Format swapchainColorFormat = GetSwapchain()->GetColorFormat();
Expand Down Expand Up @@ -1239,6 +1329,19 @@ void GraphicsBenchmarkApp::DrawExtraInfo()
ImGui::Text("%.2f GB", dataWriteInGb);
ImGui::NextColumn();

const auto bandwidthLive = GetLiveStatistics(mMetricsData.metrics[MetricsData::kTypeBandwidth]);
ImGui::Text("Write Bandwidth");
ImGui::NextColumn();
ImGui::Text(
"%.2f GB/s min=%.2f max=%.2f\n%.2f%s%.2f GB/s",
bandwidthLive.Latest(),
bandwidthLive.Min(),
bandwidthLive.Max(),
bandwidthLive.Mean(),
kUTF8PlusMinus,
bandwidthLive.StandardDeviation());
ImGui::NextColumn();

if (HasActiveMetricsRun()) {
const auto bandwidth = GetGaugeBasicStatistics(mMetricsData.metrics[MetricsData::kTypeBandwidth]);
ImGui::Text("Average Write Bandwidth");
Expand Down
13 changes: 9 additions & 4 deletions benchmarks/graphics_pipeline/GraphicsBenchmarkApp.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -436,7 +436,7 @@ class GraphicsBenchmarkApp
float3 mLightPosition = float3(10, 250, 10);
std::array<bool, TOTAL_KEY_COUNT> mPressedKeys = {0};
bool mEnableMouseMovement = true;
uint64_t mGpuWorkDuration;
uint64_t mGpuWorkDuration = 0;
grfx::SamplerPtr mLinearSampler;
grfx::DescriptorPoolPtr mDescriptorPool;
std::vector<OffscreenFrame> mOffscreenFrame;
Expand Down Expand Up @@ -480,8 +480,9 @@ class GraphicsBenchmarkApp
{
enum MetricsType : size_t
{
kTypeCPUSubmissionTime = 0,
kTypeBandwidth,
kTypeCPUSubmissionTime = 0, // in ms
kTypeBandwidth, // in GiB/s
kTypeGPUWorkDuration, // in ms
kCount
};

Expand Down Expand Up @@ -549,6 +550,7 @@ class GraphicsBenchmarkApp

// Metrics related functions
virtual void SetupMetrics() override;
virtual void SetupMetricsRun() override;
virtual void UpdateMetrics() override;

Result CompileSpherePipeline(const SpherePipelineKey& key);
Expand All @@ -571,7 +573,10 @@ class GraphicsBenchmarkApp
// Processing changed state
void ProcessInput();
void ProcessKnobs();
void ProcessQuadsKnobs();
// Process quad/sphere knobs, return if any of them has been updated.
bool ProcessSphereKnobs();
bool ProcessQuadsKnobs();
bool ProcessOffscreenRenderKnobs();

// Drawing GUI
void UpdateGUI();
Expand Down
29 changes: 22 additions & 7 deletions include/ppx/application.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -423,16 +423,17 @@ class Application
virtual void DrawGui(){}; // Draw additional project-related information to ImGui.

// Override these methods in a derived class to change the default behavior of metrics.
// Virtual for unit testing purposes.
virtual void SetupMetrics();
// Virtual for unit testing purposes.
virtual void ShutdownMetrics();
virtual void SetupMetrics(); // Called once on application startup
virtual void ShutdownMetrics(); // Called once on application shutdown
virtual void StartDefaultMetricsRun(); // Called once after SetupMetrics
virtual void SetupMetricsRun(); // Called by StartMetricsRun after metric run started

// NOTE: This function can be used for BOTH displayed AND recorded metrics.
// Thus it should always be called once per frame. Virtual for unit testing purposes.
virtual void UpdateMetrics() {}

virtual metrics::GaugeBasicStatistics GetGaugeBasicStatistics(metrics::MetricID id) const;
metrics::GaugeBasicStatistics GetGaugeBasicStatistics(metrics::MetricID id) const;
metrics::LiveStatistics GetLiveStatistics(metrics::MetricID id) const;

void TakeScreenshot();

Expand Down Expand Up @@ -523,13 +524,27 @@ class Application
// See StartMetricsRun for why this wrapper is necessary.
virtual bool HasActiveMetricsRun() const;

// Allocate a metric id to be used for a combind live/recorded metric.
metrics::MetricID AllocateMetricID();

// Add a live metric, the returned MetricID can also be used for recorded metric.
metrics::MetricID AddLiveMetric(metrics::MetricID metricID = metrics::kInvalidMetricID);

// Clear history of live metric, usually after knob changed.
void ClearLiveMetricsHistory();

// Adds a metric to the current run. If no run is active, returns metrics::kInvalidMetricID.
// See StartMetricsRun for why this wrapper is necessary.
virtual metrics::MetricID AddMetric(const metrics::MetricMetadata& metadata);
metrics::MetricID AddMetric(
const metrics::MetricMetadata& metadata,
metrics::MetricID metricID = metrics::kInvalidMetricID);

// Record data for the given metric ID. Metrics for completed runs will be discarded.
// See StartMetricsRun for why this wrapper is necessary.
virtual bool RecordMetricData(metrics::MetricID id, const metrics::MetricData& data);
bool RecordMetricData(metrics::MetricID id, const metrics::MetricData& data);

// Update live metric, if a run is active, it will also record to the run.
bool RecordLiveMetricData(metrics::MetricID id, const metrics::MetricData& data);

#if defined(PPX_BUILD_XR)
XrComponent& GetXrComponent()
Expand Down
Loading

0 comments on commit 557479c

Please sign in to comment.