Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

LiveMetric: realtime statistics for debug UI. #405

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

LiveMetric: realtime statistics for debug UI. #405

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8b98a0d
LiveMetric: realtime statistics for debug UI.
hysw Jan 16, 2024
e987c9a
Address reviewer comment and add test
hysw Jan 16, 2024
6bab0a5
Fix typo, add more comments and tests
hysw Jan 17, 2024
File filter

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
173 changes: 138 additions & 35 deletions benchmarks/graphics_pipeline/GraphicsBenchmarkApp.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
#include "GraphicsBenchmarkApp.h"
#include "SphereMesh.h"

#include "ppx/application.h"
#include "ppx/graphics_util.h"
#include "ppx/grfx/grfx_format.h"
#include "ppx/timer.h"
Expand Down Expand Up @@ -300,30 +301,51 @@ void GraphicsBenchmarkApp::Setup()
void GraphicsBenchmarkApp::SetupMetrics()
{
Application::SetupMetrics();
if (HasActiveMetricsRun()) {
ppx::metrics::MetricMetadata metadata = {ppx::metrics::MetricType::GAUGE, "CPU Submission Time", "ms", ppx::metrics::MetricInterpretation::LOWER_IS_BETTER, {0.f, 10000.f}};
mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime] = AddMetric(metadata);
PPX_ASSERT_MSG(mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime] != ppx::metrics::kInvalidMetricID, "Failed to add CPU Submission Time metric");

metadata = {ppx::metrics::MetricType::GAUGE, "Bandwidth", "GB/s", ppx::metrics::MetricInterpretation::HIGHER_IS_BETTER, {0.f, 10000.f}};
mMetricsData.metrics[MetricsData::kTypeBandwidth] = AddMetric(metadata);
PPX_ASSERT_MSG(mMetricsData.metrics[MetricsData::kTypeBandwidth] != ppx::metrics::kInvalidMetricID, "Failed to add Bandwidth metric");
}
mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime] = AllocateMetricID();
mMetricsData.metrics[MetricsData::kTypeBandwidth] = AllocateMetricID();
mMetricsData.metrics[MetricsData::kTypeGPUWorkDuration] = AllocateMetricID();

BindLiveMetric(mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime]);
BindLiveMetric(mMetricsData.metrics[MetricsData::kTypeBandwidth]);
BindLiveMetric(mMetricsData.metrics[MetricsData::kTypeGPUWorkDuration]);
}

void GraphicsBenchmarkApp::UpdateMetrics()
void GraphicsBenchmarkApp::SetupMetricsRun()
{
if (!HasActiveMetricsRun()) {
return;
Application::SetupMetricsRun();
{
ppx::metrics::MetricMetadata metadata = {
ppx::metrics::MetricType::GAUGE,
"CPU Submission Time",
"ms",
ppx::metrics::MetricInterpretation::LOWER_IS_BETTER,
{0.f, 10000.f}};
bool bindres = BindMetric(mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime], metadata);
PPX_ASSERT_MSG(bindres, "Failed to bind CPU Submission Time metric");
}
{
ppx::metrics::MetricMetadata metadata = {
ppx::metrics::MetricType::GAUGE,
"Bandwidth",
"GiB/s",
ppx::metrics::MetricInterpretation::HIGHER_IS_BETTER,
{0.f, 10000.f}};
bool bindres = BindMetric(mMetricsData.metrics[MetricsData::kTypeBandwidth], metadata);
PPX_ASSERT_MSG(bindres, "Failed to bind Bandwidth metric");
}
}

void GraphicsBenchmarkApp::UpdateMetrics()
{
uint64_t frequency = 0;
PPX_CHECKED_CALL(GetGraphicsQueue()->GetTimestampFrequency(&frequency));

ppx::metrics::MetricData data = {ppx::metrics::MetricType::GAUGE};
data.gauge.seconds = GetElapsedSeconds();
data.gauge.value = mCPUSubmissionTime;
RecordMetricData(mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime], data);
{
ppx::metrics::MetricData data = {ppx::metrics::MetricType::GAUGE};
data.gauge.seconds = GetElapsedSeconds();
data.gauge.value = mCPUSubmissionTime;
RecordLiveMetricData(mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime], data);
}

const float gpuWorkDurationInSec = static_cast<float>(mGpuWorkDuration / static_cast<double>(frequency));
const grfx::Format swapchainColorFormat = GetSwapchain()->GetColorFormat();
Expand All @@ -335,13 +357,14 @@ void GraphicsBenchmarkApp::UpdateMetrics()
const uint32_t height = isOffscreen ? mOffscreenFrame.back().height : swapchainHeight;
const uint32_t quadCount = pFullscreenQuadsCount->GetValue();

if (quadCount) {
// Skip the first kSkipFrameCount frames after the knob of quad count being changed to avoid noise
constexpr uint32_t kSkipFrameCount = 2;
if (pFullscreenQuadsCount->DigestUpdate()) {
mSkipRecordBandwidthMetricFrameCounter = kSkipFrameCount;
}
{
ppx::metrics::MetricData data = {ppx::metrics::MetricType::GAUGE};
data.gauge.seconds = GetElapsedSeconds();
data.gauge.value = gpuWorkDurationInSec * 1000.0f;
RecordLiveMetricData(mMetricsData.metrics[MetricsData::kTypeGPUWorkDuration], data);
}

if (quadCount) {
if (mSkipRecordBandwidthMetricFrameCounter == 0) {
const auto texelSize = static_cast<float>(grfx::GetFormatDescription(colorFormat)->bytesPerTexel);
const float dataWriteInGb = (static_cast<float>(width) * static_cast<float>(height) * texelSize * quadCount) / (1024.f * 1024.f * 1024.f);
Expand All @@ -350,7 +373,7 @@ void GraphicsBenchmarkApp::UpdateMetrics()
ppx::metrics::MetricData data = {ppx::metrics::MetricType::GAUGE};
data.gauge.seconds = GetElapsedSeconds();
data.gauge.value = bandwidth;
RecordMetricData(mMetricsData.metrics[MetricsData::kTypeBandwidth], data);
RecordLiveMetricData(mMetricsData.metrics[MetricsData::kTypeBandwidth], data);
}
else {
--mSkipRecordBandwidthMetricFrameCounter;
Expand Down Expand Up @@ -893,6 +916,22 @@ void GraphicsBenchmarkApp::ProcessInput()
}

void GraphicsBenchmarkApp::ProcessKnobs()
{
// Skip the first kSkipFrameCount frames after the knob of quad count being changed to avoid noise
constexpr uint32_t kSkipFrameCount = 2;

const bool sphereChanged = ProcessSphereKnobs();
const bool quadChanged = ProcessQuadsKnobs();
const bool framebufferChanged = ProcessOffscreenRenderKnobs();
const bool anyUpdate = sphereChanged || quadChanged || framebufferChanged;

if (anyUpdate) {
ClearLiveMetricsHistory();
mSkipRecordBandwidthMetricFrameCounter = kSkipFrameCount;
}
}

bool GraphicsBenchmarkApp::ProcessSphereKnobs()
{
// Detect if any knob value has been changed
// Note: DigestUpdate should be called only once per frame (DigestUpdate unflags the internal knob variable)!
Expand All @@ -901,6 +940,14 @@ void GraphicsBenchmarkApp::ProcessKnobs()
const bool depthTestWriteKnobChanged = pDepthTestWrite->DigestUpdate();
const bool enableSpheresKnobChanged = pEnableSpheres->DigestUpdate();
const bool sphereInstanceCountKnobChanged = pSphereInstanceCount->DigestUpdate();
const bool debugViewChanged = pDebugViews->DigestUpdate();
const bool anyUpdate =
(allTexturesTo1x1KnobChanged ||
alphaBlendKnobChanged ||
depthTestWriteKnobChanged ||
enableSpheresKnobChanged ||
sphereInstanceCountKnobChanged ||
debugViewChanged);

// TODO: Ideally, the `maxValue` of the drawcall-count slider knob should be changed at runtime.
// Currently, the value of the drawcall-count is adjusted to the sphere-count in case the
Expand Down Expand Up @@ -948,16 +995,20 @@ void GraphicsBenchmarkApp::ProcessKnobs()
}
}
}
return anyUpdate;
}

ProcessQuadsKnobs();
bool GraphicsBenchmarkApp::ProcessOffscreenRenderKnobs()
{
const bool offscreenChanged = pRenderOffscreen->DigestUpdate();
const bool framebufferChanged = pResolution->DigestUpdate() || pFramebufferFormat->DigestUpdate();
const bool anyUpdate = offscreenChanged || framebufferChanged;

bool offscreenChanged = pRenderOffscreen->DigestUpdate();
if (offscreenChanged) {
pBlitOffscreen->SetVisible(pRenderOffscreen->GetValue());
pFramebufferFormat->SetVisible(pRenderOffscreen->GetValue());
pResolution->SetVisible(pRenderOffscreen->GetValue());
}
bool framebufferChanged = pResolution->DigestUpdate() || pFramebufferFormat->DigestUpdate();

if ((offscreenChanged && pRenderOffscreen->GetValue()) || framebufferChanged) {
std::pair<int, int> resolution = pResolution->GetValue();
Expand All @@ -966,10 +1017,17 @@ void GraphicsBenchmarkApp::ProcessKnobs()
int fbHeight = (resolution.second > 0 ? resolution.second : GetSwapchain()->GetHeight());
UpdateOffscreenBuffer(RenderFormat(), fbWidth, fbHeight);
}
return anyUpdate;
}

void GraphicsBenchmarkApp::ProcessQuadsKnobs()
bool GraphicsBenchmarkApp::ProcessQuadsKnobs()
{
const bool countUpdated = pFullscreenQuadsCount->DigestUpdate();
const bool typeUpdated = pFullscreenQuadsType->DigestUpdate();
const bool renderpassUpdated = pFullscreenQuadsSingleRenderpass->DigestUpdate();
const bool colorUpdated = pFullscreenQuadsColor->DigestUpdate();
const bool anyUpdate = countUpdated || typeUpdated || renderpassUpdated || colorUpdated;

// Set Visibilities
if (pFullscreenQuadsCount->GetValue() > 0) {
pFullscreenQuadsType->SetVisible(true);
Expand All @@ -986,6 +1044,7 @@ void GraphicsBenchmarkApp::ProcessQuadsKnobs()
pFullscreenQuadsSingleRenderpass->SetVisible(false);
pFullscreenQuadsColor->SetVisible(false);
}
return anyUpdate;
}

#if defined(PPX_BUILD_XR)
Expand Down Expand Up @@ -1169,7 +1228,23 @@ void GraphicsBenchmarkApp::UpdateGUI()

void GraphicsBenchmarkApp::DrawExtraInfo()
{
constexpr const char* kUTF8PlusMinus = "\xC2\xB1"; // \u00B1

ImGui::Columns(2);

const auto cpuSubmissionTimeLive = GetLiveStatistics(mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime]);
ImGui::Text("CPU Submission Time");
ImGui::NextColumn();
ImGui::Text(
"%.4f ms min=%.4f max=%.4f\n%.4f%s%.4f ms",
cpuSubmissionTimeLive.Latest(),
cpuSubmissionTimeLive.Min(),
cpuSubmissionTimeLive.Max(),
cpuSubmissionTimeLive.Mean(),
kUTF8PlusMinus,
cpuSubmissionTimeLive.StandardDeviation());
ImGui::NextColumn();

if (HasActiveMetricsRun()) {
const auto cpuSubmissionTime = GetGaugeBasicStatistics(mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime]);

Expand Down Expand Up @@ -1198,17 +1273,32 @@ void GraphicsBenchmarkApp::DrawExtraInfo()

uint64_t frequency = 0;
PPX_CHECKED_CALL(GetGraphicsQueue()->GetTimestampFrequency(&frequency));
const float gpuWorkDurationInSec = static_cast<float>(mGpuWorkDuration / static_cast<double>(frequency));
const float gpuWorkDurationInMs = gpuWorkDurationInSec * 1000.0f;
const float sPerTick = 1.0f / static_cast<float>(frequency);
const float msPerTick = 1000.0f / static_cast<float>(frequency);

const auto gpuWork = GetLiveStatistics(mMetricsData.metrics[MetricsData::kTypeGPUWorkDuration]);
const double gpuWorkStd = gpuWork.StandardDeviation();
ImGui::Text("GPU Work Duration");
ImGui::NextColumn();
ImGui::Text("%.2f ms ", gpuWorkDurationInMs);
ImGui::Text(
"%.4f ms min=%.4f max=%.4f\n%.4f%s%.4f ms",
gpuWork.Latest(),
gpuWork.Min(),
gpuWork.Max(),
gpuWork.Mean(),
kUTF8PlusMinus,
gpuWork.StandardDeviation());
ImGui::NextColumn();

const float gpuFPS = static_cast<float>(frequency / static_cast<double>(mGpuWorkDuration));
const double gpuFPS = 1000.0 / gpuWork.Latest();
const double gpuAvgFPS = 1000.0 / gpuWork.Mean();
// The standard deviation of fps is a bit ill-defined, use a reasonable substitute.
const double gpuStdFPS = std::max(
1000.0 / std::max(gpuWork.Mean() - gpuWorkStd, gpuWork.Min()) - gpuAvgFPS,
gpuAvgFPS - 1000.0 / std::min(gpuWork.Mean() + gpuWorkStd, gpuWork.Max()));
ImGui::Text("GPU FPS");
ImGui::NextColumn();
ImGui::Text("%.2f fps ", gpuFPS);
ImGui::Text("%.2f fps\n%.2f%s%.2f fps", gpuFPS, gpuAvgFPS, kUTF8PlusMinus, gpuStdFPS);
ImGui::NextColumn();

const grfx::Format swapchainColorFormat = GetSwapchain()->GetColorFormat();
Expand Down Expand Up @@ -1239,21 +1329,34 @@ void GraphicsBenchmarkApp::DrawExtraInfo()
ImGui::Text("%.2f GB", dataWriteInGb);
ImGui::NextColumn();

const auto bandwidthLive = GetLiveStatistics(mMetricsData.metrics[MetricsData::kTypeBandwidth]);
ImGui::Text("Write Bandwidth");
ImGui::NextColumn();
ImGui::Text(
"%.2f GiB/s min=%.2f max=%.2f\n%.2f%s%.2f GiB/s",
bandwidthLive.Latest(),
bandwidthLive.Min(),
bandwidthLive.Max(),
bandwidthLive.Mean(),
kUTF8PlusMinus,
bandwidthLive.StandardDeviation());
ImGui::NextColumn();

if (HasActiveMetricsRun()) {
const auto bandwidth = GetGaugeBasicStatistics(mMetricsData.metrics[MetricsData::kTypeBandwidth]);
ImGui::Text("Average Write Bandwidth");
ImGui::NextColumn();
ImGui::Text("%.2f GB/s", bandwidth.average);
ImGui::Text("%.2f GiB/s", bandwidth.average);
ImGui::NextColumn();

ImGui::Text("Min Write Bandwidth");
ImGui::NextColumn();
ImGui::Text("%.2f GB/s", bandwidth.min);
ImGui::Text("%.2f GiB/s", bandwidth.min);
ImGui::NextColumn();

ImGui::Text("Max Write Bandwidth");
ImGui::NextColumn();
ImGui::Text("%.2f GB/s", bandwidth.max);
ImGui::Text("%.2f GiB/s", bandwidth.max);
ImGui::NextColumn();
}
}
Expand Down
13 changes: 9 additions & 4 deletions benchmarks/graphics_pipeline/GraphicsBenchmarkApp.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -436,7 +436,7 @@ class GraphicsBenchmarkApp
float3 mLightPosition = float3(10, 250, 10);
std::array<bool, TOTAL_KEY_COUNT> mPressedKeys = {0};
bool mEnableMouseMovement = true;
uint64_t mGpuWorkDuration;
uint64_t mGpuWorkDuration = 0;
grfx::SamplerPtr mLinearSampler;
grfx::DescriptorPoolPtr mDescriptorPool;
std::vector<OffscreenFrame> mOffscreenFrame;
Expand Down Expand Up @@ -480,8 +480,9 @@ class GraphicsBenchmarkApp
{
enum MetricsType : size_t
{
kTypeCPUSubmissionTime = 0,
kTypeBandwidth,
kTypeCPUSubmissionTime = 0, // in ms
kTypeBandwidth, // in GiB/s
kTypeGPUWorkDuration, // in ms
kCount
};

Expand Down Expand Up @@ -549,6 +550,7 @@ class GraphicsBenchmarkApp

// Metrics related functions
virtual void SetupMetrics() override;
virtual void SetupMetricsRun() override;
virtual void UpdateMetrics() override;

Result CompileSpherePipeline(const SpherePipelineKey& key);
Expand All @@ -571,7 +573,10 @@ class GraphicsBenchmarkApp
// Processing changed state
void ProcessInput();
void ProcessKnobs();
void ProcessQuadsKnobs();
// Process quad/sphere knobs, return if any of them has been updated.
bool ProcessSphereKnobs();
bool ProcessQuadsKnobs();
bool ProcessOffscreenRenderKnobs();

// Drawing GUI
void UpdateGUI();
Expand Down
30 changes: 23 additions & 7 deletions include/ppx/application.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -423,16 +423,17 @@ class Application
virtual void DrawGui(){}; // Draw additional project-related information to ImGui.

// Override these methods in a derived class to change the default behavior of metrics.
// Virtual for unit testing purposes.
virtual void SetupMetrics();
// Virtual for unit testing purposes.
virtual void ShutdownMetrics();
virtual void SetupMetrics(); // Called once on application startup
virtual void ShutdownMetrics(); // Called once on application shutdown
virtual void StartDefaultMetricsRun(); // Called once after SetupMetrics
virtual void SetupMetricsRun(); // Called by StartMetricsRun after metric run started

// NOTE: This function can be used for BOTH displayed AND recorded metrics.
// Thus it should always be called once per frame. Virtual for unit testing purposes.
virtual void UpdateMetrics() {}

virtual metrics::GaugeBasicStatistics GetGaugeBasicStatistics(metrics::MetricID id) const;
metrics::GaugeBasicStatistics GetGaugeBasicStatistics(metrics::MetricID id) const;
metrics::LiveStatistics GetLiveStatistics(metrics::MetricID id) const;

void TakeScreenshot();

Expand Down Expand Up @@ -523,13 +524,28 @@ class Application
// See StartMetricsRun for why this wrapper is necessary.
virtual bool HasActiveMetricsRun() const;

// Allocate a metric id to be used for a combined live/recorded metric.
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need to have all of these metric related functions in Application?
metrics::MetricID AllocateMetricID();

// Adds a metric to the current run. If no run is active, returns metrics::kInvalidMetricID.
// See StartMetricsRun for why this wrapper is necessary.
virtual metrics::MetricID AddMetric(const metrics::MetricMetadata& metadata);
metrics::MetricID AddMetric(const metrics::MetricMetadata& metadata);

// Bind a metric to the current run, return false if no run is active.
bool BindMetric(metrics::MetricID metricID, const metrics::MetricMetadata& metadata);

// Bind a live metric, return true on success.
bool BindLiveMetric(metrics::MetricID metricID);
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you elaborate here? What does "binding" a metric mean? Why do we need to call that? What happens if we don't?
I would ideally like to know these things without reading the code.

// Clear history of live metric, usually after knob changed.
void ClearLiveMetricsHistory();

// Record data for the given metric ID. Metrics for completed runs will be discarded.
// See StartMetricsRun for why this wrapper is necessary.
virtual bool RecordMetricData(metrics::MetricID id, const metrics::MetricData& data);
bool RecordMetricData(metrics::MetricID id, const metrics::MetricData& data);

// Update live metric, if a run is active, it will also record to the run.
bool RecordLiveMetricData(metrics::MetricID id, const metrics::MetricData& data);

#if defined(PPX_BUILD_XR)
XrComponent& GetXrComponent()
Expand Down
Loading