Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

LiveMetric: realtime statistics for debug UI. #405

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

LiveMetric: realtime statistics for debug UI. #405

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8b98a0d
LiveMetric: realtime statistics for debug UI.
hysw Jan 16, 2024
e987c9a
Address reviewer comment and add test
hysw Jan 16, 2024
6bab0a5
Fix typo, add more comments and tests
hysw Jan 17, 2024
File filter

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Fix typo, add more comments and tests
  • Loading branch information
@hysw
hysw committed Jan 17, 2024
commit 6bab0a5084c7f1c4bdc844afe255f231414eecbf
8 changes: 4 additions & 4 deletions include/ppx/application.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -524,18 +524,18 @@ class Application
// See StartMetricsRun for why this wrapper is necessary.
virtual bool HasActiveMetricsRun() const;

// Allocate a metric id to be used for a combind live/recorded metric.
// Allocate a metric id to be used for a combined live/recorded metric.
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need to have all of these metric related functions in Application?
metrics::MetricID AllocateMetricID();

// Adds a metric to the current run. If no run is active, returns metrics::kInvalidMetricID.
// See StartMetricsRun for why this wrapper is necessary.
metrics::MetricID AddMetric(const metrics::MetricMetadata& metadata);

// Bind a metric to the current run, Return false if no run is active.
// Bind a metric to the current run, return false if no run is active.
bool BindMetric(metrics::MetricID metricID, const metrics::MetricMetadata& metadata);

// Add a live metric, the returned MetricID can also be used for recorded metric.
bool BindLiveMetric(metrics::MetricID metricID = metrics::kInvalidMetricID);
// Bind a live metric, return true on success.
bool BindLiveMetric(metrics::MetricID metricID);
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you elaborate here? What does "binding" a metric mean? Why do we need to call that? What happens if we don't?
I would ideally like to know these things without reading the code.

// Clear history of live metric, usually after knob changed.
void ClearLiveMetricsHistory();
Expand Down
9 changes: 7 additions & 2 deletions include/ppx/metrics.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,7 @@ class MetricCounter final : public Metric

////////////////////////////////////////////////////////////////////////////////

// Live statistics are computerd on the fly, for recently reported metric.
// Live statistics are computed on the fly, for recently reported metric.
// The weight assigned to each entry is assigned as follows
// w_i = exp((t_i-t_now)/halfLife), default halfLife = 0.5s
// Min/Max are not affected by the weight.
Expand Down Expand Up @@ -396,8 +396,13 @@ class Manager final
MetricID metricID = AllocateID();
return BindMetric(metricID, metadata) ? metricID : kInvalidMetricID;
}

// Binds a metric to the current run. A run must be started to bind a metric.
// Failure to add a metric returns false.
bool BindMetric(MetricID metricID, const MetricMetadata& metadata);

// Binds a live metric to the metricID.
// A live metric is independent of runs, the same metricID can be used to bind a metric
// to a run, in which case RecordMetricData records to both live metric and the run.
bool BindLiveMetric(MetricID metricID, double halfLife = LiveMetric::kDefaultHalfLife);

// Records data for the given metric ID. Metrics for completed runs will be discarded.
Expand Down
84 changes: 83 additions & 1 deletion src/test/metrics_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,14 @@ TEST(MetricsTest, ManagerStartDuplicateRunFails)
}
#endif

TEST(MetricsTest, ManagerBindLiveMetricWithoutRun)
{
metrics::Manager manager;
EXPECT_FALSE(manager.HasActiveRun());
auto metricId = manager.AllocateID();
EXPECT_TRUE(manager.BindLiveMetric(metricId));
}

////////////////////////////////////////////////////////////////////////////////
// Run Tests
////////////////////////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -546,7 +554,7 @@ TEST_F(MetricsTestFixture, MetricsGaugeIgnoresSameSeconds)
EXPECT_EQ(gauge["time_series"][1][1], 11.0);
}

TEST_F(MetricsTestFixture, MetricsLiveStatistics)
TEST_F(MetricsTestFixture, MetricsLiveMetricStatistics)
{
metrics::MetricMetadata metadata;
metadata.type = metrics::MetricType::GAUGE;
Expand Down Expand Up @@ -596,4 +604,78 @@ TEST_F(MetricsTestFixture, MetricsLiveStatistics)
EXPECT_EQ(reportStats["standard_deviation"], 1.0);
}

TEST_F(MetricsTestFixture, MetricsLiveMetricAfterEndRun)
{
metrics::MetricMetadata metadata;
metadata.type = metrics::MetricType::GAUGE;
metadata.name = "gauge";

auto metricId = pManager->AllocateID();
ASSERT_TRUE(pManager->BindLiveMetric(metricId, 1.0));
ASSERT_TRUE(pManager->BindMetric(metricId, metadata));

metrics::MetricData data = {metrics::MetricType::GAUGE};
data.gauge.seconds = 1.0;
data.gauge.value = 10.0;
pManager->RecordMetricData(metricId, data);
pManager->EndRun();
data.gauge.seconds = 2.0;
data.gauge.value = 12.0;
pManager->RecordMetricData(metricId, data);

auto liveStats = pManager->GetLiveStatistics(metricId);

EXPECT_NEAR(liveStats.mean, 11.3333, 0.0010);
EXPECT_NEAR(liveStats.variance, 8.0 / 9.0, 0.0010);
}

TEST_F(MetricsTestFixture, MetricsLiveMetricClearHistory)
{
metrics::MetricMetadata metadata;
metadata.type = metrics::MetricType::GAUGE;
metadata.name = "gauge";

auto metricId = pManager->AllocateID();
ASSERT_TRUE(pManager->BindLiveMetric(metricId, 1.0));
ASSERT_TRUE(pManager->BindMetric(metricId, metadata));

metrics::MetricData data = {metrics::MetricType::GAUGE};
data.gauge.seconds = 1.0;
data.gauge.value = 10.0;
pManager->RecordMetricData(metricId, data);
pManager->ClearLiveMetricsHistory();
data.gauge.seconds = 2.0;
data.gauge.value = 12.0;
pManager->RecordMetricData(metricId, data);

auto result = pManager->CreateReport("report").GetContentString();
auto parsed = nlohmann::json::parse(result);
auto gauge = parsed["runs"][0]["gauges"][0];
EXPECT_EQ(gauge["time_series"].size(), 2);
EXPECT_EQ(gauge["time_series"][0][0], 1.0000);
EXPECT_EQ(gauge["time_series"][0][1], 10.0);
EXPECT_EQ(gauge["time_series"][1][0], 2.0);
EXPECT_EQ(gauge["time_series"][1][1], 12.0);

auto liveStats = pManager->GetLiveStatistics(metricId);

// 10.0 with weight 0.5, and 12.0 with weight 1.0
constexpr double kExpectedMean = (10.0 * 0.5 + 12.0 * 1.0) / 1.5;
constexpr double kExpectedVariance =
((10.0 - kExpectedMean) * (10.0 - kExpectedMean) * 0.5 + (12.0 - kExpectedMean) * (12.0 - kExpectedMean) * 1.0) / 1.5;
EXPECT_EQ(liveStats.latest, 12.0);
EXPECT_EQ(liveStats.seconds, 2.0);
EXPECT_EQ(liveStats.min, 12.0);
EXPECT_EQ(liveStats.max, 12.0);
EXPECT_EQ(liveStats.weight, 1.0);
EXPECT_NEAR(liveStats.mean, 12.0, 0.0010);
EXPECT_NEAR(liveStats.variance, 0.0, 0.0010);

auto reportStats = gauge["statistics"];
EXPECT_EQ(reportStats["min"], 10.0);
EXPECT_EQ(reportStats["max"], 12.0);
EXPECT_EQ(reportStats["average"], 11.0);
EXPECT_EQ(reportStats["standard_deviation"], 1.0);
}

} // namespace ppx