Add POC support for Google's WebFeatures Baseline

package.json

@@ -17,6 +17,7 @@
   },
   "homepage": "https://github.com/rum-archive/rum-insights-data#readme",
   "dependencies": {
-    "@google-cloud/bigquery": "^6.0.3"
+    "@google-cloud/bigquery": "^6.0.3",
+    "web-features": "^0.6.3"
   }
 }

pipeline.js

@@ -51,7 +51,12 @@ async function processResults( bigQueryResults ) {
             }
             else if ( query.processingtype === "groupedMetricPerDevice" ){
                 let processedData = [];
-                processedData = processing.processGroupedMetricPerDevicetype( data, query.extractmetric, query.groupby );
+
+                // quick and dirty adding full counts for the recent versions for web features baseline testing
+                if ( query.filename === "recent_useragentversion_useragentfamily_devicetype" )
+                    processedData = processing.processGroupedMetricPerDevicetype( data, query.extractmetric, query.groupby, true );
+                else 
+                    processedData = processing.processGroupedMetricPerDevicetype( data, query.extractmetric, query.groupby, false );
 
                 await fs.writeFile( outputPath, JSON.stringify(processedData), "utf8" );
             }

queries/recent_useragentversion_useragentfamily_devicetype.jsonl

@@ -0,0 +1,20 @@
+{
+    "description": "Single day version of useragentversion_useragentfamily_devicetype to more easily support WebFeatures Baseline overviews",
+    "datetype": "recent_day",
+    "processingtype": "groupedMetricPerDevice",
+    "extractmetric": "version",
+    "groupby": "family",
+    "comments": "We exclude null values here due to the already high cardinality of results. We manually select the 'interesting' user agents for the same reason.",
+    "sql": "
+        SELECT DATE as date, 
+        USERAGENTVERSION as version,
+        USERAGENTFAMILY as family, 
+        DEVICETYPE as device, 
+        COUNT(*) as rowcount, 
+        SUM(BEACONS) as beaconcount
+        FROM `akamai-mpulse-rumarchive.rumarchive.rumarchive_page_loads` 
+        WHERE DEVICETYPE IS NOT NULL and USERAGENTFAMILY IS NOT NULL AND USERAGENTVERSION IS NOT NULL AND {{DATES}}
+            AND USERAGENTFAMILY in ('Chrome', 'Chrome Mobile', 'Chrome Mobile WebView', 'Chrome Mobile iOS', 'Safari', 'Mobile Safari', 'Mobile Safari UI/WKWebView', 'Edge', 'Facebook', 'Firefox', 'Firefox Mobile', 'Opera', 'Opera Mini', 'Samsung Internet', 'Instagram')
+        GROUP BY DATE, DEVICETYPE, USERAGENTFAMILY, USERAGENTVERSION 
+        ORDER BY DATE ASC, DEVICETYPE ASC, USERAGENTFAMILY ASC, beaconcount DESC"
+}

src/processing.js

@@ -186,7 +186,7 @@ function processSingleMetricPerDevicetype(rows, metricFieldName) {
 }
 
 
-function processGroupedMetricPerDevicetype(rows, metricFieldName, groupbyFieldName) {
+function processGroupedMetricPerDevicetype(rows, metricFieldName, groupbyFieldName, includeFullCount = false) {
     /*
         This is similar to processSingleMetricPerDevicetype, but the metrics are grouped in another dimension
         For example, SingleMetric would be
@@ -305,6 +305,10 @@ function processGroupedMetricPerDevicetype(rows, metricFieldName, groupbyFieldNa
         // highcharts uses raw timestamps, so pre-calculate them
         datapoint.timestamp = "" + (new Date( row.date.value ).getTime());
 
+        if ( includeFullCount ) {
+            datapoint.count = row.beaconcount;
+        }
+
         if( percent > 0.1 ) {
             // skip entries that are 0.0 percent (especially in high-cardinality queries, these often account for many megabytes of output data)
             output.push( datapoint );