Add new query for user agent versions. Made query building with dynam…

…ic dates more robust.

pipeline.js

@@ -93,6 +93,9 @@ async function runQueries( queries ) {
                 query.cachePath = cachePath;
                 continue;
             }
+            else {
+                console.log(`Executing Query ${query.filename} via BigQuery`);
+            }
 
             await runBigQuery( query.sql, cachePath, GLOBAL_DEBUG );
             query.cachePath = cachePath;
@@ -216,12 +219,14 @@ async function getQueries() {
     }
 
     // concatenate the dates to be used in the sql queries
-    let allDatesString = "";
+    let allDatesString = "(";
     for ( const [idx, date] of allDates.entries() ) {
         allDatesString += "DATE = \"" + date + "\"";
         if ( idx != allDates.length - 1 )
             allDatesString += " OR ";
     }
+    allDatesString += ")";
+
     const lastDateString = "DATE = \"" + lastDate + "\"";
 
     // run the actual query definitions from filesystem

queries/useragentversion_useragentfamily_devicetype.jsonl

@@ -0,0 +1,20 @@
+{
+    "description": "Timeseries of the different versions per user agent per device type, ordered by occurrence seen in the RUM Archive dataset",
+    "datetype": "timeseries",
+    "processingtype": "groupedMetricPerDevice",
+    "extractmetric": "version",
+    "groupby": "family",
+    "comments": "We exclude null values here due to the already high cardinality of results. We manually select the 'interesting' user agents for the same reason.",
+    "sql": "
+        SELECT DATE as date, 
+        USERAGENTVERSION as version,
+        USERAGENTFAMILY as family, 
+        DEVICETYPE as device, 
+        COUNT(*) as rowcount, 
+        SUM(BEACONS) as beaconcount
+        FROM `akamai-mpulse-rumarchive.rumarchive.rumarchive_page_loads` 
+        WHERE DEVICETYPE IS NOT NULL and USERAGENTFAMILY IS NOT NULL AND USERAGENTVERSION IS NOT NULL AND {{TIMESERIES_DATES}}
+            AND USERAGENTFAMILY in ('Chrome', 'Chrome Mobile', 'Chrome Mobile WebView', 'Chrome Mobile iOS', 'Safari', 'Mobile Safari', 'Mobile Safari UI/WKWebView', 'Edge', 'Facebook', 'Firefox', 'Firefox Mobile', 'Opera', 'Opera Mini', 'Samsung Internet', 'Instagram')
+        GROUP BY DATE, DEVICETYPE, USERAGENTFAMILY, USERAGENTVERSION 
+        ORDER BY DATE ASC, DEVICETYPE ASC, USERAGENTFAMILY ASC, beaconcount DESC"
+}