Feature: Complete migration to CDP (#539)

* Shift service worker to use CDP completely.

* Fix JS cookies for CDP.

* Fix JS cookies.
Fix TS-lint warnings.

* Remove optional chaining for frame id

* Fix condition for frame filtering.
Fix the url parser.

* Fix frameId association.

* Fix deletion of data at correct event.

* Get all cookies for current tab via CDP.

* Add multitab support.

* Add tab data when extension is updated.

* Make multitab work

* Fix multi-tab analysis methodology for CDP.

* Keep previous frame.

* Fix the inconsistency in frame and cookie association.

* Fix removal of frameIdUrlSet.

* Add some conditions.
Add cookies by parsing network urls. Just like application tab does.

* Use loaderid to load other cookies.

* Fix frame mapping of cookies.

* Test fix.

* revert test fix

* Destructure code to make it pretty.

* Enable CDP by default

* Refactor code to make it pretty.

* Fix failing tests.

* Revert some changes

* Revert using loader id.

* More precise mapping of data.

* Collect resource urls.

* collect frame resource urls.

* Send cookies from resources.

* Reinsert adding extra frame urls from service worker.
Remove unnecessary code from the synchnorous cookie store.

* Add comments explaining each thing in the chrome.debugger.

* Fix some redundant issues remove auditsIssuesStore.

* Fix errors thrown on extension page.

* Add proper frameIds to map the cookies to frames better.

* - Remove parentChildFrameAssociation from service worker.
- Move the frameId and parent Association to the synchnourousCookieStore.
- Call from cookie provider to get additional data from service worker when devtools is opened.

* Fix some variable additions.

* Move all listeners to separate files.

* Split serviceworker into more listeners.

* Fix bug.

* Fix failing tests.

* Reintroduce webRequest API.

* Fix cli tsc problems.

* create separate functions for listeners.

* Bug fix frame id association.

* Fix frame management after extension update.

* Add service worker test.
Rename chrome listeners to listeners.
Refactor CDP migration api to use new methodology of supplementation.

* Revert jest config.

* Revert parsing method.

* Remove use of mainFrameId.

* Rearrange functions in sync cookie store.

* Refactor code.

* update frameId logic

* Revert logic of frame addition.

* Refactor code and fix js cookie.

* Move cookieDB to synchnorous cookie store.

* Fix arguments in getAndParseNetworkCookies

* Fix failing tests.

* Make non cdp environement work.
Convert non-cdp frameIds to string to accomodate non-cdp cookie analysis.

* Fix failing tests.

* Detach debugger from targets when cdp is switched off

* Revert setting cdp as default true.

* Remove exclude from exclusion reason.

* Delete cdpURLMapping.

* Add jsdoc.
Correct JSDoc.
Remove tabFramesIdMap.

* Use optional chaining.

* Fix jsdoc comment in index.ts

* Small refactoring

* Assign target in the chrome.debugger event.

* Add comments.

---------

Co-authored-by: sayedtaqui <sayedwp@gmail.com>

assets/data/open-cookie-database.json

@@ -19487,4 +19487,4 @@
       "wildcard": "0"
     }
   ]
-}
+}

packages/common/src/cookies.types.ts

@@ -13,7 +13,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 /**
  * External dependencies.
  */
@@ -64,8 +63,7 @@ export type CookieDatabase = {
 
 export type BlockedReason =
   | Protocol.Network.SetCookieBlockedReason
-  | Protocol.Network.CookieBlockedReason
-  | Protocol.Audits.CookieExclusionReason;
+  | Protocol.Network.CookieBlockedReason;
 
 export enum RESPONSE_EVENT {
   CHROME_WEBREQUEST_ON_RESPONSE_STARTED = 'CHROME_WEBREQUEST_ON_RESPONSE_STARTED',
@@ -155,8 +153,7 @@ export interface TabCookies {
 
 export interface TabFrames {
   [key: string]: {
-    frameIds: number[];
-    isOnRWS?: boolean;
+    frameIds: string[];
     frameType?: 'outermost_frame' | 'fenced_frame' | 'sub_frame';
   };
 }
@@ -192,7 +189,7 @@ export interface CookieStatsComponents {
 }
 
 export interface FramesWithCookies {
-  [key: string]: { frameIds: number[] };
+  [key: string]: { frameIds: number[] | string[] };
 }
 
 export type CookieJsonDataType = {

packages/common/src/utils/filterCookiesByFrame.ts

@@ -18,7 +18,7 @@
  */
 import { TabCookies } from '../cookies.types';
 interface TabFrames {
-  [key: string]: { frameIds: number[] };
+  [key: string]: { frameIds: string[] };
 }
 
 const filterCookiesByFrame = (
@@ -31,65 +31,11 @@ const filterCookiesByFrame = (
     return Object.values(frameFilteredCookies);
   }
 
-  let tabFramesIDMap: number[] = [];
-
-  Object.keys(tabFrames)?.forEach((url) => {
-    const frameIds = tabFrames[url].frameIds;
-
-    if (frameIds) {
-      tabFramesIDMap = [...new Set<number>([...frameIds, ...tabFramesIDMap])];
-    }
-  });
-
   Object.entries(cookies).forEach(([key, cookie]) => {
     tabFrames[frameUrl].frameIds?.forEach((frameId) => {
-      if (cookie.frameIdList?.includes(frameId)) {
+      if (frameId && cookie.frameIdList?.includes(frameId)) {
         frameFilteredCookies[key] = cookie;
       }
-
-      //For orphaned/unmapped cookies
-      let hasFrame = false;
-      cookie.frameIdList?.forEach((cookieFrameId) => {
-        if (tabFramesIDMap.includes(cookieFrameId as number)) {
-          hasFrame = true;
-        }
-      });
-
-      if (!hasFrame && cookie?.frameIdList) {
-        //For UnMapped Cookies
-        if (
-          cookie.frameIdList &&
-          cookie.frameIdList.length === 0 &&
-          cookie.parsedCookie.domain
-        ) {
-          const domainToCheck = cookie.parsedCookie.domain.startsWith('.')
-            ? cookie.parsedCookie.domain.slice(1)
-            : cookie.parsedCookie.domain;
-
-          if (frameUrl.includes(domainToCheck)) {
-            frameFilteredCookies[key] = cookie;
-          }
-          if (
-            Object.keys(tabFrames).filter((frameKey) =>
-              frameKey.includes(domainToCheck)
-            ).length === 0 &&
-            tabFrames[frameUrl].frameIds.includes(0)
-          ) {
-            frameFilteredCookies[key] = cookie;
-          }
-        }
-
-        //For Orphaned Cookies
-        if (
-          cookie.frameIdList &&
-          cookie.frameIdList.length > 0 &&
-          cookie.parsedCookie.domain
-        ) {
-          if (tabFrames[frameUrl].frameIds.includes(0)) {
-            frameFilteredCookies[key] = cookie;
-          }
-        }
-      }
     });
   });
 

packages/common/src/utils/parseRequestWillBeSentExtraInfo.ts

@@ -17,6 +17,7 @@
  * External dependencies
  */
 import type { Protocol } from 'devtools-protocol';
+
 /**
  * Internal dependencies.
  */
@@ -33,16 +34,18 @@ import isFirstParty from './isFirstParty';
  * Parses Network.requestWillBeSentExtraInfo to get extra information about a cookie.
  * @param {object} associatedCookies Cookies associated with the request being parsed.
  * @param {object} cookieDB Cookie database to find analytics from.
- * @param {object} requestMap An object for requestId to url.
+ * @param {string} requestUrl The request url.
  * @param {string} tabUrl - The top-level URL (URL in the tab's address bar).
+ * @param {string[]} frameIds The request to which the frame is associated to.
  * @param {string} requestId - The requestId of the request being processed
  * @returns {object} parsed cookies.
  */
 export default function parseRequestWillBeSentExtraInfo(
   associatedCookies: Protocol.Network.RequestWillBeSentExtraInfoEvent['associatedCookies'],
   cookieDB: CookieDatabase,
-  requestMap: { [requestId: string]: string },
+  requestUrl: string,
   tabUrl: string,
+  frameIds: string[],
   requestId: string
 ) {
   const cookies: CookieData[] = [];
@@ -52,17 +55,12 @@ export default function parseRequestWillBeSentExtraInfo(
       cookie.expires
     );
 
-    let domain,
-      url = '';
-
-    if (requestMap && requestMap[requestId]) {
-      url = requestMap[requestId] ?? '';
-    }
+    let domain;
 
     if (cookie?.domain) {
       domain = cookie?.domain;
-    } else if (!cookie?.domain && url) {
-      domain = new URL(url).hostname;
+    } else if (!cookie?.domain && requestUrl) {
+      domain = new URL(requestUrl).hostname;
     }
 
     const singleCookie: CookieData = {
@@ -78,22 +76,27 @@ export default function parseRequestWillBeSentExtraInfo(
           {
             type: REQUEST_EVENT.CDP_REQUEST_WILL_BE_SENT_EXTRA_INFO,
             requestId,
-            url: url,
+            url: requestUrl,
             blocked: blockedReasons.length !== 0,
             timeStamp: Date.now(),
           },
         ],
         responseEvents: [],
       },
       blockedReasons,
-      analytics: cookieDB ? findAnalyticsMatch(cookie.name, cookieDB) : null, // In case CDP gets cookie first.
-      url,
+      analytics: cookieDB ? findAnalyticsMatch(cookie.name, cookieDB) : null,
+      url: requestUrl,
       headerType: 'request' as CookieData['headerType'],
       isFirstParty: isFirstParty(domain, tabUrl),
       frameIdList: [],
       exemptionReason: exemptionReason ? exemptionReason : undefined,
     };
 
+    //Sometimes frameId comes empty so it shows data in other frames where cookie should not be shown.
+    if (frameIds.length > 0) {
+      singleCookie.frameIdList = [...frameIds];
+    }
+
     cookies.push(singleCookie);
   });
 

packages/common/src/utils/parseResponseReceivedExtraInfo.ts

@@ -37,9 +37,10 @@ import isFirstParty from './isFirstParty';
  * @param {object} blockedCookies Blocked Cookies associated with the response being parsed.
  * @param {object} exemptedCookies Blocked Cookies associated with the response being parsed.
  * @param {string|undefined} cookiePartitionKey Partittion key for the response.
- * @param {object} requestMap An object for requestId to url.
+ * @param {string} requestUrl The associated request URL.
  * @param {string} tabUrl - The top-level URL (URL in the tab's address bar).
  * @param {object} cookieDB Cookie database to find analytics from.
+ * @param {string[]} frameIds - The frameId the following cookies are associated to.
  * @param {string} requestId - The requestId of the request being processed
  * @returns {object} parsed cookies.
  */
@@ -48,9 +49,10 @@ export default function parseResponseReceivedExtraInfo(
   blockedCookies: Protocol.Network.ResponseReceivedExtraInfoEvent['blockedCookies'],
   exemptedCookies: Protocol.Network.ResponseReceivedExtraInfoEvent['exemptedCookies'],
   cookiePartitionKey: Protocol.Network.ResponseReceivedExtraInfoEvent['cookiePartitionKey'],
-  requestMap: { [requestId: string]: string },
+  requestUrl: string,
   tabUrl: string,
   cookieDB: CookieDatabase,
+  frameIds: string[],
   requestId: string
 ) {
   const cookies: CookieData[] = [];
@@ -86,17 +88,12 @@ export default function parseResponseReceivedExtraInfo(
       };
     }
 
-    let domain,
-      url = '';
-
-    if (requestMap && requestMap[requestId]) {
-      url = requestMap[requestId] ?? '';
-    }
+    let domain;
 
     if (parsedCookie?.domain) {
       domain = parsedCookie?.domain;
-    } else if (!parsedCookie?.domain && url) {
-      domain = new URL(url).hostname;
+    } else if (!parsedCookie?.domain && requestUrl) {
+      domain = new URL(requestUrl).hostname;
     }
 
     const singleCookie: CookieData = {
@@ -113,8 +110,8 @@ export default function parseResponseReceivedExtraInfo(
         responseEvents: [
           {
             type: RESPONSE_EVENT.CDP_RESPONSE_RECEIVED_EXTRA_INFO,
-            requestId,
-            url: url,
+            requestId: requestId,
+            url: requestUrl,
             blocked: blockedCookie ? true : false,
             timeStamp: Date.now(),
           },
@@ -123,13 +120,18 @@ export default function parseResponseReceivedExtraInfo(
       analytics: cookieDB
         ? findAnalyticsMatch(parsedCookie.name, cookieDB)
         : null,
-      url,
+      url: requestUrl,
       isFirstParty: isFirstParty(domain, tabUrl),
       headerType: 'response' as CookieData['headerType'],
       frameIdList: [],
       exemptionReason: exemptedCookie?.exemptionReason,
     };
 
+    //Sometimes frameId comes empty so it shows data in other frames where cookie should not be shown.
+    if (frameIds.length > 0) {
+      singleCookie.frameIdList = [...frameIds];
+    }
+
     cookies.push(singleCookie);
   });
 

packages/extension/src/contentScript/index.ts

@@ -25,6 +25,7 @@ import {
 } from '@ps-analysis-tool/common';
 import { computePosition, flip, shift } from '@floating-ui/core';
 import { autoUpdate, platform, arrow } from '@floating-ui/dom';
+
 /**
  * Internal dependencies.
  */
@@ -67,6 +68,11 @@ class WebpageContentScript {
    */
   tabId: number | null = null;
 
+  /**
+   * Main frame id.
+   */
+  frameId: string | null = null;
+
   /**
    * TabId of the current Tab
    */
@@ -152,6 +158,7 @@ class WebpageContentScript {
 
       if (message?.payload?.type === TABID_STORAGE) {
         this.tabId = message.payload.tabId;
+        this.frameId = message.payload.frameId;
       }
 
       if (message?.payload?.type === GET_JS_COOKIES) {
@@ -178,11 +185,16 @@ class WebpageContentScript {
    */
   async getAndProcessJSCookies(tabId: string) {
     try {
+      if (!this.frameId) {
+        return;
+      }
+
       //@ts-ignore
       const jsCookies = await cookieStore?.getAll();
       await processAndStoreDocumentCookies({
         tabUrl: window.location.href,
         tabId,
+        frameId: this.frameId,
         documentCookies: jsCookies,
         cookieDB: this.cookieDB ?? {},
       });
@@ -264,7 +276,7 @@ class WebpageContentScript {
           analytics: findAnalyticsMatch(cookie?.name, this.cookieDB),
           url: window.location.href,
           headerType: 'javascript',
-          frameIdList: [0],
+          frameIdList: [this.frameId ?? '0'],
           blockedReasons: [],
           warningReasons: [],
           isBlocked: false,

packages/extension/src/serviceWorker/attachCDP.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This function will attach the debugger to the given target.
+ * @param {{ [key: string]: number | string }} target The target where debugger needs to be attached.
+ */
+export default async function attachCDP(target: {
+  [key: string]: number | string;
+}) {
+  try {
+    await chrome.debugger.attach(target, '1.3');
+    await chrome.debugger.sendCommand(target, 'Target.setAutoAttach', {
+      // If this is set to true, debugger will be attached to every new target that is added to the current target.
+      autoAttach: true,
+      waitForDebuggerOnStart: false,
+      //Enables "flat" access to the session via specifying sessionId attribute in the commands.
+      // If this is set to true the debugger is also attached to the child targets of that the target it has been attached to.
+      flatten: true,
+    });
+    await chrome.debugger.sendCommand(target, 'Network.enable');
+    await chrome.debugger.sendCommand(target, 'Audits.enable');
+    await chrome.debugger.sendCommand(target, 'Page.enable');
+  } catch (error) {
+    //Fail silently
+  }
+}