Refactor site navigation and trip processing

README.md

@@ -5,4 +5,4 @@ Puppeteer application for crawling uber and lyft for tax information
 Start by copying options.json.example, removing the .example and filling in the options.
 Run `npm ci` inside of the directory.
 
-run src/index.js
+run `node ./src/index.js`

src/index.js

@@ -1,10 +1,9 @@
 import puppeteer from "puppeteer-core"
 import * as utils from "./utils.js"
+import { processTrip } from "./processTrip.js"
 import fs from "fs"
+import { loginToUber } from "./navigateSite.js"
 
-function sleep(ms) {
-  return new Promise((resolve) => setTimeout(resolve, ms))
-}
 async function main() {
   let options = JSON.parse(fs.readFileSync("./options.json", "utf8"))
   const browser = await puppeteer.launch({
@@ -17,6 +16,7 @@ async function main() {
 
     await page.setRequestInterception(true)
     let usefulRequestHeaders = {}
+    let testForHeaders = () => !!usefulRequestHeaders["content-type"]
     page.on("request", (request) => {
       let url = request.url()
       if (url.includes("getWebActivityFeed")) {
@@ -47,27 +47,7 @@ async function main() {
       }
     })
 
-    await page.goto("https://drivers.uber.com/earnings/activities")
+    await loginToUber(page, options, testForHeaders)
-    console.log("Went to Page")
-
-    await page.waitForSelector(`input[id="PHONE_NUMBER_or_EMAIL_ADDRESS"]`)
-    await page.type(
-      `input[id="PHONE_NUMBER_or_EMAIL_ADDRESS"]`,
-      options.uberPhoneNumber ?? "",
-    )
-    await page.click('button[type="submit"]')
-
-    for (let i = 0; i < 100; i++) {
-      await sleep(500)
-      if (usefulRequestHeaders["content-type"]) {
-        console.log("success")
-        break
-      }
-      if (i == 99) {
-        throw "fail"
-      }
-    }
-    console.log("after sleep loop")
 
     let uberJSON = []
 
@@ -108,51 +88,7 @@ async function main() {
         }
         if (activity.type == "TRIP" || activity.type == "CT") {
           // Trip or Share
-          // make sure to get the activity.tripMetaData.pickupAddress and dropOffAddress.
+          return await processTrip(activity, usefulRequestHeaders)
-          let res = await fetch(
-            "https://drivers.uber.com/earnings/api/getTrip?localeCode=en",
-            {
-              method: "POST",
-              body: JSON.stringify({
-                tripUUID: activity.uuid,
-              }),
-              headers: usefulRequestHeaders,
-            },
-          )
-          let body = await res.json()
-          let unparsedData = body
-          let cards = body?.data?.cards?.filter((card) => {
-            return card.type != "MapCard" && card.type != "TripStatsCard"
-          })
-          let breakdown =
-            cards?.find((card) => card.type == "TripAllPartiesBreakdownCard") ||
-            cards?.find((card) => card.type == "TripBreakdownCard")
-          if (breakdown) {
-            let components = breakdown.components?.filter((comp) => {
-              return (
-                comp.type != "header" &&
-                comp.type != "divider" &&
-                comp.type != "collapsableSection"
-              )
-            })
-            if (components?.length) {
-              unparsedData = components
-            }
-          }
-          if (cards?.length) {
-            unparsedData = cards
-          }
-          return {
-            uuid: activity.uuid,
-            recognizedAt: new Date(
-              (activity.recognizedAt ?? 1) * 1000,
-            ).toISOString(),
-            pickupAddress: activity.tripMetaData?.pickupAddress,
-            dropOffAddress: activity.tripMetaData?.dropOffAddress,
-            total: Number(activity.formattedTotal),
-            type: activity.activityTitle,
-            unparsedData,
-          }
         }
       })
       if (trips) {

src/navigateSite.js

@@ -0,0 +1,25 @@
+import * as utils from "./utils.js"
+
+export async function loginToUber(page, options, testForHeaders) {
+  await page.goto("https://drivers.uber.com/earnings/activities")
+  console.log("Went to Page")
+
+  await page.waitForSelector(`input[id="PHONE_NUMBER_or_EMAIL_ADDRESS"]`)
+  await page.type(
+    `input[id="PHONE_NUMBER_or_EMAIL_ADDRESS"]`,
+    options.uberPhoneNumber ?? "",
+  )
+  await page.click('button[type="submit"]')
+
+  for (let i = 0; i < 100; i++) {
+    await utils.sleep(500)
+    if (testForHeaders()) {
+      console.log("success")
+      break
+    }
+    if (i == 99) {
+      throw "Slept too long."
+    }
+  }
+  console.log("after sleep loop")
+}

src/processTrip.js

@@ -0,0 +1,45 @@
+export async function processTrip(activity, usefulRequestHeaders) {
+  // make sure to get the activity.tripMetaData.pickupAddress and dropOffAddress.
+  let res = await fetch(
+    "https://drivers.uber.com/earnings/api/getTrip?localeCode=en",
+    {
+      method: "POST",
+      body: JSON.stringify({
+        tripUUID: activity.uuid,
+      }),
+      headers: usefulRequestHeaders,
+    },
+  )
+  let body = await res.json()
+  let unparsedData = body
+  let cards = body?.data?.cards?.filter((card) => {
+    return card.type != "MapCard" && card.type != "TripStatsCard"
+  })
+  let breakdown =
+    cards?.find((card) => card.type == "TripAllPartiesBreakdownCard") ||
+    cards?.find((card) => card.type == "TripBreakdownCard")
+  if (breakdown) {
+    let components = breakdown.components?.filter((comp) => {
+      return (
+        comp.type != "header" &&
+        comp.type != "divider" &&
+        comp.type != "collapsableSection"
+      )
+    })
+    if (components?.length) {
+      unparsedData = components
+    }
+  }
+  if (cards?.length) {
+    unparsedData = cards
+  }
+  return {
+    uuid: activity.uuid,
+    recognizedAt: new Date((activity.recognizedAt ?? 1) * 1000).toISOString(),
+    pickupAddress: activity.tripMetaData?.pickupAddress,
+    dropOffAddress: activity.tripMetaData?.dropOffAddress,
+    total: Number(activity.formattedTotal),
+    type: activity.activityTitle,
+    unparsedData,
+  }
+}

src/utils.js

@@ -1,3 +1,7 @@
+export function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
 export async function settlePromises(promiseArray) {
   const settledArray = await Promise.allSettled(promiseArray)
   let errors = settledArray.filter((resolved) => resolved.reason)