diff --git a/README.md b/README.md index 548cd93..6d23700 100644 --- a/README.md +++ b/README.md @@ -5,4 +5,4 @@ Puppeteer application for crawling uber and lyft for tax information Start by copying options.json.example, removing the .example and filling in the options. Run `npm ci` inside of the directory. -run src/index.js +run `node ./src/index.js` diff --git a/src/index.js b/src/index.js index 21a0b9f..099964b 100644 --- a/src/index.js +++ b/src/index.js @@ -1,10 +1,9 @@ import puppeteer from "puppeteer-core" import * as utils from "./utils.js" +import { processTrip } from "./processTrip.js" import fs from "fs" +import { loginToUber } from "./navigateSite.js" -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)) -} async function main() { let options = JSON.parse(fs.readFileSync("./options.json", "utf8")) const browser = await puppeteer.launch({ @@ -17,6 +16,7 @@ async function main() { await page.setRequestInterception(true) let usefulRequestHeaders = {} + let testForHeaders = () => !!usefulRequestHeaders["content-type"] page.on("request", (request) => { let url = request.url() if (url.includes("getWebActivityFeed")) { @@ -47,27 +47,7 @@ async function main() { } }) - await page.goto("https://drivers.uber.com/earnings/activities") - console.log("Went to Page") - - await page.waitForSelector(`input[id="PHONE_NUMBER_or_EMAIL_ADDRESS"]`) - await page.type( - `input[id="PHONE_NUMBER_or_EMAIL_ADDRESS"]`, - options.uberPhoneNumber ?? "", - ) - await page.click('button[type="submit"]') - - for (let i = 0; i < 100; i++) { - await sleep(500) - if (usefulRequestHeaders["content-type"]) { - console.log("success") - break - } - if (i == 99) { - throw "fail" - } - } - console.log("after sleep loop") + await loginToUber(page, options, testForHeaders) let uberJSON = [] @@ -108,51 +88,7 @@ async function main() { } if (activity.type == "TRIP" || activity.type == "CT") { // Trip or Share - // make sure to get the activity.tripMetaData.pickupAddress and dropOffAddress. - let res = await fetch( - "https://drivers.uber.com/earnings/api/getTrip?localeCode=en", - { - method: "POST", - body: JSON.stringify({ - tripUUID: activity.uuid, - }), - headers: usefulRequestHeaders, - }, - ) - let body = await res.json() - let unparsedData = body - let cards = body?.data?.cards?.filter((card) => { - return card.type != "MapCard" && card.type != "TripStatsCard" - }) - let breakdown = - cards?.find((card) => card.type == "TripAllPartiesBreakdownCard") || - cards?.find((card) => card.type == "TripBreakdownCard") - if (breakdown) { - let components = breakdown.components?.filter((comp) => { - return ( - comp.type != "header" && - comp.type != "divider" && - comp.type != "collapsableSection" - ) - }) - if (components?.length) { - unparsedData = components - } - } - if (cards?.length) { - unparsedData = cards - } - return { - uuid: activity.uuid, - recognizedAt: new Date( - (activity.recognizedAt ?? 1) * 1000, - ).toISOString(), - pickupAddress: activity.tripMetaData?.pickupAddress, - dropOffAddress: activity.tripMetaData?.dropOffAddress, - total: Number(activity.formattedTotal), - type: activity.activityTitle, - unparsedData, - } + return await processTrip(activity, usefulRequestHeaders) } }) if (trips) { diff --git a/src/navigateSite.js b/src/navigateSite.js new file mode 100644 index 0000000..88c487c --- /dev/null +++ b/src/navigateSite.js @@ -0,0 +1,25 @@ +import * as utils from "./utils.js" + +export async function loginToUber(page, options, testForHeaders) { + await page.goto("https://drivers.uber.com/earnings/activities") + console.log("Went to Page") + + await page.waitForSelector(`input[id="PHONE_NUMBER_or_EMAIL_ADDRESS"]`) + await page.type( + `input[id="PHONE_NUMBER_or_EMAIL_ADDRESS"]`, + options.uberPhoneNumber ?? "", + ) + await page.click('button[type="submit"]') + + for (let i = 0; i < 100; i++) { + await utils.sleep(500) + if (testForHeaders()) { + console.log("success") + break + } + if (i == 99) { + throw "Slept too long." + } + } + console.log("after sleep loop") +} diff --git a/src/processTrip.js b/src/processTrip.js new file mode 100644 index 0000000..a942085 --- /dev/null +++ b/src/processTrip.js @@ -0,0 +1,45 @@ +export async function processTrip(activity, usefulRequestHeaders) { + // make sure to get the activity.tripMetaData.pickupAddress and dropOffAddress. + let res = await fetch( + "https://drivers.uber.com/earnings/api/getTrip?localeCode=en", + { + method: "POST", + body: JSON.stringify({ + tripUUID: activity.uuid, + }), + headers: usefulRequestHeaders, + }, + ) + let body = await res.json() + let unparsedData = body + let cards = body?.data?.cards?.filter((card) => { + return card.type != "MapCard" && card.type != "TripStatsCard" + }) + let breakdown = + cards?.find((card) => card.type == "TripAllPartiesBreakdownCard") || + cards?.find((card) => card.type == "TripBreakdownCard") + if (breakdown) { + let components = breakdown.components?.filter((comp) => { + return ( + comp.type != "header" && + comp.type != "divider" && + comp.type != "collapsableSection" + ) + }) + if (components?.length) { + unparsedData = components + } + } + if (cards?.length) { + unparsedData = cards + } + return { + uuid: activity.uuid, + recognizedAt: new Date((activity.recognizedAt ?? 1) * 1000).toISOString(), + pickupAddress: activity.tripMetaData?.pickupAddress, + dropOffAddress: activity.tripMetaData?.dropOffAddress, + total: Number(activity.formattedTotal), + type: activity.activityTitle, + unparsedData, + } +} diff --git a/src/utils.js b/src/utils.js index 28825ec..46e5d8a 100644 --- a/src/utils.js +++ b/src/utils.js @@ -1,3 +1,7 @@ +export function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + export async function settlePromises(promiseArray) { const settledArray = await Promise.allSettled(promiseArray) let errors = settledArray.filter((resolved) => resolved.reason)