Refactor site navigation and trip processing
This commit is contained in:
parent
ae67f3e3c1
commit
6aa5944f90
@ -5,4 +5,4 @@ Puppeteer application for crawling uber and lyft for tax information
|
|||||||
Start by copying options.json.example, removing the .example and filling in the options.
|
Start by copying options.json.example, removing the .example and filling in the options.
|
||||||
Run `npm ci` inside of the directory.
|
Run `npm ci` inside of the directory.
|
||||||
|
|
||||||
run src/index.js
|
run `node ./src/index.js`
|
||||||
|
74
src/index.js
74
src/index.js
@ -1,10 +1,9 @@
|
|||||||
import puppeteer from "puppeteer-core"
|
import puppeteer from "puppeteer-core"
|
||||||
import * as utils from "./utils.js"
|
import * as utils from "./utils.js"
|
||||||
|
import { processTrip } from "./processTrip.js"
|
||||||
import fs from "fs"
|
import fs from "fs"
|
||||||
|
import { loginToUber } from "./navigateSite.js"
|
||||||
|
|
||||||
function sleep(ms) {
|
|
||||||
return new Promise((resolve) => setTimeout(resolve, ms))
|
|
||||||
}
|
|
||||||
async function main() {
|
async function main() {
|
||||||
let options = JSON.parse(fs.readFileSync("./options.json", "utf8"))
|
let options = JSON.parse(fs.readFileSync("./options.json", "utf8"))
|
||||||
const browser = await puppeteer.launch({
|
const browser = await puppeteer.launch({
|
||||||
@ -17,6 +16,7 @@ async function main() {
|
|||||||
|
|
||||||
await page.setRequestInterception(true)
|
await page.setRequestInterception(true)
|
||||||
let usefulRequestHeaders = {}
|
let usefulRequestHeaders = {}
|
||||||
|
let testForHeaders = () => !!usefulRequestHeaders["content-type"]
|
||||||
page.on("request", (request) => {
|
page.on("request", (request) => {
|
||||||
let url = request.url()
|
let url = request.url()
|
||||||
if (url.includes("getWebActivityFeed")) {
|
if (url.includes("getWebActivityFeed")) {
|
||||||
@ -47,27 +47,7 @@ async function main() {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
await page.goto("https://drivers.uber.com/earnings/activities")
|
await loginToUber(page, options, testForHeaders)
|
||||||
console.log("Went to Page")
|
|
||||||
|
|
||||||
await page.waitForSelector(`input[id="PHONE_NUMBER_or_EMAIL_ADDRESS"]`)
|
|
||||||
await page.type(
|
|
||||||
`input[id="PHONE_NUMBER_or_EMAIL_ADDRESS"]`,
|
|
||||||
options.uberPhoneNumber ?? "",
|
|
||||||
)
|
|
||||||
await page.click('button[type="submit"]')
|
|
||||||
|
|
||||||
for (let i = 0; i < 100; i++) {
|
|
||||||
await sleep(500)
|
|
||||||
if (usefulRequestHeaders["content-type"]) {
|
|
||||||
console.log("success")
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if (i == 99) {
|
|
||||||
throw "fail"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
console.log("after sleep loop")
|
|
||||||
|
|
||||||
let uberJSON = []
|
let uberJSON = []
|
||||||
|
|
||||||
@ -108,51 +88,7 @@ async function main() {
|
|||||||
}
|
}
|
||||||
if (activity.type == "TRIP" || activity.type == "CT") {
|
if (activity.type == "TRIP" || activity.type == "CT") {
|
||||||
// Trip or Share
|
// Trip or Share
|
||||||
// make sure to get the activity.tripMetaData.pickupAddress and dropOffAddress.
|
return await processTrip(activity, usefulRequestHeaders)
|
||||||
let res = await fetch(
|
|
||||||
"https://drivers.uber.com/earnings/api/getTrip?localeCode=en",
|
|
||||||
{
|
|
||||||
method: "POST",
|
|
||||||
body: JSON.stringify({
|
|
||||||
tripUUID: activity.uuid,
|
|
||||||
}),
|
|
||||||
headers: usefulRequestHeaders,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
let body = await res.json()
|
|
||||||
let unparsedData = body
|
|
||||||
let cards = body?.data?.cards?.filter((card) => {
|
|
||||||
return card.type != "MapCard" && card.type != "TripStatsCard"
|
|
||||||
})
|
|
||||||
let breakdown =
|
|
||||||
cards?.find((card) => card.type == "TripAllPartiesBreakdownCard") ||
|
|
||||||
cards?.find((card) => card.type == "TripBreakdownCard")
|
|
||||||
if (breakdown) {
|
|
||||||
let components = breakdown.components?.filter((comp) => {
|
|
||||||
return (
|
|
||||||
comp.type != "header" &&
|
|
||||||
comp.type != "divider" &&
|
|
||||||
comp.type != "collapsableSection"
|
|
||||||
)
|
|
||||||
})
|
|
||||||
if (components?.length) {
|
|
||||||
unparsedData = components
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (cards?.length) {
|
|
||||||
unparsedData = cards
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
uuid: activity.uuid,
|
|
||||||
recognizedAt: new Date(
|
|
||||||
(activity.recognizedAt ?? 1) * 1000,
|
|
||||||
).toISOString(),
|
|
||||||
pickupAddress: activity.tripMetaData?.pickupAddress,
|
|
||||||
dropOffAddress: activity.tripMetaData?.dropOffAddress,
|
|
||||||
total: Number(activity.formattedTotal),
|
|
||||||
type: activity.activityTitle,
|
|
||||||
unparsedData,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
if (trips) {
|
if (trips) {
|
||||||
|
25
src/navigateSite.js
Normal file
25
src/navigateSite.js
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
import * as utils from "./utils.js"
|
||||||
|
|
||||||
|
export async function loginToUber(page, options, testForHeaders) {
|
||||||
|
await page.goto("https://drivers.uber.com/earnings/activities")
|
||||||
|
console.log("Went to Page")
|
||||||
|
|
||||||
|
await page.waitForSelector(`input[id="PHONE_NUMBER_or_EMAIL_ADDRESS"]`)
|
||||||
|
await page.type(
|
||||||
|
`input[id="PHONE_NUMBER_or_EMAIL_ADDRESS"]`,
|
||||||
|
options.uberPhoneNumber ?? "",
|
||||||
|
)
|
||||||
|
await page.click('button[type="submit"]')
|
||||||
|
|
||||||
|
for (let i = 0; i < 100; i++) {
|
||||||
|
await utils.sleep(500)
|
||||||
|
if (testForHeaders()) {
|
||||||
|
console.log("success")
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if (i == 99) {
|
||||||
|
throw "Slept too long."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
console.log("after sleep loop")
|
||||||
|
}
|
45
src/processTrip.js
Normal file
45
src/processTrip.js
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
export async function processTrip(activity, usefulRequestHeaders) {
|
||||||
|
// make sure to get the activity.tripMetaData.pickupAddress and dropOffAddress.
|
||||||
|
let res = await fetch(
|
||||||
|
"https://drivers.uber.com/earnings/api/getTrip?localeCode=en",
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify({
|
||||||
|
tripUUID: activity.uuid,
|
||||||
|
}),
|
||||||
|
headers: usefulRequestHeaders,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
let body = await res.json()
|
||||||
|
let unparsedData = body
|
||||||
|
let cards = body?.data?.cards?.filter((card) => {
|
||||||
|
return card.type != "MapCard" && card.type != "TripStatsCard"
|
||||||
|
})
|
||||||
|
let breakdown =
|
||||||
|
cards?.find((card) => card.type == "TripAllPartiesBreakdownCard") ||
|
||||||
|
cards?.find((card) => card.type == "TripBreakdownCard")
|
||||||
|
if (breakdown) {
|
||||||
|
let components = breakdown.components?.filter((comp) => {
|
||||||
|
return (
|
||||||
|
comp.type != "header" &&
|
||||||
|
comp.type != "divider" &&
|
||||||
|
comp.type != "collapsableSection"
|
||||||
|
)
|
||||||
|
})
|
||||||
|
if (components?.length) {
|
||||||
|
unparsedData = components
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (cards?.length) {
|
||||||
|
unparsedData = cards
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
uuid: activity.uuid,
|
||||||
|
recognizedAt: new Date((activity.recognizedAt ?? 1) * 1000).toISOString(),
|
||||||
|
pickupAddress: activity.tripMetaData?.pickupAddress,
|
||||||
|
dropOffAddress: activity.tripMetaData?.dropOffAddress,
|
||||||
|
total: Number(activity.formattedTotal),
|
||||||
|
type: activity.activityTitle,
|
||||||
|
unparsedData,
|
||||||
|
}
|
||||||
|
}
|
@ -1,3 +1,7 @@
|
|||||||
|
export function sleep(ms) {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms))
|
||||||
|
}
|
||||||
|
|
||||||
export async function settlePromises(promiseArray) {
|
export async function settlePromises(promiseArray) {
|
||||||
const settledArray = await Promise.allSettled(promiseArray)
|
const settledArray = await Promise.allSettled(promiseArray)
|
||||||
let errors = settledArray.filter((resolved) => resolved.reason)
|
let errors = settledArray.filter((resolved) => resolved.reason)
|
||||||
|
Loading…
x
Reference in New Issue
Block a user