Alexander_Franklin/ubercrawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add filtering

Browse Source
This commit is contained in:
PAlexanderFranklin 2023-09-02 16:54:54 -07:00
parent a1762348fe
commit 07463d3bff
1 changed files with 44 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
src/index.js
View File

@ -7,9 +7,6 @@ function sleep(ms) {
}
async function main() {
let options = JSON.parse(fs.readFileSync("./options.json", "utf8"))
if (!fs.existsSync("./uberResults.json")) {
fs.writeFileSync("./uberResults.json", "")
}
const browser = await puppeteer.launch({
headless: false,
ignoreHTTPSErrors: true,
@ -89,7 +86,7 @@ async function main() {
},
)
let body = await res.json()
let trips = body.data.activities.map(async (activity) => {
let trips = body?.data?.activities?.map(async (activity) => {
if (activity.formattedTotal == "$0.00") {
return ""
}
@ -97,6 +94,18 @@ async function main() {
// These are all duplicates of MISC items.
return ""
}
if (activity.type == "MISC" || activity.activityTitle == "Delivery") {
return {
uuid: activity.uuid,
recognizedAt: new Date(
(activity.recognizedAt ?? 1) * 1000,
).toISOString(),
pickupAddress: activity.tripMetaData?.pickupAddress,
dropOffAddress: activity.tripMetaData?.dropOffAddress,
total: Number(activity.formattedTotal),
type: activity.activityTitle,
}
}
if (activity.type == "TRIP" || activity.type == "CT") {
// Trip or Share
// make sure to get the activity.tripMetaData.pickupAddress and dropOffAddress.
@ -111,10 +120,13 @@ async function main() {
},
)
let body = await res.json()
let cards = body?.data?.cards
let breakdown = cards?.find(
(card) => card.type == "TripAllPartiesBreakdownCard",
)
let unparsedData = body
let cards = body?.data?.cards?.filter((card) => {
return card.type != "MapCard" && card.type != "TripStatsCard"
})
let breakdown =
cards?.find((card) => card.type == "TripAllPartiesBreakdownCard") ||
cards?.find((card) => card.type == "TripBreakdownCard")
if (breakdown) {
let components = breakdown.components?.filter((comp) => {
return (
@ -123,31 +135,40 @@ async function main() {
comp.type != "collapsableSection"
)
})
if (components.length) {
return components
if (components?.length) {
unparsedData = components
}
}
return body
}
if (activity.type == "MISC") {
activity.recognizedAt = (activity.recognizedAt ?? 1) * 1000
delete activity.routing
delete activity.uuid
delete activity.status
if (activity.tripMetaData?.mapUrl) {
delete activity.tripMetaData.mapUrl
if (cards?.length) {
unparsedData = cards
}
return {
uuid: activity.uuid,
recognizedAt: new Date(
(activity.recognizedAt ?? 1) * 1000,
).toISOString(),
pickupAddress: activity.tripMetaData?.pickupAddress,
dropOffAddress: activity.tripMetaData?.dropOffAddress,
total: Number(activity.formattedTotal),
type: activity.activityTitle,
unparsedData,
}
return activity
}
})
let tripResults = await utils.settlePromises(trips)
uberJSON = [...uberJSON, ...tripResults]
if (trips) {
let tripResults = await utils.settlePromises(trips)
uberJSON = [...uberJSON, ...tripResults]
} else {
let failedRequest = { failedRequest: res, body }
console.error(failedRequest)
uberJSON = [...uberJSON, failedRequest]
}
if (!body.data.pagination.hasMoreData) {
break
}
paginationOption.cursor = body.data.pagination.nextCursor
}
fs.appendFileSync("./uberResults.json", `, ${JSON.stringify(uberJSON)}`)
fs.writeFileSync("./uberResults.json", JSON.stringify(uberJSON))
} catch (err) {
console.error("Critical failure", err)
} finally {