diff --git a/index.js b/index.js index 3d7d689..3ca4d05 100644 --- a/index.js +++ b/index.js @@ -4,11 +4,21 @@ function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)) } +const settlePromises = async (promiseArray) => { + const settledArray = await Promise.allSettled(promiseArray) + let errors = settledArray.filter((resolved) => resolved.reason) + if (errors.length) { + throw errors + } + return settledArray.map((resolved) => resolved.value) +} + async function main() { try { const browser = await puppeteer.launch({ headless: false, ignoreHTTPSErrors: true, + executablePath: "/usr/bin/chromium", }) console.log("Opened Browser") const page = await browser.newPage() @@ -19,48 +29,44 @@ async function main() { page.on("request", (request) => { let url = request.url() console.log("request url", url) - if (url == "usefulrequest") { + if (url.includes("getWebActivityFeed")) { usefulRequestHeaders = { ...request.headers() } - console.log("hello!") + console.log("hello!", usefulRequestHeaders) } request.continue() }) - await page.goto("https://www.google.com") + await page.goto("https://drivers.uber.com/earnings/activities") console.log("Went to Page") - await page.waitForSelector('button[type="submit"]') - await page.type('input[name="userId"]', "username") - await page.type('input[name="password"]', "password") - console.log("Typed Credentials") - await page.click('button[type="submit"]') + await page.waitForSelector('input[aria-label="Select a date range."]') + console.log("We're in.", usefulRequestHeaders) for (let i = 0; i < 10; i++) { - if (usefulRequestHeaders.headerName) { - await browser.close() - return console.log("success") + await sleep(500) + if (usefulRequestHeaders["content-type"]) { + console.log("success") + break + } + if (i == 9) { + throw "fail" } - await sleep(300) } console.log("after sleep loop") - // const response = await fetch( - // "https://www.google.com", - // { - // method: "POST", - // headers: usefulRequestHeaders, - // body: JSON.stringify({Request - // }), - // } - // ); + const response = await fetch("https://www.google.com", { + method: "POST", + headers: usefulRequestHeaders, + body: JSON.stringify({ Request }), + }) // if (response.ok) { // console.log(await response.json()); // } - - await browser.close() } catch (err) { - return console.error("Critical failure", err) + console.error("Critical failure", err) + } finally { + await browser.close() } } diff --git a/package.json b/package.json index 2cf63b9..81d10aa 100644 --- a/package.json +++ b/package.json @@ -1,10 +1,12 @@ { + "type": "module", "name": "ubercrawler", "version": "1.0.0", "description": "Puppeteer application for crawling uber and lyft for tax information", "main": "index.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "test": "echo \"Error: no test specified\" && exit 1", + "start": "node index.js" }, "author": "", "license": "MIT",