Get basic navigation in

index.js

@@ -4,11 +4,21 @@ function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms))
 }
 
+const settlePromises = async (promiseArray) => {
+  const settledArray = await Promise.allSettled(promiseArray)
+  let errors = settledArray.filter((resolved) => resolved.reason)
+  if (errors.length) {
+    throw errors
+  }
+  return settledArray.map((resolved) => resolved.value)
+}
+
 async function main() {
   try {
     const browser = await puppeteer.launch({
       headless: false,
       ignoreHTTPSErrors: true,
+      executablePath: "/usr/bin/chromium",
     })
     console.log("Opened Browser")
     const page = await browser.newPage()
@@ -19,48 +29,44 @@ async function main() {
     page.on("request", (request) => {
       let url = request.url()
       console.log("request url", url)
-      if (url == "usefulrequest") {
+      if (url.includes("getWebActivityFeed")) {
         usefulRequestHeaders = { ...request.headers() }
-        console.log("hello!")
+        console.log("hello!", usefulRequestHeaders)
       }
       request.continue()
     })
 
-    await page.goto("https://www.google.com")
+    await page.goto("https://drivers.uber.com/earnings/activities")
     console.log("Went to Page")
 
-    await page.waitForSelector('button[type="submit"]')
-    await page.type('input[name="userId"]', "username")
-    await page.type('input[name="password"]', "password")
-    console.log("Typed Credentials")
-    await page.click('button[type="submit"]')
+    await page.waitForSelector('input[aria-label="Select a date range."]')
+    console.log("We're in.", usefulRequestHeaders)
 
     for (let i = 0; i < 10; i++) {
-      if (usefulRequestHeaders.headerName) {
-        await browser.close()
-        return console.log("success")
+      await sleep(500)
+      if (usefulRequestHeaders["content-type"]) {
+        console.log("success")
+        break
+      }
+      if (i == 9) {
+        throw "fail"
       }
-      await sleep(300)
     }
     console.log("after sleep loop")
 
-    // const response = await fetch(
-    //   "https://www.google.com",
-    //   {
-    //     method: "POST",
-    //     headers: usefulRequestHeaders,
-    //     body: JSON.stringify({Request
-    //     }),
-    //   }
-    // );
+    const response = await fetch("https://www.google.com", {
+      method: "POST",
+      headers: usefulRequestHeaders,
+      body: JSON.stringify({ Request }),
+    })
 
     // if (response.ok) {
     //   console.log(await response.json());
     // }
-
-    await browser.close()
   } catch (err) {
-    return console.error("Critical failure", err)
+    console.error("Critical failure", err)
+  } finally {
+    await browser.close()
   }
 }
 

package.json

@@ -1,10 +1,12 @@
 {
+  "type": "module",
   "name": "ubercrawler",
   "version": "1.0.0",
   "description": "Puppeteer application for crawling uber and lyft for tax information",
   "main": "index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "start": "node index.js"
   },
   "author": "",
   "license": "MIT",