ubercrawler/index.js

import puppeteer from "puppeteer-core"

function sleep(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms))
}

const settlePromises = async (promiseArray) => {
  const settledArray = await Promise.allSettled(promiseArray)
  let errors = settledArray.filter((resolved) => resolved.reason)
  if (errors.length) {
    throw errors
  }
  return settledArray.map((resolved) => resolved.value)
}

async function main() {
  try {
    const browser = await puppeteer.launch({
      headless: false,
      ignoreHTTPSErrors: true,
      executablePath: "/usr/bin/chromium",
    })
    console.log("Opened Browser")
    const page = await browser.newPage()
    console.log("Opened Page")

    await page.setRequestInterception(true)
    let usefulRequestHeaders = {}
    page.on("request", (request) => {
      let url = request.url()
      console.log("request url", url)
      if (url.includes("getWebActivityFeed")) {
        usefulRequestHeaders = { ...request.headers() }
        console.log("hello!", usefulRequestHeaders)
      }
      request.continue()
    })

    await page.goto("https://drivers.uber.com/earnings/activities")
    console.log("Went to Page")

    await page.waitForSelector('input[aria-label="Select a date range."]')
    console.log("We're in.", usefulRequestHeaders)

    for (let i = 0; i < 10; i++) {
      await sleep(500)
      if (usefulRequestHeaders["content-type"]) {
        console.log("success")
        break
      }
      if (i == 9) {
        throw "fail"
      }
    }
    console.log("after sleep loop")

    const response = await fetch("https://www.google.com", {
      method: "POST",
      headers: usefulRequestHeaders,
      body: JSON.stringify({ Request }),
    })

    // if (response.ok) {
    //   console.log(await response.json());
    // }
  } catch (err) {
    console.error("Critical failure", err)
  } finally {
    await browser.close()
  }
}

main()