diff --git a/.gitignore b/.gitignore index d27ad4c..7e8e936 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ node_modules options*.json results*.json +results*.csv uberResults*.json lyftResults*.json \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 22ba1cf..b0c7422 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "MIT", "dependencies": { + "papaparse": "^5.4.1", "puppeteer-core": "^21.1.1" } }, @@ -557,6 +558,11 @@ "node": ">= 14" } }, + "node_modules/papaparse": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.4.1.tgz", + "integrity": "sha512-HipMsgJkZu8br23pW15uvo6sib6wne/4woLZPlFf3rpDyMe9ywEXUsuD7+6K9PRkJlVT51j/sCOYDKGGS3ZJrw==" + }, "node_modules/pend": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", diff --git a/package.json b/package.json index 25562d1..e03ab81 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "author": "", "license": "MIT", "dependencies": { + "papaparse": "^5.4.1", "puppeteer-core": "^21.1.1" } } diff --git a/src/generateCSV.js b/src/generateCSV.js index 076a5a7..8f3f736 100644 --- a/src/generateCSV.js +++ b/src/generateCSV.js @@ -1,5 +1,6 @@ import fs from "fs" import * as utils from "./utils.js" +import Papa from "papaparse" function extractValuesAndLabels(data) { const result = [] @@ -25,6 +26,7 @@ async function main() { let uberResults = JSON.parse( fs.readFileSync(options.uberPath ?? "./uberResults.json", "utf8"), ) + let columns = new Set() let processedData = uberResults .filter((res) => res && !res.failedRequest) .map((result) => { @@ -41,19 +43,22 @@ async function main() { if (!result.values.length) { delete result.values } + // find all properties to make columns with + for (const [key, value] of Object.entries(result)) { + columns.add(key) + } return result }) - processedData.forEach((activity) => { - let labels = {} - activity.values?.forEach((value) => { - if (labels[value.label]) { - console.log(activity) - } else { - labels[value.label] = true - } + .map((result) => { + columns.forEach((column) => { + if (typeof result[column] == "undefined") { + result[column] = null + } + }) + return result }) - }) fs.writeFileSync("./resultsProcessed.json", JSON.stringify(processedData)) + fs.writeFileSync("./resultsProcessed.csv", Papa.unparse(processedData)) } main()