data-processing/convert.js

import fs from 'node:fs/promises';
import * as zlib from "zlib";
import readline from 'node:readline';

let currentTime = 1593561599000;

await fs.appendFile("test.csv", "timestamp,bid\n");

for await (const dayFile of fs.glob("./binance-btcusdt-futures-2020-2021/*.csv.gz")) {
  console.log("Reading", dayFile, "...");

  let outputRows = [];

  const fd = await fs.open(dayFile);

  let lineReader = readline.createInterface({
    input: fd.createReadStream().pipe(zlib.createGunzip())
  });

  let n = 0;
  for await (const line of lineReader) {
    if (n > 0) {
      let lineParts = line.split(',');
      let timestamp = parseInt(lineParts[1]);
      if (timestamp >= currentTime + 1000) {
        currentTime = Math.floor(timestamp / 1000) * 1000;
        outputRows.push([unixTime(currentTime), lineParts[3]]);
      }
    }
    n++;
  }

  console.log("Done");

  let output = "";

  outputRows.forEach((row) => {
    output += row.join(",") + "\n";
  });

  await fs.appendFile("test.csv", output);

  lineReader.close();
}


function unixTime(unixtime) {

  var u = new Date(unixtime);

  return u.getUTCFullYear() +
      '-' + ('0' + (u.getUTCMonth() + 1)).slice(-2) +
      '-' + ('0' + u.getUTCDate()).slice(-2) +
      ' ' + ('0' + u.getUTCHours()).slice(-2) +
      ':' + ('0' + u.getUTCMinutes()).slice(-2) +
      ':' + ('0' + u.getUTCSeconds()).slice(-2);
}