{"version":3,"file":"utils.mjs","sources":["../../../../src/file/providers/destination/utils.ts"],"sourcesContent":["import { Writable } from 'stream';\nimport { posix } from 'path';\nimport tar from 'tar-stream';\n\n/**\n * Create a file path factory for a given path & prefix.\n * Upon being called, the factory will return a file path for a given index\n */\nexport const createFilePathFactory =\n  (type: string) =>\n  (fileIndex = 0): string => {\n    // always write tar files with posix paths so we have a standard format for paths regardless of system\n    return posix.join(\n      // \"{type}\" directory\n      type,\n      // \"${type}_XXXXX.jsonl\" file\n      `${type}_${String(fileIndex).padStart(5, '0')}.jsonl`\n    );\n  };\n\nexport const createTarEntryStream = (\n  archive: tar.Pack,\n  pathFactory: (index?: number) => string,\n  maxSize = 2.56e8\n) => {\n  let fileIndex = 0;\n  let buffer = '';\n\n  const flush = async () => {\n    if (!buffer) {\n      return;\n    }\n\n    fileIndex += 1;\n    const name = pathFactory(fileIndex);\n    const size = buffer.length;\n\n    await new Promise<void>((resolve, reject) => {\n      archive.entry({ name, size }, buffer, (err) => {\n        if (err) {\n          reject(err);\n        }\n\n        resolve();\n      });\n    });\n\n    buffer = '';\n  };\n\n  const push = (chunk: string | Buffer) => {\n    buffer += chunk;\n  };\n\n  return new Writable({\n    async destroy(err, callback) {\n      await flush();\n      callback(err);\n    },\n\n    async write(chunk, _encoding, callback) {\n      const size = chunk.length;\n\n      if (chunk.length > maxSize) {\n        callback(new Error(`payload too large: ${chunk.length}>${maxSize}`));\n        return;\n      }\n\n      if (buffer.length + size > maxSize) {\n        await flush();\n      }\n\n      push(chunk);\n\n      callback(null);\n    },\n  });\n};\n"],"names":["createFilePathFactory","type","fileIndex","posix","join","String","padStart","createTarEntryStream","archive","pathFactory","maxSize","buffer","flush","name","size","length","Promise","resolve","reject","entry","err","push","chunk","Writable","destroy","callback","write","_encoding","Error"],"mappings":";;;AAIA;;;AAGC,IACYA,MAAAA,qBAAAA,GACX,CAACC,IACD,GAAA,CAACC,YAAY,CAAC,GAAA;;QAEZ,OAAOC,KAAAA,CAAMC,IAAI;AAEfH,QAAAA,IAAAA;QAEA,CAAC,EAAEA,IAAK,CAAA,CAAC,EAAEI,MAAAA,CAAOH,SAAWI,CAAAA,CAAAA,QAAQ,CAAC,CAAA,EAAG,GAAK,CAAA,CAAA,MAAM,CAAC,CAAA;;MAI9CC,oBAAuB,GAAA,CAClCC,OACAC,EAAAA,WAAAA,EACAC,UAAU,MAAM,GAAA;AAEhB,IAAA,IAAIR,SAAY,GAAA,CAAA;AAChB,IAAA,IAAIS,MAAS,GAAA,EAAA;AAEb,IAAA,MAAMC,KAAQ,GAAA,UAAA;AACZ,QAAA,IAAI,CAACD,MAAQ,EAAA;AACX,YAAA;AACF;QAEAT,SAAa,IAAA,CAAA;AACb,QAAA,MAAMW,OAAOJ,WAAYP,CAAAA,SAAAA,CAAAA;QACzB,MAAMY,IAAAA,GAAOH,OAAOI,MAAM;QAE1B,MAAM,IAAIC,OAAc,CAAA,CAACC,OAASC,EAAAA,MAAAA,GAAAA;AAChCV,YAAAA,OAAAA,CAAQW,KAAK,CAAC;AAAEN,gBAAAA,IAAAA;AAAMC,gBAAAA;AAAK,aAAA,EAAGH,QAAQ,CAACS,GAAAA,GAAAA;AACrC,gBAAA,IAAIA,GAAK,EAAA;oBACPF,MAAOE,CAAAA,GAAAA,CAAAA;AACT;AAEAH,gBAAAA,OAAAA,EAAAA;AACF,aAAA,CAAA;AACF,SAAA,CAAA;QAEAN,MAAS,GAAA,EAAA;AACX,KAAA;AAEA,IAAA,MAAMU,OAAO,CAACC,KAAAA,GAAAA;QACZX,MAAUW,IAAAA,KAAAA;AACZ,KAAA;AAEA,IAAA,OAAO,IAAIC,QAAS,CAAA;QAClB,MAAMC,OAAAA,CAAAA,CAAQJ,GAAG,EAAEK,QAAQ,EAAA;YACzB,MAAMb,KAAAA,EAAAA;YACNa,QAASL,CAAAA,GAAAA,CAAAA;AACX,SAAA;AAEA,QAAA,MAAMM,KAAMJ,CAAAA,CAAAA,KAAK,EAAEK,SAAS,EAAEF,QAAQ,EAAA;YACpC,MAAMX,IAAAA,GAAOQ,MAAMP,MAAM;YAEzB,IAAIO,KAAAA,CAAMP,MAAM,GAAGL,OAAS,EAAA;gBAC1Be,QAAS,CAAA,IAAIG,KAAM,CAAA,CAAC,mBAAmB,EAAEN,KAAMP,CAAAA,MAAM,CAAC,CAAC,EAAEL,OAAAA,CAAQ,CAAC,CAAA,CAAA;AAClE,gBAAA;AACF;AAEA,YAAA,IAAIC,MAAOI,CAAAA,MAAM,GAAGD,IAAAA,GAAOJ,OAAS,EAAA;gBAClC,MAAME,KAAAA,EAAAA;AACR;YAEAS,IAAKC,CAAAA,KAAAA,CAAAA;YAELG,QAAS,CAAA,IAAA,CAAA;AACX;AACF,KAAA,CAAA;AACF;;;;"}