Skip to content

Commit

Permalink
fix(vectorStores): correctly handle missing files in uploadAndPoll()
Browse files Browse the repository at this point in the history
  • Loading branch information
RobertCraigie committed Jul 9, 2024
1 parent b8a33e3 commit bc8e149
Showing 1 changed file with 9 additions and 6 deletions.
15 changes: 9 additions & 6 deletions src/resources/beta/vector-stores/file-batches.ts
Original file line number Diff line number Diff line change
Expand Up @@ -155,19 +155,22 @@ export class FileBatches extends APIResource {
{ files, fileIds = [] }: { files: Uploadable[]; fileIds?: string[] },
options?: Core.RequestOptions & { pollIntervalMs?: number; maxConcurrency?: number },
): Promise<VectorStoreFileBatch> {
if (files === null || files.length == 0) {
throw new Error('No files provided to process.');
if (files == null || files.length == 0) {
throw new Error(
`No files provided to process. If you've already uploaded files you should use \`.createAndPoll()\` instead`,
);
}

const configuredConcurrency = options?.maxConcurrency ?? 5;
//We cap the number of workers at the number of files (so we don't start any unnecessary workers)

// We cap the number of workers at the number of files (so we don't start any unnecessary workers)
const concurrencyLimit = Math.min(configuredConcurrency, files.length);

const client = this._client;
const fileIterator = files.values();
const allFileIds: string[] = [...fileIds];

//This code is based on this design. The libraries don't accommodate our environment limits.
// This code is based on this design. The libraries don't accommodate our environment limits.
// https://stackoverflow.com/questions/40639432/what-is-the-best-way-to-limit-concurrency-when-using-es6s-promise-all
async function processFiles(iterator: IterableIterator<Uploadable>) {
for (let item of iterator) {
Expand All @@ -176,10 +179,10 @@ export class FileBatches extends APIResource {
}
}

//Start workers to process results
// Start workers to process results
const workers = Array(concurrencyLimit).fill(fileIterator).map(processFiles);

//Wait for all processing to complete.
// Wait for all processing to complete.
await allSettledWithThrow(workers);

return await this.createAndPoll(vectorStoreId, {
Expand Down

0 comments on commit bc8e149

Please sign in to comment.