Add comprehensive installation and setup documentation

- Add GETTING_STARTED.md with quick start guide and development modes
- Add INSTALL.sh automated installation script
- Add INSTALLATION_CHECKLIST.md, INSTALLATION_SUCCESS.md, and INSTALLATION_SUMMARY.md
- Add QUICK_REFERENCE.md for common commands
- Add SETUP_GUIDE.md with detailed setup instructions
- Update README.md with improved project overview
- Add did-wallet app dependencies and node_modules
This commit is contained in:
Dorian
2026-01-27 17:18:21 +00:00
parent a81f655133
commit 0d073fa89e
22658 changed files with 4494151 additions and 6 deletions
+120
View File
@@ -0,0 +1,120 @@
# it-parallel
[![codecov](https://img.shields.io/codecov/c/github/achingbrain/it.svg?style=flat-square)](https://codecov.io/gh/achingbrain/it)
[![CI](https://img.shields.io/github/actions/workflow/status/achingbrain/it/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/achingbrain/it/actions/workflows/js-test-and-release.yml?query=branch%3Amain)
> Process incoming async(iterable) functions in parallel
# About
<!--
!IMPORTANT!
Everything in this README between "# About" and "# Install" is automatically
generated and will be overwritten the next time the doc generator is run.
To make changes to this section, please update the @packageDocumentation section
of src/index.js or src/index.ts
To experiment with formatting, please run "npm run docs" from the root of this
repo and examine the changes made.
-->
Takes an (async) iterable that emits promise-returning functions, invokes them in parallel up to the concurrency limit and emits the results as they become available, optionally in the same order as the input
## Example
```javascript
import parallel from 'it-parallel'
import all from 'it-all'
import delay from 'delay'
// This can also be an iterator, async iterator, generator, etc
const input = [
async () => {
console.info('start 1')
await delay(500)
console.info('end 1')
return 1
},
async () => {
console.info('start 2')
await delay(200)
console.info('end 2')
return 2
},
async () => {
console.info('start 3')
await delay(100)
console.info('end 3')
return 3
}
]
const result = await all(parallel(input, {
concurrency: 2
}))
// output:
// start 1
// start 2
// end 2
// start 3
// end 3
// end 1
console.info(result) // [2, 3, 1]
```
If order is important, pass `ordered: true` as an option:
```javascript
const result = await all(parallel(input, {
concurrency: 2,
ordered: true
}))
// output:
// start 1
// start 2
// end 2
// start 3
// end 3
// end 1
console.info(result) // [1, 2, 3]
```
# Install
```console
$ npm i it-parallel
```
## Browser `<script>` tag
Loading this module through a script tag will make its exports available as `ItParallel` in the global namespace.
```html
<script src="https://unpkg.com/it-parallel/dist/index.min.js"></script>
```
# API Docs
- <https://achingbrain.github.io/it/modules/it_parallel.html>
# License
Licensed under either of
- Apache 2.0, ([LICENSE-APACHE](https://github.com/achingbrain/it/blob/main/packages/it-parallel/LICENSE-APACHE) / <http://www.apache.org/licenses/LICENSE-2.0>)
- MIT ([LICENSE-MIT](https://github.com/achingbrain/it/blob/main/packages/it-parallel/LICENSE-MIT) / <http://opensource.org/licenses/MIT>)
# Contribution
Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions.
+4
View File
@@ -0,0 +1,4 @@
(function (root, factory) {(typeof module === 'object' && module.exports) ? module.exports = factory() : root.ItParallel = factory()}(typeof self !== 'undefined' ? self : this, function () {
"use strict";var ItParallel=(()=>{var h=Object.defineProperty;var b=Object.getOwnPropertyDescriptor;var g=Object.getOwnPropertyNames;var A=Object.prototype.hasOwnProperty;var x=(t,r)=>{for(var n in r)h(t,n,{get:r[n],enumerable:!0})},C=(t,r,n,c)=>{if(r&&typeof r=="object"||typeof r=="function")for(let l of g(r))!A.call(t,l)&&l!==n&&h(t,l,{get:()=>r[l],enumerable:!(c=b(r,l))||c.enumerable});return t};var I=t=>C(h({},"__esModule",{value:!0}),t);var O={};x(O,{default:()=>y});function a(){let t={};return t.promise=new Promise((r,n)=>{t.resolve=r,t.reject=n}),t}var f=globalThis.CustomEvent??Event;async function*y(t,r={}){let n=r.concurrency??1/0;n<1&&(n=1/0);let c=r.ordered??!1,l=new EventTarget,o=[],s=a(),u=a(),w=!1,d,v=!1;l.addEventListener("task-complete",()=>{u.resolve()}),Promise.resolve().then(async()=>{try{for await(let e of t){if(o.length===n&&(s=a(),await s.promise),v)break;let i={done:!1};o.push(i),e().then(p=>{i.done=!0,i.ok=!0,i.value=p,l.dispatchEvent(new f("task-complete"))},p=>{i.done=!0,i.err=p,l.dispatchEvent(new f("task-complete"))})}w=!0,l.dispatchEvent(new f("task-complete"))}catch(e){d=e,l.dispatchEvent(new f("task-complete"))}});function m(){return c?o[0]?.done:!!o.find(e=>e.done)}function*k(){for(;o.length>0&&o[0].done;){let e=o[0];if(o.shift(),e.ok)yield e.value;else throw v=!0,s.resolve(),e.err;s.resolve()}}function*E(){for(;m();)for(let e=0;e<o.length;e++)if(o[e].done){let i=o[e];if(o.splice(e,1),e--,i.ok)yield i.value;else throw v=!0,s.resolve(),i.err;s.resolve()}}for(;;){if(m()||(u=a(),await u.promise),d!=null||(c?yield*k():yield*E(),d!=null))throw d;if(w&&o.length===0)break}}return I(O);})();
return ItParallel}));
//# sourceMappingURL=index.min.js.map
File diff suppressed because one or more lines are too long
+85
View File
@@ -0,0 +1,85 @@
/**
* @packageDocumentation
*
* Takes an (async) iterable that emits promise-returning functions, invokes them in parallel up to the concurrency limit and emits the results as they become available, optionally in the same order as the input
*
* @example
*
* ```javascript
* import parallel from 'it-parallel'
* import all from 'it-all'
* import delay from 'delay'
*
* // This can also be an iterator, async iterator, generator, etc
* const input = [
* async () => {
* console.info('start 1')
* await delay(500)
*
* console.info('end 1')
* return 1
* },
* async () => {
* console.info('start 2')
* await delay(200)
*
* console.info('end 2')
* return 2
* },
* async () => {
* console.info('start 3')
* await delay(100)
*
* console.info('end 3')
* return 3
* }
* ]
*
* const result = await all(parallel(input, {
* concurrency: 2
* }))
*
* // output:
* // start 1
* // start 2
* // end 2
* // start 3
* // end 3
* // end 1
*
* console.info(result) // [2, 3, 1]
* ```
*
* If order is important, pass `ordered: true` as an option:
*
* ```javascript
* const result = await all(parallel(input, {
* concurrency: 2,
* ordered: true
* }))
*
* // output:
* // start 1
* // start 2
* // end 2
* // start 3
* // end 3
* // end 1
*
* console.info(result) // [1, 2, 3]
* ```
*/
export interface ParallelOptions {
/**
* How many jobs to execute in parallel (default: )
*/
concurrency?: number;
ordered?: boolean;
}
/**
* Takes an (async) iterator that emits promise-returning functions,
* invokes them in parallel and emits the results as they become available but
* in the same order as the input
*/
export default function parallel<T>(source: Iterable<() => Promise<T>> | AsyncIterable<() => Promise<T>>, options?: ParallelOptions): AsyncGenerator<T, void, undefined>;
//# sourceMappingURL=index.d.ts.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsEG;AAaH,MAAM,WAAW,eAAe;IAC9B;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,OAAO,CAAC,EAAE,OAAO,CAAA;CAClB;AAED;;;;GAIG;AACH,wBAAgC,QAAQ,CAAE,CAAC,EAAG,MAAM,EAAE,QAAQ,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,GAAE,eAAoB,GAAG,cAAc,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,CA6IrL"}
+201
View File
@@ -0,0 +1,201 @@
/**
* @packageDocumentation
*
* Takes an (async) iterable that emits promise-returning functions, invokes them in parallel up to the concurrency limit and emits the results as they become available, optionally in the same order as the input
*
* @example
*
* ```javascript
* import parallel from 'it-parallel'
* import all from 'it-all'
* import delay from 'delay'
*
* // This can also be an iterator, async iterator, generator, etc
* const input = [
* async () => {
* console.info('start 1')
* await delay(500)
*
* console.info('end 1')
* return 1
* },
* async () => {
* console.info('start 2')
* await delay(200)
*
* console.info('end 2')
* return 2
* },
* async () => {
* console.info('start 3')
* await delay(100)
*
* console.info('end 3')
* return 3
* }
* ]
*
* const result = await all(parallel(input, {
* concurrency: 2
* }))
*
* // output:
* // start 1
* // start 2
* // end 2
* // start 3
* // end 3
* // end 1
*
* console.info(result) // [2, 3, 1]
* ```
*
* If order is important, pass `ordered: true` as an option:
*
* ```javascript
* const result = await all(parallel(input, {
* concurrency: 2,
* ordered: true
* }))
*
* // output:
* // start 1
* // start 2
* // end 2
* // start 3
* // end 3
* // end 1
*
* console.info(result) // [1, 2, 3]
* ```
*/
import defer from 'p-defer';
const CustomEvent = globalThis.CustomEvent ?? Event;
/**
* Takes an (async) iterator that emits promise-returning functions,
* invokes them in parallel and emits the results as they become available but
* in the same order as the input
*/
export default async function* parallel(source, options = {}) {
let concurrency = options.concurrency ?? Infinity;
if (concurrency < 1) {
concurrency = Infinity;
}
const ordered = options.ordered ?? false;
const emitter = new EventTarget();
const ops = [];
let slotAvailable = defer();
let resultAvailable = defer();
let sourceFinished = false;
let sourceErr;
let opErred = false;
emitter.addEventListener('task-complete', () => {
resultAvailable.resolve();
});
void Promise.resolve().then(async () => {
try {
for await (const task of source) {
if (ops.length === concurrency) {
slotAvailable = defer();
await slotAvailable.promise;
}
if (opErred) {
break;
}
const op = {
done: false
};
ops.push(op);
task()
.then(result => {
op.done = true;
op.ok = true;
op.value = result;
emitter.dispatchEvent(new CustomEvent('task-complete'));
}, err => {
op.done = true;
op.err = err;
emitter.dispatchEvent(new CustomEvent('task-complete'));
});
}
sourceFinished = true;
emitter.dispatchEvent(new CustomEvent('task-complete'));
}
catch (err) {
sourceErr = err;
emitter.dispatchEvent(new CustomEvent('task-complete'));
}
});
function valuesAvailable() {
if (ordered) {
return ops[0]?.done;
}
return Boolean(ops.find(op => op.done));
}
function* yieldOrderedValues() {
while ((ops.length > 0) && ops[0].done) {
const op = ops[0];
ops.shift();
if (op.ok) {
yield op.value;
}
else {
// allow the source to exit
opErred = true;
slotAvailable.resolve();
throw op.err;
}
slotAvailable.resolve();
}
}
function* yieldUnOrderedValues() {
// more values can become available while we wait for `yield`
// to return control to this function
while (valuesAvailable()) {
for (let i = 0; i < ops.length; i++) {
if (ops[i].done) {
const op = ops[i];
ops.splice(i, 1);
i--;
if (op.ok) {
yield op.value;
}
else {
opErred = true;
slotAvailable.resolve();
throw op.err;
}
slotAvailable.resolve();
}
}
}
}
while (true) {
if (!valuesAvailable()) {
resultAvailable = defer();
await resultAvailable.promise;
}
if (sourceErr != null) {
// the source threw an error, propagate it
throw sourceErr;
}
if (ordered) {
yield* yieldOrderedValues();
}
else {
yield* yieldUnOrderedValues();
}
if (sourceErr != null) {
// if the source yields an array that is `yield *`, it can throw while the
// onward consumer is processing the array contents - make sure we
// propagate the error
// eslint-disable-next-line @typescript-eslint/only-throw-error
throw sourceErr;
}
if (sourceFinished && ops.length === 0) {
// not waiting for any results and no more tasks so we are done
break;
}
}
}
//# sourceMappingURL=index.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsEG;AAEH,OAAO,KAAK,MAAM,SAAS,CAAA;AAS3B,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,IAAI,KAAK,CAAA;AAUnD;;;;GAIG;AACH,MAAM,CAAC,OAAO,CAAC,KAAK,SAAU,CAAC,CAAC,QAAQ,CAAM,MAAoE,EAAE,UAA2B,EAAE;IAC/I,IAAI,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,QAAQ,CAAA;IAEjD,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACpB,WAAW,GAAG,QAAQ,CAAA;IACxB,CAAC;IAED,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,KAAK,CAAA;IACxC,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;IAEjC,MAAM,GAAG,GAAwB,EAAE,CAAA;IACnC,IAAI,aAAa,GAAG,KAAK,EAAE,CAAA;IAC3B,IAAI,eAAe,GAAG,KAAK,EAAE,CAAA;IAC7B,IAAI,cAAc,GAAG,KAAK,CAAA;IAC1B,IAAI,SAA4B,CAAA;IAChC,IAAI,OAAO,GAAG,KAAK,CAAA;IAEnB,OAAO,CAAC,gBAAgB,CAAC,eAAe,EAAE,GAAG,EAAE;QAC7C,eAAe,CAAC,OAAO,EAAE,CAAA;IAC3B,CAAC,CAAC,CAAA;IAEF,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;QACrC,IAAI,CAAC;YACH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;gBAChC,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;oBAC/B,aAAa,GAAG,KAAK,EAAE,CAAA;oBACvB,MAAM,aAAa,CAAC,OAAO,CAAA;gBAC7B,CAAC;gBAED,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAK;gBACP,CAAC;gBAED,MAAM,EAAE,GAAQ;oBACd,IAAI,EAAE,KAAK;iBACZ,CAAA;gBACD,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;gBAEZ,IAAI,EAAE;qBACH,IAAI,CAAC,MAAM,CAAC,EAAE;oBACb,EAAE,CAAC,IAAI,GAAG,IAAI,CAAA;oBACd,EAAE,CAAC,EAAE,GAAG,IAAI,CAAA;oBACZ,EAAE,CAAC,KAAK,GAAG,MAAM,CAAA;oBACjB,OAAO,CAAC,aAAa,CAAC,IAAI,WAAW,CAAC,eAAe,CAAC,CAAC,CAAA;gBACzD,CAAC,EAAE,GAAG,CAAC,EAAE;oBACP,EAAE,CAAC,IAAI,GAAG,IAAI,CAAA;oBACd,EAAE,CAAC,GAAG,GAAG,GAAG,CAAA;oBACZ,OAAO,CAAC,aAAa,CAAC,IAAI,WAAW,CAAC,eAAe,CAAC,CAAC,CAAA;gBACzD,CAAC,CAAC,CAAA;YACN,CAAC;YAED,cAAc,GAAG,IAAI,CAAA;YACrB,OAAO,CAAC,aAAa,CAAC,IAAI,WAAW,CAAC,eAAe,CAAC,CAAC,CAAA;QACzD,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,SAAS,GAAG,GAAG,CAAA;YACf,OAAO,CAAC,aAAa,CAAC,IAAI,WAAW,CAAC,eAAe,CAAC,CAAC,CAAA;QACzD,CAAC;IACH,CAAC,CAAC,CAAA;IAEF,SAAS,eAAe;QACtB,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAA;QACrB,CAAC;QAED,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAA;IACzC,CAAC;IAED,QAAS,CAAC,CAAC,kBAAkB;QAC3B,OAAO,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACvC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAA;YACjB,GAAG,CAAC,KAAK,EAAE,CAAA;YAEX,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;gBACV,MAAM,EAAE,CAAC,KAAK,CAAA;YAChB,CAAC;iBAAM,CAAC;gBACN,2BAA2B;gBAC3B,OAAO,GAAG,IAAI,CAAA;gBACd,aAAa,CAAC,OAAO,EAAE,CAAA;gBAEvB,MAAM,EAAE,CAAC,GAAG,CAAA;YACd,CAAC;YAED,aAAa,CAAC,OAAO,EAAE,CAAA;QACzB,CAAC;IACH,CAAC;IAED,QAAS,CAAC,CAAC,oBAAoB;QAC7B,6DAA6D;QAC7D,qCAAqC;QACrC,OAAO,eAAe,EAAE,EAAE,CAAC;YACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACpC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;oBAChB,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAA;oBACjB,GAAG,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;oBAChB,CAAC,EAAE,CAAA;oBAEH,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;wBACV,MAAM,EAAE,CAAC,KAAK,CAAA;oBAChB,CAAC;yBAAM,CAAC;wBACN,OAAO,GAAG,IAAI,CAAA;wBACd,aAAa,CAAC,OAAO,EAAE,CAAA;wBAEvB,MAAM,EAAE,CAAC,GAAG,CAAA;oBACd,CAAC;oBAED,aAAa,CAAC,OAAO,EAAE,CAAA;gBACzB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,CAAC,eAAe,EAAE,EAAE,CAAC;YACvB,eAAe,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,eAAe,CAAC,OAAO,CAAA;QAC/B,CAAC;QAED,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;YACtB,0CAA0C;YAC1C,MAAM,SAAS,CAAA;QACjB,CAAC;QAED,IAAI,OAAO,EAAE,CAAC;YACZ,KAAM,CAAC,CAAC,kBAAkB,EAAE,CAAA;QAC9B,CAAC;aAAM,CAAC;YACN,KAAM,CAAC,CAAC,oBAAoB,EAAE,CAAA;QAChC,CAAC;QAED,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;YACtB,0EAA0E;YAC1E,kEAAkE;YAClE,sBAAsB;YACtB,+DAA+D;YAC/D,MAAM,SAAS,CAAA;QACjB,CAAC;QAED,IAAI,cAAc,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvC,+DAA+D;YAC/D,MAAK;QACP,CAAC;IACH,CAAC;AACH,CAAC"}
+147
View File
@@ -0,0 +1,147 @@
{
"name": "it-parallel",
"version": "3.0.13",
"description": "Process incoming async(iterable) functions in parallel",
"author": "Alex Potsides <alex@achingbrain.net>",
"license": "Apache-2.0 OR MIT",
"homepage": "https://github.com/achingbrain/it/tree/main/packages/it-parallel#readme",
"repository": {
"type": "git",
"url": "git+https://github.com/achingbrain/it.git"
},
"bugs": {
"url": "https://github.com/achingbrain/it/issues"
},
"publishConfig": {
"access": "public",
"provenance": true
},
"type": "module",
"types": "./dist/src/index.d.ts",
"files": [
"src",
"dist",
"!dist/test",
"!**/*.tsbuildinfo"
],
"exports": {
".": {
"types": "./dist/src/index.d.ts",
"import": "./dist/src/index.js"
}
},
"release": {
"branches": [
"main"
],
"plugins": [
[
"@semantic-release/commit-analyzer",
{
"preset": "conventionalcommits",
"releaseRules": [
{
"breaking": true,
"release": "major"
},
{
"revert": true,
"release": "patch"
},
{
"type": "feat",
"release": "minor"
},
{
"type": "fix",
"release": "patch"
},
{
"type": "docs",
"release": "patch"
},
{
"type": "test",
"release": "patch"
},
{
"type": "deps",
"release": "patch"
},
{
"scope": "no-release",
"release": false
}
]
}
],
[
"@semantic-release/release-notes-generator",
{
"preset": "conventionalcommits",
"presetConfig": {
"types": [
{
"type": "feat",
"section": "Features"
},
{
"type": "fix",
"section": "Bug Fixes"
},
{
"type": "chore",
"section": "Trivial Changes"
},
{
"type": "docs",
"section": "Documentation"
},
{
"type": "deps",
"section": "Dependencies"
},
{
"type": "test",
"section": "Tests"
}
]
}
}
],
"@semantic-release/changelog",
"@semantic-release/npm",
"@semantic-release/github",
[
"@semantic-release/git",
{
"assets": [
"CHANGELOG.md",
"package.json"
]
}
]
]
},
"scripts": {
"build": "aegir build",
"lint": "aegir lint",
"dep-check": "aegir dep-check",
"clean": "aegir clean",
"test": "aegir test",
"test:node": "aegir test -t node --cov",
"test:chrome": "aegir test -t browser --cov",
"test:chrome-webworker": "aegir test -t webworker",
"test:firefox": "aegir test -t browser -- --browser firefox",
"test:firefox-webworker": "aegir test -t webworker -- --browser firefox",
"release": "aegir release"
},
"dependencies": {
"p-defer": "^4.0.1"
},
"devDependencies": {
"aegir": "^47.0.16",
"delay": "^6.0.0",
"it-all": "^3.0.0"
}
}
+238
View File
@@ -0,0 +1,238 @@
/**
* @packageDocumentation
*
* Takes an (async) iterable that emits promise-returning functions, invokes them in parallel up to the concurrency limit and emits the results as they become available, optionally in the same order as the input
*
* @example
*
* ```javascript
* import parallel from 'it-parallel'
* import all from 'it-all'
* import delay from 'delay'
*
* // This can also be an iterator, async iterator, generator, etc
* const input = [
* async () => {
* console.info('start 1')
* await delay(500)
*
* console.info('end 1')
* return 1
* },
* async () => {
* console.info('start 2')
* await delay(200)
*
* console.info('end 2')
* return 2
* },
* async () => {
* console.info('start 3')
* await delay(100)
*
* console.info('end 3')
* return 3
* }
* ]
*
* const result = await all(parallel(input, {
* concurrency: 2
* }))
*
* // output:
* // start 1
* // start 2
* // end 2
* // start 3
* // end 3
* // end 1
*
* console.info(result) // [2, 3, 1]
* ```
*
* If order is important, pass `ordered: true` as an option:
*
* ```javascript
* const result = await all(parallel(input, {
* concurrency: 2,
* ordered: true
* }))
*
* // output:
* // start 1
* // start 2
* // end 2
* // start 3
* // end 3
* // end 1
*
* console.info(result) // [1, 2, 3]
* ```
*/
import defer from 'p-defer'
interface Operation<T> {
done: boolean
ok: boolean
err: Error
value: T
}
const CustomEvent = globalThis.CustomEvent ?? Event
export interface ParallelOptions {
/**
* How many jobs to execute in parallel (default: )
*/
concurrency?: number
ordered?: boolean
}
/**
* Takes an (async) iterator that emits promise-returning functions,
* invokes them in parallel and emits the results as they become available but
* in the same order as the input
*/
export default async function * parallel <T> (source: Iterable<() => Promise<T>> | AsyncIterable<() => Promise<T>>, options: ParallelOptions = {}): AsyncGenerator<T, void, undefined> {
let concurrency = options.concurrency ?? Infinity
if (concurrency < 1) {
concurrency = Infinity
}
const ordered = options.ordered ?? false
const emitter = new EventTarget()
const ops: Array<Operation<T>> = []
let slotAvailable = defer()
let resultAvailable = defer()
let sourceFinished = false
let sourceErr: Error | undefined
let opErred = false
emitter.addEventListener('task-complete', () => {
resultAvailable.resolve()
})
void Promise.resolve().then(async () => {
try {
for await (const task of source) {
if (ops.length === concurrency) {
slotAvailable = defer()
await slotAvailable.promise
}
if (opErred) {
break
}
const op: any = {
done: false
}
ops.push(op)
task()
.then(result => {
op.done = true
op.ok = true
op.value = result
emitter.dispatchEvent(new CustomEvent('task-complete'))
}, err => {
op.done = true
op.err = err
emitter.dispatchEvent(new CustomEvent('task-complete'))
})
}
sourceFinished = true
emitter.dispatchEvent(new CustomEvent('task-complete'))
} catch (err: any) {
sourceErr = err
emitter.dispatchEvent(new CustomEvent('task-complete'))
}
})
function valuesAvailable (): boolean {
if (ordered) {
return ops[0]?.done
}
return Boolean(ops.find(op => op.done))
}
function * yieldOrderedValues (): Generator<T, void, unknown> {
while ((ops.length > 0) && ops[0].done) {
const op = ops[0]
ops.shift()
if (op.ok) {
yield op.value
} else {
// allow the source to exit
opErred = true
slotAvailable.resolve()
throw op.err
}
slotAvailable.resolve()
}
}
function * yieldUnOrderedValues (): Generator<T, void, unknown> {
// more values can become available while we wait for `yield`
// to return control to this function
while (valuesAvailable()) {
for (let i = 0; i < ops.length; i++) {
if (ops[i].done) {
const op = ops[i]
ops.splice(i, 1)
i--
if (op.ok) {
yield op.value
} else {
opErred = true
slotAvailable.resolve()
throw op.err
}
slotAvailable.resolve()
}
}
}
}
while (true) {
if (!valuesAvailable()) {
resultAvailable = defer()
await resultAvailable.promise
}
if (sourceErr != null) {
// the source threw an error, propagate it
throw sourceErr
}
if (ordered) {
yield * yieldOrderedValues()
} else {
yield * yieldUnOrderedValues()
}
if (sourceErr != null) {
// if the source yields an array that is `yield *`, it can throw while the
// onward consumer is processing the array contents - make sure we
// propagate the error
// eslint-disable-next-line @typescript-eslint/only-throw-error
throw sourceErr
}
if (sourceFinished && ops.length === 0) {
// not waiting for any results and no more tasks so we are done
break
}
}
}