FWIW this is how I implemented a proof of concept using existing web platform technologies
https://github.com/guest271314/NativeTransferableStreams.
Steps:
1. Start your local server. This can be achieves using a browser extension with Native Messaging to toggle the local server on and off.
2. Create an HTML document in the root of the server directory.
3. Turn off popup blocker at browser settings/preferences.
4. Open `Window` using `window.open()` with URL set to HTML document at 2.
5. `postMessage()` to `opener` from newly opened `Window`.
6. Transfer `ReadableStream` representing `STDIN` using `postMessage()` from `opener` to newly opened `Window`.
7. Read `ReadableStream` at newly opened Window.
8. `fetch()` localhost with `POST` body set as command to run at a local shell, for example using PHP `passthru()`.
9. Transfer `ReadableStream` of `Response.body` representing STDOUT using `postMessage()` from newly opened `Window` to `opener`.
10. Read `ReadableStream` at `opener`.
Local server
<?php
if (isset($_POST["tts"])) {
print($_GET["tts"]);
header('Vary: Origin');
header("Access-Control-Allow-Origin: *");
header("Access-Control-Allow-Methods: GET, POST, OPTIONS, HEAD");
header("Access-Control-Allow-Headers: Content-Type, Access-Control-Allow-Headers");
header("Content-Type: text/plain");
header("X-Powered-By:");
echo passthru($_POST["tts"]);
exit();
}
index.html in root of server
<!DOCTYPE html>
<html>
<body>
NativeTransferableStream
<script>
onload = async (e) => {
blur();
opener.postMessage('Ready', name);
onmessage = async ({ data }) => {
await data
.pipeThrough(new TextDecoderStream())
.pipeTo(
new WritableStream({
async write(value, c) {
const fd = new FormData();
fd.append('tts', value);
const { body } = await fetch('
http://localhost:8000', {
method: 'post',
body: fd,
});
opener.postMessage(body, name, [body]);
},
})
)
.catch(() => close());
};
};
</script>
</body>
</html>
Usage at any origin
async function audioStream(readable) {
let readOffset = 0;
let duration = 0;
let init = false;
const ac = new AudioContext({
sampleRate: 22050,
latencyHint: 0,
});
await ac.suspend();
const msd = new MediaStreamAudioDestinationNode(ac, {
channelCount: 1,
});
let inputController;
const inputStream = new ReadableStream({
async start(_) {
return (inputController = _);
},
});
const abortable = new AbortController();
const { signal } = abortable;
const inputReader = inputStream.getReader();
const { stream } = msd;
const [track] = stream.getAudioTracks();
const osc = new OscillatorNode(ac, { frequency: 0 });
const processor = new MediaStreamTrackProcessor({ track });
const generator = new MediaStreamTrackGenerator({ kind: 'audio' });
const { writable } = generator;
const { readable: audioReadable } = processor;
const audioWriter = writable.getWriter();
const mediaStream = new MediaStream([generator]);
const source = new MediaStreamAudioSourceNode(ac, { mediaStream });
source.connect(ac.destination);
osc.connect(msd);
osc.start();
track.onmute = track.onunmute = track.onended = (e) => console.log(e);
// const recorder = new MediaRecorder(mediaStream);
// recorder.ondataavailable = ({ data }) => console.log(URL.createObjectURL(data));
// recorder.start();
let channelData = [];
await Promise.all([
readable.pipeTo(
new WritableStream({
async write(value, c) {
let i = 0;
if (!init) {
init = true;
i = 44;
}
for (; i < value.buffer.byteLength; i++, readOffset++) {
if (channelData.length === 440) {
inputController.enqueue([...channelData]);
channelData.length = 0;
}
channelData.push(value[i]);
}
},
async close() {
console.log('Done writing input stream.');
if (channelData.length) {
inputController.enqueue(channelData);
}
inputController.close();
},
})
),
audioReadable.pipeTo(
new WritableStream({
async write({ timestamp }) {
if (inputController.desiredSize === 0) {
msd.disconnect();
osc.disconnect();
source.disconnect();
track.stop();
// abortable.abort();
await audioWriter.close();
await audioWriter.closed;
await inputReader.cancel();
generator.stop();
await ac.close();
console.log(
`readOffset:${readOffset}, duration:${duration}, ac.currentTime:${ac.currentTime}`,
`generator.readyState:${generator.readyState}, audioWriter.desiredSize:${audioWriter.desiredSize}`
);
return await Promise.all([
new Promise((resolve) => (stream.oninactive = resolve)),
new Promise((resolve) => (ac.onstatechange = resolve)),
]);
}
const uint8 = new Uint8Array(440);
const { value, done } = await inputReader.read();
if (!done) uint8.set(new Uint8Array(value));
const uint16 = new Uint16Array(uint8.buffer);
const floats = new Float32Array(220);
//
https://stackoverflow.com/a/35248852 for (let i = 0; i < uint16.length; i++) {
const int = uint16[i];
// If the high bit is on, then it is a negative number, and actually counts backwards.
const float =
int >= 0x8000 ? -(0x10000 - int) / 0x8000 : int / 0x7fff;
floats[i] = float;
}
const buffer = new AudioBuffer({
numberOfChannels: 1,
length: floats.length,
sampleRate: 22050,
});
buffer.getChannelData(0).set(floats);
duration += buffer.duration;
const frame = new AudioData({ timestamp, buffer });
await audioWriter.write(frame);
},
close() {
console.log('Done reading input stream.');
},
}),
{ signal, preventClose: false }
),
ac.resume(),
]);
return 'Done streaming.';
}
async function nativeTransferableStream(stdin) {
return new Promise((resolve) => {
onmessage = async (e) => {
if (e.data === 'Ready') {
const encoder = new TextEncoder();
const input = encoder.encode(stdin);
const readable = new ReadableStream({
start(c) {
c.enqueue(input);
c.close();
},
});
e.source.postMessage(readable, e.origin, [readable]);
}
if (e.data instanceof ReadableStream) {
const message = await stream(e.data);
onmessage = null;
transferableWindow.close();
resolve(message);
}
};
const transferableWindow = window.open(
'
http://localhost:8000/index.html',
location.href,
'menubar=no,location=no,resizable=no,scrollbars=no,status=no,width=100,height=100'
);
}).catch((err) => {
throw err;
});
}
let text = `... So we need people to have weird new ideas. We need more ideas to break it and make it better.
Use it
Break it
File bugs
Request features
- Real time front-end alchemy, or:
capturing, playing, altering and encoding video and audio streams, without servers or plugins!
by Soledad Penadés
von Braun believed in testing. I cannot emphasize that term enough – test, test, test.
Test to the point it breaks.
- Ed Buckbee, NASA Public Affairs Officer, Chasing the Moon
Now watch. Um, this how science works.
One researcher comes up with a result.
And that is not the truth. No, no.
A scientific emergent truth is not the
result of one experiment. What has to
happen is somebody else has to verify
it. Preferably a competitor. Preferably
someone who doesn't want you to be correct.
- Neil deGrasse Tyson, May 3, 2017 at 92nd Street Y`;
try {
await nativeTransferableStream(`espeak-ng -m --stdout -v 'Storm' "${text}"`);
} catch (err) {
console.error(err);