Executable from JSON

[guest271314]

I'm using QuickJS as a Native Messaging host to write files sent to the host from the browser, e.g., from DevTools console with externally_connectable defined in a Web extension. Native Messaging protocol uses JSON for communication between host and client.

The tricky part (that I should probably already know how to do by now) is for the case of an executable. So I have qjs in directly. Let's say I wanted to write that qjs executable to a different directory on the same machine, though I get the executable in the QuickJS script as JSON.

With Deno and Node.js (https://github.com/guest271314/native-messaging-file-writer) and Cloudflare's workerd (https://github.com/guest271314/native-messaging-workerd) I just use Native Messaging to start a local server then stream the data to the file over HTTP/HTTPS/QUIC.

Now I'm trying to do this with qjs without networking.

What am I missing here?

In console in DevTools in the browser, where file is a File or Blob object I get from somewhere I do something like this:

Start externally connectable connection (this is JSON-like over IPC)

// JSON.stringify([...new Uint8Array(209715*1.25).fill(255)]).length/1024**2
// 0.9999971389770508
// 262143
async function generateIdForPath(path) {
  return [...[...new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(path), ),), ].map( (u8) => u8.toString(16).padStart(2, "0")).join("").slice(0, 32), ].map( (hex) => String.fromCharCode(parseInt(hex, 16) + "a".charCodeAt(0))).join("", );
}

async function connectExternal() {
  if (globalThis.port) {
    globalThis.port.disconnect();

  }
  globalThis.port = chrome.runtime.connect(await generateIdForPath("/home/user/Downloads/qjs-file-writer"), {
    name: "external"
  });
  port.onMessage.addListener( (message, sender) => {
    console.log(message, sender.name);
  }
  );
  port.onDisconnect.addListener( (e) => {
    if (chrome.runtime.lastError) {
      console.log(chrome.runtimeLastError, e);
    }
    console.log(e);
  }
  );
  return port;
}

await connectExternal().catch(console.error);

await file.stream()
.pipeTo(new WritableStream({
  async write(v) {
    if (v.length >= 262143) {
      const n = (262143+1)/2;
      for (let i = 0; i < v.length; i += n) {
      await new Blob([v.subarray(i, i+n)]).stream().pipeTo(new WritableStream({
        write(v1) {
          console.log(v1.length);
          if (v1.length > 0) {
            port.postMessage([...v1]);
          }
        },
        close() {
          // console.log("Restream close");
        }
      }))
      }
    } else {
      port.postMessage([...v])
    }
  },
  close() {
    console.log("close");
  }
})).catch((e) => console.log(e)).then(() => console.log("pipe closed, probably still writing"));

Forward data to/from host native host in MV3 ServiceWorker extension script

chrome.runtime.onConnectExternal.addListener((externalConnectablePort) => {
  //globalThis.name = chrome.runtime.getManifest().short_name;
  globalThis.externalPort = externalConnectablePort;
  globalThis.port = chrome.runtime.connectNative("nm_qjs_file_writer");
  const handleNativeMessage = (message) => {
    //console.log(`Message from Native Messaging host ${port.name}`);
    globalThis.externalPort.postMessage(message);
    return true;
  }
 // port.name = "nm_deno";
  port.onMessage.addListener(handleNativeMessage);
  port.onDisconnect.addListener((message) => {
    if (chrome.runtime.lastError) {
      console.log(chrome.runtime.lastError);
      // externalPort.postMessage(chrome.runtime.lastError.message);
    };
    externalPort.disconnect();
    globalThis.externallyConnectablePort = null;
  });
  const handleExternalMessage = async (message, { sender }) => {
    //console.log(`Message from Web page ${sender.url}`);
    // console.log(message);
    await scheduler.postTask(() => {}, {delay:100, priority:"background"});
    port.postMessage(message);
    return true;
  }
  globalThis.externalPort.onMessage.addListener(handleExternalMessage);
  globalThis.externalPort.onDisconnect.addListener(async (e) => {
    console.log(e);
    globalThis.port.disconnect();
    globalThis.port = null;
    globalThis.externallyConnectablePort = null;
  });
});

QuickJS host

#!/usr/bin/env -S /home/user/bin/qjs -m --std
// QuickJS Native Messaging host
// guest271314, 5-6-2022
// import {webserver} from './webserver.so';
const err = { errno: 0 };
const file = std.open("qjs", "wb", err);
let n = 0;
let writes = 0;

function getMessage() {
  const header = new Uint32Array(1);
  std.in.read(header.buffer, 0, 4);
  const output = new Uint8Array(header[0]);
  std.in.read(output.buffer, 0, output.length);
  return output;
}

function write(data) {
  //const json = JSON.parse(String.fromCodePoint(...data));
  //const u8 = new Uint8Array([...json]);
  const buffer = data.buffer;
  const len = buffer.byteLength;
  file.write(buffer, 0, len);
  //file.close();
  //std.exit(0); 
}

function sendMessage(message) {
  const header = Uint32Array.from(
    {
      length: 4,
    },
    (_, index) => (message.length >> (index * 8)) & 0xff
  );
  const output = new Uint8Array(header.length + message.length);
  output.set(header, 0);
  output.set(message, 4);
  std.out.write(output.buffer, 0, output.length);
  std.out.flush();
}

function main() {
  while (true) {
     const message = getMessage();
     write(message);   
/*
    webserver("cat background.js", (status)=>{
      console.log(status);
      // sendMessage([...`"${status}"`].map((s) => s.charCodeAt()));  
      if (status === 'pipe broken') {
        std.exit(0);
      }
    });
*/
  }
}

try {
  main();
} catch (e) {
  std.exit(0);
}

Right now the file is written as plain text. And is 3.9 MB, where qjs is just 1.3 MB.

However, I can fetch the file as JSON in the extension with fetch(), and get back to the 1.3 MB, and read the ELF headers.

var json = await fetch("qjs").then(r => r.json())
  .then((json) => new Uint8Array(json));
  // Uint8Array(1314896) 

String.fromCodePoint(...json.subarray(0, 4));
'\x7FELF'

How do I use std or os module to convert the JSON over IPC to binary data and write the executable (application/x-executable) within the host script on the first run?

[guest271314]

I figured it out. Write a plain JavaScript object, using smaller data chunks, to avoid invalid JSON in some writes ending in ,

function main() {
  const err = { errno: 0 };
  const file = std.open("qjs", "w", err);
  let writes = 0;
  let totalBytesWritten = 0;
  while (true) {
    const message = getMessage();
    const str = String.fromCodePoint(...message);
    const { value, done } = JSON.parse(str);
    if (done) {
      file.close();
      sendMessage(
        new Uint8Array(
          [...JSON.stringify({ done, value, totalBytesWritten })].map((s) =>
            s.codePointAt()
          ),
        ),
      );
    } else {
      const u8 = new Uint8Array([...value]);
      totalBytesWritten += u8.length;
      const currentBytesWritten = u8.buffer.byteLength;
      file.write(u8.buffer, 0, currentBytesWritten);
      ++writes;
      sendMessage(
        new Uint8Array(
          [...JSON.stringify({
            done,
            writes,
            currentBytesWritten,
            totalBytesWritten,
          })].map((s) => s.codePointAt()),
        ),
      );
    }
  }
}

Writing qjs, and node to the local file system from DevTools

var file = await getFileObject();
file.stream()
.pipeTo(new WritableStream({
  async write(v) {
    //console.log(v);
    for (let i = 0; i < v.length; i += 16384) {
      port.postMessage({value: [...v.subarray(i, i + 16384)], done: false});
    }
  },
  async close() {
    port.postMessage({value: null, done: true});
    console.log("close");
  }
})).catch(console.warn);