POC for getting fetch-and-compile functioning in the browser

.eslintrc.json

@@ -9,7 +9,8 @@
     "BUNDLE_CHAIN_FILENAME": "readonly",
     "BUNDLE_ANALYTICS_FILENAME": "readonly",
     "BUNDLE_LIBRARY_FILENAME": "readonly",
-    "BUNDLE_CONSOLE_CHILD_FILENAME": "readonly"
+    "BUNDLE_CONSOLE_CHILD_FILENAME": "readonly",
+    "WORKER_CODE_BASE64": "readonly"
   },
   "extends": [
     "plugin:react-hooks/recommended"

packages/compile-solidity-tests/test/compilerSupplier/index.ts

@@ -64,7 +64,8 @@ let fakeReturn = {
 describe("CompilerSupplier", () => {
   beforeEach(function () {
     supplierOptions = {
-      events: config.events
+      events: config.events,
+      cache: "fileSystem"
     };
   });
 

packages/compile-solidity-tests/test/compilerSupplier/loadingStrategies/VersionRange.ts

@@ -8,10 +8,11 @@ import Config from "@truffle/config";
 const config = Config.default();
 let versionRangeOptions = {
   events: config.events,
-  solcConfig: config.compilers.solc
+  solcConfig: config.compilers.solc,
+  cache: "fileSystem"
 };
 const instance = new LoadingStrategies.VersionRange(versionRangeOptions);
-let fileName, expectedResult;
+let fileName: string, expectedResult: any;
 const compilerFileNames = [
   "soljson-v0.4.22+commit.124ca40d.js",
   "soljson-v0.4.23+commit.1534a40d.js",
@@ -76,11 +77,13 @@ describe("VersionRange loading strategy", () => {
       sinon.stub(instance, "getCachedSolcByVersionRange");
       sinon.stub(instance, "getSolcFromCacheOrUrl");
       sinon.stub(instance, "versionIsCached").returns("compilerFilename.js");
+      sinon.stub(instance, "compilerFromString");
     });
     afterEach(() => {
       unStub(instance, "getCachedSolcByVersionRange");
       unStub(instance, "getSolcFromCacheOrUrl");
       unStub(instance, "versionIsCached");
+      unStub(instance, "compilerFromString");
     });
 
     it("calls getCachedSolcByVersionRange when single solc is specified", async () => {
@@ -105,32 +108,32 @@ describe("VersionRange loading strategy", () => {
 
     describe("when a version constraint is specified", () => {
       beforeEach(() => {
-        sinon.stub(instance, "getAndCacheSolcByUrl");
-        sinon.stub(instance.cache, "has").returns(false);
+        sinon.stub(instance, "getAndCacheSoljsonByUrl");
+        sinon.stub(instance.cache!, "has").returns(false);
       });
       afterEach(() => {
-        unStub(instance, "getAndCacheSolcByUrl");
-        unStub(instance.cache, "has");
+        unStub(instance, "getAndCacheSoljsonByUrl");
+        unStub(instance.cache!, "has");
       });
 
       it("calls findNewstValidVersion to determine which version to fetch", async () => {
         await instance.getSolcFromCacheOrUrl("^0.5.0");
         assert.isTrue(
           // @ts-ignore
-          instance.getAndCacheSolcByUrl.calledWith(
+          instance.getAndCacheSoljsonByUrl.calledWith(
             "soljson-v0.5.4+commit.9549d8ff.js"
           ),
-          "getAndCacheSolcByUrl not called with the compiler file name"
+          "getAndCacheSoljsonByUrl not called with the compiler file name"
         );
       });
     });
 
     describe("when the version is cached", () => {
       beforeEach(() => {
-        sinon.stub(instance.cache, "has").returns(true);
+        sinon.stub(instance.cache!, "has").returns(true);
       });
       afterEach(() => {
-        unStub(instance.cache, "has");
+        unStub(instance.cache!, "has");
       });
 
       it("calls getCachedSolcByFileName", async () => {
@@ -146,27 +149,23 @@ describe("VersionRange loading strategy", () => {
 
     describe("when the version is not cached", () => {
       beforeEach(() => {
-        sinon.stub(instance.cache, "has").returns(false);
-        sinon.stub(instance.cache, "add");
-        sinon.stub(instance, "compilerFromString").returns("compiler");
+        sinon.stub(instance.cache!, "has").returns(false);
+        sinon.stub(instance, "getAndCacheSoljsonByUrl");
       });
       afterEach(() => {
-        unStub(instance.cache, "has");
-        unStub(instance.cache, "add");
-        unStub(instance, "compilerFromString");
+        unStub(instance.cache!, "has");
+        unStub(instance, "getAndCacheSoljsonByUrl");
       });
 
-      it("eventually calls add and compilerFromString", async () => {
+      it("eventually calls .getAndCacheSoljsonByUrl", async () => {
         await instance.getSolcFromCacheOrUrl("0.5.1");
         // @ts-ignore
-        assert.isTrue(instance.cache.add.called);
-        // @ts-ignore
-        assert.isTrue(instance.compilerFromString.called);
+        assert.isTrue(instance.getAndCacheSoljsonByUrl.called);
       }).timeout(60000);
     });
   });
 
-  describe(".getAndCacheSolcByUrl(fileName)", () => {
+  describe(".getAndCacheSoljsonByUrl(fileName)", () => {
     beforeEach(() => {
       fileName = "someSolcFile";
       sinon
@@ -175,24 +174,19 @@ describe("VersionRange loading strategy", () => {
         .returns(Promise.resolve({ data: "requestReturn" }));
       // @ts-ignore
       sinon.stub(instance.cache, "add").withArgs("requestReturn");
-      sinon
-        .stub(instance, "compilerFromString")
-        .withArgs("requestReturn")
-        .returns("success");
     });
     afterEach(() => {
       unStub(axios, "get");
-      unStub(instance.cache, "add");
-      unStub(instance, "compilerFromString");
+      unStub(instance.cache!, "add");
     });
 
     it("calls add with the response and the file name", async () => {
-      const result = await instance.getAndCacheSolcByUrl(fileName, 0);
+      const result = await instance.getAndCacheSoljsonByUrl(fileName, 0);
       assert.isTrue(
         // @ts-ignore
         instance.cache.add.calledWith("requestReturn", "someSolcFile")
       );
-      assert.equal(result, "success");
+      assert.equal(result, "requestReturn");
     });
   });
 
@@ -216,10 +210,10 @@ describe("VersionRange loading strategy", () => {
 
   describe("versionIsCached(version)", () => {
     beforeEach(() => {
-      sinon.stub(instance.cache, "list").returns(compilerFileNames);
+      sinon.stub(instance.cache!, "list").returns(compilerFileNames);
     });
     afterEach(() => {
-      unStub(instance.cache, "list");
+      unStub(instance.cache!, "list");
     });
 
     describe("when a cached version of the compiler is present", () => {
@@ -246,11 +240,11 @@ describe("VersionRange loading strategy", () => {
   describe("getCachedSolcByVersionRange(version)", () => {
     beforeEach(() => {
       expectedResult = "soljson-v0.4.23+commit.1534a40d.js";
-      sinon.stub(instance.cache, "list").returns(compilerFileNames);
+      sinon.stub(instance.cache!, "list").returns(compilerFileNames);
       sinon.stub(instance, "getCachedSolcByFileName");
     });
     afterEach(() => {
-      unStub(instance.cache, "list");
+      unStub(instance.cache!, "list");
       unStub(instance, "getCachedSolcByFileName");
     });
 

packages/compile-solidity/.gitignore

@@ -1,4 +1,5 @@
 dist
+src/compileInWebWorker/bundledWorker.js
 node_modules
 yarn.lock
 package-lock.json

packages/compile-solidity/package.json

@@ -46,13 +46,19 @@
     "babel-preset-env": "^1.6.1",
     "chai": "^4.2.0",
     "glob": "^7.1.6",
+    "https-browserify": "^1.0.0",
     "mocha": "10.1.0",
+    "process": "^0.11.10",
     "sinon": "^9.0.2",
+    "stream-browserify": "^3.0.0",
+    "stream-http": "^3.2.0",
     "tmp": "^0.2.1",
     "ts-node": "10.7.0",
     "ttypescript": "1.5.13",
     "typescript": "^4.7.4",
-    "typescript-transform-paths": "3.3.1"
+    "typescript-transform-paths": "3.3.1",
+    "webpack": "^5.74.0",
+    "webpack-cli": "^4.10.0"
   },
   "keywords": [
     "compile",

packages/compile-solidity/src/compileInWebWorker/index.ts

@@ -0,0 +1,25 @@
+/* eslint-env browser */
+export async function compileInWebWorker({
+  soljson,
+  compilerInput
+}): Promise<any> {
+  // @ts-ignore
+  const dataUrl = `data:text/javascript;base64,${WORKER_CODE_BASE64}`;
+  const worker = new Worker(dataUrl);
+  return new Promise(resolve => {
+    worker.addEventListener(
+      "message",
+      function (e) {
+        const output = e.data;
+        resolve({
+          compilerOutput: output,
+          // TODO: figure out where the version information is
+          solcVersion: ""
+        });
+      },
+      false
+    );
+
+    worker.postMessage({ soljson, compilerInput });
+  });
+}

packages/compile-solidity/src/compileInWebWorker/worker.ts

@@ -0,0 +1,13 @@
+/* eslint-env worker */
+import solcWrap from "solc/wrapper";
+
+self.addEventListener(
+  "message",
+  e => {
+    const { soljson, compilerInput } = e.data;
+    const solc = solcWrap(eval(soljson + "; Module;"));
+    const output = JSON.parse(solc.compile(JSON.stringify(compilerInput)));
+    self.postMessage(output);
+  },
+  false
+);

packages/compile-solidity/src/compilerSupplier/index.ts

@@ -1,11 +1,23 @@
+/* eslint-env node, browser */
 import path from "path";
 import fs from "fs";
 import semver from "semver";
 import { StrategyOptions } from "./types";
 import { Docker, Local, Native, VersionRange } from "./loadingStrategies";
-
 const defaultSolcVersion = "0.5.16";
 
+type CompilerSupplierConstructorArgs = {
+  events?: any;
+  solcConfig: {
+    version?: string;
+    docker?: boolean;
+    compilerRoots?: string[];
+    dockerTagsUrl?: string;
+    spawn?: any;
+  };
+  cache?: string;
+};
+
 type CompilerSupplierStrategy =
   | Docker
   | Native
@@ -15,10 +27,10 @@ type CompilerSupplierStrategy =
 
 export class CompilerSupplier {
   private version: string;
-  private docker: boolean;
+  private docker?: boolean;
   private strategyOptions: StrategyOptions;
 
-  constructor({ events, solcConfig }) {
+  constructor({ events, solcConfig, cache }: CompilerSupplierConstructorArgs) {
     const { version, docker, compilerRoots, dockerTagsUrl, spawn } = solcConfig;
     this.version = version ? version : defaultSolcVersion;
     this.docker = docker;
@@ -28,17 +40,22 @@ export class CompilerSupplier {
     if (compilerRoots) this.strategyOptions.compilerRoots = compilerRoots;
     if (events) this.strategyOptions.events = events;
     if (spawn) this.strategyOptions.spawn = spawn;
+    if (cache) this.strategyOptions.cache = cache;
   }
 
-  async load() {
+  getStrategy() {
     const userSpecification = this.version;
-
     let strategy: CompilerSupplierStrategy;
     const useDocker = this.docker;
     const useNative = userSpecification === "native";
-    const useSpecifiedLocal =
-      userSpecification &&
-      (fs.existsSync(userSpecification) || path.isAbsolute(userSpecification));
+    let useSpecifiedLocal: boolean | string | undefined;
+    // don't attempt file system access in browser environment
+    if (typeof window === "undefined") {
+      useSpecifiedLocal =
+        userSpecification &&
+        (fs.existsSync(userSpecification) ||
+          path.isAbsolute(userSpecification));
+    }
     const isValidVersionRange = semver.validRange(userSpecification);
 
     if (useDocker) {
@@ -50,7 +67,24 @@ export class CompilerSupplier {
     } else if (isValidVersionRange) {
       strategy = new VersionRange(this.strategyOptions);
     }
+    return {
+      strategy,
+      userSpecification
+    };
+  }
+
+  async loadSoljson() {
+    const { strategy, userSpecification } = this.getStrategy();
+    if (strategy) {
+      const soljson = await strategy.loadSoljson(userSpecification);
+      return { soljson };
+    } else {
+      throw new BadInputError(userSpecification);
+    }
+  }
 
+  async load() {
+    const { strategy, userSpecification } = this.getStrategy();
     if (strategy) {
       const solc = await strategy.load(userSpecification);
       return { solc };

packages/compile-solidity/src/compilerSupplier/loadingStrategies/Docker.ts

@@ -26,6 +26,10 @@ export class Docker {
     this.cache = new Cache();
   }
 
+  loadSoljson() {
+    throw new Error("Method loadSoljson not implemented for Docker strategy.");
+  }
+
   async load() {
     // Set a sensible limit for maxBuffer
     // See https://github.com/nodejs/node/pull/23027

packages/compile-solidity/src/compilerSupplier/loadingStrategies/Local.ts

@@ -4,6 +4,10 @@ import solcWrap from "solc/wrapper";
 import { observeListeners } from "../observeListeners";
 
 export class Local {
+  loadSoljson() {
+    throw new Error("Method loadSoljson not implemented for Local strategy.");
+  }
+
   load(localPath: string) {
     const listeners = observeListeners();
     try {

packages/compile-solidity/src/compilerSupplier/loadingStrategies/Native.ts

@@ -3,6 +3,10 @@ const { normalizeSolcVersion } = require("../normalizeSolcVersion");
 const { NoVersionError } = require("../errors");
 
 export class Native {
+  loadSoljson() {
+    throw new Error("Method loadSoljson not implemented for Native strategy.");
+  }
+
   load() {
     const versionString = this.validateAndGetSolcVersion();
     const command = "solc --standard-json";