Merge pull request #69 from 0xPolygonHermez/feature/imPolsIntegration

Adding intermediate polynomial new algorithm and new parser

src/helpers/expressionops.js

@@ -91,7 +91,7 @@ class ExpressionOps {
     number(n) {
         return {
             op: "number",
-            value: BigInt(n)
+            value: n.toString()
         }
     }
 

src/main_buildchelpers.js

@@ -10,17 +10,19 @@ const argv = require("yargs")
     .alias("c", "chelpers")
     .alias("C", "cls")
     .alias("b", "binfile")
+    .alias("g", "genericbinfile")
     .argv;
 
 async function run() {
     let cls = typeof (argv.cls) === "string" ? argv.cls.trim() : "Stark";
     const starkInfoFile = typeof (argv.starkinfo) === "string" ? argv.starkinfo.trim() : "mycircuit.starkinfo.json";
     const cHelpersFile = typeof (argv.chelpers) === "string" ? argv.chelpers.trim() : "mycircuit.chelpers";
     const binFile = typeof (argv.binfile) === "string" ? argv.binfile.trim() : "mycircuit.chelpers.bin";
+    const genericBinFile = typeof (argv.genericbinfile) === "string" ? argv.genericbinfile.trim() : undefined;
 
     const starkInfo = JSON.parse(await fs.promises.readFile(starkInfoFile, "utf8"));
 
-    await buildCHelpers(starkInfo, cHelpersFile, binFile, cls);
+    await buildCHelpers(starkInfo, cHelpersFile, cls, binFile, genericBinFile);
 
     console.log("files Generated Correctly");
 }

src/main_buildchelpers_generic.js

@@ -0,0 +1,56 @@
+const { generateParser, getAllOperations } = require("./stark/chelpers/generateParser");
+const fs = require("fs");
+
+const version = require("../package").version;
+
+const argv = require("yargs")
+    .version(version)
+    .usage("node main_buildchelpers_generic.js -c <chelpers.cpp>")
+    .alias("c", "chelpers")
+    .string("parserType")
+    .argv;
+
+async function run() {
+    const cHelpersFile = typeof (argv.chelpers) === "string" ? argv.chelpers.trim() : "mycircuit.chelpers";
+
+    let operations = getAllOperations();
+
+    let parserType = "avx";
+
+    if(argv.parserType) {
+        if(!["avx", "avx512","pack"].includes(argv.parserType)) throw new Error("Invalid parser type");
+        parserType = argv.parserType;
+    }
+
+    const parser = generateParser(operations, undefined, parserType);
+
+    const cHelpersStepsName = parserType === "avx" ? `CHELPERS_STEPS_HPP` : parserType === "avx512" ? "CHELPERS_STEPS_AVX512_HPP" : "CHELPERS_STEPS_PACK_HPP";
+
+    const cHelpersStepsClassName = parserType === "avx" ? `CHelpersSteps` : parserType === "avx512" ? "CHelpersStepsAvx512 : public CHelpersSteps" : "CHelpersStepsPack : public CHelpersSteps";
+
+    const cHelpersStepsHpp = [
+        `#ifndef ${cHelpersStepsName}`,
+        `#define ${cHelpersStepsName}`,
+        `#include "chelpers.hpp"`,
+        `${parserType !== "avx" ? `#include "chelpers_steps.hpp"` : ""}`,
+        `#include "steps.hpp"\n`,
+        `class ${cHelpersStepsClassName} {`,
+        "public:",
+    ];
+
+    cHelpersStepsHpp.push(parser);
+    cHelpersStepsHpp.push("};\n");
+    cHelpersStepsHpp.push("#endif")
+
+    await fs.promises.writeFile(cHelpersFile, cHelpersStepsHpp.join("\n"), "utf8");
+
+    console.log("Generic parser generated correctly");
+}
+
+run().then(() => {
+    process.exit(0);
+}, (err) => {
+    console.log(err.message);
+    console.log(err.stack);
+    process.exit(1);
+});

src/main_buildconsttree.js

@@ -8,14 +8,13 @@ const { buildConstTree } = require("./stark/stark_buildConstTree");
 
 const argv = require("yargs")
     .version(version)
-    .usage("node main_buildconsttree.js -c const.bin -p <pil.json> [-P <pilconfig.json>] -s <starkstruct.json> -t <consttree.bin>  -v <verification_key.json>")
+    .usage("node main_buildconsttree.js -c const.bin -p <pil.json> [-P <pilconfig.json>] -s <starkinfo.json> -t <consttree.bin>  -v <verification_key.json>")
     .alias("c", "const")
     .alias("p", "pil")
     .alias("P", "pilconfig")
-    .alias("s", "starkstruct")
+    .alias("s", "starkinfo")
     .alias("t", "consttree")
     .alias("v", "verkey")
-    .string("arity")
     .argv;
 
 async function run() {
@@ -24,18 +23,17 @@ async function run() {
     const pilFile = typeof(argv.pil) === "string" ?  argv.pil.trim() : "mycircuit.pil";
     const pilConfig = typeof(argv.pilconfig) === "string" ? JSON.parse(fs.readFileSync(argv.pilconfig.trim())) : {};
     const constFile = typeof(argv.const) === "string" ?  argv.const.trim() : "mycircuit.const";
-    const starkStructFile = typeof(argv.starkstruct) === "string" ?  argv.starkstruct.trim() : "mycircuit.stark_struct.json";
+    const starkInfoFile = typeof(argv.starkinfo) === "string" ?  argv.starkinfo.trim() : "mycircuit.stark_struct.json";
     const constTreeFile = typeof(argv.consttree) === "string" ?  argv.consttree.trim() : "mycircuit.consttree";
     const verKeyFile = typeof(argv.verkey) === "string" ?  argv.verkey.trim() : "mycircuit.verkey.json";
 
-    const starkStruct = JSON.parse(await fs.promises.readFile(starkStructFile, "utf8"));
+    const starkInfo = JSON.parse(await fs.promises.readFile(starkInfoFile, "utf8"));
     const pil = await compile(F, pilFile, null, pilConfig);
 
     const constPols = newConstantPolsArray(pil, F);
     await constPols.loadFromFile(constFile);
 
-    let arity = Number(argv.arity) || 16;
-    const {MH, constTree, verKey} = await buildConstTree(starkStruct, pil, constPols, arity);
+    const {MH, constTree, verKey} = await buildConstTree(starkInfo, pil, constPols);
 
     await fs.promises.writeFile(verKeyFile, JSONbig.stringify(verKey, null, 1), "utf8");
 

src/main_calculateimpols.js

@@ -0,0 +1,38 @@
+const fs = require("fs");
+const version = require("../package").version;
+
+const { calculateIntermediatePolynomials } = require("./pil_info/imPolsCalculation/imPolynomials");
+
+const argv = require("yargs")
+    .version(version)
+    .usage("node main_calculateimpols.js -f <infopil.json> -m <impols.json>")
+    .alias("f", "infopil")
+    .alias("m", "impols")
+    .argv;
+
+async function run() {
+    const infoPilFile = typeof(argv.infopil) === "string" ?  argv.infopil.trim() : "mycircuit.infopil.json";
+    const imPolsFile = typeof(argv.impols) === "string" ?  argv.impols.trim() : "mycircuit.impols.json";
+
+    const infoPil = JSON.parse(await fs.promises.readFile(infoPilFile, "utf8"));
+
+    const expressions = infoPil.expressions;
+    const maxDeg  = infoPil.maxDeg;
+    const cExpId = infoPil.cExpId;
+    const qDim = infoPil.qDim;
+
+    const imPols = calculateIntermediatePolynomials(expressions, cExpId, maxDeg, qDim);
+
+    await fs.promises.writeFile(imPolsFile, JSON.stringify(imPols, null, 1), "utf8");
+
+    console.log("files Generated Correctly");
+}
+
+run().then(()=> {
+    process.exit(0);
+}, (err) => {
+    console.log(err.message);
+    console.log(err.stack);
+    process.exit(1);
+});
+

src/main_genpilcode.js

@@ -0,0 +1,66 @@
+const fs = require("fs");
+const version = require("../package").version;
+
+const { generatePilCode } = require("./pil_info/generatePilCode");
+const { addIntermediatePolynomials } = require("./pil_info/imPolsCalculation/imPolynomials");
+const map = require("./pil_info/map");
+const { compile } = require("pilcom");
+const F3g = require("./helpers/f3g");
+
+const argv = require("yargs")
+    .version(version)
+    .usage("node main_genpilcode.js -f <infopil.json> -p <pil.json> -m <impols.json> -i <starkinfo.json>")
+    .alias("p", "pil")
+    .alias("f", "infopil")
+    .alias("m", "impols")
+    .alias("i", "starkinfo")
+    .argv;
+
+async function run() {
+    const F = new F3g();
+
+    const infoPilFile = typeof(argv.infopil) === "string" ?  argv.infopil.trim() : "mycircuit.infopil.json";
+    const imPolsFile = typeof(argv.impols) === "string" ?  argv.impols.trim() : "mycircuit.impols.json";
+
+    const pilFile = typeof(argv.pil) === "string" ?  argv.pil.trim() : "mycircuit.pil";
+    const pil = await compile(F, pilFile);
+
+    const starkInfoFile = typeof(argv.starkinfo) === "string" ?  argv.starkinfo.trim() : "mycircuit.starkinfo.json";
+
+    const infoPil = JSON.parse(await fs.promises.readFile(infoPilFile, "utf8"));
+    const imPols = JSON.parse(await fs.promises.readFile(imPolsFile, "utf8"));
+
+    const res = infoPil.res;
+    const expressions = imPols.newExpressions;
+    const qDeg = imPols.qDeg;
+    const imExps = imPols.imExps;
+
+    addIntermediatePolynomials(res, expressions, imExps, qDeg);
+
+    generatePilCode(res, pil, expressions);
+
+    map(res, expressions);       
+
+    console.log("--------------------- POLINOMIALS INFO ---------------------")
+    console.log(`Columns stage 1: ${res.nCm1} -> Columns in the basefield: ${res.mapSectionsN.cm1_2ns}`);
+    console.log(`Columns stage 2: ${res.nCm2} -> Columns in the basefield: ${res.mapSectionsN.cm2_2ns}`);
+    console.log(`Columns stage 3: ${res.nCm3} (${res.nImPols} intermediate polinomials) -> Columns in the basefield: ${res.mapSectionsN.cm3_2ns}`);
+    console.log(`Columns stage 4: ${res.nCm4} -> Columns in the basefield: ${res.mapSectionsN.cm4_2ns}`);
+    console.log(`Total Columns: ${res.nCm1 + res.nCm2 + res.nCm3 + res.nCm4} -> Total Columns in the basefield: ${res.mapSectionsN.cm1_2ns + res.mapSectionsN.cm2_2ns + res.mapSectionsN.cm3_2ns + res.mapSectionsN.cm4_2ns}`);
+    console.log(`Total Constraints: ${res.nConstraints}`)
+    console.log(`Number of evaluations: ${res.evMap.length}`)
+    console.log("------------------------------------------------------------")
+
+    await fs.promises.writeFile(starkInfoFile, JSON.stringify(res, null, 1), "utf8");
+
+    console.log("files Generated Correctly");
+}
+
+run().then(()=> {
+    process.exit(0);
+}, (err) => {
+    console.log(err.message);
+    console.log(err.stack);
+    process.exit(1);
+});
+

src/main_genstarkinfo.js

@@ -12,6 +12,7 @@ const argv = require("yargs")
     .alias("P", "pilconfig")
     .alias("s", "starkstruct")
     .alias("i", "starkinfo")
+    .string("arity")
     .argv;
 
 async function run() {
@@ -26,7 +27,11 @@ async function run() {
     const pil = await compile(F, pilFile, null, pilConfig);
     const starkStruct = JSON.parse(await fs.promises.readFile(starkStructFile, "utf8"));
 
-    const starkInfo = starkInfoGen(pil, starkStruct);
+    const options = {};
+    if(starkStruct.verificationHashType === "BN128") {
+        options.arity = Number(argv.arity) || 16;
+    }
+    const starkInfo = starkInfoGen(pil, starkStruct, options);
 
     await fs.promises.writeFile(starkInfoFile, JSON.stringify(starkInfo, null, 1), "utf8");
 

src/main_pil2circom.js

@@ -10,33 +10,32 @@ const JSONbig = require('json-bigint')({ useNativeBigInt: true, alwaysParseAsBig
 
 const argv = require("yargs")
     .version(version)
-    .usage("node main_pil2circom.js -o <verifier.circom> -v <verification_key.json> -s <starkinfo.json> [--skipMain] [--enableInput] [--verkeyInput] [--arity]")
+    .usage("node main_pil2circom.js -o <verifier.circom> -v <verification_key.json> -s <starkinfo.json> [--skipMain] [--enableInput] [--verkeyInput]")
     .alias("s", "starkinfo")
     .alias("v", "verkey")
     .alias("o", "output")
-    .string("arity")
     .string("index")
     .argv;
 
 async function run() {
-    const starkInfoFIle = typeof(argv.starkinfo) === "string" ?  argv.starkinfo.trim() : "starkinfo.json";
-    const verKeyFile = typeof(argv.verkey) === "string" ?  argv.verkey.trim() : "mycircuit.verkey.json";
+    const starkInfoFile = typeof(argv.starkinfo) === "string" ?  argv.starkinfo.trim() : "starkinfo.json";
     const outputFile = typeof(argv.output) === "string" ?  argv.output.trim() : "mycircuit.verifier.circom";
-
-    const verKey = JSONbig.parse(await fs.promises.readFile(verKeyFile, "utf8"));
-    const constRoot = verKey.constRoot;
-
-    const starkInfo = JSON.parse(await fs.promises.readFile(starkInfoFIle, "utf8"));
+
+    const starkInfo = JSON.parse(await fs.promises.readFile(starkInfoFile, "utf8"));
 
     const options = {
         skipMain: argv.skipMain || false,
         enableInput: argv.enableInput || false,
         verkeyInput: argv.verkeyInput || false
     }
 
-    if(starkInfo.starkStruct.verificationHashType === "BN128") {
-        options.arity =  Number(argv.arity) || 16;
-        console.log(`Arity: ${options.arity}`);
+    let constRoot;
+    if(!options.verkeyInput ) {
+        const verKeyFile = typeof(argv.verkey) === "string" ?  argv.verkey.trim() : "mycircuit.verkey.json";
+        const verKey = JSONbig.parse(await fs.promises.readFile(verKeyFile, "utf8"));
+
+        constRoot = verKey.constRoot;
+
     }
 
     if(argv.index) {

src/main_preparepil.js

@@ -0,0 +1,51 @@
+const fs = require("fs");
+const version = require("../package").version;
+
+const F3g = require("./helpers/f3g.js");
+const { compile } = require("pilcom");
+const { preparePil } = require("./pil_info/preparePil");
+
+const argv = require("yargs")
+    .version(version)
+    .usage("node main_preparepil.js -p <pil.json> [-P <pilconfig.json] -s <starkstruct.json> -f <infopil.json>")
+    .alias("p", "pil")
+    .alias("P", "pilconfig")
+    .alias("s", "starkstruct")
+    .alias("f", "infopil")
+    .argv;
+
+async function run() {
+    const F = new F3g();
+
+    const pilFile = typeof(argv.pil) === "string" ?  argv.pil.trim() : "mycircuit.pil";
+    const pilConfig = typeof(argv.pilconfig) === "string" ? JSON.parse(fs.readFileSync(argv.pilconfig.trim())) : {};
+
+    const starkStructFile = typeof(argv.starkstruct) === "string" ?  argv.starkstruct.trim() : "mycircuit.stark_struct.json";
+
+    const infoPilFile = typeof(argv.infopil) === "string" ?  argv.infopil.trim() : "mycircuit.infopil.json";
+
+    let pil = await compile(F, pilFile, null, pilConfig);
+
+    const starkStruct = JSON.parse(await fs.promises.readFile(starkStructFile, "utf8"));
+
+    const infoPil = preparePil(F, pil, starkStruct);
+
+    let maxDeg =  (1 << (starkStruct.nBitsExt - starkStruct.nBits)) + 1;
+
+    const infoPilJSON = { maxDeg, cExpId: infoPil.res.cExpId, qDim: infoPil.res.qDim, ...infoPil };
+
+    console.log("Writing file...");
+
+    await fs.promises.writeFile(infoPilFile, JSON.stringify(infoPilJSON, null, 1), "utf8");
+
+    console.log("files Generated Correctly");
+}
+
+run().then(()=> {
+    process.exit(0);
+}, (err) => {
+    console.log(err.message);
+    console.log(err.stack);
+    process.exit(1);
+});
+

src/main_prover.js

@@ -26,7 +26,6 @@ const argv = require("yargs")
     .alias("z", "zkin")
     .alias("b", "public")
     .string("proverAddr")
-    .string("arity")
     .argv;
 
 async function run() {
@@ -52,24 +51,19 @@ async function run() {
     const cmPols =  newCommitPolsArray(pil);
     await cmPols.loadFromFile(commitFile);
 
-    let options = {};
     let MH;
     if (starkInfo.starkStruct.verificationHashType == "GL") {
         MH = await buildMerklehashGL();
     } else if (starkInfo.starkStruct.verificationHashType == "BN128") {
-        let arity = Number(argv.arity) || 16;
-
-        options = {arity};
-
-        console.log(`Arity: ${arity}`);
-        MH = await buildMerkleHashBN128(arity);
+        console.log(`Merkle Tree Arity: ${starkInfo.merkleTreeArity}`);
+        MH = await buildMerkleHashBN128(starkInfo.merkleTreeArity);
     } else {
         throw new Error("Invalid Hash Type: "+ starkInfo.starkStruct.verificationHashType);
     }
 
     const constTree = await MH.readFromFile(constTreeFile);
 
-    const resP = await starkGen(cmPols, constPols, constTree, starkInfo, options);
+    const resP = await starkGen(cmPols, constPols, constTree, starkInfo);
 
     await fs.promises.writeFile(proofFile, JSONbig.stringify(resP.proof, null, 1), "utf8");