From e3582740a04bbcd24870055116aee4455183106c Mon Sep 17 00:00:00 2001 From: James Brock Date: Sun, 9 Mar 2025 17:59:02 +0900 Subject: [PATCH 1/6] license BSD-3-Clause --- library/spago.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/library/spago.yaml b/library/spago.yaml index 65ffaca..ca2ae5e 100644 --- a/library/spago.yaml +++ b/library/spago.yaml @@ -3,7 +3,7 @@ package: name: protobuf publish: - license: MIT + license: BSD-3-Clause version: 4.5.0 location: githubOwner: rowtype-yoga @@ -39,4 +39,4 @@ package: workspace: packageSet: - registry: 60.5.1 \ No newline at end of file + registry: 60.5.1 From 388bfa19017e50d7e24185bf9e75cdf4c97e9eab Mon Sep 17 00:00:00 2001 From: James Brock Date: Sun, 9 Mar 2025 18:07:24 +0900 Subject: [PATCH 2/6] upgrade parsing --- library/spago.lock | 60 ++++++++++++++++++++++++------------- library/spago.yaml | 8 +++-- plugin/spago.lock | 74 +++++++++++++++++++++++++++++++--------------- plugin/spago.yaml | 8 +++-- 4 files changed, 101 insertions(+), 49 deletions(-) diff --git a/library/spago.lock b/library/spago.lock index 9b875b0..7240289 100644 --- a/library/spago.lock +++ b/library/spago.lock @@ -54,7 +54,7 @@ "newtype": ">=5.0.0 <6.0.0" }, { - "parsing": ">=10.2.0 <11.0.0" + "parsing": ">=10.3.1 <11.0.0" }, { "parsing-dataview": ">=3.2.4 <4.0.0" @@ -146,7 +146,7 @@ }, "package_set": { "address": { - "registry": "60.5.1" + "registry": "63.3.0" }, "compiler": ">=0.15.15 <0.16.0", "content": { @@ -182,7 +182,7 @@ "ask": "1.0.0", "assert": "6.0.0", "assert-multiple": "0.4.0", - "avar": "5.0.0", + "avar": "5.0.1", "b64": "0.0.8", "barbies": "1.0.1", "barlow-lens": "0.9.0", @@ -204,7 +204,7 @@ "chameleon-transformers": "1.0.0", "channel": "1.0.0", "checked-exceptions": "3.1.1", - "choku": "1.0.1", + "choku": "1.0.2", "classless": "0.1.1", "classless-arbitrary": "0.1.1", "classless-decode-json": "0.1.1", @@ -224,6 +224,7 @@ "convertable-options": "1.0.0", "coroutines": "7.0.0", "css": "6.0.0", + "css-class-name-extractor": "0.0.4", "css-frameworks": "1.0.1", "csv-stream": "2.3.0", "data-mvc": "0.0.2", @@ -237,6 +238,7 @@ "deno": "0.0.5", "dissect": "1.0.0", "distributive": "6.0.0", + "dodo-printer": "2.2.3", "dom-filereader": "7.0.0", "dom-indexed": "12.0.0", "dom-simple": "0.4.0", @@ -263,7 +265,7 @@ "exists": "6.0.0", "exitcodes": "4.0.0", "expect-inferred": "3.0.0", - "ezfetch": "1.0.0", + "ezfetch": "1.1.0", "fahrtwind": "2.0.0", "fallback": "0.1.0", "fast-vect": "1.2.0", @@ -354,7 +356,7 @@ "js-date": "8.0.0", "js-fetch": "0.2.1", "js-fileio": "3.0.0", - "js-intl": "1.0.4", + "js-intl": "1.1.4", "js-iterators": "0.1.1", "js-maps": "0.1.2", "js-promise": "1.0.0", @@ -367,7 +369,7 @@ "justifill": "0.5.0", "jwt": "0.0.9", "labeled-data": "0.2.0", - "language-cst-parser": "0.14.0", + "language-cst-parser": "0.14.1", "lazy": "6.0.0", "lazy-joe": "1.0.0", "lcg": "4.0.0", @@ -418,20 +420,21 @@ "node-event-emitter": "3.0.0", "node-execa": "5.0.0", "node-fs": "9.2.0", - "node-glob-basic": "1.3.0", + "node-glob-basic": "2.0.0", "node-http": "9.1.0", "node-http2": "1.1.1", "node-human-signals": "1.0.0", "node-net": "5.1.0", "node-os": "5.1.0", - "node-path": "5.0.0", + "node-path": "5.0.1", "node-process": "11.2.0", "node-readline": "8.1.1", "node-sqlite3": "8.0.0", "node-stream-pipes": "2.1.6", - "node-streams": "9.0.0", + "node-streams": "9.0.1", "node-tls": "0.3.1", "node-url": "7.0.1", + "node-workerbees": "0.3.1", "node-zlib": "0.4.0", "nonempty": "7.0.0", "now": "6.0.0", @@ -473,15 +476,16 @@ "pointed-list": "0.5.1", "polymorphic-vectors": "4.0.0", "posix-types": "6.0.0", - "postgresql": "2.0.19", + "postgresql": "2.0.20", "precise": "6.0.0", "precise-datetime": "7.0.0", - "prelude": "6.0.1", + "prelude": "6.0.2", "prettier-printer": "3.0.0", + "printf": "0.1.0", "priority-queue": "0.1.2", "profunctor": "6.0.1", "profunctor-lenses": "8.0.0", - "protobuf": "4.3.0", + "protobuf": "4.4.0", "psa-utils": "8.0.0", "psci-support": "6.0.0", "punycode": "1.0.0", @@ -499,7 +503,7 @@ "react-basic": "17.0.0", "react-basic-classic": "3.0.0", "react-basic-dnd": "10.1.0", - "react-basic-dom": "6.1.0", + "react-basic-dom": "7.0.0", "react-basic-dom-beta": "0.1.1", "react-basic-emotion": "7.1.0", "react-basic-hooks": "8.2.0", @@ -538,6 +542,7 @@ "selection-foldable": "0.2.0", "selective-functors": "1.0.1", "semirings": "7.0.0", + "shuffle": "1.1.0", "signal": "13.0.0", "simple-emitter": "3.0.1", "simple-i18n": "2.0.1", @@ -551,7 +556,7 @@ "soundfonts": "4.1.0", "sparse-matrices": "2.0.1", "sparse-polynomials": "3.0.1", - "spec": "8.1.0", + "spec": "8.1.1", "spec-discovery": "8.4.0", "spec-mocha": "5.1.1", "spec-node": "0.0.3", @@ -572,6 +577,7 @@ "systemd-journald": "0.3.0", "tagged": "4.0.2", "tailrec": "6.1.0", + "tanstack-query": "2.0.0", "tecton": "0.2.1", "tecton-halogen": "0.2.0", "test-unit": "17.0.0", @@ -579,11 +585,14 @@ "thermite-dom": "0.3.1", "these": "6.0.0", "threading": "0.0.3", + "tidy": "0.11.1", + "tidy-codegen": "4.0.1", "tldr": "0.0.0", "toestand": "0.9.0", "transformation-matrix": "1.0.1", "transformers": "6.1.0", "tree-rose": "4.0.2", + "trivial-unfold": "0.1.1", "ts-bridge": "4.0.0", "tuples": "7.0.0", "two-or-more": "1.0.0", @@ -621,6 +630,7 @@ "vectors": "2.1.0", "versions": "7.0.0", "visx": "0.0.2", + "vitest": "1.0.0", "web-clipboard": "6.0.0", "web-cssom": "2.0.0", "web-cssom-view": "0.1.0", @@ -649,17 +659,25 @@ "webextension-polyfill": "0.1.0", "webgpu": "0.0.1", "which": "2.0.0", + "whine-core": "0.0.28", "xterm": "1.0.0", "yoga-fetch": "1.0.1", "yoga-json": "5.1.0", "yoga-om": "0.1.0", "yoga-postgres": "6.0.0", + "yoga-react-dom": "1.0.1", + "yoga-subtlecrypto": "0.1.0", "yoga-tree": "1.0.0", "z3": "0.0.2", "zipperarray": "2.0.0" } }, - "extra_packages": {} + "extra_packages": { + "parsing": { + "git": "https://github.com/purescript-contrib/purescript-parsing.git", + "ref": "v10.3.1" + } + } }, "packages": { "arraybuffer": { @@ -1064,9 +1082,9 @@ ] }, "parsing": { - "type": "registry", - "version": "10.2.0", - "integrity": "sha256-ZDIdMFAKkst57x6BVa1aUWJnS8smoZnXsZ339Aq1mPA=", + "type": "git", + "url": "https://github.com/purescript-contrib/purescript-parsing.git", + "rev": "3ad2900eb9a9d2b079f4b5a4547db7cf10a83eb4", "dependencies": [ "arrays", "control", @@ -1123,8 +1141,8 @@ }, "prelude": { "type": "registry", - "version": "6.0.1", - "integrity": "sha256-o8p6SLYmVPqzXZhQFd2hGAWEwBoXl1swxLG/scpJ0V0=", + "version": "6.0.2", + "integrity": "sha256-kiAPZxihtAel8uRiTNdccf4qylp/9J3jNkEHNAD0MsE=", "dependencies": [] }, "profunctor": { diff --git a/library/spago.yaml b/library/spago.yaml index ca2ae5e..36beebe 100644 --- a/library/spago.yaml +++ b/library/spago.yaml @@ -26,7 +26,7 @@ package: - lists: ">=7.0.0 <8.0.0" - maybe: ">=6.0.0 <7.0.0" - newtype: ">=5.0.0 <6.0.0" - - parsing: ">=10.2.0 <11.0.0" + - parsing: ">=10.3.1 <11.0.0" - parsing-dataview: ">=3.2.4 <4.0.0" - prelude: ">=6.0.1 <7.0.0" - record: ">=4.0.0 <5.0.0" @@ -39,4 +39,8 @@ package: workspace: packageSet: - registry: 60.5.1 + registry: 63.3.0 + extraPackages: + parsing: + git: https://github.com/purescript-contrib/purescript-parsing.git + ref: v10.3.1 # branch, tag, or commit hash diff --git a/plugin/spago.lock b/plugin/spago.lock index 042ab97..d01f140 100644 --- a/plugin/spago.lock +++ b/plugin/spago.lock @@ -11,6 +11,9 @@ "node-path", "node-process", "node-streams", + { + "parsing": ">=10.3.1 <11.0.0" + }, "protobuf", "unicode", "unsafe-coerce" @@ -128,7 +131,7 @@ }, "package_set": { "address": { - "registry": "60.5.1" + "registry": "63.3.0" }, "compiler": ">=0.15.15 <0.16.0", "content": { @@ -164,7 +167,7 @@ "ask": "1.0.0", "assert": "6.0.0", "assert-multiple": "0.4.0", - "avar": "5.0.0", + "avar": "5.0.1", "b64": "0.0.8", "barbies": "1.0.1", "barlow-lens": "0.9.0", @@ -186,7 +189,7 @@ "chameleon-transformers": "1.0.0", "channel": "1.0.0", "checked-exceptions": "3.1.1", - "choku": "1.0.1", + "choku": "1.0.2", "classless": "0.1.1", "classless-arbitrary": "0.1.1", "classless-decode-json": "0.1.1", @@ -206,6 +209,7 @@ "convertable-options": "1.0.0", "coroutines": "7.0.0", "css": "6.0.0", + "css-class-name-extractor": "0.0.4", "css-frameworks": "1.0.1", "csv-stream": "2.3.0", "data-mvc": "0.0.2", @@ -219,6 +223,7 @@ "deno": "0.0.5", "dissect": "1.0.0", "distributive": "6.0.0", + "dodo-printer": "2.2.3", "dom-filereader": "7.0.0", "dom-indexed": "12.0.0", "dom-simple": "0.4.0", @@ -245,7 +250,7 @@ "exists": "6.0.0", "exitcodes": "4.0.0", "expect-inferred": "3.0.0", - "ezfetch": "1.0.0", + "ezfetch": "1.1.0", "fahrtwind": "2.0.0", "fallback": "0.1.0", "fast-vect": "1.2.0", @@ -336,7 +341,7 @@ "js-date": "8.0.0", "js-fetch": "0.2.1", "js-fileio": "3.0.0", - "js-intl": "1.0.4", + "js-intl": "1.1.4", "js-iterators": "0.1.1", "js-maps": "0.1.2", "js-promise": "1.0.0", @@ -349,7 +354,7 @@ "justifill": "0.5.0", "jwt": "0.0.9", "labeled-data": "0.2.0", - "language-cst-parser": "0.14.0", + "language-cst-parser": "0.14.1", "lazy": "6.0.0", "lazy-joe": "1.0.0", "lcg": "4.0.0", @@ -400,20 +405,21 @@ "node-event-emitter": "3.0.0", "node-execa": "5.0.0", "node-fs": "9.2.0", - "node-glob-basic": "1.3.0", + "node-glob-basic": "2.0.0", "node-http": "9.1.0", "node-http2": "1.1.1", "node-human-signals": "1.0.0", "node-net": "5.1.0", "node-os": "5.1.0", - "node-path": "5.0.0", + "node-path": "5.0.1", "node-process": "11.2.0", "node-readline": "8.1.1", "node-sqlite3": "8.0.0", "node-stream-pipes": "2.1.6", - "node-streams": "9.0.0", + "node-streams": "9.0.1", "node-tls": "0.3.1", "node-url": "7.0.1", + "node-workerbees": "0.3.1", "node-zlib": "0.4.0", "nonempty": "7.0.0", "now": "6.0.0", @@ -455,15 +461,16 @@ "pointed-list": "0.5.1", "polymorphic-vectors": "4.0.0", "posix-types": "6.0.0", - "postgresql": "2.0.19", + "postgresql": "2.0.20", "precise": "6.0.0", "precise-datetime": "7.0.0", - "prelude": "6.0.1", + "prelude": "6.0.2", "prettier-printer": "3.0.0", + "printf": "0.1.0", "priority-queue": "0.1.2", "profunctor": "6.0.1", "profunctor-lenses": "8.0.0", - "protobuf": "4.3.0", + "protobuf": "4.4.0", "psa-utils": "8.0.0", "psci-support": "6.0.0", "punycode": "1.0.0", @@ -481,7 +488,7 @@ "react-basic": "17.0.0", "react-basic-classic": "3.0.0", "react-basic-dnd": "10.1.0", - "react-basic-dom": "6.1.0", + "react-basic-dom": "7.0.0", "react-basic-dom-beta": "0.1.1", "react-basic-emotion": "7.1.0", "react-basic-hooks": "8.2.0", @@ -520,6 +527,7 @@ "selection-foldable": "0.2.0", "selective-functors": "1.0.1", "semirings": "7.0.0", + "shuffle": "1.1.0", "signal": "13.0.0", "simple-emitter": "3.0.1", "simple-i18n": "2.0.1", @@ -533,7 +541,7 @@ "soundfonts": "4.1.0", "sparse-matrices": "2.0.1", "sparse-polynomials": "3.0.1", - "spec": "8.1.0", + "spec": "8.1.1", "spec-discovery": "8.4.0", "spec-mocha": "5.1.1", "spec-node": "0.0.3", @@ -554,6 +562,7 @@ "systemd-journald": "0.3.0", "tagged": "4.0.2", "tailrec": "6.1.0", + "tanstack-query": "2.0.0", "tecton": "0.2.1", "tecton-halogen": "0.2.0", "test-unit": "17.0.0", @@ -561,11 +570,14 @@ "thermite-dom": "0.3.1", "these": "6.0.0", "threading": "0.0.3", + "tidy": "0.11.1", + "tidy-codegen": "4.0.1", "tldr": "0.0.0", "toestand": "0.9.0", "transformation-matrix": "1.0.1", "transformers": "6.1.0", "tree-rose": "4.0.2", + "trivial-unfold": "0.1.1", "ts-bridge": "4.0.0", "tuples": "7.0.0", "two-or-more": "1.0.0", @@ -603,6 +615,7 @@ "vectors": "2.1.0", "versions": "7.0.0", "visx": "0.0.2", + "vitest": "1.0.0", "web-clipboard": "6.0.0", "web-cssom": "2.0.0", "web-cssom-view": "0.1.0", @@ -631,17 +644,24 @@ "webextension-polyfill": "0.1.0", "webgpu": "0.0.1", "which": "2.0.0", + "whine-core": "0.0.28", "xterm": "1.0.0", "yoga-fetch": "1.0.1", "yoga-json": "5.1.0", "yoga-om": "0.1.0", "yoga-postgres": "6.0.0", + "yoga-react-dom": "1.0.1", + "yoga-subtlecrypto": "0.1.0", "yoga-tree": "1.0.0", "z3": "0.0.2", "zipperarray": "2.0.0" } }, "extra_packages": { + "parsing": { + "git": "https://github.com/purescript-contrib/purescript-parsing.git", + "ref": "v10.3.1" + }, "protobuf": { "path": "../library" } @@ -1135,8 +1155,8 @@ }, "node-path": { "type": "registry", - "version": "5.0.0", - "integrity": "sha256-pd82nQ+2l5UThzaxPdKttgDt7xlsgIDLpPG0yxDEdyE=", + "version": "5.0.1", + "integrity": "sha256-ePOElFamHkffhwJcS0Ozq4A14rflnkasFU6X2B8/yXs=", "dependencies": [ "effect" ] @@ -1159,17 +1179,23 @@ }, "node-streams": { "type": "registry", - "version": "9.0.0", - "integrity": "sha256-2n6dq7YWleTDmD1Kur/ul7Cn08IvWrScgPf+0PgX2TQ=", + "version": "9.0.1", + "integrity": "sha256-7RJ6RqjOlhW+QlDFQNUHlkCG/CuYTTLT8yary5jhhsU=", "dependencies": [ "aff", + "arrays", "effect", "either", "exceptions", + "maybe", "node-buffer", "node-event-emitter", "nullable", - "prelude" + "prelude", + "refs", + "st", + "tailrec", + "unsafe-coerce" ] }, "nonempty": { @@ -1250,9 +1276,9 @@ ] }, "parsing": { - "type": "registry", - "version": "10.2.0", - "integrity": "sha256-ZDIdMFAKkst57x6BVa1aUWJnS8smoZnXsZ339Aq1mPA=", + "type": "git", + "url": "https://github.com/purescript-contrib/purescript-parsing.git", + "rev": "3ad2900eb9a9d2b079f4b5a4547db7cf10a83eb4", "dependencies": [ "arrays", "control", @@ -1318,8 +1344,8 @@ }, "prelude": { "type": "registry", - "version": "6.0.1", - "integrity": "sha256-o8p6SLYmVPqzXZhQFd2hGAWEwBoXl1swxLG/scpJ0V0=", + "version": "6.0.2", + "integrity": "sha256-kiAPZxihtAel8uRiTNdccf4qylp/9J3jNkEHNAD0MsE=", "dependencies": [] }, "profunctor": { diff --git a/plugin/spago.yaml b/plugin/spago.yaml index b1aba3b..6440255 100644 --- a/plugin/spago.yaml +++ b/plugin/spago.yaml @@ -10,6 +10,7 @@ package: - console - unicode - unsafe-coerce + - parsing: ">=10.3.1 <11.0.0" test: main: Test.Main @@ -22,7 +23,10 @@ package: workspace: packageSet: - registry: 60.5.1 + registry: 63.3.0 extraPackages: protobuf: - path: ../library \ No newline at end of file + path: ../library + parsing: + git: https://github.com/purescript-contrib/purescript-parsing.git + ref: v10.3.1 # branch, tag, or commit hash From 772a48d45e1dcb4de4bf39d35c83b98560c1f11e Mon Sep 17 00:00:00 2001 From: James Brock Date: Sun, 9 Mar 2025 20:17:32 +0900 Subject: [PATCH 3/6] nix flake update --- flake.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/flake.lock b/flake.lock index dedb19a..b27b820 100644 --- a/flake.lock +++ b/flake.lock @@ -60,11 +60,11 @@ "registry-index": "registry-index" }, "locked": { - "lastModified": 1740275730, - "narHash": "sha256-wiP+/yaL+zy8JdTHzfG0g0UdRPi3hFHVjZcLbDCFG18=", + "lastModified": 1741484263, + "narHash": "sha256-t3EgVkcA97N75UB8nv+82AtIJtLOXIRLLxrm8Fr2Hdg=", "owner": "jeslie0", "repo": "mkSpagoDerivation", - "rev": "da7426529e1f255b5033c6ff66f20810e28c4892", + "rev": "b752ee8613cc041922278df9df71351618c62832", "type": "github" }, "original": { @@ -91,11 +91,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1740463929, - "narHash": "sha256-4Xhu/3aUdCKeLfdteEHMegx5ooKQvwPHNkOgNCXQrvc=", + "lastModified": 1741332913, + "narHash": "sha256-ri1e8ZliWS3Jnp9yqpKApHaOo7KBN33W8ECAKA4teAQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "5d7db4668d7a0c6cc5fc8cf6ef33b008b2b1ed8b", + "rev": "20755fa05115c84be00b04690630cb38f0a203ad", "type": "github" }, "original": { @@ -150,11 +150,11 @@ "registry": { "flake": false, "locked": { - "lastModified": 1740209161, - "narHash": "sha256-zm459VgHJ809QjvmzZ+svKj8Eo+PTQIp2LOBfbuwj8Y=", + "lastModified": 1741415458, + "narHash": "sha256-m/8bQDH7sFmSwAmQfYXe+hmBDLfeEbSs37FMLmkB8dk=", "owner": "purescript", "repo": "registry", - "rev": "430c12069a5b15593cb100315003b480d5f2e5b1", + "rev": "2d14b19308e97159bf26713dce9a5da1ad132ccb", "type": "github" }, "original": { @@ -166,11 +166,11 @@ "registry-index": { "flake": false, "locked": { - "lastModified": 1740191161, - "narHash": "sha256-YnbVvkAhF1Bbgw048+7v1QDyfwehWsDeFSCSms7Nae8=", + "lastModified": 1741415461, + "narHash": "sha256-6ABxd+Ta53u3SPtAhXDTULNcWajQIAW5z+AuyPxmeKE=", "owner": "purescript", "repo": "registry-index", - "rev": "63511edd9043f06152eaa82940847a3682927ca7", + "rev": "7f0f8a6681e1af84f1fa5ba86927a53c0f91edf3", "type": "github" }, "original": { From 13f6518cd3a6b832afb78469b22326ce703b1cc0 Mon Sep 17 00:00:00 2001 From: James Brock Date: Sun, 9 Mar 2025 18:14:28 +0900 Subject: [PATCH 4/6] plugin docs --- plugin/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/README.md b/plugin/README.md index 33d6b59..64965d3 100644 --- a/plugin/README.md +++ b/plugin/README.md @@ -31,7 +31,7 @@ To test purescript-protobuf, run `nix develop` from the top level directory of the repo, then `cd plugin` and generate the test messages: ```shell -protoc --purescript_out=./plugin/test/Test/generated --proto_path=./test ./test/*.proto +protoc --purescript_out=./test/Test/generated --proto_path=./test ./test/*.proto ``` Or, to generate messages with the bundled plugin, not the Nix store plugin: From 1f8630b7ed4008d34ac64cfa6a3e9bf10f384076 Mon Sep 17 00:00:00 2001 From: James Brock Date: Tue, 4 Mar 2025 23:58:05 +0900 Subject: [PATCH 5/6] manyLength fail early if not enough bytes parseLenDel fail early if not enough bytes. Improve generation speed for large inputs? Actually no. --- library/src/Protobuf/Internal/Runtime.purs | 47 ++++++++++++++-------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/library/src/Protobuf/Internal/Runtime.purs b/library/src/Protobuf/Internal/Runtime.purs index 6c1f608..88fd72d 100644 --- a/library/src/Protobuf/Internal/Runtime.purs +++ b/library/src/Protobuf/Internal/Runtime.purs @@ -24,6 +24,7 @@ import Control.Monad.Trans.Class (lift) import Data.Array (snoc, foldRecM) import Data.Array as Array import Data.ArrayBuffer.Builder (DataBuff(..), PutM, subBuilder) +import Data.ArrayBuffer.DataView (byteLength) import Data.ArrayBuffer.Types (DataView, ByteLength) import Data.Enum (class BoundedEnum, fromEnum, toEnum) import Data.Foldable (foldl) @@ -39,7 +40,8 @@ import Data.UInt64 (UInt64) import Data.UInt64 as UInt64 import Effect.Class (class MonadEffect) import Parsing (ParserT, Position(..), fail, position) -import Parsing.DataView (takeN) +import Parsing.Combinators (lookAhead) +import Parsing.DataView (takeN, takeRest) import Protobuf.Internal.Common (Bytes(..), FieldNumber, WireType(..), label) import Protobuf.Internal.Decode as Decode import Protobuf.Internal.Encode as Encode @@ -76,6 +78,7 @@ type FieldNumberInt = Int -- | Call a parser repeatedly until exactly *N* bytes have been consumed. +-- | Will fail if not enough bytes remain in the DataView. -- | Will fail if too many bytes are consumed. manyLength :: forall m a. @@ -85,19 +88,24 @@ manyLength :: ByteLength -> ParserT DataView m (Array a) manyLength p len = do - Position { index: posBegin } <- position - let - go :: List a -> ParserT DataView m (Step (List a) (List a)) - go accum = do - Position { index: pos } <- position - case compare (pos - posBegin) len of - GT -> fail "manyLength consumed too many bytes." - EQ -> lift $ pure (Done accum) - LT -> do - x <- p - pure (Loop (x : accum)) - -- https://github.com/purescript-contrib/purescript-parsing/pull/199#issuecomment-1145956271 - Array.reverse <$> Array.fromFoldable <$> tailRecM go List.Nil + remaining_bytes :: Int <- byteLength <$> lookAhead takeRest + if remaining_bytes < len + then do + fail $ "manyLength " <> show len <> " not enough bytes of input " <> show remaining_bytes <> " remaining" + else do + Position { index: posBegin } <- position + let + go :: List a -> ParserT DataView m (Step (List a) (List a)) + go accum = do + Position { index: pos } <- position + case compare (pos - posBegin) len of + GT -> fail $ "manyLength " <> show len <> " consumed too many bytes " <> (show $ pos - posBegin) + EQ -> lift $ pure (Done accum) + LT -> do + x <- p + pure (Loop (x : accum)) + -- https://github.com/purescript-contrib/purescript-parsing/pull/199#issuecomment-1145956271 + Array.reverse <$> Array.fromFoldable <$> tailRecM go List.Nil -- | A message field value from an unknown `.proto` definition. -- | @@ -174,13 +182,20 @@ putFieldUnknown (UnknownLenDel fieldNumber x) = Encode.encodeBytesField fieldNum putFieldUnknown (UnknownBits32 fieldNumber x) = Encode.encodeBytesField fieldNumber x --- | Parse a length, then call a parser which takes one length as its argument. +-- | Parse a bytelength, then call a parser which takes one bytelength as its argument. parseLenDel :: forall m a. MonadEffect m => (Int -> ParserT DataView m a) -> ParserT DataView m a -parseLenDel p = p <<< UInt.toInt =<< Decode.decodeVarint32 +parseLenDel p = do + len <- UInt.toInt <$> Decode.decodeVarint32 + remaining_bytes :: Int <- byteLength <$> lookAhead takeRest + if remaining_bytes < len + then do + fail $ "parseLenDel " <> show len <> " not enough bytes of input " <> show remaining_bytes <> " remaining" + else do + p len putLenDel :: forall m a. From f0ccf73b55286f2d1d4256c7f454b2e7fda6ffd6 Mon Sep 17 00:00:00 2001 From: James Brock Date: Thu, 6 Mar 2025 09:58:26 +0900 Subject: [PATCH 6/6] parseCodeGeneratorRequestComplete Much faster plugin code generation when input CodeGenerationRequest is very large. --- plugin/src/Main.purs | 63 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 10 deletions(-) diff --git a/plugin/src/Main.purs b/plugin/src/Main.purs index 5f6bac1..57cc8d7 100644 --- a/plugin/src/Main.purs +++ b/plugin/src/Main.purs @@ -5,12 +5,13 @@ module Main (main) where import Prelude -import Control.Monad.Rec.Class (tailRecM, Step(..)) +import Control.Monad.Rec.Class (class MonadRec, Step(..), tailRecM) import Data.Array (catMaybes, concat, fold, mapWithIndex) import Data.Array as Array import Data.ArrayBuffer.ArrayBuffer as AB import Data.ArrayBuffer.Builder (execPutM) import Data.ArrayBuffer.DataView as DV +import Data.ArrayBuffer.Types as ABT import Data.CodePoint.Unicode as Unicode import Data.Either (Either(..), either) import Data.Maybe (Maybe(..), fromMaybe, maybe) @@ -22,20 +23,25 @@ import Data.String.Regex as String.Regex import Data.String.Regex.Flags as String.Regex.Flags import Data.Traversable (sequence, traverse) import Data.Tuple (Tuple(..)) +import Data.UInt as UInt import Data.UInt64 (UInt64) import Data.UInt64 as UInt64 import Effect (Effect) import Effect.Aff (runAff_, throwError, error) -import Effect.Class (liftEffect) +import Effect.Class (class MonadEffect, liftEffect) import Effect.Class.Console as Console -import Google.Protobuf.Compiler.Plugin (CodeGeneratorRequest(..), CodeGeneratorResponse, CodeGeneratorResponse_File(..), mkCodeGeneratorResponse, parseCodeGeneratorRequest, putCodeGeneratorResponse) +import Google.Protobuf.Compiler.Plugin (CodeGeneratorRequest(..), CodeGeneratorResponse, CodeGeneratorResponse_File(..), mkCodeGeneratorResponse, parseCodeGeneratorRequest, putCodeGeneratorResponse, parseVersion) import Google.Protobuf.Descriptor (DescriptorProto(..), EnumDescriptorProto(..), EnumValueDescriptorProto(..), FieldDescriptorProto(..), FieldDescriptorProto_Label(..), FieldDescriptorProto_Type(..), FieldOptions(..), FileDescriptorProto(..), OneofDescriptorProto(..), SourceCodeInfo(..), SourceCodeInfo_Location(..)) import Node.Buffer (Buffer, toArrayBuffer, fromArrayBuffer) import Node.Buffer as Buffer import Node.Path (basenameWithoutExt) import Node.Process (stdin, stdout) import Node.Stream.Aff (readSome, write) -import Parsing (runParserT) +import Parsing (ParserT, fail, runParserT) +import Parsing.DataView (takeN) +import Protobuf.Internal.Common (label) +import Protobuf.Internal.Decode (decodeString, decodeTag32) +import Protobuf.Internal.Runtime (parseLenDel, manyLength) import Unsafe.Coerce (unsafeCoerce) @@ -51,7 +57,7 @@ main = runAff_ (either (unsafeCoerce >>> Console.error) (\_ -> pure unit)) do {buffers:b,readagain} <- readSome stdin let bs' = bs <> b ab <- liftEffect $ toArrayBuffer =<< Buffer.concat bs' - runParserT (DV.whole ab) (parseCodeGeneratorRequest (AB.byteLength ab)) >>= case _ of + runParserT (DV.whole ab) (parseCodeGeneratorRequestComplete (AB.byteLength ab)) >>= case _ of Left err -> do if not readagain then do void $ throwError $ error "stdin is not readable." @@ -68,11 +74,48 @@ main = runAff_ (either (unsafeCoerce >>> Console.error) (\_ -> pure unit)) do else do void $ throwError $ error $ show err pure (Done unit) - Right request -> do - responseBuf <- execPutM $ putCodeGeneratorResponse (generate request) - buf :: Buffer <- liftEffect $ fromArrayBuffer responseBuf - write stdout [buf] - pure (Done unit) + Right unit -> do + runParserT (DV.whole ab) (parseCodeGeneratorRequest (AB.byteLength ab)) >>= case _ of + Left err -> do + void $ throwError $ error $ show err + pure (Done unit) + Right request -> do + responseBuf <- execPutM $ putCodeGeneratorResponse (generate request) + buf :: Buffer <- liftEffect $ fromArrayBuffer responseBuf + write stdout [buf] + pure (Done unit) + +-- | This is a parser which will succeed only if parseCodeGeneratorRequest will succeed. +-- | It is faster than parseCodeGeneratorRequest. +-- | We use this parser to determine if we have read the whole request from stdin. +-- | If this parser fails then we need to read more from stdin. +parseCodeGeneratorRequestComplete + :: forall m. MonadEffect m + => MonadRec m + => ABT.ByteLength + -> ParserT ABT.DataView m Unit +parseCodeGeneratorRequestComplete length = label "parseCodeGeneratorRequestComplete / " $ do + _ <- flip manyLength length do + Tuple fieldNumber _wireType <- decodeTag32 + case UInt.toInt fieldNumber of + 1 -> do -- file_to_generate + _ <- decodeString + pure unit + 2 -> do -- parameter + _ <- decodeString + pure unit + 3 -> do -- compiler_version + _ <- parseLenDel parseVersion + pure unit + 15 -> do -- proto_file + _ <- parseLenDel takeN + pure unit + 17 -> do -- source_file_descriptors + _ <- parseLenDel takeN + pure unit + _ -> do + fail "unexpected field number" + pure unit generate :: CodeGeneratorRequest -> CodeGeneratorResponse generate (CodeGeneratorRequest { proto_file }) = do