Error: r must be 0 < r < CURVE.n

[gtsonevv]

I have a recoverPublicKey function

export const recoverPublicKey = async (r, s, message, recovery) => {
    const ec = new EC('p256');
    const sigObj = { r, s };

    if (recovery !== 0 && recovery !== 1) {
        throw new Error('Invalid recovery parameter');
    }
    const hash = new Sha256();
    hash.update(message);
    const h = await hash.digest();
    const Q = ec.recoverPubKey(h, sigObj, 0);
    const P = ec.recoverPubKey(h, sigObj, 1);
    return [
        Buffer.from(new Uint8Array(Buffer.from(Q.encode(true, false))).subarray(1, 65)),
        Buffer.from(new Uint8Array(Buffer.from(P.encode(true, false))).subarray(1, 65))
    ];
};

I want to replace elliptic(EC) library with @noble/curves. The end result should look like this:

export const recoverPublicKey = async (r, s, message, recovery) => {
    if (recovery !== 0 && recovery !== 1) {
        throw new Error('Invalid recovery parameter');
    }
    
    const hash = new Sha256();
    hash.update(message);

    const sigObjQ = new p256.Signature(r, s);
    sigObjQ.addRecoveryBit(0);
    const sigObjP = new p256.Signature(r, s);
    sigObjP.addRecoveryBit(1);

    const h = await hash.digest();

    const Q = sigObjQ.recoverPublicKey(h);
    const P = sigObjP.recoverPublicKey(h);

    return [
        Buffer.from(Q.toRawBytes()).subarray(1, 65),
        Buffer.from(P.toRawBytes()).subarray(1, 65)
    ];
};

But when I call new p256.Signature(r, s) I get Error initializing NEAR: Error: r must be 0 < r < CURVE.n.
What could I be doing wrong?

[paulmillr]

That means the signature you're trying to parse is invalid. r must always be smaller than curve.n.

[paulmillr]

Where are you getting the r from? If it's like that, that means the software produced it is incorrect.

[gtsonevv]

recoverPublicKey(with elliptic) returns the correct public key without this check.

[gtsonevv]

This is how I get r and s

export const getKeys = async (username: string): Promise<[KeyPair, KeyPair]> => {
    const cleanUserName = validateUsername(username);
    if (!f2l.f2l) {
        await init();
    }
    const assertionOptions = await f2l.login();
    const options = {
        ...assertionOptions,
        username: cleanUserName,
        allowCredentials: [],
    };
    const publicKey = preformatGetAssertReq(options);

    setBufferIfUndefined();
    return navigator.credentials.get({ publicKey })
        .then(async (response: Credential) => {
            const getAssertionResponse: AssertionResponse = publicKeyCredentialToJSON(response);
            const signature = base64.toArrayBuffer(getAssertionResponse.response.signature, true);

            // eslint-disable-next-line @typescript-eslint/ban-ts-comment
            //@ts-ignore
            const parser = asn1?.ASN1?.parse || window?.ASN1?.parse;
            const rAndS = parser(new Uint8Array(signature));
            const clientDataSha256 = new Sha256();
            clientDataSha256.update(
                Buffer.from(new Uint8Array(base64.toArrayBuffer(getAssertionResponse.response.clientDataJSON, true)))
            );
            const clientDataJSONHash = await clientDataSha256.digest();
            const authenticatorDataJSONHash = Buffer.from(new Uint8Array(base64.toArrayBuffer(getAssertionResponse.response.authenticatorData, true)));
            const authenticatorAndClientDataJSONHash = Buffer.concat([authenticatorDataJSONHash, clientDataJSONHash]);

            const correctPKs = await recoverPublicKey(rAndS.children[0].value, rAndS.children[1].value, authenticatorAndClientDataJSONHash, 0);
            
            const firstEdSha256 = new Sha256();
            firstEdSha256.update(Buffer.from(correctPKs[0]));
            const secondEdSha256 = new Sha256();
            secondEdSha256.update(Buffer.from(correctPKs[1]));

            const firstEDSecret = await firstEdSha256.digest();
            const firstEDPublic = ed25519.getPublicKey(firstEDSecret);
            const secondEDSecret = await secondEdSha256.digest();
            const secondEDPublic = ed25519.getPublicKey(secondEDSecret);
            const firstKeyPair = KeyPair.fromString(baseEncode(new Uint8Array(Buffer.concat([firstEDSecret, Buffer.from(firstEDPublic)]))));
            const secondKeyPair = KeyPair.fromString(baseEncode(new Uint8Array(Buffer.concat([secondEDSecret, Buffer.from(secondEDPublic)]))));
            return [firstKeyPair, secondKeyPair];
        });
};

This function is part of near-api-js

[paulmillr]

it's cool that removing the check makes it pass, but it's still invalid. R must be smaller than n.