Clarify flag bit positions

Author: jayz22

soroban-env-common/env.json

@@ -2367,7 +2367,7 @@
                         { "name": "point2", "type": "BytesObject" }
                     ],
                     "return": "BytesObject",
-                    "docs": "Adds two BN254 G1 points. G1 encoding: 64-byte uncompressed format: be_bytes(X)||be_bytes(Y), where X and Y are 32-byte big-endian Fp field elements. The two flag bits (bit 0 and 1) of the first byte must be unset -- infinity is represented as 64 zero bytes. Points must be on curve.",
+                    "docs": "Adds two BN254 G1 points. G1 encoding: 64-byte uncompressed format: be_bytes(X)||be_bytes(Y), where X and Y are 32-byte big-endian Fp field elements. The two flag bits (0x80 and 0x40) of the first byte must be unset -- infinity is represented as 64 zero bytes. Points must be on curve.",
                     "min_supported_protocol": 25
                 },
                 {
@@ -2389,7 +2389,7 @@
                         { "name": "vp2", "type": "VecObject" }
                     ],
                     "return": "Bool",
-                    "docs": "Performs BN254 multi-pairing check over equal-length non-empty vectors of G1 and G2 points. Returns true iff the product of pairings e(G1[0],G2[0])*...*e(G1[n-1],G2[n-1]) equals 1 in Fq12. G1 encoding: 64 bytes as in bn254_g1_add. G2 encoding: 128-byte uncompressed format: be_bytes(X)||be_bytes(Y), where X and Y are Fp2 elements (64 bytes each). Fp2 element encoding: be_bytes(c1)||be_bytes(c0) where c0 is the real part and c1 is the imaginary part (each 32-byte big-endian Fp). The two flag bits (bit 0 and 1) of the first byte must be unset -- G2 infinity is 128 zero bytes. G2 points must be on curve AND in the correct subgroup.",
+                    "docs": "Performs BN254 multi-pairing check over equal-length non-empty vectors of G1 and G2 points. Returns true iff the product of pairings e(G1[0],G2[0])*...*e(G1[n-1],G2[n-1]) equals 1 in Fq12. G1 encoding: 64 bytes as in bn254_g1_add. G2 encoding: 128-byte uncompressed format: be_bytes(X)||be_bytes(Y), where X and Y are Fp2 elements (64 bytes each). Fp2 element encoding: be_bytes(c1)||be_bytes(c0) where c0 is the real part and c1 is the imaginary part (each 32-byte big-endian Fp). The two flag bits (0x80 and 0x40) of the first byte must be unset -- G2 infinity is 128 zero bytes. G2 points must be on curve AND in the correct subgroup.",
                     "min_supported_protocol": 25
                 },
                 {

soroban-env-host/src/crypto/bn254.rs

@@ -39,8 +39,9 @@ impl Host {
 
     // Arkworks and ethereum differ in terms of how flags are handled.
     //
-    // - In arkworks, the two free bits are reserved for flags: the MSB is the
-    //   Y-sign flag, and the 2nd MSB is the infinity flag
+    // - In arkworks, the two free bits are reserved for flags: the
+    //   most-significant-bit is the Y-sign flag, and the 2nd
+    //   most-significant-bit is the infinity flag
     // - In ethereum, the two free bits are always unset.
     //
     // Since encoding/decoding is in uncompressed mode (Y is included), there is
@@ -65,6 +66,11 @@ impl Host {
             ));
         }
 
+        // The incoming bytes is in big-endian, the sign bits are contained in
+        // the first byte (for G1Affine that's the MSB of the y-coordinate (Fp),
+        // for G2Affine that's the MSB of the c1 (Fp) part of the Y-coordinate
+        // (Fp2)). The highest bit (0x80) is the y-sign flag, the 2nd highest
+        // bit (0x40) is the infinity flag. We ensure both are unset.
         let flags = 0b1100_0000 & bytes[0];
         if flags != 0b0000_0000 {
             return Err(self.err(