[Kernel] Replace BCrypt RSA with portable bignum implementation
Removes the Windows-only BCrypt dependency from XeCryptBnQwNeRsaPubCrypt and replaces it with a portable modular exponentiation implementation using 64-bit arithmetic, enabling RSA signature verification on all platforms. Adds Catch2 tests validating the implementation with a 2048-bit RSA key.
This commit is contained in:
Herman S.
430
src/xenia/kernel/xboxkrnl/xecrypt_rsa.h
Normal file
430
src/xenia/kernel/xboxkrnl/xecrypt_rsa.h
Normal file
@@ -0,0 +1,430 @@
|
||||
#ifndef XECRYPT_RSA_H_
|
||||
#define XECRYPT_RSA_H_
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <vector>
|
||||
|
||||
namespace xecrypt {
|
||||
namespace bignum {
|
||||
|
||||
// Portable 128-bit unsigned helpers using only 64-bit arithmetic
|
||||
struct u128 {
|
||||
uint64_t lo;
|
||||
uint64_t hi;
|
||||
};
|
||||
|
||||
static inline u128 u128_from(uint64_t v) { return {v, 0}; }
|
||||
|
||||
static inline u128 u128_add(u128 a, uint64_t b) {
|
||||
u128 r;
|
||||
r.lo = a.lo + b;
|
||||
r.hi = a.hi + (r.lo < a.lo ? 1 : 0);
|
||||
return r;
|
||||
}
|
||||
|
||||
static inline u128 u128_sub(u128 a, uint64_t b) {
|
||||
u128 r;
|
||||
r.hi = a.hi - (a.lo < b ? 1 : 0);
|
||||
r.lo = a.lo - b;
|
||||
return r;
|
||||
}
|
||||
|
||||
static inline u128 u128_shl(u128 v, int s) {
|
||||
if (s == 0) return v;
|
||||
if (s >= 64) return {0, v.lo << (s - 64)};
|
||||
return {v.lo << s, (v.hi << s) | (v.lo >> (64 - s))};
|
||||
}
|
||||
|
||||
static inline u128 u128_shr(u128 v, int s) {
|
||||
if (s == 0) return v;
|
||||
if (s >= 64) return {v.hi >> (s - 64), 0};
|
||||
return {(v.lo >> s) | (v.hi << (64 - s)), v.hi >> s};
|
||||
}
|
||||
|
||||
static inline u128 u128_or64(u128 a, uint64_t b) { return {a.lo | b, a.hi}; }
|
||||
|
||||
// Portable 64x64 -> 128 multiply
|
||||
static inline u128 u128_mul64(uint64_t a, uint64_t b) {
|
||||
uint64_t a_lo = a & 0xFFFFFFFF;
|
||||
uint64_t a_hi = a >> 32;
|
||||
uint64_t b_lo = b & 0xFFFFFFFF;
|
||||
uint64_t b_hi = b >> 32;
|
||||
|
||||
uint64_t p0 = a_lo * b_lo;
|
||||
uint64_t p1 = a_lo * b_hi;
|
||||
uint64_t p2 = a_hi * b_lo;
|
||||
uint64_t p3 = a_hi * b_hi;
|
||||
|
||||
uint64_t mid = p1 + (p0 >> 32);
|
||||
uint64_t mid_carry = 0;
|
||||
uint64_t mid2 = mid + p2;
|
||||
if (mid < p1) mid_carry++;
|
||||
if (mid2 < mid) mid_carry++;
|
||||
|
||||
u128 r;
|
||||
r.lo = (mid2 << 32) | (p0 & 0xFFFFFFFF);
|
||||
r.hi = p3 + (mid2 >> 32) + (mid_carry << 32);
|
||||
return r;
|
||||
}
|
||||
|
||||
// 128 / 64 -> quotient and remainder
|
||||
static inline uint64_t u128_div64(u128 num, uint64_t den, uint64_t* rem) {
|
||||
if (num.hi == 0) {
|
||||
*rem = num.lo % den;
|
||||
return num.lo / den;
|
||||
}
|
||||
|
||||
uint64_t q = 0;
|
||||
uint64_t r = 0;
|
||||
for (int i = 127; i >= 0; i--) {
|
||||
r = (r << 1);
|
||||
if (i >= 64) {
|
||||
r |= (num.hi >> (i - 64)) & 1;
|
||||
} else {
|
||||
r |= (num.lo >> i) & 1;
|
||||
}
|
||||
if (r >= den) {
|
||||
r -= den;
|
||||
if (i < 64) {
|
||||
q |= (1ULL << i);
|
||||
}
|
||||
}
|
||||
}
|
||||
*rem = r;
|
||||
return q;
|
||||
}
|
||||
|
||||
// Count leading zeros - portable
|
||||
static inline int clz64(uint64_t v) {
|
||||
if (v == 0) return 64;
|
||||
int n = 0;
|
||||
if ((v & 0xFFFFFFFF00000000ULL) == 0) {
|
||||
n += 32;
|
||||
v <<= 32;
|
||||
}
|
||||
if ((v & 0xFFFF000000000000ULL) == 0) {
|
||||
n += 16;
|
||||
v <<= 16;
|
||||
}
|
||||
if ((v & 0xFF00000000000000ULL) == 0) {
|
||||
n += 8;
|
||||
v <<= 8;
|
||||
}
|
||||
if ((v & 0xF000000000000000ULL) == 0) {
|
||||
n += 4;
|
||||
v <<= 4;
|
||||
}
|
||||
if ((v & 0xC000000000000000ULL) == 0) {
|
||||
n += 2;
|
||||
v <<= 2;
|
||||
}
|
||||
if ((v & 0x8000000000000000ULL) == 0) {
|
||||
n += 1;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
static inline int clz32(uint32_t v) {
|
||||
if (v == 0) return 32;
|
||||
int n = 0;
|
||||
if ((v & 0xFFFF0000U) == 0) {
|
||||
n += 16;
|
||||
v <<= 16;
|
||||
}
|
||||
if ((v & 0xFF000000U) == 0) {
|
||||
n += 8;
|
||||
v <<= 8;
|
||||
}
|
||||
if ((v & 0xF0000000U) == 0) {
|
||||
n += 4;
|
||||
v <<= 4;
|
||||
}
|
||||
if ((v & 0xC0000000U) == 0) {
|
||||
n += 2;
|
||||
v <<= 2;
|
||||
}
|
||||
if ((v & 0x80000000U) == 0) {
|
||||
n += 1;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
class BigNum {
|
||||
public:
|
||||
std::vector<uint64_t> limbs;
|
||||
|
||||
BigNum() = default;
|
||||
|
||||
void trim() {
|
||||
while (limbs.size() > 1 && limbs.back() == 0) {
|
||||
limbs.pop_back();
|
||||
}
|
||||
}
|
||||
|
||||
static BigNum from_bytes_be(const uint8_t* data, size_t len) {
|
||||
BigNum r;
|
||||
size_t n = (len + 7) / 8;
|
||||
r.limbs.resize(n, 0);
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
size_t byte_pos = len - 1 - i;
|
||||
r.limbs[byte_pos / 8] |= static_cast<uint64_t>(data[i])
|
||||
<< (8 * (byte_pos % 8));
|
||||
}
|
||||
r.trim();
|
||||
return r;
|
||||
}
|
||||
|
||||
void to_bytes_be(uint8_t* out, size_t len) const {
|
||||
std::memset(out, 0, len);
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
size_t byte_pos = len - 1 - i;
|
||||
size_t li = byte_pos / 8;
|
||||
if (li < limbs.size()) {
|
||||
out[i] = static_cast<uint8_t>(limbs[li] >> (8 * (byte_pos % 8)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static int compare(const BigNum& a, const BigNum& b) {
|
||||
size_t an = a.limbs.size(), bn = b.limbs.size();
|
||||
size_t n = std::max(an, bn);
|
||||
for (size_t i = n; i > 0; i--) {
|
||||
uint64_t al = (i - 1 < an) ? a.limbs[i - 1] : 0;
|
||||
uint64_t bl = (i - 1 < bn) ? b.limbs[i - 1] : 0;
|
||||
if (al < bl) return -1;
|
||||
if (al > bl) return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static BigNum sub(const BigNum& a, const BigNum& b) {
|
||||
BigNum r;
|
||||
size_t n = a.limbs.size();
|
||||
r.limbs.resize(n, 0);
|
||||
uint64_t borrow = 0;
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
uint64_t bl = (i < b.limbs.size()) ? b.limbs[i] : 0;
|
||||
u128 diff = u128_sub(u128_sub(u128_from(a.limbs[i]), bl), borrow);
|
||||
r.limbs[i] = diff.lo;
|
||||
borrow = (diff.hi >> 63) ? 1 : 0;
|
||||
}
|
||||
r.trim();
|
||||
return r;
|
||||
}
|
||||
|
||||
static BigNum mul(const BigNum& a, const BigNum& b) {
|
||||
size_t an = a.limbs.size(), bn = b.limbs.size();
|
||||
BigNum r;
|
||||
r.limbs.resize(an + bn, 0);
|
||||
for (size_t i = 0; i < an; i++) {
|
||||
uint64_t carry = 0;
|
||||
for (size_t j = 0; j < bn; j++) {
|
||||
u128 prod = u128_mul64(a.limbs[i], b.limbs[j]);
|
||||
prod = u128_add(prod, r.limbs[i + j]);
|
||||
prod = u128_add(prod, carry);
|
||||
r.limbs[i + j] = prod.lo;
|
||||
carry = prod.hi;
|
||||
}
|
||||
r.limbs[i + bn] += carry;
|
||||
}
|
||||
r.trim();
|
||||
return r;
|
||||
}
|
||||
|
||||
static BigNum mod(const BigNum& a, const BigNum& m) {
|
||||
if (compare(a, m) < 0) return a;
|
||||
|
||||
size_t n = m.limbs.size();
|
||||
size_t total = a.limbs.size();
|
||||
|
||||
if (n == 0 || (n == 1 && m.limbs[0] == 0)) {
|
||||
return BigNum();
|
||||
}
|
||||
|
||||
if (n == 1) {
|
||||
uint64_t d = m.limbs[0];
|
||||
uint64_t rem = 0;
|
||||
for (size_t i = total; i > 0; i--) {
|
||||
u128 cur = u128_or64(u128_shl(u128_from(rem), 64), a.limbs[i - 1]);
|
||||
u128_div64(cur, d, &rem);
|
||||
}
|
||||
BigNum r;
|
||||
r.limbs = {rem};
|
||||
r.trim();
|
||||
return r;
|
||||
}
|
||||
|
||||
int shift = 0;
|
||||
uint64_t top = m.limbs[n - 1];
|
||||
if (top != 0) {
|
||||
shift = clz64(top);
|
||||
}
|
||||
|
||||
BigNum u, v;
|
||||
u.limbs.resize(total + 1, 0);
|
||||
if (shift > 0) {
|
||||
uint64_t carry = 0;
|
||||
for (size_t i = 0; i < total; i++) {
|
||||
u128 val = u128_or64(u128_shl(u128_from(a.limbs[i]), shift), carry);
|
||||
u.limbs[i] = val.lo;
|
||||
carry = val.hi;
|
||||
}
|
||||
u.limbs[total] = carry;
|
||||
} else {
|
||||
for (size_t i = 0; i < total; i++) u.limbs[i] = a.limbs[i];
|
||||
u.limbs[total] = 0;
|
||||
}
|
||||
|
||||
v.limbs.resize(n, 0);
|
||||
if (shift > 0) {
|
||||
uint64_t carry = 0;
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
u128 val = u128_or64(u128_shl(u128_from(m.limbs[i]), shift), carry);
|
||||
v.limbs[i] = val.lo;
|
||||
carry = val.hi;
|
||||
}
|
||||
} else {
|
||||
v.limbs = m.limbs;
|
||||
}
|
||||
|
||||
uint64_t vn_1 = v.limbs[n - 1];
|
||||
uint64_t vn_2 = (n >= 2) ? v.limbs[n - 2] : 0;
|
||||
|
||||
for (size_t j = total; j >= n; j--) {
|
||||
u128 num_top =
|
||||
u128_or64(u128_shl(u128_from(u.limbs[j]), 64), u.limbs[j - 1]);
|
||||
uint64_t rhat_val;
|
||||
uint64_t qhat_val = u128_div64(num_top, vn_1, &rhat_val);
|
||||
|
||||
while (true) {
|
||||
u128 qv2 = u128_mul64(qhat_val, vn_2);
|
||||
u128 rhs = u128_or64(u128_shl(u128_from(rhat_val), 64), u.limbs[j - 2]);
|
||||
bool gt = (qv2.hi > rhs.hi) || (qv2.hi == rhs.hi && qv2.lo > rhs.lo);
|
||||
if (!gt) break;
|
||||
qhat_val--;
|
||||
uint64_t old_rhat = rhat_val;
|
||||
rhat_val += vn_1;
|
||||
if (rhat_val < old_rhat) break;
|
||||
}
|
||||
|
||||
uint64_t carry = 0;
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
u128 prod = u128_mul64(qhat_val, v.limbs[i]);
|
||||
prod = u128_add(prod, carry);
|
||||
uint64_t prod_lo = prod.lo;
|
||||
carry = prod.hi;
|
||||
uint64_t u_val = u.limbs[j - n + i];
|
||||
u.limbs[j - n + i] = u_val - prod_lo;
|
||||
if (u_val < prod_lo) carry++;
|
||||
}
|
||||
int64_t final_diff =
|
||||
static_cast<int64_t>(u.limbs[j]) - static_cast<int64_t>(carry);
|
||||
u.limbs[j] = static_cast<uint64_t>(final_diff);
|
||||
|
||||
if (final_diff < 0) {
|
||||
uint64_t c = 0;
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
u128 sum = u128_add(u128_from(u.limbs[j - n + i]), v.limbs[i]);
|
||||
sum = u128_add(sum, c);
|
||||
u.limbs[j - n + i] = sum.lo;
|
||||
c = sum.hi;
|
||||
}
|
||||
u.limbs[j] += c;
|
||||
}
|
||||
}
|
||||
|
||||
BigNum r;
|
||||
r.limbs.resize(n, 0);
|
||||
if (shift > 0) {
|
||||
uint64_t carry = 0;
|
||||
for (size_t i = n; i > 0; i--) {
|
||||
u128 val = u128_or64(u128_shl(u128_from(carry), 64), u.limbs[i - 1]);
|
||||
r.limbs[i - 1] = u128_shr(val, shift).lo;
|
||||
carry = u.limbs[i - 1] & ((1ULL << shift) - 1);
|
||||
}
|
||||
} else {
|
||||
for (size_t i = 0; i < n; i++) r.limbs[i] = u.limbs[i];
|
||||
}
|
||||
|
||||
r.trim();
|
||||
return r;
|
||||
}
|
||||
|
||||
static BigNum modexp(const BigNum& base, uint32_t exp,
|
||||
const BigNum& mod_val) {
|
||||
BigNum result;
|
||||
result.limbs = {1};
|
||||
|
||||
if (exp == 0) {
|
||||
return mod(result, mod_val);
|
||||
}
|
||||
|
||||
int highest_bit = 31 - clz32(exp);
|
||||
BigNum b = mod(base, mod_val);
|
||||
|
||||
for (int i = highest_bit; i >= 0; i--) {
|
||||
result = mod(mul(result, result), mod_val);
|
||||
if ((exp >> i) & 1) {
|
||||
result = mod(mul(result, b), mod_val);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace bignum
|
||||
} // namespace xecrypt
|
||||
|
||||
// Performs RSA public-key encryption/decryption matching
|
||||
// XeCryptBnQwNeRsaPubCrypt. All buffers use the Xbox 360 layout: big-endian
|
||||
// uint64 limbs in little-endian limb order.
|
||||
//
|
||||
// qw_a: input data (num_qwords * 8 bytes)
|
||||
// qw_b: output buffer (num_qwords * 8 bytes)
|
||||
// modulus: RSA modulus following the XECRYPT_RSA header (num_qwords * 8 bytes)
|
||||
// num_qwords: number of 64-bit limbs
|
||||
// exponent: public exponent
|
||||
//
|
||||
// Returns 1 on success, 0 on failure.
|
||||
inline uint32_t XeCryptBnQwNeRsaPubCrypt(const uint8_t* qw_a, uint8_t* qw_b,
|
||||
const uint8_t* modulus,
|
||||
uint32_t num_qwords,
|
||||
uint32_t exponent) {
|
||||
// Reject keys below 512 bits (8 qwords) to match original BCrypt behavior.
|
||||
if (num_qwords < 8) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint32_t modulus_size = num_qwords * 8;
|
||||
|
||||
auto input_be = std::vector<uint8_t>(modulus_size);
|
||||
auto mod_be = std::vector<uint8_t>(modulus_size);
|
||||
|
||||
// Reverse qword order to produce big-endian byte arrays
|
||||
for (uint32_t i = 0; i < num_qwords; i++) {
|
||||
std::memcpy(&input_be[i * 8], &qw_a[(num_qwords - 1 - i) * 8], 8);
|
||||
std::memcpy(&mod_be[i * 8], &modulus[(num_qwords - 1 - i) * 8], 8);
|
||||
}
|
||||
|
||||
auto base =
|
||||
xecrypt::bignum::BigNum::from_bytes_be(input_be.data(), modulus_size);
|
||||
auto mod =
|
||||
xecrypt::bignum::BigNum::from_bytes_be(mod_be.data(), modulus_size);
|
||||
|
||||
auto result = xecrypt::bignum::BigNum::modexp(base, exponent, mod);
|
||||
|
||||
auto result_be = std::vector<uint8_t>(modulus_size);
|
||||
result.to_bytes_be(result_be.data(), modulus_size);
|
||||
|
||||
// Convert back to Xbox format: reverse qword order
|
||||
for (uint32_t i = 0; i < num_qwords; i++) {
|
||||
std::memcpy(&qw_b[i * 8], &result_be[(num_qwords - 1 - i) * 8], 8);
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
#endif // XECRYPT_RSA_H_
|
||||
Reference in New Issue
Block a user