From 33b4bbfc534bf8954081806164545ccd150ebc59 Mon Sep 17 00:00:00 2001 From: ndossche <7771979+ndossche@users.noreply.github.com> Date: Tue, 24 Feb 2026 23:12:19 +0100 Subject: [PATCH 1/2] Fix memory leak if ossl_bn_new() fails When that call fails, the `bn` BIGNUM is never freed in asn1integer_to_num(). To solve this, use rb_protect(). Example Valgrind report: ``` 32 (24 direct, 8 indirect) bytes in 1 blocks are definitely lost in loss record 11,113 of 25,910 malloc (at /usr/libexec/valgrind/vgpreload_memcheck-amd64-linux.so) CRYPTO_zalloc (at /usr/lib/x86_64-linux-gnu/libcrypto.so.3) BN_new (at /usr/lib/x86_64-linux-gnu/libcrypto.so.3) BN_bin2bn (at /usr/lib/x86_64-linux-gnu/libcrypto.so.3) *asn1integer_to_num (ossl_asn1.c:136) *asn1integer_to_num_i (ossl_asn1.c:165) rb_protect (at /usr/lib/x86_64-linux-gnu/libruby-3.2.so.3.2.3) *decode_int (ossl_asn1.c:356) *int_ossl_asn1_decode0_prim (ossl_asn1.c:777) *ossl_asn1_decode0 (ossl_asn1.c:936) *ossl_asn1_decode_all (ossl_asn1.c:1058) rb_vm_exec (at /usr/lib/x86_64-linux-gnu/libruby-3.2.so.3.2.3) rb_catch_obj (at /usr/lib/x86_64-linux-gnu/libruby-3.2.so.3.2.3) rb_vm_exec (at /usr/lib/x86_64-linux-gnu/libruby-3.2.so.3.2.3) rb_yield (at /usr/lib/x86_64-linux-gnu/libruby-3.2.so.3.2.3) rb_ary_each (at /usr/lib/x86_64-linux-gnu/libruby-3.2.so.3.2.3) rb_vm_exec (at /usr/lib/x86_64-linux-gnu/libruby-3.2.so.3.2.3) rb_yield (at /usr/lib/x86_64-linux-gnu/libruby-3.2.so.3.2.3) rb_ary_each (at /usr/lib/x86_64-linux-gnu/libruby-3.2.so.3.2.3) rb_vm_exec (at /usr/lib/x86_64-linux-gnu/libruby-3.2.so.3.2.3) rb_yield (at /usr/lib/x86_64-linux-gnu/libruby-3.2.so.3.2.3) rb_ary_each (at /usr/lib/x86_64-linux-gnu/libruby-3.2.so.3.2.3) rb_vm_exec (at /usr/lib/x86_64-linux-gnu/libruby-3.2.so.3.2.3) rb_catch_obj (at /usr/lib/x86_64-linux-gnu/libruby-3.2.so.3.2.3) rb_vm_exec (at /usr/lib/x86_64-linux-gnu/libruby-3.2.so.3.2.3) rb_vm_invoke_proc (at /usr/lib/x86_64-linux-gnu/libruby-3.2.so.3.2.3) rb_proc_call_kw (at /usr/lib/x86_64-linux-gnu/libruby-3.2.so.3.2.3) ``` --- ext/openssl/ossl_asn1.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ext/openssl/ossl_asn1.c b/ext/openssl/ossl_asn1.c index 71a87f046..f86068d07 100644 --- a/ext/openssl/ossl_asn1.c +++ b/ext/openssl/ossl_asn1.c @@ -118,6 +118,12 @@ asn1str_to_str(const ASN1_STRING *str) ASN1_STRING_length(str)); } +static VALUE +ossl_bn_new_wrap_i(VALUE arg) +{ + return ossl_bn_new((BIGNUM *)arg); +} + /* * ASN1_INTEGER conversions */ @@ -126,6 +132,7 @@ asn1integer_to_num(const ASN1_INTEGER *ai) { BIGNUM *bn; VALUE num; + int st = 0; if (!ai) { ossl_raise(rb_eTypeError, "ASN1_INTEGER is NULL!"); @@ -137,9 +144,11 @@ asn1integer_to_num(const ASN1_INTEGER *ai) if (!bn) ossl_raise(eOSSLError, NULL); - num = ossl_bn_new(bn); + num = rb_protect(ossl_bn_new_wrap_i, (VALUE)bn, &st); BN_free(bn); + if (st) rb_jump_tag(st); + return num; } From e887d76742e220c9968e2428d37b6fb9e49bef36 Mon Sep 17 00:00:00 2001 From: ndossche Date: Thu, 26 Feb 2026 13:51:53 +0100 Subject: [PATCH 2/2] fixup! Use wrap strategy --- ext/openssl/ossl_asn1.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/ext/openssl/ossl_asn1.c b/ext/openssl/ossl_asn1.c index f86068d07..18fa8edeb 100644 --- a/ext/openssl/ossl_asn1.c +++ b/ext/openssl/ossl_asn1.c @@ -118,12 +118,6 @@ asn1str_to_str(const ASN1_STRING *str) ASN1_STRING_length(str)); } -static VALUE -ossl_bn_new_wrap_i(VALUE arg) -{ - return ossl_bn_new((BIGNUM *)arg); -} - /* * ASN1_INTEGER conversions */ @@ -132,22 +126,21 @@ asn1integer_to_num(const ASN1_INTEGER *ai) { BIGNUM *bn; VALUE num; - int st = 0; if (!ai) { ossl_raise(rb_eTypeError, "ASN1_INTEGER is NULL!"); } + + num = ossl_bn_new(BN_value_one()); + bn = GetBNPtr(num); + if (ASN1_STRING_type(ai) == V_ASN1_ENUMERATED) - bn = ASN1_ENUMERATED_to_BN(ai, NULL); + bn = ASN1_ENUMERATED_to_BN(ai, bn); else - bn = ASN1_INTEGER_to_BN(ai, NULL); + bn = ASN1_INTEGER_to_BN(ai, bn); if (!bn) ossl_raise(eOSSLError, NULL); - num = rb_protect(ossl_bn_new_wrap_i, (VALUE)bn, &st); - BN_free(bn); - - if (st) rb_jump_tag(st); return num; }