diff --git a/pkgs/tools/misc/yubikey-manager/default.nix b/pkgs/tools/misc/yubikey-manager/default.nix
index de8c6777ae61..d2c665670128 100644
--- a/pkgs/tools/misc/yubikey-manager/default.nix
+++ b/pkgs/tools/misc/yubikey-manager/default.nix
@@ -1,4 +1,5 @@
-{ python3Packages, fetchFromGitHub, lib, yubikey-personalization, libu2f-host, libusb1, procps }:
+{ python3Packages, fetchFromGitHub, lib, yubikey-personalization, libu2f-host, libusb1, procps
+, stdenv, pyOpenSSLSupport ? !(stdenv.isDarwin && stdenv.isAarch64) }:
 
 python3Packages.buildPythonPackage rec {
   pname = "yubikey-manager";
@@ -12,25 +13,30 @@ python3Packages.buildPythonPackage rec {
     sha256 = "sha256-MwM/b1QP6pkyBjz/r6oC4sW1mKC0CKMay45a0wCktk0=";
   };
 
+  patches = lib.optionals (!pyOpenSSLSupport) [
+    ./remove-pyopenssl-tests.patch
+  ];
+
   postPatch = ''
     substituteInPlace pyproject.toml \
       --replace 'fido2 = ">=0.9, <1.0"' 'fido2 = ">*"'
     substituteInPlace "ykman/pcsc/__init__.py" \
-      --replace 'pkill' '${procps}/bin/pkill'
+      --replace 'pkill' '${if stdenv.isLinux then "${procps}" else "/usr"}/bin/pkill'
   '';
 
   nativeBuildInputs = with python3Packages; [ poetry-core ];
 
   propagatedBuildInputs =
-    with python3Packages; [
+    with python3Packages; ([
       click
       cryptography
       pyscard
       pyusb
-      pyopenssl
       six
       fido2
-    ] ++ [
+    ] ++ lib.optionals pyOpenSSLSupport [
+      pyopenssl
+    ]) ++ [
       libu2f-host
       libusb1
       yubikey-personalization
diff --git a/pkgs/tools/misc/yubikey-manager/remove-pyopenssl-tests.patch b/pkgs/tools/misc/yubikey-manager/remove-pyopenssl-tests.patch
new file mode 100644
index 000000000000..5be08f4ddbb9
--- /dev/null
+++ b/pkgs/tools/misc/yubikey-manager/remove-pyopenssl-tests.patch
@@ -0,0 +1,41 @@
+diff --git a/pyproject.toml b/pyproject.toml
+index 65a5943..e6932e0 100644
+--- a/pyproject.toml
++++ b/pyproject.toml
+@@ -30,7 +30,6 @@ packages = [
+ python = "^3.6"
+ dataclasses = {version = "^0.8", python = "<3.7"}
+ cryptography = ">=2.1, <39"
+-pyOpenSSL = {version = ">=0.15.1", optional = true}
+ pyscard = "^1.9 || ^2.0"
+ fido2 = ">=0.9, <2.0"
+ click = "^7.0 || ^8.0"
+diff --git a/tests/test_util.py b/tests/test_util.py
+index 6ccda6c..b4460e4 100644
+--- a/tests/test_util.py
++++ b/tests/test_util.py
+@@ -8,7 +8,6 @@ from ykman.util import _parse_pkcs12_pyopenssl, _parse_pkcs12_cryptography
+ from ykman.otp import format_oath_code, generate_static_pw, time_challenge
+ from .util import open_file
+ from cryptography.hazmat.primitives.serialization import pkcs12
+-from OpenSSL import crypto
+ 
+ import unittest
+ 
+@@ -114,16 +113,6 @@ class TestUtilityFunctions(unittest.TestCase):
+         ) as rsa_2048_key_cert_encrypted_pfx:
+             self.assertTrue(is_pkcs12(rsa_2048_key_cert_encrypted_pfx.read()))
+ 
+-    def test_parse_pkcs12(self):
+-        with open_file("rsa_2048_key_cert.pfx") as rsa_2048_key_cert_pfx:
+-            data = rsa_2048_key_cert_pfx.read()
+-
+-        key1, certs1 = _parse_pkcs12_cryptography(pkcs12, data, None)
+-        key2, certs2 = _parse_pkcs12_pyopenssl(crypto, data, None)
+-        self.assertEqual(key1.private_numbers(), key2.private_numbers())
+-        self.assertEqual(1, len(certs1))
+-        self.assertEqual(certs1, certs2)
+-
+     def test_is_pem(self):
+         self.assertFalse(is_pem(b"just a byte string"))
+         self.assertFalse(is_pem(None))