diff --git a/src/lib/prov/pkcs11/p11_ecc_key.cpp b/src/lib/prov/pkcs11/p11_ecc_key.cpp
index da8d12a1863..6162e69bf2a 100644
--- a/src/lib/prov/pkcs11/p11_ecc_key.cpp
+++ b/src/lib/prov/pkcs11/p11_ecc_key.cpp
@@ -13,16 +13,21 @@
 #if defined(BOTAN_HAS_ECC_PUBLIC_KEY_CRYPTO)
 
    #include <botan/ber_dec.h>
+   #include <botan/internal/ec_key_data.h>
    #include <botan/internal/workfactor.h>
 
 namespace Botan::PKCS11 {
+
 namespace {
+
 /// Converts a DER-encoded ANSI X9.62 ECPoint to EC_Point
-EC_Point decode_public_point(const secure_vector<uint8_t>& ec_point_data, const EC_Group& group) {
-   secure_vector<uint8_t> ec_point;
+EC_AffinePoint decode_public_point(const EC_Group& group, std::span<const uint8_t> ec_point_data) {
+   std::vector<uint8_t> ec_point;
    BER_Decoder(ec_point_data).decode(ec_point, ASN1_Type::OctetString);
-   return group.OS2ECP(ec_point);
+   // Throws if invalid
+   return EC_AffinePoint(group, ec_point);
 }
+
 }  // namespace
 
 EC_PublicKeyGenerationProperties::EC_PublicKeyGenerationProperties(const std::vector<uint8_t>& ec_params) :
@@ -38,20 +43,19 @@ EC_PublicKeyImportProperties::EC_PublicKeyImportProperties(const std::vector<uin
 }
 
 PKCS11_EC_PublicKey::PKCS11_EC_PublicKey(Session& session, ObjectHandle handle) : Object(session, handle) {
-   secure_vector<uint8_t> ec_parameters = get_attribute_value(AttributeType::EcParams);
-   m_domain_params = EC_Group(unlock(ec_parameters));
-   m_public_key = decode_public_point(get_attribute_value(AttributeType::EcPoint), m_domain_params);
-   m_domain_encoding = EC_Group_Encoding::NamedCurve;
+   auto ec_parameters = get_attribute_value(AttributeType::EcParams);
+   auto pt_bytes = get_attribute_value(AttributeType::EcPoint);
+
+   EC_Group group(ec_parameters);
+   auto pt = decode_public_point(group, pt_bytes);
+   m_public_key = std::make_shared<EC_PublicKey_Data>(std::move(group), std::move(pt));
 }
 
 PKCS11_EC_PublicKey::PKCS11_EC_PublicKey(Session& session, const EC_PublicKeyImportProperties& props) :
       Object(session, props) {
-   m_domain_params = EC_Group(props.ec_params());
-
-   secure_vector<uint8_t> ec_point;
-   BER_Decoder(props.ec_point()).decode(ec_point, ASN1_Type::OctetString);
-   m_public_key = m_domain_params.OS2ECP(ec_point);
-   m_domain_encoding = EC_Group_Encoding::NamedCurve;
+   EC_Group group(props.ec_params());
+   auto pt = decode_public_point(group, props.ec_point());
+   m_public_key = std::make_shared<EC_PublicKey_Data>(std::move(group), std::move(pt));
 }
 
 EC_PrivateKeyImportProperties::EC_PrivateKeyImportProperties(const std::vector<uint8_t>& ec_params,
@@ -61,8 +65,7 @@ EC_PrivateKeyImportProperties::EC_PrivateKeyImportProperties(const std::vector<u
    add_binary(AttributeType::Value, m_value.serialize());
 }
 
-PKCS11_EC_PrivateKey::PKCS11_EC_PrivateKey(Session& session, ObjectHandle handle) :
-      Object(session, handle), m_domain_params(), m_public_key() {
+PKCS11_EC_PrivateKey::PKCS11_EC_PrivateKey(Session& session, ObjectHandle handle) : Object(session, handle) {
    secure_vector<uint8_t> ec_parameters = get_attribute_value(AttributeType::EcParams);
    m_domain_params = EC_Group(unlock(ec_parameters));
 }
@@ -96,9 +99,10 @@ PKCS11_EC_PrivateKey::PKCS11_EC_PrivateKey(Session& session,
                                        &priv_key_handle);
 
    this->reset_handle(priv_key_handle);
-
    Object public_key(session, pub_key_handle);
-   m_public_key = decode_public_point(public_key.get_attribute_value(AttributeType::EcPoint), m_domain_params);
+
+   auto pt_bytes = public_key.get_attribute_value(AttributeType::EcPoint);
+   m_public_key = decode_public_point(m_domain_params, pt_bytes).to_legacy_point();
 }
 
 size_t PKCS11_EC_PrivateKey::key_length() const {
diff --git a/src/lib/pubkey/ec_group/ec_apoint.h b/src/lib/pubkey/ec_group/ec_apoint.h
index 592e06e5d4a..5fce5d902d2 100644
--- a/src/lib/pubkey/ec_group/ec_apoint.h
+++ b/src/lib/pubkey/ec_group/ec_apoint.h
@@ -28,7 +28,14 @@ class EC_AffinePoint_Data;
 
 class BOTAN_UNSTABLE_API EC_AffinePoint final {
    public:
+      /// Point deserialization. Throws if wrong length or not a valid point
+      ///
+      /// This accepts SEC1 compressed or uncompressed formats
+      EC_AffinePoint(const EC_Group& group, std::span<const uint8_t> bytes);
+
       /// Point deserialization. Returns nullopt if wrong length or not a valid point
+      ///
+      /// This accepts SEC1 compressed or uncompressed formats
       static std::optional<EC_AffinePoint> deserialize(const EC_Group& group, std::span<const uint8_t> bytes);
 
       /// Multiply by the group generator returning a complete point
@@ -133,8 +140,6 @@ class BOTAN_UNSTABLE_API EC_AffinePoint final {
       EC_AffinePoint& operator=(const EC_AffinePoint& other);
       EC_AffinePoint& operator=(EC_AffinePoint&& other) noexcept;
 
-      EC_AffinePoint(const EC_Group& group, std::span<const uint8_t> bytes);
-
       /**
       * Deprecated conversion
       */
diff --git a/src/lib/pubkey/ec_group/ec_group.cpp b/src/lib/pubkey/ec_group/ec_group.cpp
index 54ddc9bd4b1..a06239d318e 100644
--- a/src/lib/pubkey/ec_group/ec_group.cpp
+++ b/src/lib/pubkey/ec_group/ec_group.cpp
@@ -745,9 +745,6 @@ bool EC_Group::verify_group(RandomNumberGenerator& rng, bool strong) const {
    return true;
 }
 
-EC_Group::Mul2Table::Mul2Table(const EC_Group& group, const EC_Point& h) :
-      EC_Group::Mul2Table(EC_AffinePoint(group, h)) {}
-
 EC_Group::Mul2Table::Mul2Table(const EC_AffinePoint& h) : m_tbl(h._group()->make_mul2_table(h._inner())) {}
 
 EC_Group::Mul2Table::~Mul2Table() = default;
diff --git a/src/lib/pubkey/ec_group/ec_group.h b/src/lib/pubkey/ec_group/ec_group.h
index 4c4fa9f28c9..82003159958 100644
--- a/src/lib/pubkey/ec_group/ec_group.h
+++ b/src/lib/pubkey/ec_group/ec_group.h
@@ -229,13 +229,6 @@ class BOTAN_PUBLIC_API(2, 0) EC_Group final {
       /// Table for computing g*x + h*y
       class Mul2Table final {
          public:
-            /**
-            * Internal transition function
-            *
-            * @warning this will be removed in 3.6.0, NOT COVERED BY SEMVER
-            */
-            Mul2Table(const EC_Group& group, const EC_Point& h);
-
             /**
             * Create a table for computing g*x + h*y
             */
diff --git a/src/lib/pubkey/ec_group/ec_scalar.cpp b/src/lib/pubkey/ec_group/ec_scalar.cpp
index 7021b76b601..e805d9d61ab 100644
--- a/src/lib/pubkey/ec_group/ec_scalar.cpp
+++ b/src/lib/pubkey/ec_group/ec_scalar.cpp
@@ -63,6 +63,12 @@ EC_Scalar EC_Scalar::from_bigint(const EC_Group& group, const BigInt& bn) {
    return EC_Scalar(group._data()->scalar_from_bigint(bn));
 }
 
+BigInt EC_Scalar::to_bigint() const {
+   secure_vector<uint8_t> bytes(m_scalar->bytes());
+   m_scalar->serialize_to(bytes);
+   return BigInt::from_bytes(bytes);
+}
+
 EC_Scalar EC_Scalar::gk_x_mod_order(const EC_Scalar& scalar, RandomNumberGenerator& rng, std::vector<BigInt>& ws) {
    const auto& group = scalar._inner().group();
    return EC_Scalar(group->gk_x_mod_order(scalar.inner(), rng, ws));
@@ -106,6 +112,13 @@ std::optional<EC_Scalar> EC_Scalar::deserialize(const EC_Group& group, std::span
    }
 }
 
+EC_Scalar::EC_Scalar(const EC_Group& group, std::span<const uint8_t> bytes) {
+   m_scalar = group._data()->scalar_deserialize(bytes);
+   if(!m_scalar) {
+      throw Decoding_Error("EC_Scalar::from_bytes is not a valid scalar value");
+   }
+}
+
 bool EC_Scalar::is_zero() const {
    return inner().is_zero();
 }
diff --git a/src/lib/pubkey/ec_group/ec_scalar.h b/src/lib/pubkey/ec_group/ec_scalar.h
index fc8e21b426d..6ee1e5342db 100644
--- a/src/lib/pubkey/ec_group/ec_scalar.h
+++ b/src/lib/pubkey/ec_group/ec_scalar.h
@@ -54,6 +54,14 @@ class BOTAN_UNSTABLE_API EC_Scalar final {
       */
       static EC_Scalar from_bytes_mod_order(const EC_Group& group, std::span<const uint8_t> bytes);
 
+      /**
+      * Convert a bytestring to an EC_Scalar
+      *
+      * This is similar to deserialize but instead of returning nullopt if the input
+      * is invalid, it will throw an exception.
+      */
+      EC_Scalar(const EC_Group& group, std::span<const uint8_t> bytes);
+
       /**
       * Deserialize a pair of scalars
       *
@@ -180,6 +188,11 @@ class BOTAN_UNSTABLE_API EC_Scalar final {
       */
       bool is_eq(const EC_Scalar& x) const;
 
+      /**
+      * Convert *this to a BigInt
+      */
+      BigInt to_bigint() const;
+
       friend EC_Scalar operator+(const EC_Scalar& x, const EC_Scalar& y) { return x.add(y); }
 
       friend EC_Scalar operator-(const EC_Scalar& x, const EC_Scalar& y) { return x.sub(y); }
diff --git a/src/lib/pubkey/ecc_key/ec_key_data.cpp b/src/lib/pubkey/ecc_key/ec_key_data.cpp
new file mode 100644
index 00000000000..3f68f7d7ba4
--- /dev/null
+++ b/src/lib/pubkey/ecc_key/ec_key_data.cpp
@@ -0,0 +1,51 @@
+/*
+* (C) 2024 Jack Lloyd
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#include <botan/internal/ec_key_data.h>
+
+#include <botan/rng.h>
+
+namespace Botan {
+
+EC_PublicKey_Data::EC_PublicKey_Data(EC_Group group, std::span<const uint8_t> bytes) :
+      m_group(std::move(group)), m_point(m_group, bytes), m_legacy_point(m_point.to_legacy_point()) {}
+
+EC_PrivateKey_Data::EC_PrivateKey_Data(EC_Group group, RandomNumberGenerator& rng) :
+      m_group(std::move(group)), m_scalar(EC_Scalar::random(m_group, rng)), m_legacy_x(m_scalar.to_bigint()) {}
+
+EC_PrivateKey_Data::EC_PrivateKey_Data(EC_Group group, BigInt x) :
+      m_group(std::move(group)), m_scalar(EC_Scalar::from_bigint(m_group, std::move(x))), m_legacy_x(m_scalar.to_bigint()) {}
+
+EC_PrivateKey_Data::EC_PrivateKey_Data(EC_Group group, EC_Scalar x) :
+      m_group(std::move(group)), m_scalar(std::move(x)), m_legacy_x(m_scalar.to_bigint()) {}
+
+EC_PrivateKey_Data::EC_PrivateKey_Data(EC_Group group, std::span<const uint8_t> bytes) :
+      m_group(std::move(group)), m_scalar(m_group, bytes), m_legacy_x(m_scalar.to_bigint()) {}
+
+std::shared_ptr<EC_PublicKey_Data> EC_PrivateKey_Data::public_key(RandomNumberGenerator& rng,
+                                                                  bool with_modular_inverse) const {
+   auto public_point = [&] {
+      std::vector<BigInt> ws;
+      if(with_modular_inverse) {
+         return EC_AffinePoint::g_mul(m_scalar.invert(), rng, ws);
+      } else {
+         return EC_AffinePoint::g_mul(m_scalar, rng, ws);
+      }
+   };
+
+   return std::make_shared<EC_PublicKey_Data>(m_group, public_point());
+}
+
+std::shared_ptr<EC_PublicKey_Data> EC_PrivateKey_Data::public_key(bool with_modular_inverse) const {
+   Null_RNG null_rng;
+   return this->public_key(null_rng, with_modular_inverse);
+}
+
+void EC_PrivateKey_Data::serialize_to(std::span<uint8_t> output) const {
+   m_scalar.serialize_to(output);
+}
+
+}  // namespace Botan
diff --git a/src/lib/pubkey/ecc_key/ec_key_data.h b/src/lib/pubkey/ecc_key/ec_key_data.h
new file mode 100644
index 00000000000..9c4382562ec
--- /dev/null
+++ b/src/lib/pubkey/ecc_key/ec_key_data.h
@@ -0,0 +1,78 @@
+/*
+* (C) 2024 Jack Lloyd
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#ifndef BOTAN_EC_KEY_DATA_H_
+#define BOTAN_EC_KEY_DATA_H_
+
+#include <botan/ec_apoint.h>
+#include <botan/ec_group.h>
+#include <botan/ec_scalar.h>
+
+#include <botan/bigint.h>
+#include <botan/ec_point.h>
+
+namespace Botan {
+
+class RandomNumberGenerator;
+
+class EC_PublicKey_Data final {
+   public:
+      EC_PublicKey_Data(EC_Group group, EC_AffinePoint pt) :
+            m_group(std::move(group)), m_point(std::move(pt)), m_legacy_point(m_point.to_legacy_point()) {}
+
+      EC_PublicKey_Data(EC_Group group, std::span<const uint8_t> bytes);
+
+      const EC_Group& group() const { return m_group; }
+
+      const EC_AffinePoint& public_key() const { return m_point; }
+
+      const EC_Point& legacy_point() const { return m_legacy_point; }
+
+   private:
+      EC_Group m_group;
+      EC_AffinePoint m_point;
+      EC_Point m_legacy_point;
+};
+
+class EC_PrivateKey_Data final {
+   public:
+      EC_PrivateKey_Data(EC_Group group, RandomNumberGenerator& rng);
+
+      EC_PrivateKey_Data(EC_Group group, BigInt x);
+
+      EC_PrivateKey_Data(EC_Group group, EC_Scalar x);
+
+      EC_PrivateKey_Data(EC_Group group, std::span<const uint8_t> bytes);
+
+      std::shared_ptr<EC_PublicKey_Data> public_key(RandomNumberGenerator& rng, bool with_modular_inverse) const;
+
+      std::shared_ptr<EC_PublicKey_Data> public_key(bool with_modular_inverse) const;
+
+      void serialize_to(std::span<uint8_t> output) const;
+
+      template <typename T>
+      T serialize() const {
+         T bytes(this->group().get_order_bytes());
+         this->serialize_to(bytes);
+         return bytes;
+      }
+
+      const EC_Group& group() const { return m_group; }
+
+      const EC_Scalar& private_key() const { return m_scalar; }
+
+      const BigInt& legacy_bigint() const { return m_legacy_x; }
+
+   private:
+      EC_Group m_group;
+
+      EC_Scalar m_scalar;
+      BigInt m_legacy_x;
+};
+
+}  // namespace Botan
+
+#endif
diff --git a/src/lib/pubkey/ecc_key/ecc_key.cpp b/src/lib/pubkey/ecc_key/ecc_key.cpp
index 0a11728520b..dd952b6e6cc 100644
--- a/src/lib/pubkey/ecc_key/ecc_key.cpp
+++ b/src/lib/pubkey/ecc_key/ecc_key.cpp
@@ -14,6 +14,7 @@
 #include <botan/ec_point.h>
 #include <botan/numthry.h>
 #include <botan/secmem.h>
+#include <botan/internal/ec_key_data.h>
 #include <botan/internal/fmt.h>
 #include <botan/internal/workfactor.h>
 
@@ -29,7 +30,7 @@ size_t EC_PublicKey::estimated_strength() const {
 
 namespace {
 
-EC_Group_Encoding default_encoding_for(EC_Group& group) {
+EC_Group_Encoding default_encoding_for(const EC_Group& group) {
    if(group.get_curve_oid().empty()) {
       return EC_Group_Encoding::Explicit;
    } else {
@@ -39,16 +40,39 @@ EC_Group_Encoding default_encoding_for(EC_Group& group) {
 
 }  // namespace
 
-EC_PublicKey::EC_PublicKey(const EC_Group& dom_par, const EC_Point& pub_point) :
-      m_domain_params(dom_par), m_public_key(pub_point), m_domain_encoding(default_encoding_for(m_domain_params)) {}
+EC_PublicKey::EC_PublicKey(EC_Group group, const EC_Point& pub_point) {
+   auto pt = EC_AffinePoint(group, pub_point);
+   m_public_key = std::make_shared<const EC_PublicKey_Data>(std::move(group), std::move(pt));
+   m_domain_encoding = default_encoding_for(domain());
+}
+
+EC_PublicKey::EC_PublicKey(EC_Group group, EC_AffinePoint pub_point) {
+   m_public_key = std::make_shared<const EC_PublicKey_Data>(std::move(group), std::move(pub_point));
+   m_domain_encoding = default_encoding_for(domain());
+}
+
+EC_PublicKey::EC_PublicKey(const AlgorithmIdentifier& alg_id, std::span<const uint8_t> key_bits) {
+   m_public_key = std::make_shared<const EC_PublicKey_Data>(EC_Group(alg_id.parameters()), key_bits);
+   m_domain_encoding = default_encoding_for(domain());
+}
 
-EC_PublicKey::EC_PublicKey(const AlgorithmIdentifier& alg_id, std::span<const uint8_t> key_bits) :
-      m_domain_params{EC_Group(alg_id.parameters())},
-      m_public_key{domain().OS2ECP(key_bits)},
-      m_domain_encoding(default_encoding_for(m_domain_params)) {}
+const EC_Group& EC_PublicKey::domain() const {
+   BOTAN_STATE_CHECK(m_public_key != nullptr);
+   return m_public_key->group();
+}
+
+const EC_Point& EC_PublicKey::public_point() const {
+   BOTAN_STATE_CHECK(m_public_key != nullptr);
+   return m_public_key->legacy_point();
+}
+
+const EC_AffinePoint& EC_PublicKey::_public_key() const {
+   BOTAN_STATE_CHECK(m_public_key != nullptr);
+   return m_public_key->public_key();
+}
 
 bool EC_PublicKey::check_key(RandomNumberGenerator& rng, bool /*strong*/) const {
-   return m_domain_params.verify_group(rng) && m_domain_params.verify_public_element(public_point());
+   return domain().verify_group(rng) && domain().verify_public_element(public_point());
 }
 
 AlgorithmIdentifier EC_PublicKey::algorithm_identifier() const {
@@ -76,7 +100,7 @@ void EC_PublicKey::set_point_encoding(EC_Point_Format enc) {
 }
 
 void EC_PublicKey::set_parameter_encoding(EC_Group_Encoding form) {
-   if(form == EC_Group_Encoding::NamedCurve && m_domain_params.get_curve_oid().empty()) {
+   if(form == EC_Group_Encoding::NamedCurve && domain().get_curve_oid().empty()) {
       throw Invalid_Argument("Cannot used NamedCurve encoding for a curve without an OID");
    }
 
@@ -84,53 +108,58 @@ void EC_PublicKey::set_parameter_encoding(EC_Group_Encoding form) {
 }
 
 const BigInt& EC_PrivateKey::private_value() const {
-   if(m_private_key == 0) {
-      throw Invalid_State("EC_PrivateKey::private_value - uninitialized");
-   }
+   BOTAN_STATE_CHECK(m_private_key != nullptr);
+   return m_private_key->legacy_bigint();
+}
 
-   return m_private_key;
+const EC_Scalar& EC_PrivateKey::_private_key() const {
+   BOTAN_STATE_CHECK(m_private_key != nullptr);
+   return m_private_key->private_key();
 }
 
 /**
 * EC_PrivateKey constructor
 */
 EC_PrivateKey::EC_PrivateKey(RandomNumberGenerator& rng,
-                             const EC_Group& ec_group,
-                             const BigInt& x,
+                             EC_Group ec_group,
+                             BigInt x,
                              bool with_modular_inverse) {
-   m_domain_params = ec_group;
-   m_domain_encoding = default_encoding_for(m_domain_params);
-
    if(x == 0) {
-      m_private_key = ec_group.random_scalar(rng);
+      m_private_key = std::make_shared<EC_PrivateKey_Data>(std::move(ec_group), rng);
    } else {
-      BOTAN_ARG_CHECK(x > 0 && x < ec_group.get_order(), "ECC private key out of range");
-      m_private_key = x;
+      m_private_key = std::make_shared<EC_PrivateKey_Data>(std::move(ec_group), std::move(x));
    }
 
-   std::vector<BigInt> ws;
+   m_public_key = m_private_key->public_key(rng, with_modular_inverse);
+   m_domain_encoding = default_encoding_for(domain());
+}
 
-   if(with_modular_inverse) {
-      // ECKCDSA
-      m_public_key = domain().blinded_base_point_multiply(m_domain_params.inverse_mod_order(m_private_key), rng, ws);
-   } else {
-      m_public_key = domain().blinded_base_point_multiply(m_private_key, rng, ws);
-   }
+EC_PrivateKey::EC_PrivateKey(RandomNumberGenerator& rng, EC_Group ec_group, bool with_modular_inverse) {
+   m_private_key = std::make_shared<EC_PrivateKey_Data>(std::move(ec_group), rng);
+   m_public_key = m_private_key->public_key(rng, with_modular_inverse);
+   m_domain_encoding = default_encoding_for(domain());
+}
 
-   BOTAN_ASSERT(m_public_key.on_the_curve(), "Generated public key point was on the curve");
+EC_PrivateKey::EC_PrivateKey(EC_Group ec_group, EC_Scalar x, bool with_modular_inverse) {
+   m_private_key = std::make_shared<EC_PrivateKey_Data>(std::move(ec_group), std::move(x));
+   m_public_key = m_private_key->public_key(with_modular_inverse);
+   m_domain_encoding = default_encoding_for(domain());
 }
 
 secure_vector<uint8_t> EC_PrivateKey::raw_private_key_bits() const {
-   return m_private_key.serialize<secure_vector<uint8_t>>(domain().get_order_bytes());
+   BOTAN_STATE_CHECK(m_private_key != nullptr);
+   return m_private_key->serialize<secure_vector<uint8_t>>();
 }
 
 secure_vector<uint8_t> EC_PrivateKey::private_key_bits() const {
+   BOTAN_STATE_CHECK(m_private_key != nullptr && m_public_key != nullptr);
+
    return DER_Encoder()
       .start_sequence()
       .encode(static_cast<size_t>(1))
       .encode(raw_private_key_bits(), ASN1_Type::OctetString)
       .start_explicit_context_specific(1)
-      .encode(m_public_key.encode(EC_Point_Format::Uncompressed), ASN1_Type::BitString)
+      .encode(m_public_key->public_key().serialize_uncompressed(), ASN1_Type::BitString)
       .end_cons()
       .end_cons()
       .get_contents();
@@ -139,41 +168,33 @@ secure_vector<uint8_t> EC_PrivateKey::private_key_bits() const {
 EC_PrivateKey::EC_PrivateKey(const AlgorithmIdentifier& alg_id,
                              std::span<const uint8_t> key_bits,
                              bool with_modular_inverse) {
-   m_domain_params = EC_Group(alg_id.parameters());
-   m_domain_encoding = default_encoding_for(m_domain_params);
+   EC_Group group(alg_id.parameters());
 
    OID key_parameters;
+   secure_vector<uint8_t> private_key_bits;
    secure_vector<uint8_t> public_key_bits;
 
    BER_Decoder(key_bits)
       .start_sequence()
       .decode_and_check<size_t>(1, "Unknown version code for ECC key")
-      .decode_octet_string_bigint(m_private_key)
+      .decode(private_key_bits, ASN1_Type::OctetString)
       .decode_optional(key_parameters, ASN1_Type(0), ASN1_Class::ExplicitContextSpecific)
       .decode_optional_string(public_key_bits, ASN1_Type::BitString, 1, ASN1_Class::ExplicitContextSpecific)
       .end_cons();
 
-   if(m_private_key < 1 || m_private_key >= m_domain_params.get_order()) {
-      throw Decoding_Error("Invalid EC private key");
-   }
+   m_private_key = std::make_shared<EC_PrivateKey_Data>(group, private_key_bits);
 
    if(public_key_bits.empty()) {
-      if(with_modular_inverse) {
-         // ECKCDSA
-         m_public_key = domain().get_base_point() * m_domain_params.inverse_mod_order(m_private_key);
-      } else {
-         m_public_key = domain().get_base_point() * m_private_key;
-      }
-
-      BOTAN_ASSERT(m_public_key.on_the_curve(), "Public point derived from loaded key was on the curve");
+      m_public_key = m_private_key->public_key(with_modular_inverse);
    } else {
-      m_public_key = domain().OS2ECP(public_key_bits);
-      // OS2ECP verifies that the point is on the curve
+      m_public_key = std::make_shared<EC_PublicKey_Data>(group, public_key_bits);
    }
+
+   m_domain_encoding = default_encoding_for(domain());
 }
 
 bool EC_PrivateKey::check_key(RandomNumberGenerator& rng, bool strong) const {
-   if(m_private_key < 1 || m_private_key >= m_domain_params.get_order()) {
+   if(!m_private_key) {
       return false;
    }
 
diff --git a/src/lib/pubkey/ecc_key/ecc_key.h b/src/lib/pubkey/ecc_key/ecc_key.h
index d2eb4da54ae..6717664155b 100644
--- a/src/lib/pubkey/ecc_key/ecc_key.h
+++ b/src/lib/pubkey/ecc_key/ecc_key.h
@@ -12,9 +12,13 @@
 
 #include <botan/ec_group.h>
 #include <botan/pk_keys.h>
+#include <memory>
 
 namespace Botan {
 
+class EC_PublicKey_Data;
+class EC_PrivateKey_Data;
+
 /**
 * This class represents abstract ECC public keys. When encoding a key
 * via an encoder that can be accessed via the corresponding member
@@ -29,6 +33,8 @@ class BOTAN_PUBLIC_API(2, 0) EC_PublicKey : public virtual Public_Key {
    public:
       EC_PublicKey(const EC_PublicKey& other) = default;
       EC_PublicKey& operator=(const EC_PublicKey& other) = default;
+      EC_PublicKey(EC_PublicKey&& other) = default;
+      EC_PublicKey& operator=(EC_PublicKey&& other) = default;
       ~EC_PublicKey() override = default;
 
       /**
@@ -37,7 +43,7 @@ class BOTAN_PUBLIC_API(2, 0) EC_PublicKey : public virtual Public_Key {
       * domain parameters of this point are not set
       * @result the public point of this key
       */
-      const EC_Point& public_point() const { return m_public_key; }
+      const EC_Point& public_point() const;
 
       AlgorithmIdentifier algorithm_identifier() const override;
 
@@ -53,7 +59,7 @@ class BOTAN_PUBLIC_API(2, 0) EC_PublicKey : public virtual Public_Key {
       * domain parameters of this point are not set
       * @result the domain parameters of this key
       */
-      const EC_Group& domain() const { return m_domain_params; }
+      const EC_Group& domain() const;
 
       /**
       * Set the domain parameter encoding to be used when encoding this key.
@@ -94,13 +100,24 @@ class BOTAN_PUBLIC_API(2, 0) EC_PublicKey : public virtual Public_Key {
 
       const BigInt& get_int_field(std::string_view field) const override;
 
+      const EC_AffinePoint& _public_key() const;
+
    protected:
       /**
-      * Create a public key.
-      * @param dom_par EC domain parameters
+      * Load a public key from the point.
+      *
+      * @param group EC domain parameters
+      * @param pub_point public point on the curve
+      */
+      EC_PublicKey(EC_Group group, const EC_Point& pub_point);
+
+      /**
+      * Load a public key from the point.
+      *
+      * @param group EC domain parameters
       * @param pub_point public point on the curve
       */
-      EC_PublicKey(const EC_Group& dom_par, const EC_Point& pub_point);
+     EC_PublicKey(EC_Group group, EC_AffinePoint pub_point);
 
       /**
       * Load a public key.
@@ -109,11 +126,10 @@ class BOTAN_PUBLIC_API(2, 0) EC_PublicKey : public virtual Public_Key {
       */
       EC_PublicKey(const AlgorithmIdentifier& alg_id, std::span<const uint8_t> key_bits);
 
-      EC_PublicKey() : m_domain_params{}, m_public_key{}, m_domain_encoding(EC_Group_Encoding::Explicit) {}
+      EC_PublicKey() = default;
 
-      EC_Group m_domain_params;
-      EC_Point m_public_key;
-      EC_Group_Encoding m_domain_encoding;
+      std::shared_ptr<const EC_PublicKey_Data> m_public_key;
+      EC_Group_Encoding m_domain_encoding = EC_Group_Encoding::NamedCurve;
       EC_Point_Format m_point_encoding = EC_Point_Format::Uncompressed;
 };
 
@@ -141,12 +157,16 @@ class BOTAN_PUBLIC_API(2, 0) EC_PrivateKey : public virtual EC_PublicKey,
 
       EC_PrivateKey(const EC_PrivateKey& other) = default;
       EC_PrivateKey& operator=(const EC_PrivateKey& other) = default;
+      EC_PrivateKey(EC_PrivateKey&& other) = default;
+      EC_PrivateKey& operator=(EC_PrivateKey&& other) = default;
       ~EC_PrivateKey() override = default;
 
       const BigInt& get_int_field(std::string_view field) const final;
 
+      const EC_Scalar& _private_key() const;
+
    protected:
-      /*
+      /**
       * If x=0, creates a new private key in the domain
       * using the given rng. If with_modular_inverse is set,
       * the public key will be calculated by multiplying
@@ -155,10 +175,28 @@ class BOTAN_PUBLIC_API(2, 0) EC_PrivateKey : public virtual EC_PublicKey,
       * multiplying directly with x (as in ECDSA).
       */
       EC_PrivateKey(RandomNumberGenerator& rng,
-                    const EC_Group& domain,
-                    const BigInt& x,
+                    EC_Group domain,
+                    BigInt x,
                     bool with_modular_inverse = false);
 
+      /**
+      * Creates a new private key
+      *
+      * If @p with_modular_inverse is set, the public key will be calculated by
+      * multiplying the base point with the modular inverse of x (as in ECGDSA
+      * and ECKCDSA), otherwise by multiplying directly with x (as in ECDSA).
+      */
+      EC_PrivateKey(RandomNumberGenerator& rng, EC_Group group, bool with_modular_inverse = false);
+
+      /**
+      * Load a EC private key from the secret scalar
+      *
+      * If @p with_modular_inverse is set, the public key will be calculated by
+      * multiplying the base point with the modular inverse of x (as in ECGDSA
+      * and ECKCDSA), otherwise by multiplying directly with x (as in ECDSA).
+      */
+      EC_PrivateKey(EC_Group group, EC_Scalar scalar, bool with_modular_inverse = false);
+
       /*
       * Creates a new private key object from the
       * ECPrivateKey structure given in key_bits.
@@ -174,7 +212,7 @@ class BOTAN_PUBLIC_API(2, 0) EC_PrivateKey : public virtual EC_PublicKey,
 
       EC_PrivateKey() = default;
 
-      BigInt m_private_key;
+      std::shared_ptr<const EC_PrivateKey_Data> m_private_key;
 };
 
 BOTAN_DIAGNOSTIC_POP
diff --git a/src/lib/pubkey/ecc_key/info.txt b/src/lib/pubkey/ecc_key/info.txt
index 329bb06c3ff..982413ea900 100644
--- a/src/lib/pubkey/ecc_key/info.txt
+++ b/src/lib/pubkey/ecc_key/info.txt
@@ -18,3 +18,7 @@ numbertheory
 <header:public>
 ecc_key.h
 </header:public>
+
+<header:internal>
+ec_key_data.h
+</header:internal>
diff --git a/src/lib/pubkey/ecdh/ecdh.cpp b/src/lib/pubkey/ecdh/ecdh.cpp
index dcdada0bf06..b1c50fdb36c 100644
--- a/src/lib/pubkey/ecdh/ecdh.cpp
+++ b/src/lib/pubkey/ecdh/ecdh.cpp
@@ -28,7 +28,7 @@ class ECDH_KA_Operation final : public PK_Ops::Key_Agreement_with_KDF {
       ECDH_KA_Operation(const ECDH_PrivateKey& key, std::string_view kdf, RandomNumberGenerator& rng) :
             PK_Ops::Key_Agreement_with_KDF(kdf),
             m_group(key.domain()),
-            m_l_times_priv(mul_cofactor_inv(m_group, key.private_value())),
+            m_l_times_priv(mul_cofactor_inv(m_group, key._private_key())),
             m_rng(rng) {}
 
       size_t agreed_value_size() const override { return m_group.get_p_bytes(); }
@@ -47,18 +47,16 @@ class ECDH_KA_Operation final : public PK_Ops::Key_Agreement_with_KDF {
       }
 
    private:
-      static EC_Scalar mul_cofactor_inv(const EC_Group& group, const BigInt& bn) {
+      static EC_Scalar mul_cofactor_inv(const EC_Group& group, const EC_Scalar& x) {
          // We implement BSI TR-03111 ECKAEG which only matters in the (rare/deprecated)
          // case of a curve with cofactor.
 
-         auto private_key = EC_Scalar::from_bigint(group, bn);
-
          if(group.has_cofactor()) {
             // We could precompute this but cofactors are rare
             auto l = group.inverse_mod_order(group.get_cofactor());
-            return private_key * EC_Scalar::from_bigint(group, l);
+            return x * EC_Scalar::from_bigint(group, l);
          } else {
-            return private_key;
+            return x;
          }
       }
 
diff --git a/src/lib/pubkey/ecdsa/ecdsa.cpp b/src/lib/pubkey/ecdsa/ecdsa.cpp
index cc43078a9be..896eafa32c4 100644
--- a/src/lib/pubkey/ecdsa/ecdsa.cpp
+++ b/src/lib/pubkey/ecdsa/ecdsa.cpp
@@ -122,12 +122,12 @@ class ECDSA_Signature_Operation final : public PK_Ops::Signature_with_Hash {
       ECDSA_Signature_Operation(const ECDSA_PrivateKey& ecdsa, std::string_view padding, RandomNumberGenerator& rng) :
             PK_Ops::Signature_with_Hash(padding),
             m_group(ecdsa.domain()),
-            m_x(EC_Scalar::from_bigint(m_group, ecdsa.private_value())),
+            m_x(ecdsa._private_key()),
             m_b(EC_Scalar::random(m_group, rng)),
             m_b_inv(m_b.invert()) {
 #if defined(BOTAN_HAS_RFC6979_GENERATOR)
          m_rfc6979 = std::make_unique<RFC6979_Nonce_Generator>(
-            this->rfc6979_hash_function(), m_group.get_order(), ecdsa.private_value());
+            this->rfc6979_hash_function(), m_group.get_order_bits(), ecdsa._private_key());
 #endif
       }
 
@@ -194,12 +194,12 @@ std::vector<uint8_t> ECDSA_Signature_Operation::raw_sign(std::span<const uint8_t
 class ECDSA_Verification_Operation final : public PK_Ops::Verification_with_Hash {
    public:
       ECDSA_Verification_Operation(const ECDSA_PublicKey& ecdsa, std::string_view padding) :
-            PK_Ops::Verification_with_Hash(padding), m_group(ecdsa.domain()), m_gy_mul(m_group, ecdsa.public_point()) {}
+            PK_Ops::Verification_with_Hash(padding), m_group(ecdsa.domain()), m_gy_mul(ecdsa._public_key()) {}
 
       ECDSA_Verification_Operation(const ECDSA_PublicKey& ecdsa, const AlgorithmIdentifier& alg_id) :
             PK_Ops::Verification_with_Hash(alg_id, "ECDSA", true),
             m_group(ecdsa.domain()),
-            m_gy_mul(m_group, ecdsa.public_point()) {}
+            m_gy_mul(ecdsa._public_key()) {}
 
       bool verify(std::span<const uint8_t> msg, std::span<const uint8_t> sig) override;
 
diff --git a/src/lib/pubkey/ecgdsa/ecgdsa.cpp b/src/lib/pubkey/ecgdsa/ecgdsa.cpp
index 812b2315551..6ef01ac3eee 100644
--- a/src/lib/pubkey/ecgdsa/ecgdsa.cpp
+++ b/src/lib/pubkey/ecgdsa/ecgdsa.cpp
@@ -37,9 +37,7 @@ namespace {
 class ECGDSA_Signature_Operation final : public PK_Ops::Signature_with_Hash {
    public:
       ECGDSA_Signature_Operation(const ECGDSA_PrivateKey& ecgdsa, std::string_view emsa) :
-            PK_Ops::Signature_with_Hash(emsa),
-            m_group(ecgdsa.domain()),
-            m_x(EC_Scalar::from_bigint(m_group, ecgdsa.private_value())) {}
+            PK_Ops::Signature_with_Hash(emsa), m_group(ecgdsa.domain()), m_x(ecgdsa._private_key()) {}
 
       std::vector<uint8_t> raw_sign(std::span<const uint8_t> msg, RandomNumberGenerator& rng) override;
 
@@ -82,14 +80,12 @@ std::vector<uint8_t> ECGDSA_Signature_Operation::raw_sign(std::span<const uint8_
 class ECGDSA_Verification_Operation final : public PK_Ops::Verification_with_Hash {
    public:
       ECGDSA_Verification_Operation(const ECGDSA_PublicKey& ecgdsa, std::string_view padding) :
-            PK_Ops::Verification_with_Hash(padding),
-            m_group(ecgdsa.domain()),
-            m_gy_mul(m_group, ecgdsa.public_point()) {}
+            PK_Ops::Verification_with_Hash(padding), m_group(ecgdsa.domain()), m_gy_mul(ecgdsa._public_key()) {}
 
       ECGDSA_Verification_Operation(const ECGDSA_PublicKey& ecgdsa, const AlgorithmIdentifier& alg_id) :
             PK_Ops::Verification_with_Hash(alg_id, "ECGDSA"),
             m_group(ecgdsa.domain()),
-            m_gy_mul(m_group, ecgdsa.public_point()) {}
+            m_gy_mul(ecgdsa._public_key()) {}
 
       bool verify(std::span<const uint8_t> msg, std::span<const uint8_t> sig) override;
 
diff --git a/src/lib/pubkey/ecies/ecies.cpp b/src/lib/pubkey/ecies/ecies.cpp
index 681060ed296..877b9440487 100644
--- a/src/lib/pubkey/ecies/ecies.cpp
+++ b/src/lib/pubkey/ecies/ecies.cpp
@@ -66,9 +66,8 @@ class ECIES_ECDH_KA_Operation final : public PK_Ops::Key_Agreement_with_KDF {
 
       secure_vector<uint8_t> raw_agree(const uint8_t w[], size_t w_len) override {
          const EC_Group& group = m_key.domain();
-         const auto x = EC_Scalar::from_bigint(group, m_key.private_value());
          if(auto input_point = EC_AffinePoint::deserialize(group, {w, w_len})) {
-            return input_point->mul(x, m_rng, m_ws).x_bytes();
+            return input_point->mul(m_key._private_key(), m_rng, m_ws).x_bytes();
          } else {
             throw Decoding_Error("ECIES - Invalid elliptic curve point");
          }
diff --git a/src/lib/pubkey/eckcdsa/eckcdsa.cpp b/src/lib/pubkey/eckcdsa/eckcdsa.cpp
index 0bad8824a33..51e30153c75 100644
--- a/src/lib/pubkey/eckcdsa/eckcdsa.cpp
+++ b/src/lib/pubkey/eckcdsa/eckcdsa.cpp
@@ -71,8 +71,8 @@ std::unique_ptr<HashFunction> eckcdsa_signature_hash(const AlgorithmIdentifier&
    return HashFunction::create_or_throw(oid_info[1]);
 }
 
-std::vector<uint8_t> eckcdsa_prefix(const EC_Point& point, size_t hash_block_size) {
-   auto prefix = concat<std::vector<uint8_t>>(point.x_bytes(), point.y_bytes());
+std::vector<uint8_t> eckcdsa_prefix(const EC_AffinePoint& point, size_t hash_block_size) {
+   auto prefix = point.xy_bytes<std::vector<uint8_t>>();
 
    // Either truncate or zero-extend to match the hash block size
    prefix.resize(hash_block_size);
@@ -117,10 +117,10 @@ class ECKCDSA_Signature_Operation final : public PK_Ops::Signature {
    public:
       ECKCDSA_Signature_Operation(const ECKCDSA_PrivateKey& eckcdsa, std::string_view padding) :
             m_group(eckcdsa.domain()),
-            m_x(EC_Scalar::from_bigint(m_group, eckcdsa.private_value())),
+            m_x(eckcdsa._private_key()),
             m_hash(eckcdsa_signature_hash(padding)),
             m_prefix_used(false) {
-         m_prefix = eckcdsa_prefix(eckcdsa.public_point(), m_hash->hash_block_size());
+         m_prefix = eckcdsa_prefix(eckcdsa._public_key(), m_hash->hash_block_size());
       }
 
       void update(std::span<const uint8_t> input) override {
@@ -188,18 +188,18 @@ class ECKCDSA_Verification_Operation final : public PK_Ops::Verification {
    public:
       ECKCDSA_Verification_Operation(const ECKCDSA_PublicKey& eckcdsa, std::string_view padding) :
             m_group(eckcdsa.domain()),
-            m_gy_mul(m_group, eckcdsa.public_point()),
+            m_gy_mul(eckcdsa._public_key()),
             m_hash(eckcdsa_signature_hash(padding)),
             m_prefix_used(false) {
-         m_prefix = eckcdsa_prefix(eckcdsa.public_point(), m_hash->hash_block_size());
+         m_prefix = eckcdsa_prefix(eckcdsa._public_key(), m_hash->hash_block_size());
       }
 
       ECKCDSA_Verification_Operation(const ECKCDSA_PublicKey& eckcdsa, const AlgorithmIdentifier& alg_id) :
             m_group(eckcdsa.domain()),
-            m_gy_mul(m_group, eckcdsa.public_point()),
+            m_gy_mul(eckcdsa._public_key()),
             m_hash(eckcdsa_signature_hash(alg_id)),
             m_prefix_used(false) {
-         m_prefix = eckcdsa_prefix(eckcdsa.public_point(), m_hash->hash_block_size());
+         m_prefix = eckcdsa_prefix(eckcdsa._public_key(), m_hash->hash_block_size());
       }
 
       void update(std::span<const uint8_t> msg) override;
diff --git a/src/lib/pubkey/gost_3410/gost_3410.cpp b/src/lib/pubkey/gost_3410/gost_3410.cpp
index cce48ce5cbf..1747b321fd3 100644
--- a/src/lib/pubkey/gost_3410/gost_3410.cpp
+++ b/src/lib/pubkey/gost_3410/gost_3410.cpp
@@ -11,6 +11,7 @@
 
 #include <botan/ber_dec.h>
 #include <botan/der_enc.h>
+#include <botan/internal/ec_key_data.h>
 #include <botan/internal/fmt.h>
 #include <botan/internal/pk_ops_impl.h>
 
@@ -59,14 +60,14 @@ GOST_3410_PublicKey::GOST_3410_PublicKey(const AlgorithmIdentifier& alg_id, std:
    // The parameters also includes hash and cipher OIDs
    BER_Decoder(alg_id.parameters()).start_sequence().decode(ecc_param_id);
 
-   m_domain_params = EC_Group::from_OID(ecc_param_id);
+   auto group = EC_Group::from_OID(ecc_param_id);
 
-   const size_t p_bits = m_domain_params.get_p_bits();
+   const size_t p_bits = group.get_p_bits();
    if(p_bits != 256 && p_bits != 512) {
       throw Decoding_Error(fmt("GOST-34.10-2012 is not defined for parameters of size {}", p_bits));
    }
 
-   secure_vector<uint8_t> bits;
+   std::vector<uint8_t> bits;
    BER_Decoder(key_bits).decode(bits, ASN1_Type::OctetString);
 
    if(bits.size() != 2 * (p_bits / 8)) {
@@ -76,22 +77,18 @@ GOST_3410_PublicKey::GOST_3410_PublicKey(const AlgorithmIdentifier& alg_id, std:
    const size_t part_size = bits.size() / 2;
 
    // Keys are stored in little endian format (WTF)
-   for(size_t i = 0; i != part_size / 2; ++i) {
-      std::swap(bits[i], bits[part_size - 1 - i]);
-      std::swap(bits[part_size + i], bits[2 * part_size - 1 - i]);
-   }
-
-   BigInt x(bits.data(), part_size);
-   BigInt y(&bits[part_size], part_size);
-
-   m_public_key = domain().point(x, y);
+   std::vector<uint8_t> encoding;
+   encoding.reserve(bits.size() + 1);
+   encoding.push_back(0x04);
+   encoding.insert(encoding.end(), bits.rbegin() + part_size, bits.rend());
+   encoding.insert(encoding.end(), bits.rbegin(), bits.rend() - part_size);
 
-   BOTAN_ASSERT(m_public_key.on_the_curve(), "Loaded GOST 34.10 public key is on the curve");
+   m_public_key = std::make_shared<EC_PublicKey_Data>(std::move(group), encoding);
 }
 
 GOST_3410_PrivateKey::GOST_3410_PrivateKey(RandomNumberGenerator& rng, const EC_Group& domain, const BigInt& x) :
       EC_PrivateKey(rng, domain, x) {
-   const size_t p_bits = m_domain_params.get_p_bits();
+   const size_t p_bits = domain.get_p_bits();
    if(p_bits != 256 && p_bits != 512) {
       throw Decoding_Error(fmt("GOST-34.10-2012 is not defined for parameters of size {}", p_bits));
    }
@@ -120,9 +117,7 @@ EC_Scalar gost_msg_to_scalar(const EC_Group& group, std::span<const uint8_t> msg
 class GOST_3410_Signature_Operation final : public PK_Ops::Signature_with_Hash {
    public:
       GOST_3410_Signature_Operation(const GOST_3410_PrivateKey& gost_3410, std::string_view emsa) :
-            PK_Ops::Signature_with_Hash(emsa),
-            m_group(gost_3410.domain()),
-            m_x(EC_Scalar::from_bigint(m_group, gost_3410.private_value())) {}
+            PK_Ops::Signature_with_Hash(emsa), m_group(gost_3410.domain()), m_x(gost_3410._private_key()) {}
 
       size_t signature_length() const override { return 2 * m_group.get_order_bytes(); }
 
@@ -201,12 +196,12 @@ std::string gost_hash_from_algid(const AlgorithmIdentifier& alg_id) {
 class GOST_3410_Verification_Operation final : public PK_Ops::Verification_with_Hash {
    public:
       GOST_3410_Verification_Operation(const GOST_3410_PublicKey& gost, std::string_view padding) :
-            PK_Ops::Verification_with_Hash(padding), m_group(gost.domain()), m_gy_mul(m_group, gost.public_point()) {}
+            PK_Ops::Verification_with_Hash(padding), m_group(gost.domain()), m_gy_mul(gost._public_key()) {}
 
       GOST_3410_Verification_Operation(const GOST_3410_PublicKey& gost, const AlgorithmIdentifier& alg_id) :
             PK_Ops::Verification_with_Hash(gost_hash_from_algid(alg_id)),
             m_group(gost.domain()),
-            m_gy_mul(m_group, gost.public_point()) {}
+            m_gy_mul(gost._public_key()) {}
 
       bool verify(std::span<const uint8_t> msg, std::span<const uint8_t> sig) override;
 
diff --git a/src/lib/pubkey/rfc6979/rfc6979.cpp b/src/lib/pubkey/rfc6979/rfc6979.cpp
index cccf3448750..84a39c24172 100644
--- a/src/lib/pubkey/rfc6979/rfc6979.cpp
+++ b/src/lib/pubkey/rfc6979/rfc6979.cpp
@@ -1,6 +1,6 @@
 /*
 * RFC 6979 Deterministic Nonce Generator
-* (C) 2014,2015 Jack Lloyd
+* (C) 2014,2015,2024 Jack Lloyd
 *
 * Botan is released under the Simplified BSD License (see license.txt)
 */
@@ -13,20 +13,18 @@
 
 namespace Botan {
 
-RFC6979_Nonce_Generator::RFC6979_Nonce_Generator(std::string_view hash, const BigInt& order, const BigInt& x) :
-      m_order(order),
-      m_qlen(m_order.bits()),
-      m_rlen(m_qlen / 8 + (m_qlen % 8 ? 1 : 0)),
-      m_rng_in(m_rlen * 2),
-      m_rng_out(m_rlen) {
+RFC6979_Nonce_Generator::~RFC6979_Nonce_Generator() = default;
+
+RFC6979_Nonce_Generator::RFC6979_Nonce_Generator(std::string_view hash, size_t order_bits, const BigInt& x) :
+      m_qlen(order_bits), m_rlen((m_qlen + 7) / 8), m_rng_in(m_rlen * 2), m_rng_out(m_rlen) {
    m_hmac_drbg = std::make_unique<HMAC_DRBG>(MessageAuthenticationCode::create_or_throw(fmt("HMAC({})", hash)));
 
    x.serialize_to(std::span{m_rng_in}.first(m_rlen));
 }
 
-RFC6979_Nonce_Generator::~RFC6979_Nonce_Generator() = default;
+BigInt RFC6979_Nonce_Generator::nonce_for(const BigInt& order, const BigInt& m) {
+   BOTAN_DEBUG_ASSERT(order.bits() == m_qlen);
 
-BigInt RFC6979_Nonce_Generator::nonce_for(const BigInt& m) {
    m.serialize_to(std::span{m_rng_in}.last(m_rlen));
 
    m_hmac_drbg->initialize_with(m_rng_in);
@@ -43,12 +41,19 @@ BigInt RFC6979_Nonce_Generator::nonce_for(const BigInt& m) {
       if(shift > 0) {
          k >>= shift;
       }
-   } while(k == 0 || k >= m_order);
+   } while(k == 0 || k >= order);
 
    return k;
 }
 
 #if defined(BOTAN_HAS_ECC_GROUP)
+RFC6979_Nonce_Generator::RFC6979_Nonce_Generator(std::string_view hash, size_t order_bits, const EC_Scalar& scalar) :
+      m_qlen(order_bits), m_rlen((m_qlen + 7) / 8), m_rng_in(m_rlen * 2), m_rng_out(m_rlen) {
+   m_hmac_drbg = std::make_unique<HMAC_DRBG>(MessageAuthenticationCode::create_or_throw(fmt("HMAC({})", hash)));
+
+   scalar.serialize_to(std::span{m_rng_in}.first(m_rlen));
+}
+
 EC_Scalar RFC6979_Nonce_Generator::nonce_for(const EC_Group& group, const EC_Scalar& m) {
    m.serialize_to(std::span{m_rng_in}.last(m_rlen));
 
diff --git a/src/lib/pubkey/rfc6979/rfc6979.h b/src/lib/pubkey/rfc6979/rfc6979.h
index 8dabc7f269e..32f8637d1c6 100644
--- a/src/lib/pubkey/rfc6979/rfc6979.h
+++ b/src/lib/pubkey/rfc6979/rfc6979.h
@@ -1,6 +1,6 @@
 /*
 * RFC 6979 Deterministic Nonce Generator
-* (C) 2014,2015 Jack Lloyd
+* (C) 2014,2015,2024 Jack Lloyd
 *
 * Botan is released under the Simplified BSD License (see license.txt)
 */
@@ -10,7 +10,8 @@
 
 #include <botan/bigint.h>
 #include <memory>
-#include <string>
+#include <span>
+#include <string_view>
 
 #if defined(BOTAN_HAS_ECC_GROUP)
    #include <botan/ec_scalar.h>
@@ -22,24 +23,24 @@ class HMAC_DRBG;
 
 class BOTAN_TEST_API RFC6979_Nonce_Generator final {
    public:
-      /**
-      * Note: keeps persistent reference to order
-      */
-      RFC6979_Nonce_Generator(std::string_view hash, const BigInt& order, const BigInt& x);
+      RFC6979_Nonce_Generator(std::string_view hash, size_t order_bits, const BigInt& x);
 
-      ~RFC6979_Nonce_Generator();
+      BigInt nonce_for(const BigInt& group_order, const BigInt& m);
 
 #if defined(BOTAN_HAS_ECC_GROUP)
+      RFC6979_Nonce_Generator(std::string_view hash, size_t order_bits, const EC_Scalar& scalar);
+
       EC_Scalar nonce_for(const EC_Group& group, const EC_Scalar& m);
 #endif
 
-      BigInt nonce_for(const BigInt& m);
+      ~RFC6979_Nonce_Generator();
 
    private:
-      const BigInt& m_order;
-      size_t m_qlen, m_rlen;
+      size_t m_qlen;
+      size_t m_rlen;
       std::unique_ptr<HMAC_DRBG> m_hmac_drbg;
-      secure_vector<uint8_t> m_rng_in, m_rng_out;
+      secure_vector<uint8_t> m_rng_in;
+      secure_vector<uint8_t> m_rng_out;
 };
 
 /**
@@ -49,8 +50,8 @@ class BOTAN_TEST_API RFC6979_Nonce_Generator final {
 * @param hash the hash function used to generate h
 */
 inline BigInt generate_rfc6979_nonce(const BigInt& x, const BigInt& q, const BigInt& h, std::string_view hash) {
-   RFC6979_Nonce_Generator gen(hash, q, x);
-   return gen.nonce_for(h);
+   RFC6979_Nonce_Generator gen(hash, q.bits(), x);
+   return gen.nonce_for(q, h);
 }
 
 }  // namespace Botan
diff --git a/src/lib/pubkey/sm2/sm2.cpp b/src/lib/pubkey/sm2/sm2.cpp
index 7ecabdde913..5b67d85297c 100644
--- a/src/lib/pubkey/sm2/sm2.cpp
+++ b/src/lib/pubkey/sm2/sm2.cpp
@@ -32,8 +32,10 @@ bool SM2_PrivateKey::check_key(RandomNumberGenerator& rng, bool strong) const {
 
    // SM2 has an oddity in private key generation when compared to
    // other EC*DSA style signature algorithms described in ISO14888-3:
-   // the private key x MUST be in ]0, q-1[ instead of ]0, q[.
-   if(m_private_key < 1 || m_private_key >= m_domain_params.get_order() - 1) {
+   // the private key x MUST be in [0, q-1) instead of [0, q).
+   //
+   // The lower bound is already checked by the default impl
+   if(private_value() >= domain().get_order() - 1) {
       return false;
    }
 
@@ -45,19 +47,21 @@ bool SM2_PrivateKey::check_key(RandomNumberGenerator& rng, bool strong) const {
 }
 
 SM2_PrivateKey::SM2_PrivateKey(const AlgorithmIdentifier& alg_id, std::span<const uint8_t> key_bits) :
-      EC_PrivateKey(alg_id, key_bits) {
-   m_da_inv = domain().inverse_mod_order(m_private_key + 1);
-}
+      EC_PrivateKey(alg_id, key_bits),
+      m_da_inv((this->_private_key() + EC_Scalar::one(domain())).invert()),
+      m_da_inv_legacy(m_da_inv.to_bigint()) {}
 
-SM2_PrivateKey::SM2_PrivateKey(RandomNumberGenerator& rng, const EC_Group& domain, const BigInt& x) :
-      EC_PrivateKey(rng, domain, x) {
-   m_da_inv = domain.inverse_mod_order(m_private_key + 1);
-}
+SM2_PrivateKey::SM2_PrivateKey(RandomNumberGenerator& rng, EC_Group group, BigInt x) :
+      EC_PrivateKey(rng, std::move(group), std::move(x)),
+      m_da_inv((this->_private_key() + EC_Scalar::one(domain())).invert()),
+      m_da_inv_legacy(m_da_inv.to_bigint()) {}
+
+namespace {
 
 std::vector<uint8_t> sm2_compute_za(HashFunction& hash,
                                     std::string_view user_id,
-                                    const EC_Group& domain,
-                                    const EC_Point& pubkey) {
+                                    const EC_Group& group,
+                                    const EC_AffinePoint& pubkey) {
    if(user_id.size() >= 8192) {
       throw Invalid_Argument("SM2 user id too long to represent");
    }
@@ -68,34 +72,30 @@ std::vector<uint8_t> sm2_compute_za(HashFunction& hash,
    hash.update(get_byte<1>(uid_len));
    hash.update(user_id);
 
-   const size_t p_bytes = domain.get_p_bytes();
+   const size_t p_bytes = group.get_p_bytes();
 
-   hash.update(domain.get_a().serialize(p_bytes));
-   hash.update(domain.get_b().serialize(p_bytes));
-   hash.update(domain.get_g_x().serialize(p_bytes));
-   hash.update(domain.get_g_y().serialize(p_bytes));
+   hash.update(group.get_a().serialize(p_bytes));
+   hash.update(group.get_b().serialize(p_bytes));
+   hash.update(group.get_g_x().serialize(p_bytes));
+   hash.update(group.get_g_y().serialize(p_bytes));
    hash.update(pubkey.xy_bytes());
 
    return hash.final<std::vector<uint8_t>>();
 }
 
-namespace {
-
 /**
 * SM2 signature operation
 */
 class SM2_Signature_Operation final : public PK_Ops::Signature {
    public:
       SM2_Signature_Operation(const SM2_PrivateKey& sm2, std::string_view ident, std::string_view hash) :
-            m_group(sm2.domain()),
-            m_x(EC_Scalar::from_bigint(m_group, sm2.private_value())),
-            m_da_inv(EC_Scalar::from_bigint(m_group, sm2.get_da_inv())) {
+            m_group(sm2.domain()), m_x(sm2._private_key()), m_da_inv(sm2._get_da_inv()) {
          if(hash == "Raw") {
             // m_hash is null, m_za is empty
          } else {
             m_hash = HashFunction::create_or_throw(hash);
             // ZA=H256(ENTLA || IDA || a || b || xG || yG || xA || yA)
-            m_za = sm2_compute_za(*m_hash, ident, m_group, sm2.public_point());
+            m_za = sm2_compute_za(*m_hash, ident, m_group, sm2._public_key());
             m_hash->update(m_za);
          }
       }
@@ -153,13 +153,13 @@ std::vector<uint8_t> SM2_Signature_Operation::sign(RandomNumberGenerator& rng) {
 class SM2_Verification_Operation final : public PK_Ops::Verification {
    public:
       SM2_Verification_Operation(const SM2_PublicKey& sm2, std::string_view ident, std::string_view hash) :
-            m_group(sm2.domain()), m_gy_mul(m_group, sm2.public_point()) {
+            m_group(sm2.domain()), m_gy_mul(sm2._public_key()) {
          if(hash == "Raw") {
             // m_hash is null, m_za is empty
          } else {
             m_hash = HashFunction::create_or_throw(hash);
             // ZA=H256(ENTLA || IDA || a || b || xG || yG || xA || yA)
-            m_za = sm2_compute_za(*m_hash, ident, m_group, sm2.public_point());
+            m_za = sm2_compute_za(*m_hash, ident, m_group, sm2._public_key());
             m_hash->update(m_za);
          }
       }
@@ -241,6 +241,14 @@ std::unique_ptr<Private_Key> SM2_PublicKey::generate_another(RandomNumberGenerat
    return std::make_unique<SM2_PrivateKey>(rng, domain());
 }
 
+std::vector<uint8_t> sm2_compute_za(HashFunction& hash,
+                                    std::string_view user_id,
+                                    const EC_Group& group,
+                                    const EC_Point& point) {
+   auto apoint = EC_AffinePoint(group, point);
+   return sm2_compute_za(hash, user_id, group, apoint);
+}
+
 std::unique_ptr<PK_Ops::Verification> SM2_PublicKey::create_verification_op(std::string_view params,
                                                                             std::string_view provider) const {
    if(provider == "base" || provider.empty()) {
diff --git a/src/lib/pubkey/sm2/sm2.h b/src/lib/pubkey/sm2/sm2.h
index ef9d3061aef..e8911d197cc 100644
--- a/src/lib/pubkey/sm2/sm2.h
+++ b/src/lib/pubkey/sm2/sm2.h
@@ -82,7 +82,7 @@ class BOTAN_PUBLIC_API(2, 2) SM2_PrivateKey final : public SM2_PublicKey,
       * @param domain parameters to used for this key
       * @param x the private key (if zero, generate a new random key)
       */
-      SM2_PrivateKey(RandomNumberGenerator& rng, const EC_Group& domain, const BigInt& x = BigInt::zero());
+      SM2_PrivateKey(RandomNumberGenerator& rng, EC_Group domain, BigInt x = BigInt::zero());
 
       bool check_key(RandomNumberGenerator& rng, bool) const override;
 
@@ -96,16 +96,25 @@ class BOTAN_PUBLIC_API(2, 2) SM2_PrivateKey final : public SM2_PublicKey,
                                                                std::string_view params,
                                                                std::string_view provider) const override;
 
-      const BigInt& get_da_inv() const { return m_da_inv; }
+      BOTAN_DEPRECATED("Deprecated no replacement") const BigInt& get_da_inv() const { return m_da_inv_legacy; }
+
+      const EC_Scalar& _get_da_inv() const { return m_da_inv; }
 
    private:
-      BigInt m_da_inv;
+      EC_Scalar m_da_inv;
+      BigInt m_da_inv_legacy;
 };
 
 BOTAN_DIAGNOSTIC_POP
 
 class HashFunction;
 
+/*
+* This is deprecated because it's not clear what it is useful for
+*
+* Open an issue on GH if you are using this
+*/
+BOTAN_DEPRECATED("Deprecated unclear usage")
 std::vector<uint8_t> BOTAN_PUBLIC_API(2, 5)
    sm2_compute_za(HashFunction& hash, std::string_view user_id, const EC_Group& domain, const EC_Point& pubkey);
 
diff --git a/src/lib/pubkey/sm2/sm2_enc.cpp b/src/lib/pubkey/sm2/sm2_enc.cpp
index b1d10057440..552dd849cb4 100644
--- a/src/lib/pubkey/sm2/sm2_enc.cpp
+++ b/src/lib/pubkey/sm2/sm2_enc.cpp
@@ -22,7 +22,7 @@ namespace {
 class SM2_Encryption_Operation final : public PK_Ops::Encryption {
    public:
       SM2_Encryption_Operation(const SM2_Encryption_PublicKey& key, std::string_view kdf_hash) :
-            m_group(key.domain()), m_peer(m_group, key.public_point()), m_ws(EC_Point::WORKSPACE_SIZE) {
+            m_group(key.domain()), m_peer(key._public_key()), m_ws(EC_Point::WORKSPACE_SIZE) {
          m_hash = HashFunction::create_or_throw(kdf_hash);
          m_kdf = KDF::create_or_throw(fmt("KDF2({})", kdf_hash));
       }
@@ -88,7 +88,7 @@ class SM2_Decryption_Operation final : public PK_Ops::Decryption {
       SM2_Decryption_Operation(const SM2_Encryption_PrivateKey& key,
                                RandomNumberGenerator& rng,
                                std::string_view kdf_hash) :
-            m_group(key.domain()), m_x(EC_Scalar::from_bigint(m_group, key.private_value())), m_rng(rng) {
+            m_group(key.domain()), m_x(key._private_key()), m_rng(rng) {
          m_hash = HashFunction::create_or_throw(kdf_hash);
 
          const std::string kdf_name = fmt("KDF2({})", kdf_hash);
diff --git a/src/tests/test_ecdsa.cpp b/src/tests/test_ecdsa.cpp
index d20d90d3bbc..5aaee63c2f3 100644
--- a/src/tests/test_ecdsa.cpp
+++ b/src/tests/test_ecdsa.cpp
@@ -234,8 +234,12 @@ class ECDSA_Invalid_Key_Tests final : public Text_Based_Test {
             return result;
          }
 
-         auto key = std::make_unique<Botan::ECDSA_PublicKey>(group, *public_point);
-         result.test_eq("public key fails check", key->check_key(this->rng(), false), false);
+         try {
+            auto key = std::make_unique<Botan::ECDSA_PublicKey>(group, *public_point);
+            result.test_failure("Invalid public key was deserialized");
+         } catch(Botan::Decoding_Error&) {
+            result.test_success("public key fails to deserialize");
+         }
          return result;
       }
 };
diff --git a/src/tests/test_rfc6979.cpp b/src/tests/test_rfc6979.cpp
index 2ad7eead9c1..4f7d462d749 100644
--- a/src/tests/test_rfc6979.cpp
+++ b/src/tests/test_rfc6979.cpp
@@ -38,11 +38,11 @@ class RFC6979_KAT_Tests final : public Text_Based_Test {
 
          result.test_eq("vector matches", Botan::generate_rfc6979_nonce(X, Q, H, hash), K);
 
-         Botan::RFC6979_Nonce_Generator gen(hash, Q, X);
+         Botan::RFC6979_Nonce_Generator gen(hash, Q.bits(), X);
 
-         result.test_eq("vector matches", gen.nonce_for(H), K);
-         result.test_ne("different output for H+1", gen.nonce_for(H + 1), K);
-         result.test_eq("vector matches when run again", gen.nonce_for(H), K);
+         result.test_eq("vector matches", gen.nonce_for(Q, H), K);
+         result.test_ne("different output for H+1", gen.nonce_for(Q, H + 1), K);
+         result.test_eq("vector matches when run again", gen.nonce_for(Q, H), K);
 
          return result;
       }