diff --git a/src/core/layout/qgslayoutitemgroup.cpp b/src/core/layout/qgslayoutitemgroup.cpp
index 48041cd30f7a..b9896a4a4054 100644
--- a/src/core/layout/qgslayoutitemgroup.cpp
+++ b/src/core/layout/qgslayoutitemgroup.cpp
@@ -206,7 +206,8 @@ void QgsLayoutItemGroup::attemptResize( const QgsLayoutSize &size, bool includes
       command->saveBeforeState();
     }
 
-    QRectF itemRect = mapRectFromItem( item, item->rect() );
+    const QRectF originalItemRect = item->rect();
+    QRectF itemRect = mapRectFromItem( item, originalItemRect );
     QgsLayoutUtils::relativeResizeRect( itemRect, oldRect, newRect );
 
     itemRect = itemRect.normalized();
diff --git a/src/core/layout/qgslayoutitemmap.cpp b/src/core/layout/qgslayoutitemmap.cpp
index 00f034a76e08..0dc32e14bf82 100644
--- a/src/core/layout/qgslayoutitemmap.cpp
+++ b/src/core/layout/qgslayoutitemmap.cpp
@@ -228,12 +228,14 @@ void QgsLayoutItemMap::setExtent( const QgsRectangle &extent )
   //recalculate data defined scale and extents, since that may override extent
   refreshMapExtents();
 
-  //adjust height
-  QRectF currentRect = rect();
-
-  double newHeight = currentRect.width() * mExtent.height() / mExtent.width();
-
-  attemptSetSceneRect( QRectF( pos().x(), pos().y(), currentRect.width(), newHeight ) );
+  //adjust height, if possible
+  if ( mExtent.isFinite() && !mExtent.isEmpty() )
+  {
+    const QRectF currentRect = rect();
+    const double newHeight = mExtent.width() == 0 ? 0
+                             : currentRect.width() * mExtent.height() / mExtent.width();
+    attemptSetSceneRect( QRectF( pos().x(), pos().y(), currentRect.width(), newHeight ) );
+  }
   update();
 }
 
diff --git a/src/core/layout/qgslayoutpoint.cpp b/src/core/layout/qgslayoutpoint.cpp
index e590befed650..652e93be7aa5 100644
--- a/src/core/layout/qgslayoutpoint.cpp
+++ b/src/core/layout/qgslayoutpoint.cpp
@@ -26,7 +26,9 @@ QgsLayoutPoint::QgsLayoutPoint( const double x, const double y, const Qgis::Layo
   , mY( y )
   , mUnits( units )
 {
-
+#ifdef QGISDEBUG
+  Q_ASSERT_X( !std::isnan( mX ) && !std::isnan( mY ), "QgsLayoutPoint", "Layout point with NaN coordinates created" );
+#endif
 }
 
 QgsLayoutPoint::QgsLayoutPoint( const QPointF point, const Qgis::LayoutUnit units )
@@ -65,7 +67,16 @@ QgsLayoutPoint QgsLayoutPoint::decodePoint( const QString &string )
   {
     return QgsLayoutPoint();
   }
-  return QgsLayoutPoint( parts[0].toDouble(), parts[1].toDouble(), QgsUnitTypes::decodeLayoutUnit( parts[2] ) );
+  const double x = parts[0].toDouble();
+  const double y = parts[1].toDouble();
+
+  // don't restore corrupted coordinates from xml. This can happen when eg a broken item size causes a nan position,
+  // which breaks the layout size calculation and results in nan or massive x/y values. Restoring these leads to a broken
+  // layout which cannot be interacted with.
+  if ( std::isnan( x ) || std::isnan( y ) || x > 9.99998e+06 || y > 9.99998e+06 )
+    return QgsLayoutPoint();
+
+  return QgsLayoutPoint( x, y, QgsUnitTypes::decodeLayoutUnit( parts[2] ) );
 }
 
 bool QgsLayoutPoint::operator==( const QgsLayoutPoint &other ) const
diff --git a/src/core/layout/qgslayoutpoint.h b/src/core/layout/qgslayoutpoint.h
index ad67318773c5..5a5230d6c8de 100644
--- a/src/core/layout/qgslayoutpoint.h
+++ b/src/core/layout/qgslayoutpoint.h
@@ -20,6 +20,7 @@
 
 #include "qgis_core.h"
 #include "qgis.h"
+#include "qgsconfig.h"
 #include <QPointF>
 
 /**
@@ -76,7 +77,13 @@ class CORE_EXPORT QgsLayoutPoint
      * \see x()
      * \see setY()
     */
-    void setX( const double x ) { mX = x; }
+    void setX( const double x )
+    {
+#ifdef QGISDEBUG
+      Q_ASSERT_X( !std::isnan( x ), "QgsLayoutPoint", "Layout point with NaN coordinates created" );
+#endif
+      mX = x;
+    }
 
     /**
      * Returns y coordinate of point.
@@ -90,7 +97,13 @@ class CORE_EXPORT QgsLayoutPoint
      * \see y()
      * \see setX()
     */
-    void setY( const double y ) { mY = y; }
+    void setY( const double y )
+    {
+#ifdef QGISDEBUG
+      Q_ASSERT_X( !std::isnan( y ), "QgsLayoutPoint", "Layout point with NaN coordinates created" );
+#endif
+      mY = y;
+    }
 
     /**
      * Returns the units for the point.
diff --git a/src/core/layout/qgslayoutsize.cpp b/src/core/layout/qgslayoutsize.cpp
index a23769f1ab4b..a69c269ecde6 100644
--- a/src/core/layout/qgslayoutsize.cpp
+++ b/src/core/layout/qgslayoutsize.cpp
@@ -25,6 +25,9 @@ QgsLayoutSize::QgsLayoutSize( const double width, const double height, const Qgi
   , mHeight( height )
   , mUnits( units )
 {
+#ifdef QGISDEBUG
+  Q_ASSERT_X( !std::isnan( width ) && !std::isnan( height ), "QgsLayoutSize", "Layout size with NaN dimensions created" );
+#endif
 }
 
 QgsLayoutSize::QgsLayoutSize( const QSizeF size, const Qgis::LayoutUnit units )
@@ -62,8 +65,13 @@ QgsLayoutSize QgsLayoutSize::decodeSize( const QString &string )
   {
     return QgsLayoutSize();
   }
-  return QgsLayoutSize( parts[0].toDouble(), parts[1].toDouble(), QgsUnitTypes::decodeLayoutUnit( parts[2] ) );
 
+  const double width = parts[0].toDouble();
+  const double height = parts[1].toDouble();
+  if ( std::isnan( width ) || std::isnan( height ) )
+    return QgsLayoutSize();
+
+  return QgsLayoutSize( width, height, QgsUnitTypes::decodeLayoutUnit( parts[2] ) );
 }
 
 bool QgsLayoutSize::operator==( const QgsLayoutSize &other ) const
diff --git a/src/core/layout/qgslayoutsize.h b/src/core/layout/qgslayoutsize.h
index 2f78a389e9ed..63316a813753 100644
--- a/src/core/layout/qgslayoutsize.h
+++ b/src/core/layout/qgslayoutsize.h
@@ -20,6 +20,7 @@
 
 #include "qgis_core.h"
 #include "qgis.h"
+#include "qgsconfig.h"
 #include <QSizeF>
 
 
@@ -66,7 +67,14 @@ class CORE_EXPORT QgsLayoutSize
      * \see setHeight()
      * \see setUnits()
     */
-    void setSize( const double width, const double height ) { mWidth = width; mHeight = height; }
+    void setSize( const double width, const double height )
+    {
+      mWidth = width;
+      mHeight = height;
+#ifdef QGISDEBUG
+      Q_ASSERT_X( !std::isnan( width ) && !std::isnan( height ), "QgsLayoutSize", "Layout size with NaN dimensions created" );
+#endif
+    }
 
     /**
      * Returns the width of the size.
@@ -80,7 +88,13 @@ class CORE_EXPORT QgsLayoutSize
      * \see width()
      * \see setHeight()
     */
-    void setWidth( const double width ) { mWidth = width; }
+    void setWidth( const double width )
+    {
+      mWidth = width;
+#ifdef QGISDEBUG
+      Q_ASSERT_X( !std::isnan( width ), "QgsLayoutSize", "Layout size with NaN dimensions created" );
+#endif
+    }
 
     /**
      * Returns the height of the size.
@@ -94,7 +108,13 @@ class CORE_EXPORT QgsLayoutSize
      * \see height()
      * \see setWidth()
     */
-    void setHeight( const double height ) { mHeight = height; }
+    void setHeight( const double height )
+    {
+      mHeight = height;
+#ifdef QGISDEBUG
+      Q_ASSERT_X( !std::isnan( height ), "QgsLayoutSize", "Layout size with NaN dimensions created" );
+#endif
+    }
 
     /**
      * Returns the units for the size.
diff --git a/src/core/layout/qgslayoututils.cpp b/src/core/layout/qgslayoututils.cpp
index 73810b8f825a..6cf09cae422b 100644
--- a/src/core/layout/qgslayoututils.cpp
+++ b/src/core/layout/qgslayoututils.cpp
@@ -156,10 +156,18 @@ QgsRenderContext QgsLayoutUtils::createRenderContextForLayout( QgsLayout *layout
 void QgsLayoutUtils::relativeResizeRect( QRectF &rectToResize, const QRectF &boundsBefore, const QRectF &boundsAfter )
 {
   //linearly scale rectToResize relative to the scaling from boundsBefore to boundsAfter
-  double left = relativePosition( rectToResize.left(), boundsBefore.left(), boundsBefore.right(), boundsAfter.left(), boundsAfter.right() );
-  double right = relativePosition( rectToResize.right(), boundsBefore.left(), boundsBefore.right(), boundsAfter.left(), boundsAfter.right() );
-  double top = relativePosition( rectToResize.top(), boundsBefore.top(), boundsBefore.bottom(), boundsAfter.top(), boundsAfter.bottom() );
-  double bottom = relativePosition( rectToResize.bottom(), boundsBefore.top(), boundsBefore.bottom(), boundsAfter.top(), boundsAfter.bottom() );
+  const double left = !qgsDoubleNear( boundsBefore.left(), boundsBefore.right() )
+                      ? relativePosition( rectToResize.left(), boundsBefore.left(), boundsBefore.right(), boundsAfter.left(), boundsAfter.right() )
+                      : boundsAfter.left();
+  const double right = !qgsDoubleNear( boundsBefore.left(), boundsBefore.right() )
+                       ? relativePosition( rectToResize.right(), boundsBefore.left(), boundsBefore.right(), boundsAfter.left(), boundsAfter.right() )
+                       : boundsAfter.right();
+  const double top = !qgsDoubleNear( boundsBefore.top(), boundsBefore.bottom() )
+                     ? relativePosition( rectToResize.top(), boundsBefore.top(), boundsBefore.bottom(), boundsAfter.top(), boundsAfter.bottom() )
+                     : boundsAfter.top();
+  const double bottom = !qgsDoubleNear( boundsBefore.top(), boundsBefore.bottom() )
+                        ? relativePosition( rectToResize.bottom(), boundsBefore.top(), boundsBefore.bottom(), boundsAfter.top(), boundsAfter.bottom() )
+                        : boundsAfter.bottom();
 
   rectToResize.setRect( left, top, right - left, bottom - top );
 }
diff --git a/src/core/scalebar/qgsscalebarrenderer.cpp b/src/core/scalebar/qgsscalebarrenderer.cpp
index d499f1b80ed3..9444c0b4b74f 100644
--- a/src/core/scalebar/qgsscalebarrenderer.cpp
+++ b/src/core/scalebar/qgsscalebarrenderer.cpp
@@ -246,7 +246,7 @@ QSizeF QgsScaleBarRenderer::calculateBoxSize( QgsRenderContext &context, const Q
 
   //consider last number and label
   const double largestLabelNumber = settings.numberOfSegments() * settings.unitsPerSegment() / settings.mapUnitsPerScaleBarUnit();
-  const QString largestNumberLabel = settings.numericFormat()->formatDouble( largestLabelNumber, QgsNumericFormatContext() );
+  const QString largestNumberLabel = std::isnan( largestLabelNumber ) ? QString() : settings.numericFormat()->formatDouble( largestLabelNumber, QgsNumericFormatContext() );
   const QString largestLabel = largestNumberLabel + ' ' + settings.unitLabel();
   double largestLabelWidth;
   if ( settings.labelHorizontalPlacement() == QgsScaleBarSettings::LabelCenteredSegment )
diff --git a/tests/src/core/testqgslayoutunits.cpp b/tests/src/core/testqgslayoutunits.cpp
index 3c62f4b9fc8d..70c894ca0229 100644
--- a/tests/src/core/testqgslayoutunits.cpp
+++ b/tests/src/core/testqgslayoutunits.cpp
@@ -380,6 +380,12 @@ void TestQgsLayoutUnits::sizeEncodeDecode()
   //test with bad string
   result = QgsLayoutSize::decodeSize( QStringLiteral( "1,2" ) );
   QCOMPARE( result, QgsLayoutSize() );
+
+  //test with nan values
+  result = QgsLayoutSize::decodeSize( QStringLiteral( "nan,2,mm" ) );
+  QCOMPARE( result, QgsLayoutSize() );
+  result = QgsLayoutSize::decodeSize( QStringLiteral( "2,nan,mm" ) );
+  QCOMPARE( result, QgsLayoutSize() );
 }
 
 void TestQgsLayoutUnits::createPoint()
@@ -520,6 +526,14 @@ void TestQgsLayoutUnits::pointEncodeDecode()
   //test with bad string
   result = QgsLayoutPoint::decodePoint( QStringLiteral( "1,2" ) );
   QCOMPARE( result, QgsLayoutPoint() );
+
+  //test with nan values
+  result = QgsLayoutPoint::decodePoint( QStringLiteral( "nan,2,mm" ) );
+  QCOMPARE( result, QgsLayoutPoint() );
+  result = QgsLayoutPoint::decodePoint( QStringLiteral( "2,nan,mm" ) );
+  QCOMPARE( result, QgsLayoutPoint() );
+  result = QgsLayoutPoint::decodePoint( QStringLiteral( "9.99999e+06,1e+07,mm" ) );
+  QCOMPARE( result, QgsLayoutPoint() );
 }
 
 void TestQgsLayoutUnits::converterCreate()