diff --git a/decompiler/level_extractor/extract_level.cpp b/decompiler/level_extractor/extract_level.cpp
index 4aa68323a9f..773d23f259a 100644
--- a/decompiler/level_extractor/extract_level.cpp
+++ b/decompiler/level_extractor/extract_level.cpp
@@ -112,10 +112,12 @@ void extract_art_groups_from_level(const ObjectFileDB& db,
                                    std::map<std::string, level_tools::ArtData>& art_group_data) {
   if (db.obj_files_by_dgo.count(dgo_name)) {
     const auto& files = db.obj_files_by_dgo.at(dgo_name);
+    MercSwapInfo swapped_info;
     for (const auto& file : files) {
       if (file.name.length() > 3 && !file.name.compare(file.name.length() - 3, 3, "-ag")) {
         const auto& ag_file = db.lookup_record(file);
-        extract_merc(ag_file, tex_db, db.dts, tex_remap, level_data, false, db.version());
+        extract_merc(ag_file, tex_db, db.dts, tex_remap, level_data, false, db.version(),
+                     swapped_info);
         extract_joint_group(ag_file, db.dts, db.version(), art_group_data);
       }
     }
diff --git a/decompiler/level_extractor/extract_level.h b/decompiler/level_extractor/extract_level.h
index ac4712f9409..202923a617f 100644
--- a/decompiler/level_extractor/extract_level.h
+++ b/decompiler/level_extractor/extract_level.h
@@ -10,6 +10,40 @@
 
 namespace decompiler {
 
+// info about what models have been replaced/added per level
+struct MercSwapInfo {
+  std::map<std::string, std::vector<std::string>> per_level_merc_swaps;
+  std::map<std::string, std::vector<std::string>> per_level_custom_mdls;
+
+  bool already_swapped(const std::string& model, const std::string& level) {
+    auto mdls_it = per_level_merc_swaps.find(level);
+    if (mdls_it != per_level_merc_swaps.end()) {
+      auto& mdls = mdls_it->second;
+      auto mdl = std::find(mdls.begin(), mdls.end(), model);
+      return mdl != mdls.end();
+    }
+    return false;
+  }
+
+  bool already_added(const std::string& model, const std::string& level) {
+    auto mdls_it = per_level_custom_mdls.find(level);
+    if (mdls_it != per_level_custom_mdls.end()) {
+      auto& mdls = mdls_it->second;
+      auto mdl = std::find(mdls.begin(), mdls.end(), model);
+      return mdl != mdls.end();
+    }
+    return false;
+  }
+
+  void add_to_swapped_list(const std::string& model, const std::string& level) {
+    per_level_merc_swaps[level].push_back(model);
+  }
+
+  void add_to_custom_list(const std::string& model, const std::string& level) {
+    per_level_custom_mdls[level].push_back(model);
+  }
+};
+
 // extract everything
 void extract_all_levels(const ObjectFileDB& db,
                         const TextureDB& tex_db,
diff --git a/decompiler/level_extractor/extract_merc.cpp b/decompiler/level_extractor/extract_merc.cpp
index f5f5e1fa5d5..e88b4fd0aa3 100644
--- a/decompiler/level_extractor/extract_merc.cpp
+++ b/decompiler/level_extractor/extract_merc.cpp
@@ -1669,7 +1669,8 @@ void extract_merc(const ObjectFileData& ag_data,
                   const std::vector<level_tools::TextureRemap>& map,
                   tfrag3::Level& out,
                   bool dump_level,
-                  GameVersion version) {
+                  GameVersion version,
+                  MercSwapInfo& swapped_info) {
   if (dump_level) {
     file_util::create_dir_if_needed(file_util::get_file_path({"debug_out/merc"}));
   }
@@ -1795,31 +1796,43 @@ void extract_merc(const ObjectFileData& ag_data,
     }
   }
 
-  // do model replacements if present
-  auto merc_replacement_folder = file_util::get_jak_project_dir() / "custom_assets" /
-                                 game_version_names[version] / "merc_replacements";
-  if (file_util::file_exists(merc_replacement_folder.string())) {
-    auto merc_replacements =
-        file_util::find_files_in_dir(merc_replacement_folder, std::regex(".*\\.glb"));
-    for (auto& path : merc_replacements) {
-      auto name = path.stem().string();
-      auto it = std::find_if(out.merc_data.models.begin(), out.merc_data.models.end(),
-                             [&](const auto& m) { return m.name == name; });
-      if (it != out.merc_data.models.end()) {
-        auto& model = *it;
-        replace_model(out, model, path);
+  // do model replacement if present
+  for (auto& ctrl : ctrls) {
+    auto merc_replacements_path = file_util::get_jak_project_dir() / "custom_assets" /
+                                  game_version_names[version] / "merc_replacements";
+    if (!swapped_info.already_swapped(ctrl.name, out.level_name)) {
+      if (file_util::file_exists(merc_replacements_path.string())) {
+        std::string file_name(ctrl.name + ".glb");
+        auto mdl_path = merc_replacements_path / file_name;
+        if (file_util::file_exists(mdl_path.string())) {
+          auto it = std::find_if(out.merc_data.models.begin(), out.merc_data.models.end(),
+                                 [&](const auto& m) { return m.name == ctrl.name; });
+          if (it != out.merc_data.models.end()) {
+            auto& model = *it;
+            replace_model(out, model, mdl_path);
+            swapped_info.add_to_swapped_list(ctrl.name, out.level_name);
+          }
+        }
+      } else {
+        lg::info("{} in level {} was already swapped, skipping", ctrl.name, out.level_name);
       }
     }
-  }
 
-  // add custom models if present
-  auto lvl_name = out.level_name == "" ? "common" : out.level_name;
-  auto models_folder = file_util::get_jak_project_dir() / "custom_assets" /
-                       game_version_names[version] / "models" / lvl_name;
-  if (file_util::file_exists(models_folder.string())) {
-    auto custom_models = file_util::find_files_in_dir(models_folder, std::regex(".*\\.glb"));
-    for (auto& mdl : custom_models) {
-      add_custom_model_to_level(out, mdl.stem().string(), mdl);
+    // add custom models if present
+    auto lvl_name = out.level_name == "" ? "common" : out.level_name;
+    auto models_folder = file_util::get_jak_project_dir() / "custom_assets" /
+                         game_version_names[version] / "models" / lvl_name;
+    if (file_util::file_exists(models_folder.string())) {
+      auto custom_models = file_util::find_files_in_dir(models_folder, std::regex(".*\\.glb"));
+      for (auto& mdl : custom_models) {
+        auto name = mdl.stem().string();
+        if (!swapped_info.already_added(name, lvl_name)) {
+          add_custom_model_to_level(out, name, mdl);
+          swapped_info.add_to_custom_list(name, lvl_name);
+        } else {
+          lg::info("custom model {} was already added to level {}, skipping", name, lvl_name);
+        }
+      }
     }
   }
 }
diff --git a/decompiler/level_extractor/extract_merc.h b/decompiler/level_extractor/extract_merc.h
index 0332c88e574..3a64620a356 100644
--- a/decompiler/level_extractor/extract_merc.h
+++ b/decompiler/level_extractor/extract_merc.h
@@ -1,5 +1,7 @@
 #pragma once
 
+#include "extract_level.h"
+
 #include "common/custom_data/Tfrag3Data.h"
 
 #include "decompiler/ObjectFile/ObjectFileDB.h"
@@ -14,5 +16,6 @@ void extract_merc(const ObjectFileData& ag_data,
                   const std::vector<level_tools::TextureRemap>& map,
                   tfrag3::Level& out,
                   bool dump_level,
-                  GameVersion version);
+                  GameVersion version,
+                  MercSwapInfo& swapped_info);
 }  // namespace decompiler
\ No newline at end of file
diff --git a/decompiler/level_extractor/merc_replacement.cpp b/decompiler/level_extractor/merc_replacement.cpp
index 00565a7b560..d79a1acbf86 100644
--- a/decompiler/level_extractor/merc_replacement.cpp
+++ b/decompiler/level_extractor/merc_replacement.cpp
@@ -81,7 +81,9 @@ void extract(const std::string& name,
   tfrag3::MercEffect envmap_eff;
   envmap_eff.has_envmap = false;
   out.new_model.name = name;
-  out.new_model.max_bones = joints;
+  // if we have a skeleton, use that joint count, otherwise use a high default value since the model
+  // we replace can have more
+  out.new_model.max_bones = joints != 3 ? joints : 100;
   out.new_model.max_draws = 0;
 
   auto process_normal_draw = [&](tfrag3::MercEffect& eff, int mat_idx, const tfrag3::MercDraw& d_) {
diff --git a/goalc/build_level/jak1/build_level.cpp b/goalc/build_level/jak1/build_level.cpp
index 810df3e7c78..8395940fbb1 100644
--- a/goalc/build_level/jak1/build_level.cpp
+++ b/goalc/build_level/jak1/build_level.cpp
@@ -234,8 +234,9 @@ bool run_build_level(const std::string& input_file,
           if (ag.name.length() > 3 && !ag.name.compare(ag.name.length() - 3, 3, "-ag")) {
             const auto& ag_file = db.lookup_record(ag);
             lg::info("custom level: extracting art group {}", ag_file.name_in_dgo);
+            decompiler::MercSwapInfo info;
             decompiler::extract_merc(ag_file, tex_db, db.dts, tex_remap, pc_level, false,
-                                     db.version());
+                                     db.version(), info);
           }
         }
       }
diff --git a/goalc/build_level/jak2/build_level.cpp b/goalc/build_level/jak2/build_level.cpp
index 7ab7db5bd7f..ec0f6f9ae33 100644
--- a/goalc/build_level/jak2/build_level.cpp
+++ b/goalc/build_level/jak2/build_level.cpp
@@ -150,8 +150,9 @@ bool run_build_level(const std::string& input_file,
           if (ag.name.length() > 3 && !ag.name.compare(ag.name.length() - 3, 3, "-ag")) {
             const auto& ag_file = db.lookup_record(ag);
             lg::print("custom level: extracting art group {}\n", ag_file.name_in_dgo);
+            decompiler::MercSwapInfo info;
             decompiler::extract_merc(ag_file, tex_db, db.dts, tex_remap, pc_level, false,
-                                     db.version());
+                                     db.version(), info);
           }
         }
       }
diff --git a/goalc/build_level/jak3/build_level.cpp b/goalc/build_level/jak3/build_level.cpp
index 57185b736b1..fd997cfa68f 100644
--- a/goalc/build_level/jak3/build_level.cpp
+++ b/goalc/build_level/jak3/build_level.cpp
@@ -148,8 +148,9 @@ bool run_build_level(const std::string& input_file,
           if (ag.name.length() > 3 && !ag.name.compare(ag.name.length() - 3, 3, "-ag")) {
             const auto& ag_file = db.lookup_record(ag);
             lg::print("custom level: extracting art group {}\n", ag_file.name_in_dgo);
+            decompiler::MercSwapInfo info;
             decompiler::extract_merc(ag_file, tex_db, db.dts, tex_remap, pc_level, false,
-                                     db.version());
+                                     db.version(), info);
           }
         }
       }