fixed deduper problems

autorun.template.lua

@@ -1,7 +1,7 @@
 fm.autorun = {
     HD = false,
     day = true,                     -- note: snapshots in the same timeline should have the same day/night settings for now (this will be possible in the future)
-    night = true,
+    night = false,
     around_build_range = 5.2,		-- max range from buildings that images will be saved. Feel free to crank this up to very large numbers, it will only render chunks that already exist, it will not generate new ones.
     around_smaller_range = 1.2,		-- same as above, but smaller range for the following entity types:
     smaller_types = {"lamp", "electric-pole", "radar", "straight-rail", "curved-rail", "rail-signal", "rail-chain-signal", "locomotive", "cargo-wagon", "fluid-wagon", "car"},

index.html.template

@@ -208,6 +208,9 @@
 	<div id="map" style="background: #1B2D33;"></div>
 	<script>
 		"use strict";
+
+
+		var DEBUG = false;
 
 
 		//var _getTileUrl = L.TileLayer.prototype.getTileUrl;
@@ -227,6 +230,10 @@
 		var globalTileIndex = {};
 		var globalTileNightIndex = {};
 		for (var i = 0; i < mapInfo.maps.length; i++) {
+			if (DEBUG) {
+				globalTileIndex = {};
+				globalTileNightIndex = {};
+			}
 			var map = mapInfo.maps[i];
 			var layer = map.surfaces[surface];
 			if (!globalTileNightIndex[surface]) {
@@ -307,7 +314,7 @@
 					var LLayer = L.tileLayer(undefined, {
 						id: layer.path,
 						attribution: '<a href="https://github.com/L0laapk3/FactorioMods_FactorioMaps">FactorioMaps</a>',
-						minNativeZoom: layer.zoom.min,
+						minNativeZoom: DEBUG ? 20 : layer.zoom.min,
 						maxNativeZoom: layer.zoom.max,
 						minZoom: layer.zoom.min >= 1 ? layer.zoom.min - 1 : 1,
 						maxZoom: 23, //layer.zoom.max + 2,

info.json

@@ -1,6 +1,6 @@
 {
     "name": "L0laapk3_FactorioMaps",
-    "version": "2.2.1",
+    "version": "2.3.0",
     "title": "FactorioMaps",
     "author": "L0laapk3",
     "contact": "https://github.com/L0laapk3/",

ref.py

@@ -5,23 +5,21 @@
 
 
 
-
 ext = ".bmp"
 outext = ".jpg"
 
 
 def compare(path, basePath, new, treshold):
 
     try:
-        #test = path[1:-1] + (path[-1].split(".")[0] + "dub.png",)
-        #print(test)
-        #diff = ImageChops.difference(Image.open(os.path.join(basePath, new, *path[1:])), Image.open(os.path.join(basePath, *path)))
-        #Image.open(os.path.join(basePath, *path)).save(os.path.join(basePath, new, *test))
-        #print(ImageStat.Stat(ImageChops.difference(Image.open(os.path.join(basePath, new, *path[1:])), Image.open(os.path.join(basePath, *path)))).sum2)
-        diff = ImageChops.difference(Image.open(os.path.join(basePath, new, *path[1:]), mode='r').replace(outext, ext), Image.open(os.path.join(basePath, *path), mode='r'))
-        #if sum(ImageStat.Stat(diff.copy().point(lambda x: 255 if x >= 16 else x ** 2)).sum2) + 256 * sum(ImageStat.Stat(diff.point(lambda x: x ** 2 >> 8)).sum2) > treshold:
+        newImg = Image.open(os.path.join(basePath, new, *path[1:]), mode='r')
+        oldImg = Image.open(os.path.join(basePath, *path).replace(ext, outext), mode='r')
+        size = (oldImg.size[0] / 8, oldImg.size[0] / 8)
+        newImg.thumbnail(size, Image.BILINEAR)
+        oldImg.thumbnail(size, Image.BILINEAR)
+        diff = ImageChops.difference(newImg, oldImg)
+
         if sum(ImageStat.Stat(diff).sum2) > treshold:
-            #print("%s %s" % (total, path))
             return (True, path[1:])
     except IOError:
         print("error")
@@ -148,7 +146,7 @@ def getBase64(number, isNight): #coordinate to 18 bit value (3 char base64)
                             path = os.path.join(toppath, "Images", data["maps"][old]["path"], surfaceName, daytime, str(z))
                             for x in os.listdir(path):
                                 for y in os.listdir(os.path.join(path, x)):
-                                    oldImages[(x, y)] = data["maps"][old]["path"]
+                                    oldImages[(x, y.replace(ext, outext))] = data["maps"][old]["path"]
 
 
                     if daytime != "day":
@@ -173,7 +171,7 @@ def getBase64(number, isNight): #coordinate to 18 bit value (3 char base64)
                             if (x, os.path.splitext(y)[0]) in dayImages or (x, y.replace(ext, outext)) not in oldImages:
                                 keepList.append((surfaceName, daytime, str(z), x, y))
                             elif (x, y.replace(ext, outext)) in oldImages:
-                                compareList.append((oldImages[(x, y)], surfaceName, daytime, str(z), x, y))
+                                compareList.append((oldImages[(x, y.replace(ext, outext))], surfaceName, daytime, str(z), x, y))
 
 
 
@@ -185,7 +183,11 @@ def getBase64(number, isNight): #coordinate to 18 bit value (3 char base64)
         print("found %s new images" % len(keepList))
         if len(compareList) > 0:
             print("comparing %s existing images" % len(compareList))
-            resultList = pool.map(partial(compare, treshold=20*Image.open(os.path.join(toppath, "Images", *compareList[0])).size[0] ** 2, basePath=os.path.join(toppath, "Images"), new=str(newMap["path"])), compareList, 128)
+            treshold = .3 * Image.open(os.path.join(toppath, "Images", *compareList[0]).replace(ext, outext)).size[0] ** 2
+            print(treshold)
+            #compare(compareList[0], treshold=treshold, basePath=os.path.join(toppath, "Images"), new=str(newMap["path"]))
+            resultList = pool.map(partial(compare, treshold=treshold, basePath=os.path.join(toppath, "Images"), new=str(newMap["path"])), compareList, 128)
+
             newList = map(lambda x: x[1], filter(lambda x: x[0], resultList))
             firstRemoveList += map(lambda x: x[1], filter(lambda x: not x[0], resultList))
             print("found %s changed in %s images" % (len(newList), len(compareList)))

updates.json

@@ -8,5 +8,6 @@
     "2.0.0": "Added HD mode! Also allow more zoom levels in general.",
     "2.1.0": "Block all 404 request, The client now knows exactly which images exist. (Breaks old maps)",
     "2.2.0": "Significantly increased image quality and file size at the cost of some processing time.",
-    "2.2.1": "Fixed problem with missing graphics."
+    "2.2.1": "Fixed problem with missing graphics.",
+    "2.3.0": "Fixed problems with the deduplicator."
 }

zoom.py

@@ -48,8 +48,10 @@ def work(basepath, pathList, surfaceName, daytime, start, stop, last, chunk):
 						except OSError:
 							pass
 
+					isOriginal = []
 					for m in range(len(coords)):
-						if not os.path.isfile(paths[m]):
+						isOriginal.append(os.path.isfile(paths[m]))
+						if not isOriginal[m]:
 							for n in range(1, len(pathList)):
 								paths[m] = os.path.join(basepath, pathList[n], surfaceName, daytime, str(k), str(i+coords[m][0]), str(j+coords[m][1]) + outext)
 								if os.path.isfile(paths[m]):
@@ -60,15 +62,16 @@ def work(basepath, pathList, surfaceName, daytime, start, stop, last, chunk):
 					size = 0
 
 					imgs = []
-					for m in range(4):
+					for m in range(len(coords)):
 						if (os.path.isfile(paths[m])):
 							img = Image.open(paths[m], mode='r').convert("RGB")
 							if size == 0:
 								size = img.size[0]
 								result = Image.new('RGB', (size, size), (27, 45, 51))
 							result.paste(box=(coords[m][0]*size/2, coords[m][1]*size/2), im=img.resize((size/2, size/2), Image.ANTIALIAS))
 
-							imgs.append((img, paths[m]))
+							if isOriginal[m]:
+								imgs.append((img, paths[m]))
 
 
 					if outext != ext and k == last+1:
@@ -108,7 +111,7 @@ def thread(basepath, pathList, surfaceName, daytime, start, stop, last, allChunk
 	maxthreads = mp.cpu_count()
 
 
-	#print(basepath)
+	print(basepath)
 
 
 	with open(datapath, "r") as f: