From 7975cef124902b332099aba18c8c2e98a9866ef8 Mon Sep 17 00:00:00 2001
From: L0laapk3 <L0laapk3@gmail.com>
Date: Sat, 22 Dec 2018 00:36:51 +0100
Subject: [PATCH] fixed deduper problems

---
 autorun.template.lua |  2 +-
 index.html.template  |  9 ++++++++-
 info.json            |  2 +-
 ref.py               | 26 ++++++++++++++------------
 updates.json         |  3 ++-
 zoom.py              | 11 +++++++----
 6 files changed, 33 insertions(+), 20 deletions(-)

diff --git a/autorun.template.lua b/autorun.template.lua
index d81b91c..f9ab4cf 100644
--- a/autorun.template.lua
+++ b/autorun.template.lua
@@ -1,7 +1,7 @@
 fm.autorun = {
     HD = false,
     day = true,                     -- note: snapshots in the same timeline should have the same day/night settings for now (this will be possible in the future)
-    night = true,
+    night = false,
     around_build_range = 5.2,		-- max range from buildings that images will be saved. Feel free to crank this up to very large numbers, it will only render chunks that already exist, it will not generate new ones.
     around_smaller_range = 1.2,		-- same as above, but smaller range for the following entity types:
     smaller_types = {"lamp", "electric-pole", "radar", "straight-rail", "curved-rail", "rail-signal", "rail-chain-signal", "locomotive", "cargo-wagon", "fluid-wagon", "car"},
diff --git a/index.html.template b/index.html.template
index 19baa98..87ee748 100644
--- a/index.html.template
+++ b/index.html.template
@@ -208,6 +208,9 @@
 	<div id="map" style="background: #1B2D33;"></div>
 	<script>
 		"use strict";
+
+		
+		var DEBUG = false;
 		
 
 		//var _getTileUrl = L.TileLayer.prototype.getTileUrl;
@@ -227,6 +230,10 @@
 		var globalTileIndex = {};
 		var globalTileNightIndex = {};
 		for (var i = 0; i < mapInfo.maps.length; i++) {
+			if (DEBUG) {
+				globalTileIndex = {};
+				globalTileNightIndex = {};
+			}
 			var map = mapInfo.maps[i];
 			var layer = map.surfaces[surface];
 			if (!globalTileNightIndex[surface]) {
@@ -307,7 +314,7 @@
 					var LLayer = L.tileLayer(undefined, {
 						id: layer.path,
 						attribution: '<a href="https://github.com/L0laapk3/FactorioMods_FactorioMaps">FactorioMaps</a>',
-						minNativeZoom: layer.zoom.min,
+						minNativeZoom: DEBUG ? 20 : layer.zoom.min,
 						maxNativeZoom: layer.zoom.max,
 						minZoom: layer.zoom.min >= 1 ? layer.zoom.min - 1 : 1,
 						maxZoom: 23, //layer.zoom.max + 2,
diff --git a/info.json b/info.json
index 6d3a3e0..104c502 100644
--- a/info.json
+++ b/info.json
@@ -1,6 +1,6 @@
 {
     "name": "L0laapk3_FactorioMaps",
-    "version": "2.2.1",
+    "version": "2.3.0",
     "title": "FactorioMaps",
     "author": "L0laapk3",
     "contact": "https://github.com/L0laapk3/",
diff --git a/ref.py b/ref.py
index 6d268a4..3802f21 100644
--- a/ref.py
+++ b/ref.py
@@ -5,7 +5,6 @@
 
 
 
-
 ext = ".bmp"
 outext = ".jpg"
 
@@ -13,15 +12,14 @@
 def compare(path, basePath, new, treshold):
     
     try:
-        #test = path[1:-1] + (path[-1].split(".")[0] + "dub.png",)
-        #print(test)
-        #diff = ImageChops.difference(Image.open(os.path.join(basePath, new, *path[1:])), Image.open(os.path.join(basePath, *path)))
-        #Image.open(os.path.join(basePath, *path)).save(os.path.join(basePath, new, *test))
-        #print(ImageStat.Stat(ImageChops.difference(Image.open(os.path.join(basePath, new, *path[1:])), Image.open(os.path.join(basePath, *path)))).sum2)
-        diff = ImageChops.difference(Image.open(os.path.join(basePath, new, *path[1:]), mode='r').replace(outext, ext), Image.open(os.path.join(basePath, *path), mode='r'))
-        #if sum(ImageStat.Stat(diff.copy().point(lambda x: 255 if x >= 16 else x ** 2)).sum2) + 256 * sum(ImageStat.Stat(diff.point(lambda x: x ** 2 >> 8)).sum2) > treshold:
+        newImg = Image.open(os.path.join(basePath, new, *path[1:]), mode='r')
+        oldImg = Image.open(os.path.join(basePath, *path).replace(ext, outext), mode='r')
+        size = (oldImg.size[0] / 8, oldImg.size[0] / 8)
+        newImg.thumbnail(size, Image.BILINEAR)
+        oldImg.thumbnail(size, Image.BILINEAR)
+        diff = ImageChops.difference(newImg, oldImg)
+        
         if sum(ImageStat.Stat(diff).sum2) > treshold:
-            #print("%s %s" % (total, path))
             return (True, path[1:])
     except IOError:
         print("error")
@@ -148,7 +146,7 @@ def getBase64(number, isNight): #coordinate to 18 bit value (3 char base64)
                             path = os.path.join(toppath, "Images", data["maps"][old]["path"], surfaceName, daytime, str(z))
                             for x in os.listdir(path):
                                 for y in os.listdir(os.path.join(path, x)):
-                                    oldImages[(x, y)] = data["maps"][old]["path"]
+                                    oldImages[(x, y.replace(ext, outext))] = data["maps"][old]["path"]
 
 
                     if daytime != "day":
@@ -173,7 +171,7 @@ def getBase64(number, isNight): #coordinate to 18 bit value (3 char base64)
                             if (x, os.path.splitext(y)[0]) in dayImages or (x, y.replace(ext, outext)) not in oldImages:
                                 keepList.append((surfaceName, daytime, str(z), x, y))
                             elif (x, y.replace(ext, outext)) in oldImages:
-                                compareList.append((oldImages[(x, y)], surfaceName, daytime, str(z), x, y))
+                                compareList.append((oldImages[(x, y.replace(ext, outext))], surfaceName, daytime, str(z), x, y))
 
                
 
@@ -185,7 +183,11 @@ def getBase64(number, isNight): #coordinate to 18 bit value (3 char base64)
         print("found %s new images" % len(keepList))
         if len(compareList) > 0:
             print("comparing %s existing images" % len(compareList))
-            resultList = pool.map(partial(compare, treshold=20*Image.open(os.path.join(toppath, "Images", *compareList[0])).size[0] ** 2, basePath=os.path.join(toppath, "Images"), new=str(newMap["path"])), compareList, 128)
+            treshold = .3 * Image.open(os.path.join(toppath, "Images", *compareList[0]).replace(ext, outext)).size[0] ** 2
+            print(treshold)
+            #compare(compareList[0], treshold=treshold, basePath=os.path.join(toppath, "Images"), new=str(newMap["path"]))
+            resultList = pool.map(partial(compare, treshold=treshold, basePath=os.path.join(toppath, "Images"), new=str(newMap["path"])), compareList, 128)
+
             newList = map(lambda x: x[1], filter(lambda x: x[0], resultList))
             firstRemoveList += map(lambda x: x[1], filter(lambda x: not x[0], resultList))
             print("found %s changed in %s images" % (len(newList), len(compareList)))
diff --git a/updates.json b/updates.json
index f2b3734..7590390 100644
--- a/updates.json
+++ b/updates.json
@@ -8,5 +8,6 @@
     "2.0.0": "Added HD mode! Also allow more zoom levels in general.",
     "2.1.0": "Block all 404 request, The client now knows exactly which images exist. (Breaks old maps)",
     "2.2.0": "Significantly increased image quality and file size at the cost of some processing time.",
-    "2.2.1": "Fixed problem with missing graphics."
+    "2.2.1": "Fixed problem with missing graphics.",
+    "2.3.0": "Fixed problems with the deduplicator."
 }
\ No newline at end of file
diff --git a/zoom.py b/zoom.py
index 98ac116..14fdefd 100644
--- a/zoom.py
+++ b/zoom.py
@@ -48,8 +48,10 @@ def work(basepath, pathList, surfaceName, daytime, start, stop, last, chunk):
 						except OSError:
 							pass
 
+					isOriginal = []
 					for m in range(len(coords)):
-						if not os.path.isfile(paths[m]):
+						isOriginal.append(os.path.isfile(paths[m]))
+						if not isOriginal[m]:
 							for n in range(1, len(pathList)):
 								paths[m] = os.path.join(basepath, pathList[n], surfaceName, daytime, str(k), str(i+coords[m][0]), str(j+coords[m][1]) + outext)
 								if os.path.isfile(paths[m]):
@@ -60,7 +62,7 @@ def work(basepath, pathList, surfaceName, daytime, start, stop, last, chunk):
 					size = 0
 
 					imgs = []
-					for m in range(4):
+					for m in range(len(coords)):
 						if (os.path.isfile(paths[m])):
 							img = Image.open(paths[m], mode='r').convert("RGB")
 							if size == 0:
@@ -68,7 +70,8 @@ def work(basepath, pathList, surfaceName, daytime, start, stop, last, chunk):
 								result = Image.new('RGB', (size, size), (27, 45, 51))
 							result.paste(box=(coords[m][0]*size/2, coords[m][1]*size/2), im=img.resize((size/2, size/2), Image.ANTIALIAS))
 
-							imgs.append((img, paths[m]))
+							if isOriginal[m]:
+								imgs.append((img, paths[m]))
 
 
 					if outext != ext and k == last+1:
@@ -108,7 +111,7 @@ def thread(basepath, pathList, surfaceName, daytime, start, stop, last, allChunk
 	maxthreads = mp.cpu_count()
 
 
-	#print(basepath)
+	print(basepath)
 
 
 	with open(datapath, "r") as f: