From 8738a9d40f916ab8bbc868801bb99fbfd921b415 Mon Sep 17 00:00:00 2001 From: Zhao Zuohong Date: Sun, 14 Jul 2024 23:40:21 +0800 Subject: [PATCH] =?UTF-8?q?=E7=89=B9=E5=BE=81=E5=8C=B9=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- _includes/common/mathjax.liquid | 4 +- _includes/feature-matching.html | 10 +- _includes/matcher.html | 441 ++++++++++++++++++++++++++++++++ dev/feature-matching.md | 44 +++- 4 files changed, 489 insertions(+), 10 deletions(-) create mode 100644 _includes/matcher.html diff --git a/_includes/common/mathjax.liquid b/_includes/common/mathjax.liquid index 4291f9cb..85f379d2 100644 --- a/_includes/common/mathjax.liquid +++ b/_includes/common/mathjax.liquid @@ -1,6 +1,6 @@ -{%- if page.content contains "$$" -%} +{%- if page.content contains "$" -%} {%- endif %} diff --git a/_includes/feature-matching.html b/_includes/feature-matching.html index c766b681..0af5e690 100644 --- a/_includes/feature-matching.html +++ b/_includes/feature-matching.html @@ -6,7 +6,7 @@ 
import cv2
 from matplotlib import pyplot as plt
 from arknights_mower.utils.image import loadimg
-from arknights_mower.utils.matcher import ORB, flann
+from arknights_mower.utils.matcher import GOOD_DISTANCE_LIMIT, ORB, flann
@@ -33,7 +33,7 @@
-
2024-07-13 19:59:06,194 - DEBUG - /home/zhao/Documents/arknights-mower/arknights_mower/utils/image.py:44 - loadimg - /home/zhao/Documents/hot_update/hortus/terminal.jpg
+
2024-07-14 11:28:16,539 - DEBUG - /home/zhao/Documents/arknights-mower/arknights_mower/utils/image.py:44 - loadimg - /home/zhao/Documents/hot_update/hortus/terminal.jpg
@@ -73,7 +73,7 @@
-
2024-07-13 19:59:06,644 - DEBUG - /home/zhao/Documents/arknights-mower/arknights_mower/utils/image.py:44 - loadimg - /home/zhao/Documents/mower-profile/screenshot/501/20240705031952.png
+
2024-07-14 11:28:16,986 - DEBUG - /home/zhao/Documents/arknights-mower/arknights_mower/utils/image.py:44 - loadimg - /home/zhao/Documents/mower-profile/screenshot/501/20240705031952.png
@@ -98,7 +98,6 @@ 
matches = flann.knnMatch(des1, des2, k=2)
-GOOD_DISTANCE_LIMIT = 0.7
 good = []
 for pair in matches:
     if (len_pair := len(pair)) == 2:
@@ -189,7 +188,7 @@
 

 

 

-
2024-07-13 19:59:08,194 - DEBUG - /home/zhao/Documents/arknights-mower/arknights_mower/utils/image.py:44 - loadimg - /home/zhao/Documents/mower-profile/screenshot/501/20240705071219.png
+
2024-07-14 11:28:18,374 - DEBUG - /home/zhao/Documents/arknights-mower/arknights_mower/utils/image.py:44 - loadimg - /home/zhao/Documents/mower-profile/screenshot/501/20240705071219.png
 

 

 

@@ -214,7 +213,6 @@
 

 

 
matches = flann.knnMatch(des1, des2, k=2)
-GOOD_DISTANCE_LIMIT = 0.7
 good = []
 for pair in matches:
     if (len_pair := len(pair)) == 2:
diff --git a/_includes/matcher.html b/_includes/matcher.html
new file mode 100644
index 00000000..c99e1f36
--- /dev/null
+++ b/_includes/matcher.html
@@ -0,0 +1,441 @@
+

+

+
In [1]:

+

+

+
from matplotlib import pyplot as plt
+from arknights_mower.utils.image import cropimg, loadimg, loadres
+from arknights_mower.utils.matcher import Matcher
+

+

+

+

+

+

+

+
In [2]:

+

+

+
sc = loadimg("/home/zhao/Documents/mower-profile/screenshot/202/20240714174003.png", True)
+plt.imshow(sc, cmap="gray", vmin=0, vmax=255)
+plt.show()
+

+

+

+

+

+

+

+

+

+
2024-07-14 23:25:31,654 - DEBUG - /home/zhao/Documents/arknights-mower/arknights_mower/utils/image.py:44 - loadimg - /home/zhao/Documents/mower-profile/screenshot/202/20240714174003.png
+

+

+

+

+

+

+No description has been provided for this image
+

+

+

+

+

+

+

+
In [3]:

+

+

+
res = loadres("infra_collect_factory", True)
+plt.imshow(res, cmap="gray", vmin=0, vmax=255)
+plt.show()
+

+

+

+

+

+

+

+

+

+
2024-07-14 23:25:31,938 - DEBUG - /home/zhao/Documents/arknights-mower/arknights_mower/utils/image.py:44 - loadimg - /home/zhao/Documents/arknights-mower/arknights_mower/resources/infra_collect_factory.png
+

+

+

+

+

+

+No description has been provided for this image
+

+

+

+

+

+

+

+
In [4]:

+

+

+
matcher = Matcher(sc)
+

+

+

+

+

+

+

+

+

+
2024-07-14 23:25:32,060 - DEBUG - /home/zhao/Documents/arknights-mower/arknights_mower/utils/matcher.py:66 - __init__ - Matcher init: shape ((1080, 1920))
+

+

+

+

+

+

+

+

+
In [5]:

+

+

+
matcher.match(res, draw=True)
+

+

+

+

+

+

+

+

+

+No description has been provided for this image
+

+

+

+

+

+
2024-07-14 23:25:32,428 - DEBUG - /home/zhao/Documents/arknights-mower/arknights_mower/utils/matcher.py:192 - score - transform matrix: [[0.9981798972146724, 0.0034091767564587566, 242.9226171323746], [-0.00882626456769677, 1.00219164365151, 985.2447691724211]]
+

+

+

+

+

+

+No description has been provided for this image
+

+

+

+

+

+No description has been provided for this image
+

+

+

+

+

+
2024-07-14 23:25:32,828 - DEBUG - /home/zhao/Documents/arknights-mower/arknights_mower/utils/matcher.py:106 - match - match success: ([[243, 985], [321, 1063]], (0.26649076517150394, 0.7210257114508213, 1.0, 0.9736551440651242))
+

+

+

+

+
Out[5]:

+

+
[[243, 985], [321, 1063]]

+

+

+

+

+

+

+

+
In [6]:

+

+

+
sc = loadimg("/home/zhao/Documents/mower-profile/screenshot/201/20240714214339.png", True)
+plt.imshow(sc, cmap="gray", vmin=0, vmax=255)
+plt.show()
+

+

+

+

+

+

+

+

+

+
2024-07-14 23:25:32,838 - DEBUG - /home/zhao/Documents/arknights-mower/arknights_mower/utils/image.py:44 - loadimg - /home/zhao/Documents/mower-profile/screenshot/201/20240714214339.png
+

+

+

+

+

+

+No description has been provided for this image
+

+

+

+

+

+

+

+
In [7]:

+

+

+
res = loadres("control_central", True)
+plt.imshow(res, cmap="gray", vmin=0, vmax=255)
+plt.show()
+

+

+

+

+

+

+

+

+

+
2024-07-14 23:25:33,067 - DEBUG - /home/zhao/Documents/arknights-mower/arknights_mower/utils/image.py:44 - loadimg - /home/zhao/Documents/arknights-mower/arknights_mower/resources/control_central.png
+

+

+

+

+

+

+No description has been provided for this image
+

+

+

+

+

+

+

+
In [8]:

+

+

+
matcher = Matcher(sc)
+

+

+

+

+

+

+

+

+

+
2024-07-14 23:25:33,212 - DEBUG - /home/zhao/Documents/arknights-mower/arknights_mower/utils/matcher.py:66 - __init__ - Matcher init: shape ((1080, 1920))
+

+

+

+

+

+

+

+

+
In [9]:

+

+

+
matcher.match(res, draw=True, dpi_aware=True)
+

+

+

+

+

+

+

+

+

+No description has been provided for this image
+

+

+

+

+

+
2024-07-14 23:25:33,683 - DEBUG - /home/zhao/Documents/arknights-mower/arknights_mower/utils/matcher.py:192 - score - transform matrix: [[0.7544874125196873, -0.002213000681994066, 1097.881567848667], [0.0007849747819058742, 0.7567520230790792, 238.5087977188498]]
+

+

+

+

+

+

+No description has been provided for this image
+

+

+

+

+

+No description has been provided for this image
+

+

+

+

+

+
2024-07-14 23:25:34,088 - DEBUG - /home/zhao/Documents/arknights-mower/arknights_mower/utils/matcher.py:106 - match - match success: ([[1098, 239], [1294, 359]], (0.1488994389296504, 0.8881538977989784, 1.0, 0.8984694173297328))
+

+

+

+

+
Out[9]:

+

+
[[1098, 239], [1294, 359]]

+

+

+

+

+

+

+

+
In [10]:

+

+

+
sc = loadimg("/home/zhao/Documents/mower-profile/screenshot/1105/20240712065207.png", True)
+plt.imshow(sc, cmap="gray", vmin=0, vmax=255)
+plt.show()
+

+

+

+

+

+

+

+

+

+
2024-07-14 23:25:34,097 - DEBUG - /home/zhao/Documents/arknights-mower/arknights_mower/utils/image.py:44 - loadimg - /home/zhao/Documents/mower-profile/screenshot/1105/20240712065207.png
+

+

+

+

+

+

+No description has been provided for this image
+

+

+

+

+

+

+

+
In [11]:

+

+

+
res = loadimg("/home/zhao/Documents/mower-profile/screenshot/1106/20240712065212.png", True)
+plt.imshow(res, cmap="gray", vmin=0, vmax=255)
+plt.show()
+

+

+

+

+

+

+

+

+

+
2024-07-14 23:25:34,344 - DEBUG - /home/zhao/Documents/arknights-mower/arknights_mower/utils/image.py:44 - loadimg - /home/zhao/Documents/mower-profile/screenshot/1106/20240712065212.png
+

+

+

+

+

+

+No description has been provided for this image
+

+

+

+

+

+

+

+
In [12]:

+

+

+
res = cropimg(res, ((1020, 230), (1210, 280)))
+plt.imshow(res, cmap="gray", vmin=0, vmax=255)
+plt.show()
+

+

+

+

+

+

+

+

+

+No description has been provided for this image
+

+

+

+

+

+

+

+
In [13]:

+

+

+
matcher = Matcher(sc)
+

+

+

+

+

+

+

+

+

+
2024-07-14 23:25:34,672 - DEBUG - /home/zhao/Documents/arknights-mower/arknights_mower/utils/matcher.py:66 - __init__ - Matcher init: shape ((1080, 1920))
+

+

+

+

+

+

+

+

+
In [14]:

+

+

+
matcher.match(res, draw=True)
+

+

+

+

+

+

+

+

+

+No description has been provided for this image
+

+

+

+

+

+
2024-07-14 23:25:35,040 - DEBUG - /home/zhao/Documents/arknights-mower/arknights_mower/utils/matcher.py:192 - score - transform matrix: [[1.1197843731842343, -0.0005674608950607882, 1013.6984387283818], [-0.0015298378674758022, 1.12178520790043, 610.6445517137909]]
+

+

+

+

+

+

+No description has been provided for this image
+

+

+

+

+

+No description has been provided for this image
+

+

+

+

+

+
2024-07-14 23:25:35,396 - DEBUG - /home/zhao/Documents/arknights-mower/arknights_mower/utils/matcher.py:106 - match - match success: ([[1014, 611], [1227, 667]], (0.09854604200323101, 0.6821052752043072, 0.9375, 0.7308784325378576))
+

+

+

+

+
Out[14]:

+

+
[[1014, 611], [1227, 667]]

+

+

+

+

+

diff --git a/dev/feature-matching.md b/dev/feature-matching.md
index afbace4b..ddcbdbd0 100644
--- a/dev/feature-matching.md
+++ b/dev/feature-matching.md
@@ -8,7 +8,7 @@ sort: 5
 
 ## 特征与特征点
 
-可以直接使用 `arknights_mower.utils.matcher` 中的 `ORB` 提取特征点。
+`arknights_mower.utils.matcher` 中的 `ORB` 可用于提取特征点。
 
 一般而言，在图像中，拐角、复杂的图案与纹理（包括文字）处可以提取到较多的特征点；在空白处，很难或无法提取到特征点。从复杂的图像中可以提取到更多的特征点。
 
@@ -19,9 +19,49 @@ sort: 5
 
 ## 特征点的匹配
 
-可以直接使用 `arknights_mower.utils.matcher` 中的 `flann` 对特征点进行匹配。
+`arknights_mower.utils.matcher` 中的 `flann` 可用于匹配特征点。
+
+对于目标图像中的每个特征点 $A$，使用 `flann.knnMatch(k=2)`找出截图中与之距离最近的和第二近的两个特征点 $B_1$、$B_2$. 如果 $A$ 与 $B_1$ 的距离 $\text{d}(A, B_1)$ 和 $A$ 与 $B_2$ 的距离 $\text{d}(A, B_2)$ 的比值小于 `GOOD_DISTANCE_LIMIT`，就认为 $A$ 与 $B_1$ 是一对“好”的匹配。
+
+下面的例子展示了如何利用特征匹配在终端页面定位活动入口。
 
 

 使用示例
 {% include feature-matching.html %}
 

+
+## 利用特征匹配定位目标图像
+
+`arknights_mower.utils.matcher` 中的 `Matcher` 类可用于定位目标图片。
+
+实例化 `Matcher` 类时，需传入灰度图像 `origin`。实例化过程中计算 `origin` 图像的特征点。
+
+`Matcher` 类的实例方法 `match()` 接受 6 个参数，其中 `query` 必选：
+
+- `query`：灰度目标图像；
+- `draw`：控制是否绘制并显示匹配过程；
+- `scope`：`origin` 的特征点中，在此区域内的特征点参与匹配；
+- `dpi_aware`：大部分元素的尺寸不变，`dpi_aware` 默认为 `False`，若匹配结果与目标图像的尺寸相差较大时拒绝结果。对于尺寸有变化的元素，将此选项设置为 `True`；
+- `prescore`：SSIM 分数阈值。如果此参数为正，则根据 SSIM 分数决定是否接受匹配结果。
+- `judge`：在 `prescore` 为 0 时生效。如果为 `True`，则使用支持向量机判断是否接受匹配结果，否则直接接受匹配结果。
+
+如果匹配成功，`match()` 返回目标图片在截图中匹配到的区域；否则返回 `None`。
+
+

+使用示例
+{% include matcher.html %}
+

+
+## 注意事项
+
+### 性能
+
+从截图提取特征点一般会花费数十毫秒，一次 FLANN 匹配也会花费十几毫秒到几十毫秒。大量使用特征匹配会导致脚本很慢。
+
+### 目标图像截取
+
+目标图像应尽量满足特征点数量多、尺寸小、与其它目标图像有较大区别。例如对于按钮，只截取按钮中独特的图案，或在文字中截取 3-5 字的关键词，相比截取整个按钮，往往能得到更好的效果。
+
+### 随机性
+
+FLANN 和 RANSAC 算法具有一定的随机性。若结果变化较大，可考虑重新截取目标图像，或换用其它匹配方式。