Skip to content

Commit

Permalink
feat(core): use trim strategy by default
Browse files Browse the repository at this point in the history
  • Loading branch information
@marcincichocki
marcincichocki authored Dec 17, 2023
1 parent 6aeb633 commit 64cc08c
Show file tree
Hide file tree
Showing 15 changed files with 125 additions and 257 deletions.
15 changes: 9 additions & 6 deletions src/common/node/sharp-image-container.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,14 @@ class SharpFragmentContainer implements FragmentContainer<sharp.Sharp> {
return new SharpFragmentContainer(this.original);
}

async toBase64({ trim }: { trim?: boolean } = {}) {
async toTrimmedBase64(threshold: number) {
const buffer = await this.instance.toBuffer();

if (trim) {
return this.trim(buffer);
}
return this.trim(buffer, threshold);
}

async toBase64() {
const buffer = await this.instance.toBuffer();
const { width, height, format } = await this.instance.metadata();
const data = buffer.toString('base64');
const uri = toBase64DataUri(format, data);
Expand Down Expand Up @@ -54,12 +55,14 @@ class SharpFragmentContainer implements FragmentContainer<sharp.Sharp> {
return this;
}

private async trim(buffer: Buffer) {
private async trim(buffer: Buffer, threshold: number) {
try {
const {
data,
info: { format, width, height },
} = await sharp(buffer).trim().toBuffer({ resolveWithObject: true });
} = await sharp(buffer)
.trim({ threshold })
.toBuffer({ resolveWithObject: true });
const uri = toBase64DataUri(format, data.toString('base64'));
const dimensions = { width, height };

Expand Down
Binary file added src/core/bp-registry/2560x1440/patch-2.1g-default.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added src/core/bp-registry/2560x1440/patch-2.1g-max.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added src/core/bp-registry/2560x1440/patch-2.1g-min.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
62 changes: 61 additions & 1 deletion src/core/bp-registry/registry.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,9 @@
"thresholdGrid": 155,
"thresholdGridAuto": false,
"thresholdDaemons": 140,
"thresholdDaemonsAuto": false
"thresholdDaemonsAuto": false,
"thresholdBufferSizeAuto": false,
"thresholdBufferSize": 10
}
}
],
Expand Down Expand Up @@ -421,6 +423,64 @@
"BD", "1C", "1C", "1C", "55"
],
"bufferSize": 7
},
{
"path": "2560x1440/patch-2.1g-default.png",
"daemons": [
["BD", "7A"],
["7A", "BD"],
["55", "7A", "7A"]
],
"grid": [
"1C", "7A", "55", "7A", "1C", "55", "BD",
"55", "BD", "E9", "55", "E9", "BD", "1C",
"55", "FF", "E9", "7A", "E9", "BD", "FF",
"55", "BD", "7A", "FF", "7A", "E9", "BD",
"E9", "7A", "E9", "55", "1C", "7A", "7A",
"E9", "1C", "55", "BD", "E9", "7A", "BD",
"7A", "7A", "7A", "7A", "1C", "55", "1C"
],
"bufferSize": 8
},
{
"path": "2560x1440/patch-2.1g-max.png",
"daemons": [
["FF", "7A"],
["1C", "E9"],
["1C", "55", "1C"]
],
"grid": [
"7A", "1C", "7A", "1C", "1C", "55", "7A",
"1C", "55", "E9", "55", "55", "FF", "1C",
"FF", "7A", "7A", "55", "E9", "BD", "55",
"55", "1C", "7A", "FF", "55", "7A", "55",
"BD", "1C", "BD", "7A", "1C", "BD", "7A",
"FF", "55", "FF", "1C", "FF", "1C", "E9",
"FF", "1C", "55", "1C", "7A", "1C", "55"
],
"bufferSize": 8,
"settings": {
"thresholdDaemonsAuto": false,
"thresholdDaemons": 200
}
},
{
"path": "2560x1440/patch-2.1g-min.png",
"daemons": [
["1C", "7A"],
["E9", "7A"],
["7A", "E9", "E9"]
],
"grid": [
"BD", "1C", "55", "7A", "FF", "BD", "E9",
"1C", "E9", "E9", "E9", "1C", "1C", "7A",
"E9", "BD", "FF", "BD", "1C", "1C", "55",
"1C", "1C", "1C", "1C", "7A", "E9", "FF",
"1C", "55", "BD", "55", "7A", "7A", "7A",
"FF", "E9", "55", "1C", "7A", "E9", "E9",
"E9", "7A", "E9", "E9", "1C", "BD", "7A"
],
"bufferSize": 8
}
],
"3440x1440": [
Expand Down
199 changes: 20 additions & 179 deletions src/core/ocr/fragments/buffer-size-fragment.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@ import { BufferSize, BUFFER_SIZE_MAX, BUFFER_SIZE_MIN } from '../../common';
import { BreachProtocolFragment, BreachProtocolFragmentResult } from './base';
import { FragmentId, FragmentStatus } from './fragment';

export abstract class BreachProtocolBufferSizeBase<
export type BreachProtocolBufferSizeFragmentResult =
BreachProtocolFragmentResult<BufferSize, FragmentId.BufferSize>;

export class BreachProtocolBufferSizeFragment<
TImage
> extends BreachProtocolFragment<BufferSize, TImage, FragmentId.BufferSize> {
readonly id = FragmentId.BufferSize;
Expand All @@ -16,11 +19,6 @@ export abstract class BreachProtocolBufferSizeBase<

readonly boundingBox = this.getFragmentBoundingBox();

protected readonly fragment = this.container.toFragmentContainer({
boundingBox: this.boundingBox,
colors: 2,
});

/** Percentage that padding in buffer box takes. */
protected readonly padding = 0.00937;

Expand All @@ -30,191 +28,34 @@ export abstract class BreachProtocolBufferSizeBase<
/** Percentage that gap between buffer squares takes. */
protected readonly gap = 0.00546;

getStatus(n: number) {
if (
!Number.isInteger(n) ||
n < BUFFER_SIZE_MIN ||
(!this.options.extendedBufferSizeRecognitionRange && n > BUFFER_SIZE_MAX)
) {
return FragmentStatus.InvalidSize;
}

return FragmentStatus.Valid;
}
}

class BufferSizeControlGroup {
/** Thickness of control group in pixels. */
// 7 pixels is max on lowest resolution, anything more will
// interfere with boxes, and control group will always fail.
private readonly size = 7;

constructor(
private start: number,
private end: number,
private value: number
) {}

private getLineIndexes(length: number) {
return [...Array(this.size)].map((x, i) => i * length);
}

/** Check if every pixel in control group has given value. */
verify(data: Uint8Array, rowLength: number) {
const startIndex = Math.round(this.start * rowLength);
const endIndex = Math.round(this.end * rowLength);

return this.getLineIndexes(rowLength)
.map((n) => data.subarray(startIndex + n, endIndex + n))
.every((line) => line.every((p) => p === this.value));
}
}

export type BreachProtocolBufferSizeFragmentResult =
BreachProtocolFragmentResult<BufferSize, FragmentId.BufferSize>;

export class BreachProtocolBufferSizeFragment<
TImage
> extends BreachProtocolBufferSizeBase<TImage> {
private readonly controlGroups = [
// Buffer boxes.
new BufferSizeControlGroup(
this.options.extendedBufferSizeRecognitionRange ? 0.102 : 0.12,
this.options.extendedBufferSizeRecognitionRange ? 0.189 : 0.22,
255
),
// End of fragment.
new BufferSizeControlGroup(
this.options.extendedBufferSizeRecognitionRange ? 0.9 : 0.7,
1,
0
),
];

private static cachedThreshold: number = null;

private async checkControlGroupsForThreshold(threshold: number) {
const data = await this.fragment.clone().threshold(threshold).toPixelData();

return this.controlGroups.map((cg) =>
cg.verify(data, this.boundingBox.width)
) as [boolean, boolean];
}

// Run binary search and check control groups to narrow
// correct result. Since thresholds only depend on gamma,
// values below 128 are excluded to speed up search(lowest
// gamma 0.5 require ~160 threshold).
private async findThreshold() {
const base = 128;
let start = 0;
let end = base - 1;
let i = 0;

do {
let m = Math.ceil((start + end) / 2);
const threshold = m + base;
const [cg1, cg2] = await this.checkControlGroupsForThreshold(threshold);

if (cg1 && cg2) {
return threshold;
}

if (!cg1) {
// First control group has some black pixels, threshold is too high.
end = m - 1;
} else if (!cg2) {
// Second control group has some white pixels, threshold is too low.
start = m + 1;
}
} while (++i < Math.log2(base));

// No threshold found.
return base;
}

async recognize(
fixedThreshold = BreachProtocolBufferSizeFragment.cachedThreshold,
useFallback = true
): Promise<BreachProtocolBufferSizeFragmentResult> {
const threshold = fixedThreshold ?? (await this.findThreshold());
const { uri } = await this.fragment.threshold(threshold).toBase64();
const data = await this.fragment.toPixelData();
const bufferSize = this.getBufferSizeFromPixels(data);

if (this.getStatus(bufferSize) !== FragmentStatus.Valid) {
// In rare cases where given value is wrong, repeat with
// binary search. For example when user changes gamma mid
// game, saved value will be wrong. This allows to re-calibrate
// threshold on the fly.
// One side effect of this behavior is that --threshold-buffer-size
// flag is quite useless, because even it fails, fallback will be used.
if (useFallback && fixedThreshold !== null) {
return this.recognize(null);
}
}

// Cache valid threshold to limit amount of computation required on following breach protocols.
BreachProtocolBufferSizeFragment.cachedThreshold = threshold;

return this.getFragmentResult(null, bufferSize, uri, threshold);
}

private verifyControlGroups(data: Uint8Array, length: number) {
return this.controlGroups.every((cg) => cg.verify(data, length));
}

private getSizeOfBufferBox(data: Uint8Array, width: number) {
const row = data.subarray(0, width);
let size = 0;

if (!this.verifyControlGroups(data, width)) {
return 0;
}

for (let i = 0; i < row.length; i++) {
if (row[i] === 255) {
size += 1;
}
}

return size;
}

private getBufferSizeFromPixels(data: Uint8Array) {
const { width, innerWidth } = this.boundingBox;
let size = this.getSizeOfBufferBox(data, width) / innerWidth;
let bufferSize = 0;

size -= 2 * this.padding;

while (size > 0) {
size -= this.square + this.gap;
bufferSize += 1;
}

return bufferSize as BufferSize;
}
}

export class BreachProtocolBufferSizeTrimFragment<
TImage
> extends BreachProtocolBufferSizeBase<TImage> {
override readonly fragment = this.container.toFragmentContainer({
readonly fragment = this.container.toFragmentContainer({
boundingBox: this.boundingBox,
flop: true,
});

// Ensure compatibility with current api.
async recognize(
threshold?: number
): Promise<BreachProtocolBufferSizeFragmentResult> {
const { uri, dimensions } = await this.fragment.toBase64({ trim: true });
const { uri, dimensions } = await this.fragment.toTrimmedBase64(
threshold ?? 30
);
const bufferSize = await this.getBufferSizeFromPixels(dimensions.width);

return this.getFragmentResult(null, bufferSize, uri, null);
}

getStatus(n: number) {
if (
!Number.isInteger(n) ||
n < BUFFER_SIZE_MIN ||
(!this.options.extendedBufferSizeRecognitionRange && n > BUFFER_SIZE_MAX)
) {
return FragmentStatus.InvalidSize;
}

return FragmentStatus.Valid;
}

private async getBufferSizeFromPixels(width: number) {
const { innerWidth } = this.boundingBox;

Expand Down
7 changes: 4 additions & 3 deletions src/core/ocr/image-container.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,11 @@ export interface FragmentContainer<T> {
/** Applies threshold transformation to fragment. */
threshold(threshold: number, grayscale?: boolean): this;

/** Returns fragment as trimmed base64 data uri. */
toTrimmedBase64(threshold: number): Promise<EncodedFragmentContainerResult>;

/** Returns fragment as base64 data uri. */
toBase64(options?: {
trim?: boolean;
}): Promise<EncodedFragmentContainerResult>;
toBase64(): Promise<EncodedFragmentContainerResult>;

/** Returns fragment as raw pixel data. */
toPixelData(): Promise<Uint8Array>;
Expand Down
Loading

0 comments on commit 64cc08c

Please sign in to comment.