forked from Kernel360/f1-JDON-Backend
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
69060c7
commit ba8fa0a
Showing
3 changed files
with
116 additions
and
35 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
40 changes: 40 additions & 0 deletions
40
...in/java/kernel/jdon/crawler/wanted/service/infrastructure/JobDetailProcessingCounter.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| package kernel.jdon.crawler.wanted.service.infrastructure; | ||
|
|
||
| import kernel.jdon.crawler.config.ScrapingWantedConfig; | ||
|
|
||
| public class JobDetailProcessingCounter { | ||
| private final int thresholdCount; | ||
| private final int failLimitCount; | ||
| private int sleepCount = 0; | ||
| private int consecutiveFailCount = 0; | ||
|
|
||
| public JobDetailProcessingCounter(ScrapingWantedConfig scrapingWantedConfig) { | ||
| this.thresholdCount = scrapingWantedConfig.getSleep().getThresholdCount(); | ||
| this.failLimitCount = scrapingWantedConfig.getLimit().getFailCount(); | ||
| } | ||
|
|
||
| public void incrementSleepCounter() { | ||
| sleepCount++; | ||
| } | ||
|
|
||
| public void resetSleepCounter() { | ||
| sleepCount = 0; | ||
| } | ||
|
|
||
| public void incrementFailCount() { | ||
| consecutiveFailCount++; | ||
| } | ||
|
|
||
| public void resetFailCount() { | ||
| consecutiveFailCount = 0; | ||
| } | ||
|
|
||
| public boolean isSleepRequired() { | ||
| return sleepCount == thresholdCount; | ||
| } | ||
|
|
||
| public boolean isBreakRequired() { | ||
| return consecutiveFailCount == failLimitCount; | ||
| } | ||
| } | ||
|
|
43 changes: 43 additions & 0 deletions
43
...main/java/kernel/jdon/crawler/wanted/service/infrastructure/JobListProcessingCounter.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| package kernel.jdon.crawler.wanted.service.infrastructure; | ||
|
|
||
| import java.util.LinkedHashSet; | ||
| import java.util.List; | ||
| import java.util.Set; | ||
|
|
||
| import kernel.jdon.crawler.config.ScrapingWantedConfig; | ||
|
|
||
| public class JobListProcessingCounter { | ||
| private final int maxFetchJDListSize; | ||
| private final int maxFetchJDListOffset; | ||
| private int offset = 0; | ||
| private Set<Long> fetchedJobIds = new LinkedHashSet<>(); | ||
|
|
||
| public JobListProcessingCounter(ScrapingWantedConfig scrapingWantedConfig) { | ||
| this.maxFetchJDListSize = scrapingWantedConfig.getMaxFetchJdList().getSize(); | ||
| this.maxFetchJDListOffset = scrapingWantedConfig.getMaxFetchJdList().getOffset(); | ||
| } | ||
|
|
||
| public boolean isBelowSizeLimit() { | ||
| return fetchedJobIds.size() < maxFetchJDListSize; | ||
| } | ||
|
|
||
| public boolean isBelowOffsetLimit(int jobIdListSize) { | ||
| return jobIdListSize < maxFetchJDListOffset; | ||
| } | ||
|
|
||
| public void incrementOffset() { | ||
| offset += maxFetchJDListOffset; | ||
| } | ||
|
|
||
| public int getOffset() { | ||
| return this.offset; | ||
| } | ||
|
|
||
| public Set<Long> getFetchedJobIds() { | ||
| return fetchedJobIds; | ||
| } | ||
|
|
||
| public void addFetchedJobIds(List<Long> jobIdList) { | ||
| fetchedJobIds.addAll(jobIdList); | ||
| } | ||
| } |