From 7b59e7597fefbd57a7a9032860dc24591dc8766c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=9D=B4=EC=9C=A4=EC=84=A0?= Date: Fri, 26 Jan 2024 17:48:06 +0900 Subject: [PATCH] =?UTF-8?q?Temp:=20=EB=A9=98=ED=86=A0=EB=A7=81=20=EC=A7=88?= =?UTF-8?q?=EB=AC=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- module-crawler/build.gradle | 1 + .../kernel/jdon/ModuleCrawlerApplication.java | 2 + .../jdon/crawler/config/ScrapingConfig.java | 26 +++++++++++++ .../config/mangkyu/DataSourceProperties.java | 18 +++++++++ .../wanted/service/WantedCrawlerService.java | 38 +++++++++++-------- 5 files changed, 69 insertions(+), 16 deletions(-) create mode 100644 module-crawler/src/main/java/kernel/jdon/crawler/config/ScrapingConfig.java create mode 100644 module-crawler/src/main/java/kernel/jdon/crawler/config/mangkyu/DataSourceProperties.java diff --git a/module-crawler/build.gradle b/module-crawler/build.gradle index 365cb6a86..ee2f4945d 100644 --- a/module-crawler/build.gradle +++ b/module-crawler/build.gradle @@ -32,6 +32,7 @@ dependencies { compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' + annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor' implementation group: 'org.jsoup', name: 'jsoup', version: '1.17.1' diff --git a/module-crawler/src/main/java/kernel/jdon/ModuleCrawlerApplication.java b/module-crawler/src/main/java/kernel/jdon/ModuleCrawlerApplication.java index 85f097c22..a3c5fc5f6 100644 --- a/module-crawler/src/main/java/kernel/jdon/ModuleCrawlerApplication.java +++ b/module-crawler/src/main/java/kernel/jdon/ModuleCrawlerApplication.java @@ -2,8 +2,10 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.context.properties.ConfigurationPropertiesScan; @SpringBootApplication +@ConfigurationPropertiesScan public class ModuleCrawlerApplication { public static void main(String[] args) { diff --git a/module-crawler/src/main/java/kernel/jdon/crawler/config/ScrapingConfig.java b/module-crawler/src/main/java/kernel/jdon/crawler/config/ScrapingConfig.java new file mode 100644 index 000000000..a97efcd27 --- /dev/null +++ b/module-crawler/src/main/java/kernel/jdon/crawler/config/ScrapingConfig.java @@ -0,0 +1,26 @@ +package kernel.jdon.crawler.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@Getter +@RequiredArgsConstructor +@ConfigurationProperties(prefix = "scraping.wanted") +public class ScrapingConfig { + private final MaxFetchJdListConfig maxFetchJdList; + private final SleepConfig sleep; + + @Getter + public static class MaxFetchJdListConfig { + private int size; + private int offset; + } + + @Getter + public static class SleepConfig { + private int thresholdCount; + private int timeMillis; + } +} diff --git a/module-crawler/src/main/java/kernel/jdon/crawler/config/mangkyu/DataSourceProperties.java b/module-crawler/src/main/java/kernel/jdon/crawler/config/mangkyu/DataSourceProperties.java new file mode 100644 index 000000000..771a6f005 --- /dev/null +++ b/module-crawler/src/main/java/kernel/jdon/crawler/config/mangkyu/DataSourceProperties.java @@ -0,0 +1,18 @@ +package kernel.jdon.crawler.config.mangkyu; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@Getter +@RequiredArgsConstructor +@ConfigurationProperties(prefix = "spring.datasource") +public class DataSourceProperties { + + private final String driverClassName; + private final String url; + private final String username; + private final String password; + +} diff --git a/module-crawler/src/main/java/kernel/jdon/crawler/wanted/service/WantedCrawlerService.java b/module-crawler/src/main/java/kernel/jdon/crawler/wanted/service/WantedCrawlerService.java index 769075060..2132062a9 100644 --- a/module-crawler/src/main/java/kernel/jdon/crawler/wanted/service/WantedCrawlerService.java +++ b/module-crawler/src/main/java/kernel/jdon/crawler/wanted/service/WantedCrawlerService.java @@ -7,11 +7,11 @@ import java.util.List; import java.util.Set; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.client.RestTemplate; +import kernel.jdon.crawler.config.ScrapingConfig; import kernel.jdon.crawler.config.UrlConfig; import kernel.jdon.crawler.global.error.code.WantedErrorCode; import kernel.jdon.crawler.global.error.exception.CrawlerException; @@ -48,24 +48,30 @@ public class WantedCrawlerService { private final SkillRepository skillRepository; private final SkillHistoryRepository skillHistoryRepository; private final JobCategoryRepository jobCategoryRepository; - @Value("${scraping.wanted.max_fetch_jd_list.size}") - private int MAX_FETCH_JD_LIST_SIZE; - @Value("${scraping.wanted.max_fetch_jd_list.offset}") - private int MAX_FETCH_JD_LIST_OFFSET; - @Value("${scraping.wanted.sleep.time_millis}") - private int SLEEP_TIME_MILLIS; - @Value("${scraping.wanted.sleep.threshold_count}") - private int THRESHOLD_COUNT; + private final ScrapingConfig scrapingConfig; + private final int MAX_FETCH_JD_LIST_SIZE = 0; + private final int MAX_FETCH_JD_LIST_OFFSET = 0; + private final int SLEEP_TIME_MILLIS = 0; + private final int THRESHOLD_COUNT = 0; + // @Value("${scraping.wanted.max_fetch_jd_list.size}") + // private int MAX_FETCH_JD_LIST_SIZE; + // @Value("${scraping.wanted.max_fetch_jd_list.offset}") + // private int MAX_FETCH_JD_LIST_OFFSET; + // @Value("${scraping.wanted.sleep.time_millis}") + // private int SLEEP_TIME_MILLIS; + // @Value("${scraping.wanted.sleep.threshold_count}") + // private int THRESHOLD_COUNT; @Transactional public void fetchJd() throws InterruptedException { - for (JobSearchJobPosition jobPosition : JobSearchJobPosition.getAllPositions()) { - Set fetchJobIds = fetchJobIdList(jobPosition); - - JobCategory findJobCategory = findByJobPosition(jobPosition); - - createJobDetail(jobPosition, findJobCategory, fetchJobIds); - } + System.out.println("scrapingConfig : " + scrapingConfig); + // for (JobSearchJobPosition jobPosition : JobSearchJobPosition.getAllPositions()) { + // Set fetchJobIds = fetchJobIdList(jobPosition); + // + // JobCategory findJobCategory = findByJobPosition(jobPosition); + // + // createJobDetail(jobPosition, findJobCategory, fetchJobIds); + // } } private JobCategory findByJobPosition(JobSearchJobPosition jobPosition) {