Skip to content

Commit

Permalink
Temp: 멘토링 질문
Browse files Browse the repository at this point in the history
  • Loading branch information
yoonseon12 committed Jan 26, 2024
1 parent 0b55e5c commit 7b59e75
Show file tree
Hide file tree
Showing 5 changed files with 69 additions and 16 deletions.
1 change: 1 addition & 0 deletions module-crawler/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ dependencies {
compileOnly 'org.projectlombok:lombok'

annotationProcessor 'org.projectlombok:lombok'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'

implementation group: 'org.jsoup', name: 'jsoup', version: '1.17.1'

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.ConfigurationPropertiesScan;

@SpringBootApplication
@ConfigurationPropertiesScan
public class ModuleCrawlerApplication {

public static void main(String[] args) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package kernel.jdon.crawler.config;

import org.springframework.boot.context.properties.ConfigurationProperties;

import lombok.Getter;
import lombok.RequiredArgsConstructor;

@Getter
@RequiredArgsConstructor
@ConfigurationProperties(prefix = "scraping.wanted")
public class ScrapingConfig {
private final MaxFetchJdListConfig maxFetchJdList;
private final SleepConfig sleep;

@Getter
public static class MaxFetchJdListConfig {
private int size;
private int offset;
}

@Getter
public static class SleepConfig {
private int thresholdCount;
private int timeMillis;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package kernel.jdon.crawler.config.mangkyu;

import org.springframework.boot.context.properties.ConfigurationProperties;

import lombok.Getter;
import lombok.RequiredArgsConstructor;

@Getter
@RequiredArgsConstructor
@ConfigurationProperties(prefix = "spring.datasource")
public class DataSourceProperties {

private final String driverClassName;
private final String url;
private final String username;
private final String password;

}
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@
import java.util.List;
import java.util.Set;

import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.client.RestTemplate;

import kernel.jdon.crawler.config.ScrapingConfig;
import kernel.jdon.crawler.config.UrlConfig;
import kernel.jdon.crawler.global.error.code.WantedErrorCode;
import kernel.jdon.crawler.global.error.exception.CrawlerException;
Expand Down Expand Up @@ -48,24 +48,30 @@ public class WantedCrawlerService {
private final SkillRepository skillRepository;
private final SkillHistoryRepository skillHistoryRepository;
private final JobCategoryRepository jobCategoryRepository;
@Value("${scraping.wanted.max_fetch_jd_list.size}")
private int MAX_FETCH_JD_LIST_SIZE;
@Value("${scraping.wanted.max_fetch_jd_list.offset}")
private int MAX_FETCH_JD_LIST_OFFSET;
@Value("${scraping.wanted.sleep.time_millis}")
private int SLEEP_TIME_MILLIS;
@Value("${scraping.wanted.sleep.threshold_count}")
private int THRESHOLD_COUNT;
private final ScrapingConfig scrapingConfig;
private final int MAX_FETCH_JD_LIST_SIZE = 0;
private final int MAX_FETCH_JD_LIST_OFFSET = 0;
private final int SLEEP_TIME_MILLIS = 0;
private final int THRESHOLD_COUNT = 0;
// @Value("${scraping.wanted.max_fetch_jd_list.size}")
// private int MAX_FETCH_JD_LIST_SIZE;
// @Value("${scraping.wanted.max_fetch_jd_list.offset}")
// private int MAX_FETCH_JD_LIST_OFFSET;
// @Value("${scraping.wanted.sleep.time_millis}")
// private int SLEEP_TIME_MILLIS;
// @Value("${scraping.wanted.sleep.threshold_count}")
// private int THRESHOLD_COUNT;

@Transactional
public void fetchJd() throws InterruptedException {
for (JobSearchJobPosition jobPosition : JobSearchJobPosition.getAllPositions()) {
Set<Long> fetchJobIds = fetchJobIdList(jobPosition);

JobCategory findJobCategory = findByJobPosition(jobPosition);

createJobDetail(jobPosition, findJobCategory, fetchJobIds);
}
System.out.println("scrapingConfig : " + scrapingConfig);
// for (JobSearchJobPosition jobPosition : JobSearchJobPosition.getAllPositions()) {
// Set<Long> fetchJobIds = fetchJobIdList(jobPosition);
//
// JobCategory findJobCategory = findByJobPosition(jobPosition);
//
// createJobDetail(jobPosition, findJobCategory, fetchJobIds);
// }
}

private JobCategory findByJobPosition(JobSearchJobPosition jobPosition) {
Expand Down

0 comments on commit 7b59e75

Please sign in to comment.