Scrape code repos with paper id. #125 First character pressing of inp…

…ut boxes is blocked? #123
Future-Scholars · Aug 6, 2022 · 53db7ee · 53db7ee
1 parent f5159b2
commit 53db7ee
Show file tree

Hide file tree

Showing 5 changed files with 107 additions and 12 deletions.
diff --git a/packages/preload/repositories/scraper-repository/scrapers/paperwithcode.ts b/packages/preload/repositories/scraper-repository/scrapers/paperwithcode.ts
@@ -1,6 +1,7 @@
-import { Response } from "got";
+import got, { Response } from "got";
+import { HttpProxyAgent, HttpsProxyAgent } from "hpagent";
 
-import { Scraper, ScraperRequestType } from "./scraper";
+import { Scraper, ScraperRequestType, ScraperType } from "./scraper";
 import { formatString } from "../../../utils/string";
 import { PaperEntityDraft } from "../../../models/PaperEntityDraft";
 
@@ -37,9 +38,64 @@ export class PwCScraper extends Scraper {
     entityDraft: PaperEntityDraft
   ): PaperEntityDraft {
     const response = JSON.parse(rawResponse.body) as {
+      count?: number;
+      results: {
+        url: string;
+        is_official: boolean;
+      }[];
+    };
+    if (response.count) {
+      let codeList: string[] = [];
+      for (const result of response.results) {
+        codeList.push(
+          JSON.stringify({
+            url: result.url,
+            isOfficial: result.is_official,
+          })
+        );
+      }
+      codeList = codeList.sort((a, b) => {
+        const aIsOfficial = JSON.parse(a).isOfficial;
+        const bIsOfficial = JSON.parse(b).isOfficial;
+        if (aIsOfficial && !bIsOfficial) {
+          return -1;
+        } else if (!aIsOfficial && bIsOfficial) {
+          return 1;
+        } else {
+          return 0;
+        }
+      });
+      entityDraft.setValue("codes", codeList);
+    }
+    return entityDraft;
+  }
+
+  scrapeImpl = scrapeImpl;
+}
+
+async function scrapeImpl(
+  this: ScraperType,
+  entityDraft: PaperEntityDraft
+): Promise<PaperEntityDraft> {
+  const { scrapeURL, headers, enable } = this.preProcess(
+    entityDraft
+  ) as ScraperRequestType;
+
+  if (enable) {
+    const agent = this.getProxyAgent();
+    let options = {
+      headers: headers,
+      retry: 1,
+      timeout: 10000,
+      agent: agent,
+    };
+    const rawSearchResponse = await got(scrapeURL, options);
+
+    const searchResponse = JSON.parse(rawSearchResponse.body) as {
       count?: number;
       results: {
         paper: {
+          id: string;
           title: string;
         };
         repository: {
@@ -55,9 +111,10 @@ export class PwCScraper extends Scraper {
       lowercased: true,
     });
 
-    if (response.count) {
+    let id = "";
+    if (searchResponse.count) {
       const codeList: string[] = [];
-      for (const result of response.results) {
+      for (const result of searchResponse.results) {
         const hitTitle = formatString({
           str: result.paper.title,
           removeStr: "&amp",
@@ -66,16 +123,26 @@ export class PwCScraper extends Scraper {
         });
 
         if (hitTitle === targetTitle && result.repository) {
-          codeList.push(
-            JSON.stringify({
-              url: result.repository.url,
-              isOfficial: result.is_official,
-            })
-          );
+          id = result.paper.id;
+          break;
         }
       }
-      entityDraft.setValue("codes", codeList);
     }
+
+    if (id) {
+      const rawRepoResponse = await got(
+        `https://paperswithcode.com/api/v1/papers/${id}/repositories/`,
+        options
+      );
+
+      return this.parsingProcess(
+        rawRepoResponse,
+        entityDraft
+      ) as PaperEntityDraft;
+    } else {
+      return entityDraft;
+    }
+  } else {
     return entityDraft;
   }
 }
diff --git a/packages/renderer/src/ui/app-view.vue b/packages/renderer/src/ui/app-view.vue
@@ -14,7 +14,6 @@ import PreferenceView from "./preference-view/preference-view.vue";
 import FeedEditView from "./edit-view/feed-edit-view.vue";
 
 import { PreferenceStore } from "../../../preload/utils/preference";
-import { createModalView } from "./components/modal-view";
 
 const sortBy = ref("addTime");
 const sortOrder = ref("desc");

diff --git a/packages/renderer/src/ui/edit-view/edit-view.vue b/packages/renderer/src/ui/edit-view/edit-view.vue
@@ -31,6 +31,16 @@ const entityDraft = ref(new PaperEntityDraft());
 const pubTypes = ["Article", "Conference", "Others", "Book"];
 
 const keyDownListener = (e: KeyboardEvent) => {
+  if (
+    e.target instanceof HTMLInputElement ||
+    e.target instanceof HTMLTextAreaElement
+  ) {
+    if (e.key === "Escape") {
+      onCloseClicked();
+    }
+    return true;
+  }
+
   e.preventDefault();
   if (e.key === "Escape") {
     onCloseClicked();

diff --git a/packages/renderer/src/ui/edit-view/feed-edit-view.vue b/packages/renderer/src/ui/edit-view/feed-edit-view.vue
@@ -9,6 +9,15 @@ const isFeedEditViewShown = ref(false);
 const feedDraft = ref(new FeedDraft());
 
 const keyDownListener = (e: KeyboardEvent) => {
+  if (
+    e.target instanceof HTMLInputElement ||
+    e.target instanceof HTMLTextAreaElement
+  ) {
+    if (e.key === "Escape") {
+      onCloseClicked();
+    }
+    return true;
+  }
   e.preventDefault();
   if (e.key === "Escape") {
     onCloseClicked();

diff --git a/packages/renderer/src/ui/preference-view/preference-view.vue b/packages/renderer/src/ui/preference-view/preference-view.vue
@@ -43,6 +43,16 @@ const onCloseClicked = () => {
 };
 
 const keyDownListener = (e: KeyboardEvent) => {
+  if (
+    e.target instanceof HTMLInputElement ||
+    e.target instanceof HTMLTextAreaElement
+  ) {
+    if (e.key === "Escape") {
+      onCloseClicked();
+    }
+    return true;
+  }
+
   e.preventDefault();
   if (e.key === "Escape") {
     onCloseClicked();