Enhance: scraping logic for papers from feeds.

Future-Scholars · Feb 26, 2024 · 15de8a9 · 15de8a9
1 parent d10ea33
commit 15de8a9
Show file tree

Hide file tree

Showing 3 changed files with 47 additions and 26 deletions.
diff --git a/app/renderer/services/command-service.ts b/app/renderer/services/command-service.ts
@@ -92,6 +92,15 @@ export class CommandService extends Eventable<{}> {
         });
       },
     });
+
+    this.register({
+      id: "scrape_preprints",
+      description: "Scrape metadata for all preprint papers in the library.",
+      priority: 99996,
+      handler: () => {
+        paperService.scrapePreprint();
+      },
+    });
   }
 
   @errorcatching("Failed to register command.", true, "CommandService")

diff --git a/app/renderer/services/feed-service.ts b/app/renderer/services/feed-service.ts
@@ -461,19 +461,15 @@ export class FeedService extends Eventable<IFeedServiceState> {
       "Feed"
     );
 
-    const paperEntityDrafts = await this._scrapeService.scrape(
-      feedEntities.map((feedEntityDraft: IFeedEntityObject) => {
+    const paperEntityDrafts = feedEntities.map(
+      (feedEntityDraft: IFeedEntityObject) => {
         const paperEntityDraft = new PaperEntity({}, true).fromFeed(
           feedEntityDraft
         );
         // NOTE: we don't want to download the PDFs when adding to library.
         paperEntityDraft.mainURL = "";
-        return {
-          type: "PaperEntity",
-          value: paperEntityDraft,
-        };
-      }),
-      ["semanticscholar"]
+        return paperEntityDraft;
+      }
     );
 
     // NOTE: here we decide to not download the PDFs when adding to library.

diff --git a/app/renderer/services/paper-service.ts b/app/renderer/services/paper-service.ts
@@ -180,7 +180,7 @@ export class PaperService extends Eventable<IPaperServiceState> {
       this._schedulerService.createTask(
         "paperServiceScrapePreprint",
         () => {
-          this.scrapePreprint();
+          this._routineScrapePreprint();
         },
         7 * 86400,
         undefined,
@@ -601,6 +601,38 @@ export class PaperService extends Eventable<IPaperServiceState> {
     "PaperService"
   )
   async scrapePreprint() {
+    if (this._databaseCore.getState("dbInitializing")) {
+      return;
+    }
+    this._logService.info(
+      `Scraping metadata of preprint paper(s)...`,
+      "",
+      true,
+      "PaperService"
+    );
+    const preprintPaperEntities = this._paperEntityRepository.load(
+      await this._databaseCore.realm(),
+      '(publication contains[c] "arXiv") OR (publication contains[c] "openreview") OR publication == ""',
+      "addTime",
+      "desc"
+    );
+    await this.scrape(
+      preprintPaperEntities.map((paperEntity) => {
+        return new PaperEntity(paperEntity);
+      })
+    );
+  }
+
+  /**
+   * Scrape preprint paper entities.
+   */
+  @processing(ProcessingKey.General)
+  @errorcatching(
+    "Failed to scrape metadata of preprints.",
+    true,
+    "PaperService"
+  )
+  async _routineScrapePreprint() {
     if (this._databaseCore.getState("dbInitializing")) {
       return;
     }
@@ -612,23 +644,7 @@ export class PaperService extends Eventable<IPaperServiceState> {
       ) {
         return;
       }
-      this._logService.info(
-        `Scraping metadata of preprint paper(s)...`,
-        "",
-        true,
-        "PaperService"
-      );
-      const preprintPaperEntities = this._paperEntityRepository.load(
-        await this._databaseCore.realm(),
-        '(publication contains[c] "arXiv") OR (publication contains[c] "openreview") OR publication == ""',
-        "addTime",
-        "desc"
-      );
-      await this.scrape(
-        preprintPaperEntities.map((paperEntity) => {
-          return new PaperEntity(paperEntity);
-        })
-      );
+      await this.scrapePreprint();
       this._preferenceService.set({
         lastRematchTime: Math.round(Date.now() / 1000),
       });