Bump up to v1.10.2

.github/workflows/electron_deploy.yml

@@ -210,6 +210,8 @@ jobs:
 
       - name: Upload to Oracle Storage
         run: |
+          powershell Compress-Archive './release/${{ env.VERSION }}/Paperlib_${{ env.VERSION }}.exe' './latest.zip'
+          oci os object put --bucket-name ${{ secrets.OCI_OS_BUCKET }} --file './latest.zip' --namespace ${{ secrets.OCI_OS_NAMESPACE }} --force --name 'distribution/electron-win/latest.zip'
           oci os object put --bucket-name ${{ secrets.OCI_OS_BUCKET }} --file './release/${{ env.VERSION }}/Paperlib_${{ env.VERSION }}.exe' --namespace ${{ secrets.OCI_OS_NAMESPACE }} --force --name 'distribution/electron-win/Paperlib_${{ env.VERSION }}.exe'
           oci os object put --bucket-name ${{ secrets.OCI_OS_BUCKET }} --file './release/${{ env.VERSION }}/Paperlib_${{ env.VERSION }}.exe' --namespace ${{ secrets.OCI_OS_NAMESPACE }} --force --name distribution/electron-win/latest.exe
           oci os object put --bucket-name ${{ secrets.OCI_OS_BUCKET }} --file './release/${{ env.VERSION }}/Paperlib_${{ env.VERSION }}.exe.blockmap' --namespace ${{ secrets.OCI_OS_NAMESPACE }} --force --name 'distribution/electron-win/Paperlib_${{ env.VERSION }}.exe.blockmap'

CHANGELOG_CN.md

@@ -1,3 +1,8 @@
+## v1.10.2 05/09 2022
+
+1. 新的元数据搜刮器: crossref.org
+2. 修复漏洞。
+
 ## v1.10.1 01/09 2022
 
 1. 修复漏洞。

CHANGELOG_EN.md

@@ -1,3 +1,9 @@
+## v1.10.2 Sep 05 2022
+
+1. New metadata scraper: corssref.org
+2. Fixed some bugs.
+
+
 ## v1.10.1 Sep 01 2022
 
 1. Fixed some bugs.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "paperlib",
-  "version": "1.10.1",
+  "version": "1.10.2",
   "description": "A simple academic paper management tool for CSer.",
   "productName": "Paperlib",
   "author": "Geoffrey Chen <[email protected]>",
@@ -53,6 +53,7 @@
     "@headlessui/vue": "1.5.0",
     "bootstrap-icons-vue": "1.8.1",
     "bson": "4.6.3",
+    "check-internet-connected": "2.0.6",
     "chokidar": "3.5.3",
     "citation-js": "^0.6.4",
     "drag-drop": "7.2.0",
@@ -66,7 +67,6 @@
     "keytar": "7.9.0",
     "markdown-it": "13.0.1",
     "markdown-it-texmath": "1.0.0",
-    "no-internet": "1.6.1",
     "node-html-parser": "5.3.3",
     "realm": "10.13.0",
     "rss-parser": "3.12.0",

packages/preload/repositories/db-repository/db-init.ts

@@ -2,8 +2,6 @@ import Realm from "realm";
 import path from "path";
 import { existsSync, promises as fsPromise } from "fs";
 import keytar from "keytar";
-// @ts-ignore
-import noInternet from "no-internet";
 
 import { PaperEntitySchema } from "../../models/PaperEntity";
 import {
@@ -195,7 +193,12 @@ export async function loginCloud(
     });
   }
 
-  if (await noInternet()) {
+  const checkInternetConnected = require("check-internet-connected");
+  if (
+    !(await checkInternetConnected({
+      domain: "https://httpbin.org/get",
+    }))
+  ) {
     console.log("No internet!");
     return this.app.currentUser;
   }

packages/preload/repositories/scraper-repository/scraper-repository.ts

@@ -14,6 +14,7 @@ import { CVFScraper } from "./scrapers/cvf";
 import { PwCScraper } from "./scrapers/paperwithcode";
 import { OpenreviewScraper } from "./scrapers/openreview";
 import { GoogleScholarScraper } from "./scrapers/google-scholar";
+import { CrossRefScraper } from "./scrapers/crossref";
 import { CustomScraper } from "./scrapers/custom";
 
 import { Preference, ScraperPreference } from "../../utils/preference";
@@ -83,6 +84,12 @@ export class ScraperRepository {
           case "pdf":
             scraperInstance = new PDFScraper(this.sharedState, this.preference);
             break;
+          case "crossref":
+            scraperInstance = new CrossRefScraper(
+              this.sharedState,
+              this.preference
+            );
+            break;
           case "doi":
             scraperInstance = new DOIScraper(this.sharedState, this.preference);
             break;

packages/preload/repositories/scraper-repository/scrapers/crossref.ts

@@ -0,0 +1,71 @@
+import { Response } from "got";
+
+import { Scraper, ScraperRequestType } from "./scraper";
+import { formatString } from "../../../utils/string";
+import { Preference } from "../../../utils/preference";
+import { SharedState } from "../../../utils/appstate";
+import { PaperEntityDraft } from "../../../models/PaperEntityDraft";
+
+export class CrossRefScraper extends Scraper {
+  constructor(sharedState: SharedState, preference: Preference) {
+    super(sharedState, preference);
+  }
+
+  preProcess(entityDraft: PaperEntityDraft): ScraperRequestType {
+    const enable =
+      this.getEnable("crossref") &&
+      entityDraft.title !== "" &&
+      entityDraft.doi === "";
+
+    const scrapeURL = `https://api.crossref.org/works?query.title=${entityDraft.title}`;
+
+    const headers = {};
+    if (enable) {
+      this.sharedState.set(
+        "viewState.processInformation",
+        `Scraping metadata from crossref.org ...`
+      );
+    }
+
+    return { scrapeURL, headers, enable };
+  }
+
+  parsingProcess(
+    rawResponse: Response<string>,
+    entityDraft: PaperEntityDraft
+  ): PaperEntityDraft {
+    const parsedResponse = JSON.parse(rawResponse.body) as {
+      message: {
+        items: [
+          {
+            title: string[];
+            DOI: string;
+          }
+        ];
+      };
+    };
+    console.log(parsedResponse);
+    for (const item of parsedResponse.message.items) {
+      const plainHitTitle = formatString({
+        str: item.title[0],
+        removeStr: "&amp",
+        removeSymbol: true,
+        lowercased: true,
+      });
+
+      const existTitle = formatString({
+        str: entityDraft.title,
+        removeStr: "&amp",
+        removeSymbol: true,
+        lowercased: true,
+      });
+
+      if (plainHitTitle === existTitle && item.DOI && entityDraft.doi === "") {
+        entityDraft.doi = item.DOI;
+        break;
+      }
+    }
+
+    return entityDraft;
+  }
+}

packages/preload/repositories/scraper-repository/scrapers/pdf.ts

@@ -140,6 +140,10 @@ export class PDFScraper extends Scraper {
       }
     }
 
+    if (entityDraft.doi.endsWith(",")) {
+      entityDraft.setValue("doi", entityDraft.doi.slice(0, -1));
+    }
+
     if (entityDraft.title === "" || entityDraft.title === "untitled") {
       entityDraft.setValue("title", rawResponse.largestText.slice(0, 400));
     }

packages/preload/utils/preference.ts

@@ -206,6 +206,17 @@ const defaultPreferences: PreferenceStore = {
       parsingProcessCode: "",
       scrapeImplCode: "",
     },
+    {
+      name: "crossref",
+      description: "crossref.org",
+      enable: true,
+      custom: false,
+      args: "",
+      priority: 8.5,
+      preProcessCode: "",
+      parsingProcessCode: "",
+      scrapeImplCode: "",
+    },
     {
       name: "doi",
       description: "DOI.org",
@@ -349,6 +360,18 @@ export class Preference {
       }
     }
 
+    const existingScrapers = this.store
+      .get("scrapers")
+      .map((scraper) => scraper.name);
+
+    for (const defaultPref of defaultPreferences.scrapers) {
+      if (!existingScrapers.includes(defaultPref.name)) {
+        const existingPreference = this.store.get("scrapers");
+        existingPreference.push(defaultPref);
+        this.store.set("scrapers", existingPreference);
+      }
+    }
+
     if (migrationRequired) {
       let defaultScraperPrefs = defaultPreferences.scrapers;
       if (!this.store.get("pdfBuiltinScraper")) {

packages/renderer/src/ui/preference-view/components/scraper.vue

@@ -42,7 +42,7 @@ const onApplyUpdate = () => {
         </div>
 
         <div
-          class="text-right text-neutral-600 w-[5%] dark:text-neutral-500 overflow-hidden text-ellipsis pr-1 my-auto"
+          class="text-right text-neutral-600 w-[6%] dark:text-neutral-500 overflow-hidden text-ellipsis pr-1 my-auto"
         >
           {{ scraperPref.priority }}
         </div>

packages/renderer/src/ui/preference-view/scraper-view.vue

@@ -84,7 +84,7 @@ const onClickGuide = () => {
     <div class="flex text-xxs font-semibold">
       <div class="pl-4 w-[17%]">Scraper</div>
       <div>Priority</div>
-      <div class="pl-2">Description</div>
+      <div class="pl-4">Description</div>
     </div>
     <hr class="mx-2 mb-1 dark:border-neutral-600" />
     <div class="flex flex-col px-2 rounded-md max-h-[450px] overflow-scroll">

packages/renderer/src/ui/whats-new-view.vue

@@ -34,11 +34,15 @@ onMounted(() => {
       <div class="w-[45rem] h-screen px-3 mx-auto">
         <img class="w-20 mx-auto mb-2" src="../assets/icon.png" />
         <p class="text-center text-2xl font-bold mb-8">
-          What's New in Paperlib 1.10.1
+          What's New in Paperlib 1.10.2
         </p>
+        <p class="mt-10 mb-4"><b>New Features</b></p>
+        <ul class="list-disc mb-20">
+          <li>New metadata scraper: crossref.org</li>
+        </ul>
         <p class="mt-10 mb-4"><b>Improvements and fixed Bugs</b></p>
         <ul class="list-disc mb-20">
-          <li>Fixed some bugs: #134 #135.</li>
+          <li>Fixed some bugs in the sync cloud login.</li>
         </ul>
 
         <div

release-notes.md

@@ -1 +1,2 @@
-1. Fixed some bugs.
+1. New metadata scraper: corssref.org
+2. Fixed some bugs.