fix: 优化知网抓取代码

l0o0 · l0o0 · commit 5817f58cb415 · 2026-03-26T14:32:17.000+08:00
1. 增加异常信息日志输出
2. 优化任务执行,控制任务并行,使用串行检索,避免并发被反爬风控
diff --git a/src/modules/services/cnki.ts b/src/modules/services/cnki.ts
@@ -270,55 +270,23 @@ async function getSnapshotItem(
   return undefined;
 }
 
-// Update addtional information to the item.
-// Citations from CNKI, Use keyword: CNKICite
-async function updateItem(
-  item: Zotero.Item | null,
-  searchResult: ScrapeSearchResult,
-): Promise<Zotero.Item | null> {
-  if (item) {
-    if (searchResult.citation) {
-      ztoolkit.ExtraField.setExtraField(
-        item,
-        "CNKICite",
-        `${searchResult.citation}`,
-      );
-    }
-
-    if (searchResult.netFirst) {
-      ztoolkit.ExtraField.setExtraField(
-        item,
-        "Status",
-        "advance online publication",
-      );
-    }
-
-    // Remove unmatched Zotero fields note.
-    if (item.getNotes().length > 0) {
-      item.getNotes().forEach(async (nid) => {
-        const nItem = Zotero.Items.get(nid);
-        await nItem.eraseTx();
-      });
-    }
-
-    if (!item.getField("date") && searchResult.date) {
-      item.setField("date", searchResult.date);
-    }
-  }
-  return item;
-}
-
 export class CNKI implements ScrapeService {
   async search(
     searchOption: SearchOption,
   ): Promise<ScrapeSearchResult[] | null> {
     ztoolkit.log("serch options: ", searchOption);
     const postOption = createSearchPostOptions(searchOption);
     let responseText: string;
+    const cookieBox = await addon.data.myCookieSandbox.getCNKIHomeCookieBox();
+    ztoolkit.log("Cookies in sandbox: ", cookieBox._cookies);
+    ztoolkit.log(addon.taskRunner.runningTask);
+    addon.taskRunner.runningTask?.addMsg(
+      `CNKI site info: ${Object.keys(cookieBox._cookies).length}`,
+    );
     const resp = await Zotero.HTTP.request("POST", postOption.url, {
       headers: postOption.headers,
       body: postOption.data,
-      cookieSandbox: await addon.data.myCookieSandbox.getCNKIHomeCookieBox(),
+      cookieSandbox: cookieBox,
       timeout: 10000,
       successCodes: [200, 403],
     });
diff --git a/src/modules/services/index.ts b/src/modules/services/index.ts
@@ -164,7 +164,11 @@ export async function metaTranslate(task: ScraperTask): Promise<void> {
 
     if (translatedItems.length === 1) {
       // if (addon.data.env != "development")
-      const translatedItem = await globalItemFix(task.item, translatedItems[0]);
+      const translatedItem = await globalItemFix(
+        task.item,
+        translatedItems[0],
+        task,
+      );
       if (task.type == "attachment") {
         task.item.parentID = translatedItem.id;
       } else if (task.type == "snapshot") {
@@ -203,9 +207,11 @@ export async function metaTranslate(task: ScraperTask): Promise<void> {
 }
 
 // Need to update data in item returned by translator.
+// Add some extra data.
 async function globalItemFix(
   oldItem: Zotero.Item,
   newItem: Zotero.Item,
+  task: ScraperTask,
 ): Promise<Zotero.Item> {
   if (Zotero.Prefs.get("extensions.zotero.automaticTags", true)) {
     // Keyword tag type is automatic.
@@ -224,5 +230,36 @@ async function globalItemFix(
   // Preserve collections
   oldItem.getCollections().forEach((cid) => newItem!.addToCollection(cid));
   await newItem.saveTx();
+
+  // CNKI extra data fix
+  const searchResult = task.searchResults[task.resultIndex!];
+  if (searchResult.citation) {
+    ztoolkit.ExtraField.setExtraField(
+      newItem,
+      "CNKICite",
+      `${searchResult.citation}`,
+    );
+  }
+
+  if (searchResult.netFirst) {
+    ztoolkit.ExtraField.setExtraField(
+      newItem,
+      "Status",
+      "advance online publication",
+    );
+  }
+
+  // Remove unmatched Zotero fields note.
+  if (newItem.getNotes().length > 0) {
+    newItem.getNotes().forEach(async (nid) => {
+      const nItem = Zotero.Items.get(nid);
+      await nItem.eraseTx();
+    });
+  }
+
+  if (!newItem.getField("date") && searchResult.date) {
+    newItem.setField("date", searchResult.date);
+  }
+
   return newItem;
 }
diff --git a/src/utils/task.ts b/src/utils/task.ts
@@ -203,8 +203,7 @@ export class TaskRunner {
   ): Promise<string> {
     const task = this.createTask(item, type, silent);
     task.addMsg(getString("task-msg-header"));
-    task.addMsg(`Zotero version: ${Zotero.version}`);
-    task.addMsg(`Addon version: ${version}`);
+    task.addMsg(`Zotero: ${Zotero.version}, Jasminum: ${version}`);
     await this.addTask(task);
     return task.id;
   }
@@ -215,9 +214,9 @@ export class TaskRunner {
   async runTask(task: AttachmentTask | ScraperTask): Promise<void> {
     this.runningTask = task;
     if (this.getTaskType(task) === "attachmentScraper") {
-      this.runAttachmentTask(task as AttachmentTask);
+      await this.runAttachmentTask(task as AttachmentTask);
     } else {
-      this.runScrapeTask(task as ScraperTask);
+      await this.runScrapeTask(task as ScraperTask);
     }
     this.runningTask = null;
   }
