added extract version

Author: azhong-git

src/converters.ts

@@ -70,20 +70,27 @@ export function htmlToMarkdown(
   return turndownService.turndown(processedHtml);
 }
 
-// Adapted from https://github.com/adbar/trafilatura/blob/9338dffe4478f91639952fa3b3105d6c0a461587/trafilatura/xpaths.py
-// Changes:
-// - Commented out tags
+// Adapted from https://github.com/adbar/trafilatura/blob/c7e00f3a31e436c7b6ce666b44712e16e30908c0/trafilatura/xpaths.py#L100
+// Added:
 // - Add contains(@id, "filter") to remove filter menus
+// Removed (because user might want to extract them):
+// - Commented out tags
+// - Commented out author
+// - Commented out rating
+// - Commented out attachment
+// - Commented out timestamp
+// - Commented out user-info and user-profile
+// - Commented out comment or hidden section
 const OVERALL_DISCARD_XPATH = [
   // navigation + footers, news outlets related posts, sharing, jp-post-flair jp-relatedposts
   `.//*[(self::div or self::item or self::list
            or self::p or self::section or self::span)][
   contains(translate(@id, "F","f"), "footer") or contains(translate(@class, "F","f"), "footer")
   or contains(@id, "related") or contains(translate(@class, "R", "r"), "related") or
   contains(@id, "viral") or contains(@class, "viral") or
+  contains(@id, "filter") or
   starts-with(@id, "shar") or starts-with(@class, "shar") or
   contains(@class, "share-") or
-  contains(@id, "filter") or
   contains(translate(@id, "S", "s"), "share") or
   contains(@id, "social") or contains(@class, "social") or contains(@class, "sociable") or
   contains(@id, "syndication") or contains(@class, "syndication") or
@@ -93,6 +100,7 @@ const OVERALL_DISCARD_XPATH = [
   or contains(@class, "subnav") or
   contains(@id, "cookie") or contains(@class, "cookie") or ` +
     // `contains(@id, "tags") or contains(@class, "tags") or ` +
+    `contains(@id, "sidebar") or contains(@class, "sidebar") or ` +
     `contains(@id, "banner") or contains(@class, "banner")
   or contains(@class, "meta") or
   contains(@id, "menu") or contains(@class, "menu") or
@@ -105,7 +113,8 @@ const OVERALL_DISCARD_XPATH = [
     `contains(@id, "button") or contains(@class, "button")
   or contains(translate(@class, "B", "b"), "byline") or ` +
     // contains(@class, "rating") or ` +
-    // contains(@class, "timestamp") or
+    `starts-with(@class, "widget") or ` +
+    // contains(@class, "attachment") or contains(@class, "timestamp") or
     // contains(@class, "user-info") or contains(@class, "user-profile") or
     `contains(@class, "-ad-") or contains(@class, "-icon")
   or contains(@class, "article-infos") or
@@ -118,11 +127,13 @@ const OVERALL_DISCARD_XPATH = [
   or contains(@id, "premium-") or contains(@id, "paywall")
   or contains(@class, "obfuscated") or contains(@class, "blurred")
   or contains(@class, " ad ")
-  or contains(@class, "next-post")
+  or contains(@class, "next-post") or contains(@class, "side-stories")
+  or contains(@class, "related-stories") or contains(@class, "most-popular")
+  or contains(@class, "mol-factbox") or starts-with(@class, "ZendeskForm")
   or contains(@class, "message-container") or contains(@id, "message_container")
   or contains(@class, "yin") or contains(@class, "zlylin") or
   contains(@class, "xg1") or contains(@id, "bmdh")
-  or @data-lp-replacement-content]`,
+  or @data-lp-replacement-content or @data-testid]`,
 
   // comment debris + hidden parts
   // `.//*[@class="comments-title" or contains(@class, "comments-title") or
@@ -135,6 +146,9 @@ const OVERALL_DISCARD_XPATH = [
   // or contains(@class, "notloaded")]`,
 ];
 
+// Adapted from https://github.com/adbar/trafilatura/blob/c7e00f3a31e436c7b6ce666b44712e16e30908c0/trafilatura/xpaths.py#L179
+// Removed:
+// - contains(@style, "border")
 const PRECISION_DISCARD_XPATH = [
   ".//header",
   `.//*[(self::div or self::item or self::list
@@ -144,6 +158,10 @@ const PRECISION_DISCARD_XPATH = [
   ]`,
 ];
 
+// Adatpted from https://github.com/adbar/trafilatura/blob/c7e00f3a31e436c7b6ce666b44712e16e30908c0/trafilatura/settings.py#L55
+// Removed (because user might want to extract them):
+// - form
+// - fieldset
 const MANUALLY_CLEANED = [
   // important
   "aside",
@@ -159,7 +177,9 @@ const MANUALLY_CLEANED = [
   "applet",
   "audio",
   "canvas",
+  "figure",
   "map",
+  "picture",
   "svg",
   "video",
   // secondary
@@ -190,13 +210,10 @@ const MANUALLY_CLEANED = [
   "rt",
   "rtc",
   "select",
+  "source",
   "style",
   "track",
   "textarea",
   "time",
   "use",
-  // images
-  "figure",
-  "picture",
-  "source",
 ];