Added disableThumbnail option in wp

package.json

@@ -1,6 +1,6 @@
 {
  "name": "@unistudents/saffron",
- "version": "6.1.5",
+ "version": "6.2.0",
  "description": "A fairly intuitive & powerful framework that enables you to collect & save articles and news from all over the web. ",
  "license": "MIT",
  "homepage": "https://github.com/unistudents/saffron#readme",
@@ -19,31 +19,31 @@
  "build": "tsc -d"
  },
  "dependencies": {
- "axios": "1.6.8",
+ "axios": "1.7.1",
  "chalk": "^4.1.2",
  "cheerio": "^1.0.0-rc.12",
- "fast-xml-parser": "^4.3.2",
- "glob": "^10.3.10",
+ "fast-xml-parser": "^4.4.0",
+ "glob": "^10.3.15",
  "lodash": "^4.17.21",
- "luxon": "^3.4.3",
+ "luxon": "^3.4.4",
  "rss-parser": "^3.13.0",
- "socket.io": "^4.7.2",
- "socket.io-client": "^4.7.2",
+ "socket.io": "^4.7.5",
+ "socket.io-client": "^4.7.5",
  "striptags": "^3.2.0"
  },
  "devDependencies": {
- "@types/chai": "^4.3.9",
- "@types/cheerio": "^0.22.33",
- "@types/express": "^4.17.20",
+ "@types/chai": "^4.3.16",
+ "@types/cheerio": "^0.22.35",
+ "@types/express": "^4.17.21",
  "@types/glob": "^8.1.0",
- "@types/lodash": "^4.14.200",
- "@types/luxon": "^3.3.3",
- "@types/mocha": "^10.0.3",
- "chai": "^4.3.10",
- "express": "^4.18.2",
- "mocha": "^10.2.0",
- "ts-node": "^10.9.1",
- "typescript": "^5.2.2",
+ "@types/lodash": "^4.17.4",
+ "@types/luxon": "^3.4.2",
+ "@types/mocha": "^10.0.6",
+ "chai": "^4.4.1",
+ "express": "^4.19.2",
+ "mocha": "^10.4.0",
+ "ts-node": "^10.9.2",
+ "typescript": "^5.4.5",
  "http-proxy-middleware": "^2.0.6"
  }
 }

src/components/parser.type.ts

@@ -71,6 +71,7 @@ export type ScrapeWordPressV2 = {
  size?: number; // per_page
  };
  thumbnail?: string;
+ disableThumbnail?: boolean;
  };
 };
 

src/index.ts

@@ -11,4 +11,5 @@ export {RequestsResult, SourceFile, ParserResult, SourceScrape, MergedConfig, Co
 // TODO: Add source file templates
 // A template folder, where a source file can extend to. Will contain the same fields
 // as a source file, but not all fields are required.
+// TODO: Add debug mode, where file cache exists and if already fetched then do not repeat the request
 

src/parsers/wordpress.v2.parser.ts

@@ -60,6 +60,12 @@ export class WordpressV2Parser extends Parser {
 
  if (scrape.articles.thumbnail && typeof scrape.articles.thumbnail !== 'string')
  throw new Error("articles.thumbnail must be a string");
+
+ if (typeof scrape.articles.disableThumbnail !== 'undefined' && typeof scrape.articles.disableThumbnail !== 'boolean')
+ throw new Error("articles.disableThumbnail must be a boolean");
+
+ if(scrape.articles.disableThumbnail && scrape.articles.thumbnail)
+ throw new Error("articles.thumbnail cannot be populated");
  }
  }
  }
@@ -120,7 +126,7 @@ export class WordpressV2Parser extends Parser {
  const instructions = utils.source.instructions;
 
  const categoriesUrl = `${utils.url}/${instructions.wp.paths!.categories}`;
- let postsUrl = `${utils.url}/${instructions.wp.paths!.posts}?_embed&per_page=${instructions.amount}`;
+ let postsUrl = `${utils.url}/${instructions.wp.paths!.posts}?${instructions.wp.articles?.disableThumbnail ? '' : '_embed&'}per_page=${instructions.amount}`;
 
  const filters = instructions.wp.articles!.filter!;
  if (filters.search) postsUrl += `&search=${encodeURIComponent(filters.search)}`;
@@ -199,8 +205,10 @@ export class WordpressV2Parser extends Parser {
  }
 
  // Thumbnail
- const thumbnailSize = instructions.wp.articles!.thumbnail!;
- article.thumbnail = p._embedded?.['wp:featuredmedia']?.[0]?.media_details?.sizes?.[thumbnailSize]?.source_url;
+ if(!instructions.wp.articles!.disableThumbnail) {
+ const thumbnailSize = instructions.wp.articles!.thumbnail!;
+ article.thumbnail = p._embedded?.['wp:featuredmedia']?.[0]?.media_details?.sizes?.[thumbnailSize]?.source_url;
+ }
 
  let include: string[] = instructions.wp.articles!.include!;
  // The date the object was last modified.