Merge pull request #214 from jshemas/new-tags

adding extra og tags for music and video and JSON LD
jshemas · Mar 3, 2024 · ba270db · ba270db
2 parents 497cfec + 2da007a
commit ba270db
Show file tree

Hide file tree

Showing 26 changed files with 776 additions and 227 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,12 @@
 # Change Log
 
+## 6.5.0
+
+- Adding support for JSON LD
+- Adding support for `og:image:alt`, `twitterAccount`, `fbAppId` and extra og tags for `music` and `video`
+- Fixing jsdoc param name
+- Updating dependencies
+
 ## 6.4.0
 
 - Add character encoding detection and decoding logic using `iconv-lite`

diff --git a/README.md b/README.md
@@ -3,9 +3,9 @@
 [![Node.js CI](https://github.com/jshemas/openGraphScraper/workflows/Node.js%20CI/badge.svg?branch=master)](https://github.com/jshemas/openGraphScraper/actions?query=branch%3Amaster)
 [![Known Vulnerabilities](https://snyk.io/test/github/jshemas/openGraphScraper/badge.svg)](https://snyk.io/test/github/jshemas/openGraphScraper)
 
-A simple node module(with TypeScript declarations) for scraping Open Graph and Twitter Card info off a site.
+A simple node module(with TypeScript declarations) for scraping Open Graph and Twitter Card and other metadata off a site.
 
-Note: `open-graph-scraper` doesn't support browser usage at this time.
+Note: `open-graph-scraper` doesn't support browser usage at this time but you can use `open-graph-scraper-lite` if you already have the `HTML` and can't use Node's [Fetch API](https://nodejs.org/dist/latest-v18.x/docs/api/globals.html#fetch).
 
 ## Installation
 

diff --git a/dist/lib/extract.js b/dist/lib/extract.js
@@ -56,6 +56,11 @@ function extractMetaTags(body, options) {
     // if onlyGetOpenGraphInfo isn't set, run the open graph fallbacks
     if (!options.onlyGetOpenGraphInfo) {
         ogObject = (0, fallback_1.default)(ogObject, options, $, body);
+        $('script').each((index, script) => {
+            if (script.attribs.type && script.attribs.type === 'application/ld+json') {
+                ogObject.jsonLD = JSON.parse($(script).text());
+            }
+        });
     }
     return ogObject;
 }

diff --git a/dist/lib/fields.js b/dist/lib/fields.js
@@ -22,6 +22,11 @@ const fields = [
         property: 'og:logo',
         fieldName: 'ogLogo',
     },
+    {
+        multiple: false,
+        property: 'og:website',
+        fieldName: 'ogWebsite',
+    },
     {
         multiple: true,
         property: 'og:image',
@@ -162,11 +167,61 @@ const fields = [
         property: 'og:video:secure_url',
         fieldName: 'ogVideoSecureURL',
     },
+    {
+        multiple: true,
+        property: 'og:movie',
+        fieldName: 'ogMovie',
+    },
+    {
+        multiple: true,
+        property: 'og:episode',
+        fieldName: 'ogEpisode',
+    },
+    {
+        multiple: true,
+        property: 'og:video:actor',
+        fieldName: 'ogVideoActor',
+    },
     {
         multiple: true,
         property: 'og:video:actor:id',
         fieldName: 'ogVideoActorId',
     },
+    {
+        multiple: true,
+        property: 'og:video:actor:role',
+        fieldName: 'ogVideoActorRole',
+    },
+    {
+        multiple: true,
+        property: 'og:video:director',
+        fieldName: 'ogVideoDirector',
+    },
+    {
+        multiple: true,
+        property: 'og:video:writer',
+        fieldName: 'ogVideoWriter',
+    },
+    {
+        multiple: true,
+        property: 'og:video:duration',
+        fieldName: 'ogVideoDuration',
+    },
+    {
+        multiple: true,
+        property: 'og:video:release_date',
+        fieldName: 'ogVideoReleaseDate',
+    },
+    {
+        multiple: true,
+        property: 'og:video:Tag',
+        fieldName: 'ogVideoTag',
+    },
+    {
+        multiple: true,
+        property: 'og:video:series',
+        fieldName: 'ogVideoSeries',
+    },
     {
         multiple: true,
         property: 'og:video:width',
@@ -182,6 +237,16 @@ const fields = [
         property: 'og:video:type',
         fieldName: 'ogVideoType',
     },
+    {
+        multiple: true,
+        property: 'og:video:tv_show',
+        fieldName: 'ogVideoTvShow',
+    },
+    {
+        multiple: true,
+        property: 'og:video:other',
+        fieldName: 'ogVideoOther',
+    },
     {
         multiple: false,
         property: 'twitter:card',
@@ -212,6 +277,11 @@ const fields = [
         property: 'twitter:creator:id',
         fieldName: 'twitterCreatorId',
     },
+    {
+        multiple: false,
+        property: 'twitter:account',
+        fieldName: 'twitterAccount',
+    },
     {
         multiple: false,
         property: 'twitter:title',
@@ -317,6 +387,11 @@ const fields = [
         property: 'twitter:app:url:googleplay',
         fieldName: 'twitterAppUrlGooglePlay',
     },
+    {
+        multiple: true,
+        property: 'music:playlist',
+        fieldName: 'musicPlaylist',
+    },
     {
         multiple: true,
         property: 'music:song',
@@ -352,6 +427,11 @@ const fields = [
         property: 'music:duration',
         fieldName: 'musicDuration',
     },
+    {
+        multiple: true,
+        property: 'music:radio_station',
+        fieldName: 'musicRadioStation',
+    },
     {
         multiple: true,
         property: 'music:creator',
@@ -857,5 +937,10 @@ const fields = [
         property: 'al:web:should_fallback',
         fieldName: 'alWebShouldFallback',
     },
+    {
+        multiple: false,
+        property: 'fb:app_id',
+        fieldName: 'fbAppId',
+    },
 ];
 exports.default = fields;
diff --git a/dist/lib/types.d.ts b/dist/lib/types.d.ts
@@ -160,6 +160,8 @@ export type OgObjectInteral = {
     error?: string;
     errorDetails?: Error;
     favicon?: string;
+    fbAppId?: string;
+    jsonLD?: object;
     modifiedTime?: string;
     musicAlbum?: string;
     musicAlbumDisc?: string;
@@ -168,6 +170,8 @@ export type OgObjectInteral = {
     musicCreator?: string;
     musicDuration?: string;
     musicMusician?: string;
+    musicPlaylist?: string;
+    musicRadioStation?: string;
     musicReleaseDate?: string;
     musicSong?: MusicSongObject[];
     musicSongDisc?: string | string[] | null[];
@@ -189,17 +193,19 @@ export type OgObjectInteral = {
     ogDate?: string;
     ogDescription?: string;
     ogDeterminer?: string;
+    ogEpisode?: string;
     ogImage?: ImageObject[];
+    ogImageAlt?: string | string[] | null[];
     ogImageHeight?: string | string[] | null[];
     ogImageProperty?: string | string[] | null[];
     ogImageSecureURL?: string | string[] | null[];
     ogImageType?: string | string[] | null[];
     ogImageURL?: string | string[] | null[];
     ogImageWidth?: string | string[] | null[];
-    ogImageAlt?: string | string[] | null[];
     ogLocale?: string;
     ogLocaleAlternate?: string;
     ogLogo?: string;
+    ogMovie?: string;
     ogPriceAmount?: string;
     ogPriceCurrency?: string;
     ogProductAvailability?: string;
@@ -212,12 +218,23 @@ export type OgObjectInteral = {
     ogType?: string;
     ogUrl?: string;
     ogVideo?: VideoObject[];
+    ogVideoActor?: string;
     ogVideoActorId?: string;
+    ogVideoActorRole?: string;
+    ogVideoDirector?: string;
+    ogVideoDuration?: string;
     ogVideoHeight?: string | string[] | null[];
+    ogVideoOther?: string;
     ogVideoProperty?: string | string[] | null[];
+    ogVideoReleaseDate?: string;
     ogVideoSecureURL?: string;
+    ogVideoSeries?: string;
+    ogVideoTag?: string;
+    ogVideoTvShow?: string;
     ogVideoType?: string | string[] | null[];
     ogVideoWidth?: string | string[] | null[];
+    ogVideoWriter?: string;
+    ogWebsite?: string;
     placeLocationLatitude?: string;
     placeLocationLongitude?: string;
     profileFirstName?: string;
@@ -241,6 +258,7 @@ export type OgObjectInteral = {
     restaurantVariationPriceAmount?: string;
     restaurantVariationPriceCurrency?: string;
     success?: boolean;
+    twitterAccount?: string;
     twitterAppIdGooglePlay?: string;
     twitterAppIdiPad?: string;
     twitterAppIdiPhone?: string;

diff --git a/lib/extract.ts b/lib/extract.ts
@@ -56,6 +56,12 @@ export default function extractMetaTags(body: string, options: OpenGraphScraperO
   // if onlyGetOpenGraphInfo isn't set, run the open graph fallbacks
   if (!options.onlyGetOpenGraphInfo) {
     ogObject = fallback(ogObject, options, $, body);
+
+    $('script').each((index, script) => {
+      if (script.attribs.type && script.attribs.type === 'application/ld+json') {
+        ogObject.jsonLD = JSON.parse($(script).text());
+      }
+    });
   }
 
   return ogObject;

diff --git a/lib/fields.ts b/lib/fields.ts
@@ -20,6 +20,11 @@ const fields = [
     property: 'og:logo',
     fieldName: 'ogLogo',
   },
+  {
+    multiple: false,
+    property: 'og:website',
+    fieldName: 'ogWebsite',
+  },
   {
     multiple: true,
     property: 'og:image',
@@ -160,11 +165,61 @@ const fields = [
     property: 'og:video:secure_url',
     fieldName: 'ogVideoSecureURL',
   },
+  {
+    multiple: true,
+    property: 'og:movie',
+    fieldName: 'ogMovie',
+  },
+  {
+    multiple: true,
+    property: 'og:episode',
+    fieldName: 'ogEpisode',
+  },
+  {
+    multiple: true,
+    property: 'og:video:actor',
+    fieldName: 'ogVideoActor',
+  },
   {
     multiple: true,
     property: 'og:video:actor:id',
     fieldName: 'ogVideoActorId',
   },
+  {
+    multiple: true,
+    property: 'og:video:actor:role',
+    fieldName: 'ogVideoActorRole',
+  },
+  {
+    multiple: true,
+    property: 'og:video:director',
+    fieldName: 'ogVideoDirector',
+  },
+  {
+    multiple: true,
+    property: 'og:video:writer',
+    fieldName: 'ogVideoWriter',
+  },
+  {
+    multiple: true,
+    property: 'og:video:duration',
+    fieldName: 'ogVideoDuration',
+  },
+  {
+    multiple: true,
+    property: 'og:video:release_date',
+    fieldName: 'ogVideoReleaseDate',
+  },
+  {
+    multiple: true,
+    property: 'og:video:Tag',
+    fieldName: 'ogVideoTag',
+  },
+  {
+    multiple: true,
+    property: 'og:video:series',
+    fieldName: 'ogVideoSeries',
+  },
   {
     multiple: true,
     property: 'og:video:width',
@@ -180,6 +235,16 @@ const fields = [
     property: 'og:video:type',
     fieldName: 'ogVideoType',
   },
+  {
+    multiple: true,
+    property: 'og:video:tv_show',
+    fieldName: 'ogVideoTvShow',
+  },
+  {
+    multiple: true,
+    property: 'og:video:other',
+    fieldName: 'ogVideoOther',
+  },
   {
     multiple: false,
     property: 'twitter:card',
@@ -210,6 +275,11 @@ const fields = [
     property: 'twitter:creator:id',
     fieldName: 'twitterCreatorId',
   },
+  {
+    multiple: false,
+    property: 'twitter:account',
+    fieldName: 'twitterAccount',
+  },
   {
     multiple: false,
     property: 'twitter:title',
@@ -315,6 +385,11 @@ const fields = [
     property: 'twitter:app:url:googleplay',
     fieldName: 'twitterAppUrlGooglePlay',
   },
+  {
+    multiple: true,
+    property: 'music:playlist',
+    fieldName: 'musicPlaylist',
+  },
   {
     multiple: true,
     property: 'music:song',
@@ -350,6 +425,11 @@ const fields = [
     property: 'music:duration',
     fieldName: 'musicDuration',
   },
+  {
+    multiple: true,
+    property: 'music:radio_station',
+    fieldName: 'musicRadioStation',
+  },
   {
     multiple: true,
     property: 'music:creator',
@@ -855,6 +935,11 @@ const fields = [
     property: 'al:web:should_fallback',
     fieldName: 'alWebShouldFallback',
   },
+  {
+    multiple: false,
+    property: 'fb:app_id',
+    fieldName: 'fbAppId',
+  },
 ];
 
 export default fields;