Merge pull request #1877 from openzim/1876-update-mw-action-api-to-ve…

…rsion-2

1876 update mw action api to version 2

src/Downloader.ts

@@ -415,6 +415,7 @@ class Downloader {
       ...MediaWiki.queryOpts,
       prop: MediaWiki.queryOpts.prop.concat(prop),
       rdnamespace: validNamespaceIds.join('|'),
+      formatversion: '2',
       redirects: redirects ? true : undefined,
     }
   }
@@ -669,7 +670,7 @@ class Downloader {
 
     // Saving, as a js module, the jsconfigvars that are set in the header of a wikipedia page
     // the script below extracts the config with a regex executed on the page header returned from the api
-    const scriptTags = domino.createDocument(`${headhtml['*']}</body></html>`).getElementsByTagName('script')
+    const scriptTags = domino.createDocument(`${headhtml}</body></html>`).getElementsByTagName('script')
     const regex = /mw\.config\.set\(\{.*?\}\);/gm
     // eslint-disable-next-line @typescript-eslint/prefer-for-of
     for (let i = 0; i < scriptTags.length; i += 1) {

src/MediaWiki.ts

@@ -20,6 +20,7 @@ export interface QueryOpts {
   rdlimit: string
   rdnamespace: string | number
   redirects?: boolean
+  formatversion: string
 }
 
 class MediaWiki {
@@ -115,6 +116,7 @@ class MediaWiki {
       rdlimit: 'max',
       rdnamespace: 0,
       redirects: false,
+      formatversion: '2',
     }
 
     this.#hasWikimediaDesktopRestApi = null
@@ -151,7 +153,7 @@ class MediaWiki {
       }
 
       const resp = await downloader.getJSON<MwApiResponse>(this.apiUrlDirector.buildQueryURL(reqOpts))
-      const isCoordinateWarning = resp.warnings && resp.warnings.query && (resp.warnings.query['*'] || '').includes('coordinates')
+      const isCoordinateWarning = JSON.stringify(resp?.warnings?.query ?? '').includes('coordinates')
       if (isCoordinateWarning) {
         logger.info('Coordinates not available on this wiki')
         return (this.#hasCoordinates = false)
@@ -183,7 +185,7 @@ class MediaWiki {
       }
 
       // Getting token to login.
-      const { content, responseHeaders } = await downloader.downloadContent(url + 'action=query&meta=tokens&type=login&format=json')
+      const { content, responseHeaders } = await downloader.downloadContent(url + 'action=query&meta=tokens&type=login&format=json&formatversion=2')
 
       // Logging in
       await axios(this.apiUrl.href, {
@@ -221,10 +223,10 @@ class MediaWiki {
       const entries = json.query[type]
       Object.keys(entries).forEach((key) => {
         const entry = entries[key]
-        const name = entry['*']
+        const name = type === 'namespaces' ? entry.name : entry.alias
         const num = entry.id
         const allowedSubpages = 'subpages' in entry
-        const isContent = !!(entry.content !== undefined || util.contains(addNamespaces, num))
+        const isContent = type === 'namespaces' ? !!(entry.content || util.contains(addNamespaces, num)) : !!(entry.content !== undefined || util.contains(addNamespaces, num))
         const canonical = entry.canonical ? entry.canonical : ''
         const details = { num, allowedSubpages, isContent }
         /* Namespaces in local language */

src/types.d.ts

@@ -124,7 +124,7 @@ interface MwApiQueryResponse {
 }
 
 interface MwApiResponse {
-  batchcomplete: string
+  batchcomplete: boolean
   query: MwApiQueryResponse
   continue?: {
     [key: string]: string

src/util/builders/url/api.director.ts

@@ -19,6 +19,7 @@ export default class ApiURLDirector {
         cmtype: 'subcat',
         cmlimit: 'max',
         format: 'json',
+        formatversion: '2',
         cmtitle: articleId,
         cmcontinue: continueStr,
       })
@@ -28,7 +29,7 @@ export default class ApiURLDirector {
   buildSiteInfoQueryURL() {
     return urlBuilder
       .setDomain(this.baseDomain)
-      .setQueryParams({ action: 'query', meta: 'siteinfo', format: 'json', siprop: 'general|namespaces|statistics|variables|category|wikidesc' })
+      .setQueryParams({ action: 'query', meta: 'siteinfo', format: 'json', formatversion: '2', siprop: 'general|namespaces|statistics|variables|category|wikidesc' })
       .build()
   }
 
@@ -37,15 +38,21 @@ export default class ApiURLDirector {
   }
 
   buildNamespacesURL() {
-    return urlBuilder.setDomain(this.baseDomain).setQueryParams({ action: 'query', meta: 'siteinfo', siprop: 'namespaces|namespacealiases', format: 'json' }).build()
+    return urlBuilder
+      .setDomain(this.baseDomain)
+      .setQueryParams({ action: 'query', meta: 'siteinfo', siprop: 'namespaces|namespacealiases', format: 'json', formatversion: '2' })
+      .build()
   }
 
   buildSiteInfoURL() {
-    return urlBuilder.setDomain(this.baseDomain).setQueryParams({ action: 'query', meta: 'siteinfo', format: 'json' }).build()
+    return urlBuilder.setDomain(this.baseDomain).setQueryParams({ action: 'query', meta: 'siteinfo', format: 'json', formatversion: '2' }).build()
   }
 
   buildVisualEditorURL() {
-    return urlBuilder.setDomain(this.baseDomain).setQueryParams({ action: 'visualeditor', mobileformat: 'html', format: 'json', paction: 'parse', page: '' }).build(true)
+    return urlBuilder
+      .setDomain(this.baseDomain)
+      .setQueryParams({ action: 'visualeditor', mobileformat: 'html', format: 'json', paction: 'parse', formatversion: '2', page: '' })
+      .build(true)
   }
 
   buildArticleApiURL(articleId: string) {
@@ -55,6 +62,6 @@ export default class ApiURLDirector {
   }
 
   private buildBaseArticleURL() {
-    return urlBuilder.setDomain(this.baseDomain).setQueryParams({ action: 'parse', format: 'json', prop: 'modules|jsconfigvars|headhtml' }).build()
+    return urlBuilder.setDomain(this.baseDomain).setQueryParams({ action: 'parse', format: 'json', prop: 'modules|jsconfigvars|headhtml', formatversion: '2' }).build()
   }
 }

test/unit/builders/url/api.director.test.ts

@@ -7,15 +7,17 @@ describe('ApiURLDirector', () => {
     it('should return a string URL to get article sub categories', () => {
       const url = apiUrlDirector.buildSubCategoriesURL('article-123')
 
-      expect(url).toBe('https://en.wikipedia.org/w/api.php?action=query&list=categorymembers&cmtype=subcat&cmlimit=max&format=json&cmtitle=article-123&cmcontinue=')
+      expect(url).toBe('https://en.wikipedia.org/w/api.php?action=query&list=categorymembers&cmtype=subcat&cmlimit=max&format=json&formatversion=2&cmtitle=article-123&cmcontinue=')
     })
   })
 
   describe('buildSiteInfoQueryURL', () => {
     it('should return string URL to get site info', () => {
       const url = apiUrlDirector.buildSiteInfoQueryURL()
 
-      expect(url).toBe('https://en.wikipedia.org/w/api.php?action=query&meta=siteinfo&format=json&siprop=general%7Cnamespaces%7Cstatistics%7Cvariables%7Ccategory%7Cwikidesc')
+      expect(url).toBe(
+        'https://en.wikipedia.org/w/api.php?action=query&meta=siteinfo&format=json&formatversion=2&siprop=general%7Cnamespaces%7Cstatistics%7Cvariables%7Ccategory%7Cwikidesc',
+      )
     })
   })
 
@@ -31,31 +33,31 @@ describe('ApiURLDirector', () => {
     it('should return a string URL with predefined query params and provided page for retrieving article', () => {
       const url = apiUrlDirector.buildArticleApiURL('article-123')
 
-      expect(url).toBe('https://en.wikipedia.org/w/api.php?action=parse&format=json&prop=modules%7Cjsconfigvars%7Cheadhtml&page=article-123')
+      expect(url).toBe('https://en.wikipedia.org/w/api.php?action=parse&format=json&prop=modules%7Cjsconfigvars%7Cheadhtml&formatversion=2&page=article-123')
     })
   })
 
   describe('buildNamespacesURL', () => {
     it('should return a string URL with predefined query params to get article namespaces', () => {
       const url = apiUrlDirector.buildNamespacesURL()
 
-      expect(url).toBe('https://en.wikipedia.org/w/api.php?action=query&meta=siteinfo&siprop=namespaces%7Cnamespacealiases&format=json')
+      expect(url).toBe('https://en.wikipedia.org/w/api.php?action=query&meta=siteinfo&siprop=namespaces%7Cnamespacealiases&format=json&formatversion=2')
     })
   })
 
   describe('buildSiteInfoURL', () => {
     it('should return a string URL with predefined query params for retrieving site info', () => {
       const url = apiUrlDirector.buildSiteInfoURL()
 
-      expect(url).toBe('https://en.wikipedia.org/w/api.php?action=query&meta=siteinfo&format=json')
+      expect(url).toBe('https://en.wikipedia.org/w/api.php?action=query&meta=siteinfo&format=json&formatversion=2')
     })
   })
 
   describe('buildVisualEditorURL', () => {
     it('should return base visual editor URL object with default query params', () => {
       const url = apiUrlDirector.buildVisualEditorURL()
 
-      expect(url.href).toBe('https://en.wikipedia.org/w/api.php?action=visualeditor&mobileformat=html&format=json&paction=parse&page=')
+      expect(url.href).toBe('https://en.wikipedia.org/w/api.php?action=visualeditor&mobileformat=html&format=json&paction=parse&formatversion=2&page=')
     })
   })
 })

test/unit/downloader.test.ts

@@ -37,6 +37,12 @@ describe('Downloader class', () => {
     await downloader.setBaseUrls()
   })
 
+  test('Test Action API version 2 response in comparison with version 1', async () => {
+    const actionAPIResV1 = await downloader.getJSON('https://en.wikipedia.org/w/api.php?action=parse&format=json&prop=modules|jsconfigvars|headhtml&page=Potato')
+    const actionAPIResV2 = await downloader.getJSON('https://en.wikipedia.org/w/api.php?action=parse&format=json&prop=modules|jsconfigvars|headhtml&formatversion=2&page=Potato')
+    expect(actionAPIResV1).not.toEqual(actionAPIResV2)
+  })
+
   test('downloader.query returns valid JSON', async () => {
     const queryRet = await downloader.query()
     expect(typeof queryRet).toBe('object')
@@ -88,7 +94,7 @@ describe('Downloader class', () => {
     expect(Paris).toBeDefined()
     expect(Zürich).toBeDefined()
 
-    expect(THISARTICLEDOESNTEXIST.missing).toBe('')
+    expect(THISARTICLEDOESNTEXIST.missing).toBe(true)
   })
 
   test("getArticleDetailsNS query returns 'gapContinue' or 'multiple articles', ", async () => {
@@ -235,7 +241,7 @@ describe('Downloader class', () => {
     })
 
     test('Url is not image type', async () => {
-      const isnotImage = isImageUrl('https://en.wikipedia.org/w/api.php?action=query&meta=siteinfo&format=json')
+      const isnotImage = isImageUrl('https://en.wikipedia.org/w/api.php?action=query&meta=siteinfo&format=json&formatversion=2')
       expect(isnotImage).not.toBeTruthy()
     })
 
@@ -339,7 +345,7 @@ describe('Downloader class', () => {
 
   async function getRandomImageUrl(): Promise<string> {
     const resp = await Axios(
-      'https://commons.wikimedia.org/w/api.php?action=query&generator=random&grnnamespace=6&prop=imageinfo&iiprop=url&formatversion=2&iiurlwidth=100&format=json',
+      'https://commons.wikimedia.org/w/api.php?action=query&generator=random&grnnamespace=6&prop=imageinfo&iiprop=url&formatversion=2&iiurlwidth=100&format=json&formatversion=2',
     )
     const url = resp.data.query.pages[0].imageinfo[0].url
     return isImageUrl(url) ? url : getRandomImageUrl()

test/unit/util.test.ts

@@ -313,7 +313,7 @@ describe('Utils', () => {
 
   test('No title normalisation', async () => {
     const resp = await axios.get<MwApiResponse>(
-      'https://en.wiktionary.org/w/api.php?action=query&format=json&prop=redirects|revisions|pageimages&rdlimit=max&rdnamespace=0&redirects=true&titles=constructor',
+      'https://en.wiktionary.org/w/api.php?action=query&format=json&prop=redirects|revisions|pageimages&rdlimit=max&rdnamespace=0&redirects=true&titles=constructor&formatversion=2',
       { responseType: 'json' },
     )
     const normalizedObject = normalizeMwResponse(resp.data.query)