lib/*-verifier.js - improve abstraction

- add pypi verifier (untested)

lib/npm-verifier.js

@@ -3,17 +3,20 @@
 
 const axios = require('axios');
 
-const {repoParts, BaseVerifier} = require('./base-verifier');
+const {repoPartsKey, BaseVerifier} = require('./base-verifier');
 
-const npmUrlTemplate = 'https://registry.npmjs.com/${package}';
+const packageUrlTemplate = 'https://registry.npmjs.com/${package}';
 
 
 // must be able to fetch the package metadata or nothing else can happen.
 // a constructor can't return a promise so this is a construct-initialize
 // pattern.
 async function makeNpmVerifier (packageName, options = {}) {
   const npmVerifier = new NpmVerifier(packageName, options);
-  await npmVerifier.initialize();
+  const versions = await npmVerifier.initialize();
+  if (versions instanceof Error) {
+    return versions;
+  }
   return npmVerifier;
 }
 
@@ -22,39 +25,64 @@ class NpmVerifier extends BaseVerifier {
 
   constructor (packageName, options) {
     super('npm', packageName, options);
-    this.npmPackageMetadataUrl = npmUrlTemplate.replace('${package}', packageName);
+    this.packageMetadataUrl = packageUrlTemplate.replace('${package}', packageName);
     this.repoVersionTag = this.options.repoVersionTag || 'v${tag}';
   }
 
   async initialize () {
-    this.packageMetadata = await this.fetchPackageMetadata();
+    try {
+      this.packageMetadata = await this.fetchPackageMetadata();
+    } catch (e) {
+      return e;
+    }
     const vMetadata = this.vMetadata = this.packageMetadata.versions;
     const versions = this.versions = Object.keys(vMetadata);
 
     this.makeRequiredDirectories();
 
-    // decode the repo info for all the versions now so if there are any of an unrecognized format
-    // it will throw an error at the start.
-    // TODO BAM - allow skipping those that don't match
+    // decode the repo info for each version so if any formats can't be handled it's known
+    // at the outset.
     for (let i = 0; i < versions.length; i++) {
-      vMetadata[versions[i]][repoParts] = this.extractRepoInfo(vMetadata[versions[i]]);
+      // get the version-specific data
+      const vsd = vMetadata[versions[i]];
+      let repoParts;
+      // if there is a sourceUrl option use it else verify that there is a repository of
+      // the right type.
+      let sourceUrl;
+      if (this.options.sourceUrl) {
+        sourceUrl = this.options.sourceUrl;
+      } else if (!vsd.repository) {
+        this.warn(`no repository information for version ${vsd.version}`);
+      } else if (vsd.repository.type !== 'git') {
+        this.warn(`ignoring repository type ${vsd.repository.type} for version ${vsd.version}`);
+      } else {
+        sourceUrl = vsd.repository.url;
+      }
+
+      if (sourceUrl) {
+        repoParts = this.extractRepoInfo(sourceUrl);
+        if (!repoParts) {
+          this.warn(`invalid repository ${sourceUrl} for version ${vsd.version}`);
+        }
+      }
+
+      vMetadata[versions[i]][repoPartsKey] = repoParts;
     }
+    // versions isn't used by makeNpmVerifier but maybe in the future. need to return something
+    // that isn't an error.
+    return versions;
   }
 
   async fetchPackageMetadata (options) {
-    return axios.get(this.npmPackageMetadataUrl)
+    return axios.get(this.packageMetadataUrl)
       .then(info => this.normalize(info.data))
   }
 
-  // in theory this can normalize across ruby and python once i discover what metadata
-  // the supply.
+  // normalize the metadata so it's like node, at least for the versions.
   normalize (metadata) {
     return metadata;
   }
 
-  //const gitUrl = `https://api.github.com/repos/${agentMap.gitUser}/${agentMap.gitRepo}/tarball/${tag}`;
-  //const gitTarget = `git/${agentMap.gitRepo}-${tag}.tar.gz`;
-
   //
   // package-dependent information follows
   //
@@ -74,25 +102,20 @@ class NpmVerifier extends BaseVerifier {
     return this.unpackedPkgDir;
   }
 
-  // i think this is npm-specific but am not sure. maybe just the type property?
-  // 'git+https://github.com/appoptics/appoptics-apm-node.git'
-  extractRepoInfo (versionData) {
-    const repository = versionData.repository;
-    if (!repository) {
-      this.warn(`no repository information for version ${versionData.version}`);
-      return undefined;
-    }
-    if (repository.type !== 'git') {
-      this.warn(`ignoring repository type ${repository.type} for version ${versionData.version}`);
-      return undefined;
+  // repository: git+https://github.com/appoptics/appoptics-apm-node.git
+  // this treats the 'git+' and '.git' as optional as they're not used by
+  // the code.
+  extractRepoInfo (repository) {
+    if (repository.endsWith('.git')) {
+      repository = repository.slice(0, -'.git'.length);
     }
-    //                                                 1         2         3
-    const match = repository.url.match(/git\+https:\/\/([^\/]+)\/([^\/]+)\/(.+)\.git/);
-
+    // match groups                 x                   1         2         3
+    const match = repository.match(/(?:git\+)?https:\/\/([^\/]+)\/([^\/]+)\/(.+)/);
     if (!match) {
       // hmmm. there is some format we don't handle.
-      throw new TypeError(`unexpected repository.url format ${repository.url}`);
+      return match;
     }
+
     return {
       host: match[1],
       user: match[2],

lib/pypi-verifier.js

@@ -0,0 +1,163 @@
+'use strict';
+
+
+const axios = require('axios');
+
+const {repoPartsKey, BaseVerifier} = require('./base-verifier');
+
+// the url used to fetch metadata for a package
+const packageUrlTemplate = 'https://pypi.org/pypi/${package}/json';
+
+
+// must be able to fetch the package metadata or nothing else can happen.
+// a constructor can't return a promise so this is a construct-initialize
+// pattern.
+async function makePypiVerifier (packageName, options = {}) {
+  const pypiVerifier = new PypiVerifier(packageName, options);
+  const versions = await pypiVerifier.initialize();
+  if (versions instanceof Error) {
+    return versions;
+  }
+  return pypiVerifier;
+}
+
+
+class PypiVerifier extends BaseVerifier {
+
+  constructor (packageName, options) {
+    super('pypi', packageName, options);
+    this.packageMetadataUrl = packageUrlTemplate.replace('${package}', packageName);
+    this.repoVersionTag = this.options.repoVersionTag || 'v${tag}';
+  }
+
+  async initialize () {
+    try {
+      this.packageMetadata = await this.fetchPackageMetadata();
+    } catch (e) {
+      return e;
+    }
+    const vMetadata = this.vMetadata = this.packageMetadata.releases;
+    const versions = this.versions = Object.keys(vMetadata);
+
+    this.makeRequiredDirectories();
+
+    // decode the repo for each version so if any formats can't be handled it's known
+    // at the outset.
+    for (let i = 0; i < versions.length; i++) {
+      // get the version-specific data
+      const vsd = vMetadata[versions[i]];
+      let repoParts;
+      // if there is a sourceUrl option use it else verify that there is a repository of
+      // the right type. try to use the home page properties but what they point to depends
+      // on the publisher so they might not be right.
+      let sourceUrl;
+      if (this.options.sourceUrl) {
+        sourceUrl = this.options.sourceUrl;
+      } else if (this.packageMetadata.info.project_urls.Homepage) {
+        sourceUrl = this.packageMetadata.info.project_urls.Homepage;
+      } else if (this.packageMetadata.info.home_page) {
+        sourceUrl = this.packageMetadata.info.home_page;
+      }
+
+      if (sourceUrl) {
+        repoParts = this.extractRepoInfo(sourceUrl);
+        if (!repoParts) {
+          this.warn(`invalid repository ${sourceUrl} for version ${vsd.version}`);
+        }
+      }
+
+      vMetadata[versions[i]][repoPartsKey] = repoParts;
+    }
+
+    return versions;
+  }
+
+  async fetchPackageMetadata (options) {
+    return axios.get(this.packageMetadataUrl)
+      .then(info => this.normalize(info.data))
+  }
+
+  normalize (metadata) {
+    // fix up the versions so they don't point to arrays and only point to the
+    // objects referencing .tar.gz files.
+    const releases = {};
+    const versions = Object.keys(metadata.releases);
+
+    for (let i = 0; i < versions.length; i++) {
+      // only store the version if there is entry for a .tar.gz file
+      const entries = metadata.releases[versions[i]];
+      for (let j = 0; j < entries.length; j++) {
+        if (entries[j].url && entries[j].url.endsWith('.tar.gz')) {
+          releases[versions[i]] = entries[j];
+          break;
+        }
+      }
+    }
+    metadata.releases = releases;
+
+    return metadata;
+  }
+
+  //
+  // package-dependent information follows
+  //
+  getPkgUrl (version) {
+    return this.vMetadata[version].url;
+  }
+
+  getPkgTarget (version) {
+    return `${this.pkgDir}/${this.package}-${this.versionTag(version)}.tar.gz`;
+  }
+
+  getPkgUnpackCommand (pkgTarget, unpackedPkgDir) {
+    return `tar --strip-components=1 -zvxf ${pkgTarget} -C ${unpackedPkgDir}`;
+  }
+
+  getDownloadedUnpackedPkgDir (version) {
+    return this.unpackedPkgDir;
+  }
+
+  // repository expected: https://github.com/<user>/<package>'
+  extractRepoInfo (repository) {
+    // match groups                               1         2         3
+    const match = repository.match(/https:\/\/([^\/]+)\/([^\/]+)\/(.+)/);
+    if (!match) {
+      // hmmm. there is some format we don't handle.
+      return match;
+    }
+
+    return {
+      host: match[1],
+      user: match[2],
+      name: match[3],
+    }
+  }
+
+}
+
+module.exports = {makePypiVerifier};
+
+if (!module.parent) {
+  (async function tester () {
+    const nv = await makePypiVerifier('appoptics-apm');
+
+    const versions = nv.getMatchingVersions('>= 6.5.1 <= 6.6.0');
+
+    const results = await nv.verifyThese(versions);
+
+    // eslint-disable-next-line no-console
+    console.log('');
+
+    results.forEach(result => {
+      const differences = nv.extractDifferences(result, {differences: 'important'});
+      if (differences) {
+        // eslint-disable-next-line no-console
+        console.log(`unexpected differences for ${result.version}:\n${differences.join('\n')}`);
+      }
+
+    })
+  })().then(r =>
+    // eslint-disable-next-line no-console
+    console.log('done')
+  );
+}