From 9451aeea10c51b1400bcb13e5a22d245ce65f7fc Mon Sep 17 00:00:00 2001 From: Dar Malovani Date: Thu, 9 Nov 2017 14:29:07 +0200 Subject: [PATCH] feat: rewrite raw endpoints to /contents api for GH Since GH enterprise 2.11 the `/raw` endpoint is not supported. In order for us to make a gradual change without breaking our users broker clients I added logic that rewrites /raw to /repos/owner/repo/contents endpoint. Once all of our GHE users will use the we could change all our system code that touches GH files, or hopefully will do it on the GH agent, and delete this code. I know that this code makes our filter logic GH aware, and this was not the intention of the broker, but this was the solution for non breaking change. --- README.md | 3 -- client-templates/github/.env.sample | 4 --- client-templates/github/accept.json.sample | 40 +++++++++++----------- dockerfiles/github-com/Dockerfile | 3 -- dockerfiles/github-enterprise/Dockerfile | 3 -- lib/relay.js | 29 +++++++++++++++- test/fixtures/client/filters.json | 12 +++++-- test/functional/server-client.test.js | 28 ++++++++++++++- test/utils.js | 5 +++ 9 files changed, 89 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index cc2cc50ec..6b95f6dea 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,6 @@ To use the the broker client with a GitHub Enterprise deployment, run `docker pu - `GITHUB_TOKEN` - a personal access token with full `repo` and `admin:repo_hook` scopes. - `GITHUB` - the hostname of your GitHub Enterprise deployment, such as `your.ghe.domain.com`. - `GITHUB_API` - the API endpoint of your GitHub Enterprise deployment. Should be `$GITHUB/api/v3`. -- `GITHUB_RAW` - the raw file access endpoint of your GitHub Enterprise deployment. Should be `$GITHUB/raw`. - `PORT` - the local port at which the broker client accepts connections. Default is 7341. - `BROKER_CLIENT_URL` - the full URL of the broker client as it will be accessible by your GitHub Enterprise deployment webhooks, such as `http://my.broker.client:7341` @@ -75,7 +74,6 @@ docker run --restart=always \ -e GITHUB_TOKEN=secret-github-token \ -e GITHUB=your.ghe.domain.com \ -e GITHUB_API=your.ghe.domain.com/api/v3 \ - -e GITHUB_RAW=your.ghe.domain.com/raw \ -e PORT=8000 \ -e BROKER_CLIENT_URL=http://my.broker.client:8000 \ snyk/broker:github-enterprise @@ -92,7 +90,6 @@ ENV BROKER_TOKEN secret-broker-token ENV GITHUB_TOKEN secret-github-token ENV GITHUB your.ghe.domain.com ENV GITHUB_API your.ghe.domain.com/api/v3 -ENV GITHUB_RAW your.ghe.domain.com/raw ENV PORT 8000 ENV BROKER_CLIENT_URL http://my.broker.client:8000 ``` diff --git a/client-templates/github/.env.sample b/client-templates/github/.env.sample index a613ea7f2..0b5c2f533 100644 --- a/client-templates/github/.env.sample +++ b/client-templates/github/.env.sample @@ -12,10 +12,6 @@ GITHUB= # changed to "api.github.com" GITHUB_API=$GITHUB/api/v3 -# the url where raw file content is accessed, excluding scheme. For github.com -# this should be changed to "raw.githubusercontent.com" -GITHUB_RAW=$GITHUB/raw - # the url of your broker client (including scheme and port) # BROKER_CLIENT_URL= diff --git a/client-templates/github/accept.json.sample b/client-templates/github/accept.json.sample index 09ddaea5e..b5ca4e923 100644 --- a/client-templates/github/accept.json.sample +++ b/client-templates/github/accept.json.sample @@ -124,62 +124,62 @@ { "//": "used to determine the full dependency tree", "method": "GET", - "path": "/:name/:repo/:branch*/package.json", - "origin": "https://${GITHUB_TOKEN}@${GITHUB_RAW}" + "path": "/repos/:name/:repo/contents/:path*/package.json", + "origin": "https://${GITHUB_TOKEN}@${GITHUB_API}" }, { "//": "used to determine the full dependency tree", "method": "GET", - "path": "/:name/:repo/:branch*/package-lock.json", - "origin": "https://${GITHUB_TOKEN}@${GITHUB_RAW}" + "path": "/repos/:name/:repo/contents/:path*/package-lock.json", + "origin": "https://${GITHUB_TOKEN}@${GITHUB_API}" }, { "//": "used to determine the full dependency tree", "method": "GET", - "path": "/:name/:repo/:branch*/Gemfile.lock", - "origin": "https://${GITHUB_TOKEN}@${GITHUB_RAW}" + "path": "/repos/:name/:repo/contents/:path*/Gemfile.lock", + "origin": "https://${GITHUB_TOKEN}@${GITHUB_API}" }, { "//": "used to determine the full dependency tree", "method": "GET", - "path": "/:name/:repo/:branch*/Gemfile", - "origin": "https://${GITHUB_TOKEN}@${GITHUB_RAW}" + "path": "/repos/:name/:repo/contents/:path*/Gemfile", + "origin": "https://${GITHUB_TOKEN}@${GITHUB_API}" }, { "//": "used to determine the full dependency tree", "method": "GET", - "path": "/:name/:repo/:branch*/pom.xml", - "origin": "https://${GITHUB_TOKEN}@${GITHUB_RAW}" + "path": "/repos/:name/:repo/contents/:path*/pom.xml", + "origin": "https://${GITHUB_TOKEN}@${GITHUB_API}" }, { "//": "used to determine the full dependency tree", "method": "GET", - "path": "/:name/:repo/:branch*/requirements.txt", - "origin": "https://${GITHUB_TOKEN}@${GITHUB_RAW}" + "path": "/repos/:name/:repo/contents/:path*/requirements.txt", + "origin": "https://${GITHUB_TOKEN}@${GITHUB_API}" }, { "//": "used to determine the full dependency tree", "method": "GET", - "path": "/:name/:repo/:branch*/yarn.lock", - "origin": "https://${GITHUB_TOKEN}@${GITHUB_RAW}" + "path": "/repos/:name/:repo/contents/:path*/yarn.lock", + "origin": "https://${GITHUB_TOKEN}@${GITHUB_API}" }, { "//": "used to determine the full dependency tree", "method": "GET", - "path": "/:name/:repo/:branch*/build.gradle", - "origin": "https://${GITHUB_TOKEN}@${GITHUB_RAW}" + "path": "/repos/:name/:repo/contents/:path*/build.gradle", + "origin": "https://${GITHUB_TOKEN}@${GITHUB_API}" }, { "//": "used to determine the full dependency tree", "method": "GET", - "path": "/:name/:repo/:branch*/build.sbt", - "origin": "https://${GITHUB_TOKEN}@${GITHUB_RAW}" + "path": "/repos/:name/:repo/contents/:path*/build.sbt", + "origin": "https://${GITHUB_TOKEN}@${GITHUB_API}" }, { "//": "used to check if there's any ignore rules or existing patches", "method": "GET", - "path": "/:name/:repo/:branch*/.snyk", - "origin": "https://${GITHUB_TOKEN}@${GITHUB_RAW}" + "path": "/repos/:name/:repo/contents/:path*/.snyk", + "origin": "https://${GITHUB_TOKEN}@${GITHUB_API}" }, { diff --git a/dockerfiles/github-com/Dockerfile b/dockerfiles/github-com/Dockerfile index 8829c0b29..93517ed57 100644 --- a/dockerfiles/github-com/Dockerfile +++ b/dockerfiles/github-com/Dockerfile @@ -46,9 +46,6 @@ ENV GITHUB github.com # The URL that the github API should be accessed, excluding scheme. ENV GITHUB_API api.github.com -# The URL where raw file content is accessed, excluding scheme. -ENV GITHUB_RAW raw.githubusercontent.com - # The URL of the Snyk broker server ENV BROKER_SERVER_URL https://broker.snyk.io diff --git a/dockerfiles/github-enterprise/Dockerfile b/dockerfiles/github-enterprise/Dockerfile index 15f3520b8..04441d31b 100644 --- a/dockerfiles/github-enterprise/Dockerfile +++ b/dockerfiles/github-enterprise/Dockerfile @@ -32,9 +32,6 @@ ENV GITHUB=your.ghe.domain.com # The URL that the github API should be accessed at. ENV GITHUB_API $GITHUB/api/v3 -# The URL where raw file content is accessed, excluding scheme. -ENV GITHUB_RAW $GITHUB/raw - # The port used by the broker client to accept webhooks # Default value is 7341 ENV PORT 7341 diff --git a/lib/relay.js b/lib/relay.js index fa199b9d3..2dd261aa1 100644 --- a/lib/relay.js +++ b/lib/relay.js @@ -55,8 +55,35 @@ function requestHandler(filterRules) { function responseHandler(filterRules, config) { const filters = Filters(filterRules); + const supportedFiles = (filterRules || []) + .filter(f => f.path && f.path.includes('/contents/')) + .map(f => f.path.split('/').pop()); + return (brokerToken) => ({ url, headers, method, body = null } = {}, emit) => { - // run the request through the filter + + // Changing old /raw endpoint to /contents api for GH clients. + // Since GH enterprise 2.11 version the /raw endpoint is deprecated + // This piece of code allow us to move to the new /content API without + // breaking old broker clients. + // Once all our GHE client will move to this new version of the broker + // client we could remove this code. + function rewriteRawApi() { + // find urls that ends with manifest files + const file = supportedFiles.find(f => url.endsWith(f)); + if (file && url) { + headers.accept = 'application/vnd.github.2.11.raw'; + // switching urls from: /:name/:repo/:branch*/:path + // to /repos/:name/:repo/contents/:path?ref=:branch + // $1 - repo/owner, $2 - branch, $3 - path + url = url + .replace(/(\/.+?\/.+?\/)(.+?)(\/.*)/, `/repos$1contents$3?ref=$2`); + } + } + + if (config.GITHUB) { + rewriteRawApi(); + } + logger.info({ method, url, headers }, 'request captured'); filters({ url, method, body, headers }, (error, result) => { if (error) { diff --git a/test/fixtures/client/filters.json b/test/fixtures/client/filters.json index f46e0010e..48c75daba 100644 --- a/test/fixtures/client/filters.json +++ b/test/fixtures/client/filters.json @@ -61,6 +61,13 @@ ] }, + + { + "path": "/repos/:repo/:owner/contents/folder*/package.json", + "method": "GET", + "origin": "http://localhost:${originPort}" + }, + { "path": "/nested/path-with/wild*/to/file.ext", "method": "GET", @@ -107,10 +114,9 @@ "valid": [ { "queryParam": "proxyMe", - "values": ["please"], + "values": ["please"] } ] - }, - + } ] } diff --git a/test/functional/server-client.test.js b/test/functional/server-client.test.js index e9d6a73c2..9f2b74bb0 100644 --- a/test/functional/server-client.test.js +++ b/test/functional/server-client.test.js @@ -27,6 +27,7 @@ test('proxy requests originating from behind the broker server', t => { process.chdir(path.resolve(root, '../fixtures/client')); process.env.BROKER_TYPE = 'client'; + process.env.GITHUB = 'github.com'; process.env.BROKER_TOKEN = '12345'; process.env.BROKER_SERVER_URL = `http://localhost:${serverPort}`; process.env.ORIGIN_PORT = echoServerPort; @@ -35,7 +36,7 @@ test('proxy requests originating from behind the broker server', t => { // wait for the client to successfully connect to the server and identify itself server.io.on('connection', socket => { socket.on('identify', token => { - t.plan(19); + t.plan(21); t.test('successfully broker POST', t => { const url = `http://localhost:${serverPort}/broker/${token}/echo-body`; @@ -245,6 +246,31 @@ test('proxy requests originating from behind the broker server', t => { }); }); + t.test('/raw is rewritten to /contents for GH users', t => { + const url = `http://localhost:${serverPort}/broker/${token}/owner/repo/HEAD/folder/package.json`; + const headers = {}; + request({ url, method: 'get', headers, json: true }, (err, res) => { + t.equal(res.statusCode, 200, '200 statusCode'); + t.equal(res.body.headers.accept, 'application/vnd.github.2.11.raw', + 'injected raw headers'); + t.equal(res.body.query.ref, 'HEAD', + 'extracted ref as query parameter'); + t.equal(res.body.url, + '/repos/owner/repo/contents/folder/package.json?ref=HEAD', + 'get correct full url'); + t.end(); + }); + }); + + t.test('/raw is not rewritten to /contents for unsupported manifest file', t => { + const url = `http://localhost:${serverPort}/broker/${token}/owner/repo/HEAD/folder/unsupportedFile.ext`; + const headers = {}; + request({ url, method: 'get', headers }, (err, res) => { + t.equal(res.statusCode, 401, '401 statusCode'); + t.end(); + }); + }); + t.test('clean up', t => { client.close(); setTimeout(() => { diff --git a/test/utils.js b/test/utils.js index 9cd7358e8..71d6a3840 100644 --- a/test/utils.js +++ b/test/utils.js @@ -40,6 +40,11 @@ echoServer.get( res.send(req.params.filename); }); +echoServer.get('/repos/owner/repo/contents/folder/package.json', + (req, res) => { + res.json({headers: req.headers, query: req.query, url: req.url}); + }); + echoServer.all('*', (req, res) => { res.send(false); });