Support auto directory creation in PUT, allow header differs based on…

… writability, added canonical link header, added ETag tracking version, removed index.json extension, added .well-known/dat, added X-Resolve:none for disabling resloving index.json, added X-Blocks and X-Blocks-Downloaded

README.md

@@ -51,6 +51,14 @@ After you've created it, `fetch` will be have like it does in [browsers](https:/
 
 Closes resources for the Dat SDK. This does nothing if you specified the Hyperdrive and `resolveName` options.
 
+### Common Headers
+
+Each response will contain a header for the canonical URL represented as a `Link` header with `rel=canonical`.
+
+Each response will also contain the `Allow` header of all the methods currently allowed. If the archive is writable, this will contain `PUT`.
+
+There is also an `ETag` header which will be a JSON string containging the drive's current `version`. This will change only when the drive has gotten an update of some sort and is monotonically incrementing.
+
 ### `fetch('hyper://NAME/example.txt', {method: 'GET'})`
 
 This will attempt to load `example.txt` from the archive labeled by `NAME`.
@@ -60,13 +68,13 @@ You can find the details about how resolution works in the [resolve-dat-path](ht
 
 `NAME` can either be the 64 character hex key for an archive, a domain to parse with [dat-dns](https://www.npmjs.com/package/dat-dns), or a name for an archive which allows you to write to it.
 
-### `fetch('hyper://NAME/index.json', {method: 'GET'})`
+The response headers will contain `X-Blocks` for the number of blocks of data this file represents on disk, and `X-Blocks-Downloaded` which is the number of blocks from this file that have been downloaded locally.
 
-The `index.json` file is special in that it will be modified to contain some extra parameters in the JSON content.
+### `fetch('hyper://NAME/.well-known/dat', {method: 'GET'})`
 
-This extends from the [Index.json Manifest](https://docs.beakerbrowser.com/developers/index.json-manifest) spec in Beaker.
+This is used by the dat-dns module for resoving dns domains to `dat://` URLs.
 
-`url` will get set to the `hyper://` URL of the archive. This will resolve the `NAME` to always be the 64 character hex key.
+This will return some text which will have a `dat://` URL of your archive, followed by a newline and a TTL for the DNS record.
 
 ### `fetch('hyper://NAME/example/', {method: 'GET'})`
 
@@ -76,8 +84,27 @@ By default it will render out an HTML page with links to files within that direc
 
 You can set the `Accept` header to `application/json` in order to have it return a JSON array with file names.
 
+e.g.
+
+```json
+["example.txt", "posts/", "example2.md"]
+```
+
+Files in the directory will be listed under their name, sub-directories will have a `/` appended to them.
+
 `NAME` can either be the 64 character hex key for an archive, a domain to parse with [dat-dns](https://www.npmjs.com/package/dat-dns), or a name for an archive which allows you to write to it.
 
+### `fetch('hyper://NAME/example.txt', {method: 'GET', headers: {'X-Resolve': 'none'}})`
+
+Setting the `X-Resolve` header to `none` will prevent resolving `index.html` files and will attempt to load the path as is.
+This can be useful for list files in a directory that would normally render as a page.
+
+You should omit the header for the default behavior, different values may be supported in the future.
+
+`NAME` can either be the 64 character hex key for an archive, a domain to parse with [dat-dns](https://www.npmjs.com/package/dat-dns), or a name for an archive which allows you to write to it.
+
+The response headers will contain `X-Blocks` for the number of blocks of data this file represents on disk, and `X-Blocks-Downloaded` which is the number of blocks from this file that have been downloaded locally.
+
 ### `fetch('hyper://NAME/example.txt', {method: 'PUT', body: 'Hello World'})`
 
 You can add files to archives using a `PUT` method along with a `body`.

index.js

@@ -7,10 +7,15 @@ const { Readable } = require('stream')
 const parseRange = require('range-parser')
 const bodyToStream = require('fetch-request-body-to-stream')
 const pump = require('pump-promise')
+const makeDir = require('make-dir')
 
 const DAT_REGEX = /\w+:\/\/([^/]+)\/?([^#?]*)?/
 const NOT_WRITABLE_ERROR = 'Archive not writable'
 
+const READABLE_ALLOW = ['GET', 'HEAD']
+const WRITABLE_ALLOW = ['PUT', 'DELETE']
+const ALL_ALLOW = READABLE_ALLOW.concat(WRITABLE_ALLOW)
+
 module.exports = function makeFetch (opts = {}) {
   let { Hyperdrive, resolveName, base, session, writable = false } = opts
 
@@ -100,11 +105,25 @@ module.exports = function makeFetch (opts = {}) {
 
       await archive.ready()
 
+      const canonical = `hyper://${archive.key.toString('hex')}/${path || ''}`
+      responseHeaders.append('Link', `<${canonical}>; rel="canonical"`)
+
+      const isWritable = writable && archive.writable
+      const allowHeaders = isWritable ? ALL_ALLOW : READABLE_ALLOW
+      responseHeaders.set('Allow', allowHeaders.join(', '))
+
+      // We can say the file hasn't changed if the drive version hasn't changed
+      responseHeaders.set('ETag', `"${archive.version}"`)
+
       if (method === 'PUT') {
         checkWritable(archive)
         if (path.endsWith('/')) {
-          await archive.mkdir(path)
+          await makeDir(path, { fs: archive })
         } else {
+          const parentDir = path.split('/').slice(0, -1).join('/')
+          if (parentDir) {
+            await makeDir(parentDir, { fs: archive })
+          }
           // Create a new file from the request body
           const { body } = opts
           const source = bodyToStream(body, session)
@@ -128,30 +147,22 @@ module.exports = function makeFetch (opts = {}) {
 
         return new FakeResponse(200, 'OK', responseHeaders, intoStream(''), url)
       } else if ((method === 'GET') || (method === 'HEAD')) {
-        let resolved = null
+        let stat = null
         let finalPath = path
 
-        if (finalPath === 'index.json') {
-          const resolvedURL = `hyper://${archive.key.toString('hex')}`
-          const { writable } = archive
-          let content = { url: resolvedURL, writable }
-          try {
-            const string = await archive.readFile(finalPath, 'utf8')
-            const parsed = JSON.parse(string)
-            content = { parsed, ...content }
-          } catch (e) {
-            // Probably a parsing error or something
-          }
-
-          const stringified = JSON.stringify(content, null, '\t')
-
-          responseHeaders.set('Content-Type', 'application/json')
-
-          return new FakeResponse(200, 'OK', responseHeaders, intoStream(stringified), url)
+        if (finalPath === '.well-known/dat') {
+          const { key } = archive
+          const entry = `dat://${key.toString('hex')}\nttl=3600`
+          return new FakeResponse(200, 'OK', responseHeaders, intoStream(entry), url)
         }
         try {
-          resolved = await resolveDatPathAwait(archive, path)
-          finalPath = resolved.path
+          if (headers.get('X-Resolve') === 'none') {
+            stat = await archive.stat(path)
+          } else {
+            const resolved = await resolveDatPathAwait(archive, path)
+            finalPath = resolved.path
+            stat = resolved.stat
+          }
         } catch (e) {
           return new FakeResponse(
             404,
@@ -169,7 +180,7 @@ module.exports = function makeFetch (opts = {}) {
         const isRanged = headers.get('Range') || headers.get('range')
         let statusCode = 200
 
-        if (resolved.type === 'directory') {
+        if (stat.isDirectory()) {
           const stats = await archive.readdir(finalPath, { includeStats: true })
           const files = stats.map(({ stat, name }) => (stat.isDirectory() ? `${name}/` : name))
 
@@ -194,8 +205,12 @@ module.exports = function makeFetch (opts = {}) {
           }
         } else {
           responseHeaders.set('Accept-Ranges', 'bytes')
+
+          const { blocks, downloadedBlocks } = await archive.stats(finalPath)
+          responseHeaders.set('X-Blocks', `${blocks}`)
+          responseHeaders.set('X-Blocks-Downloaded', `${downloadedBlocks}`)
+
           if (isRanged) {
-            const { stat } = resolved
             const { size } = stat
             const range = parseRange(size, isRanged)[0]
             if (range && range.type === 'bytes') {
@@ -224,7 +239,6 @@ module.exports = function makeFetch (opts = {}) {
           return new FakeResponse(statusCode, 'ok', responseHeaders, stream, url)
         }
       } else {
-        responseHeaders.set('Allow', 'GET, HEAD, PUT, DELETE')
         return new FakeResponse(405, 'Method Not Allowed', responseHeaders, intoStream('Method Not Allowed'), url)
       }
     } catch (e) {

package.json

@@ -29,6 +29,7 @@
     "end-of-stream-promise": "^1.0.0",
     "fetch-headers": "^2.0.0",
     "fetch-request-body-to-stream": "^1.0.0",
+    "make-dir": "^3.1.0",
     "mime": "^2.4.4",
     "pump-promise": "^1.0.0",
     "range-parser": "^1.2.1",

test.js

@@ -14,7 +14,8 @@ async function test () {
 
   const fetch = require('./')({
     Hyperdrive,
-    resolveName
+    resolveName,
+    writable: true
   })
 
   const url = `dat://${archive.key.toString('hex')}${FILE_LOCATION}`
@@ -29,13 +30,14 @@ async function test () {
 
   console.log(contentType)
   console.log(text)
+  console.log([...response.headers.entries()])
 
   const url2 = 'hyper://example/example.txt'
   const contents = 'Hello World'
 
   console.log('Putting into', url2, contents)
 
-  await fetch(url2, { method: 'PUT', body: contents })
+  await checkOK(await fetch(url2, { method: 'PUT', body: contents }))
 
   console.log('Wrote to archive')
 
@@ -65,13 +67,23 @@ async function test () {
   console.log('Directory after delete')
   console.log(text4)
 
-  const url4 = 'hyper://example/index.json'
+  const url4 = 'hyper://example/.well-known/dat'
 
   const response5 = await fetch(url4)
   await checkOK(response5)
-  const json = await response5.json()
+  const json = await response5.text()
 
-  console.log('Created archive info', json)
+  console.log('Archive well-known URL', json)
+
+  const url5 = 'hyper://example/foo/bar/'
+  await checkOK(await fetch(url5, { method: 'PUT' }))
+
+  console.log('Created multiple folders')
+
+  const url6 = 'hyper://example/fizz/buzz/example.txt'
+  await checkOK(await fetch(url6, { method: 'PUT', contents }))
+
+  console.log('Created file along with parent folders')
 
   await close()
 }