feat: storage and orm

.changeset/mighty-terms-shake.md

@@ -0,0 +1,13 @@
+---
+'micro-stacks': patch
+'@micro-stacks/react': patch
+'@micro-stacks/storage': patch
+---
+
+This update introduces new storage primitives in the package `@micro-stacks/storage`.
+
+- a new `Storage` class, which consumes an instance of the `MicroStacksClient` OR takes a `privateKey`
+- a new `Model` class, which takes a storage _adapter_ and makes working with Gaia much easier
+
+Additionally, there have been some non-breaking changes done to `micro-stacks` to allow for passing fetch functions to
+the inner workings of various functions.

.github/workflows/release.yml

@@ -16,7 +16,7 @@ jobs:
         uses: styfle/[email protected]
         with:
           access_token: ${{ github.token }}
-          
+
       - name: Wait for tests to succeed
         uses: lewagon/[email protected]
         with:

apps/docs/pages/docs/storage/meta.en-US.json

@@ -1,5 +1,6 @@
 {
   "saving-data": "Saving data",
   "fetching-data": "Fetching data",
+  "modelling-data": "Modeling data",
   "encryption": "Encryption"
 }

apps/docs/pages/docs/storage/modelling-data.en-US.mdx

@@ -0,0 +1,77 @@
+import * as Integrations from 'components/integrations-tabs';
+
+# Modeling data for Gaia
+
+<Integrations.Tabs />
+
+Gaia is a pretty simple key-value store. There are endpoints for writing data, reading data, listing
+files, and deleting files. In order to build more advanced applications, it's important to think
+about how we can model out our data such that we don't lose files randomly, and can easily know the
+state of our data.
+
+Micro-stacks exposes some helpful abstractions to make this really easy.
+
+## Model
+
+Each framework specific library will expose a function you can use to create new Models:
+
+```tsx
+import { useModel } from '@micro-stacks/react';
+
+interface ModelType {
+  firstName: string;
+  lastName: string;
+  emailAddress: string;
+}
+
+const model = useModel<ModelType>('MyModelType');
+```
+
+### Listing Model IDs
+
+When interacting with Gaia, it's often best to create a file that serves as an index for our Model,
+which would allow us to fetch many entries, or list out specific entries of your Model. Because of
+how the underlying Model class names files, we are also able to do some ordering without knowing the
+contents of the file.
+
+#### Ordering
+
+#### Paginating
+
+### Saving an entry
+
+Below we can see an example for saving a model with the built in `save` function:
+
+```tsx
+import * as React from 'react';
+import { useModel } from '@micro-stacks/react';
+
+interface ModelType {
+  firstName: string;
+  lastName: string;
+  emailAddress: string;
+}
+
+const SaveButton = (props: ModelType) => {
+  const [isLoading, setIsLoading] = React.useState(false);
+  const model = useModel<ModelType>('MyModelType');
+
+  const onClick = async () => {
+    setIsLoading(true);
+    await model.save({
+      firstName: props.firstName,
+      lastName: props.lastName,
+      emailAddress: props.emailAddress,
+    });
+    setIsLoading(false);
+  };
+
+  const buttonLabel = isLoading ? 'saving...' : 'save model';
+
+  return <button onClick={onClick}>{buttonLabel}</button>;
+};
+```
+
+#### Saving many entries
+
+### Deleting an entry

package.json

@@ -9,7 +9,7 @@
   "scripts": {
     "lint": "turbo run lint",
     "build": "turbo run build --parallel",
-    "test": "turbo run test --parallel",
+    "test": "pnpm run -r test",
     "build:packages": "turbo run build --parallel --filter='./packages/*'",
     "build:docs": "turbo run build --filter=@micro-stacks/docs",
     "dev:docs": "turbo run dev --filter=@micro-stacks/docs",

packages/client/package.json

@@ -65,7 +65,9 @@
     "react-dom": ">=18.2.0",
     "tsup": "6.2.3",
     "type-fest": "^2.19.0",
-    "typescript": "4.8.2"
+    "typescript": "4.8.2",
+    "vite": "3.0.3",
+    "vitest": "0.19.1"
   },
   "dependencies": {
     "fast-deep-equal": "3.1.3",

packages/client/src/common/types.ts

@@ -11,6 +11,7 @@ import { Status, StatusKeys, TxType } from './constants';
 import { ChainID } from 'micro-stacks/network';
 import { SignedOptionsWithOnHandlers } from 'micro-stacks/connect';
 import { ClarityValue } from 'micro-stacks/clarity';
+import { GaiaHubConfig } from 'micro-stacks/storage/gaia/types';
 
 export interface AppDetails {
   /** A human-readable name for your application */
@@ -115,6 +116,16 @@ export interface ClientConfig {
    * instance of `StacksProvider` (user needs to install a wallet)
    */
   onNoWalletFound?: () => void | Promise<void>;
+  /**
+   * gaiaConfig
+   * Configuration options for any optional storage functionality
+   */
+  gaiaConfig?: {
+    gaiaHubUrl?: string;
+    gaiaReadUrl?: string;
+    gaiaHubConfig?: GaiaHubConfig;
+    privateKey?: string;
+  };
   fetcher?: (input: RequestInfo, init?: RequestInit) => Promise<Response>;
 }
 

packages/client/src/micro-stacks-client.ts

@@ -223,6 +223,13 @@ export class MicroStacksClient {
     this.setState(store.state);
   }
 
+  /** ------------------------------------------------------------------------------------------------------------------
+   *   Gaia config
+   *  ------------------------------------------------------------------------------------------------------------------
+   */
+
+  getGaiaConfig = () => this.config.gaiaConfig;
+
   /** ------------------------------------------------------------------------------------------------------------------
    *   State selectors
    *  ------------------------------------------------------------------------------------------------------------------

packages/core/src/crypto/token-signer/token-signer.ts

@@ -5,6 +5,17 @@ import { derToJoseES256 } from './ecdsa-sig-formatter';
 import { Json, SignedToken } from './types';
 import { createSigningInput } from './create-signing-input';
 import { MissingParametersError, utf8ToBytes } from 'micro-stacks/common';
+import * as secp256k1 from '@noble/secp256k1';
+import { hmac } from '@noble/hashes/hmac';
+import { sha256 } from '@noble/hashes/sha256';
+
+function handleSetHasher() {
+  secp256k1.utils.hmacSha256Sync = (key: Uint8Array, ...msgs: Uint8Array[]) => {
+    const h = hmac.create(sha256, key);
+    msgs.forEach(msg => h.update(msg));
+    return h.digest();
+  };
+}
 
 export class TokenSigner {
   tokenType: string;
@@ -97,6 +108,7 @@ export class TokenSigner {
     signingInput: string,
     signingInputHash: Uint8Array
   ): SignedToken | string {
+    if (typeof secp256k1.utils.hmacSha256Sync === 'undefined') handleSetHasher();
     const sig = signSync(signingInputHash, this.rawPrivateKey, {
       // whether a signature s should be no more than 1/2 prime order.
       // true makes signatures compatible with libsecp256k1

packages/core/src/storage/common/types.ts

@@ -1,6 +1,6 @@
 import type { GaiaHubConfig } from '../gaia/types';
 import type { EncryptionOptions } from 'micro-stacks/crypto';
-
+export type FetcherFn = (input: RequestInfo, init?: RequestInit) => Promise<Response>;
 /**
  * Specify a valid MIME type, encryption options, and whether to sign the [[UserSession.putFile]].
  */
@@ -21,6 +21,7 @@ export interface PutFileOptions extends EncryptionOptions {
   encrypt?: boolean | string;
   gaiaHubConfig: GaiaHubConfig;
   privateKey?: string;
+  fetcher?: FetcherFn;
 }
 
 export interface GetFileUrlOptions {
@@ -60,10 +61,12 @@ export interface GetFileOptions extends GetFileUrlOptions {
   verify?: boolean;
   gaiaHubConfig: GaiaHubConfig;
   privateKey?: string;
+  fetcher?: FetcherFn;
 }
 
 export interface ProfileLookupOptions {
   username: string;
   verify?: boolean;
   zoneFileLookupURL?: string;
+  fetcher?: FetcherFn;
 }

packages/core/src/storage/gaia/config.ts

@@ -8,6 +8,7 @@ import type {
   ScopedGaiaTokenOptions,
   GenerateGaiaHubConfigOptions,
 } from './types';
+import { FetcherFn } from '../common/types';
 
 /**
  * Gaia takes the key 'domain' but 'path' makes more sense to someone implementing this
@@ -51,26 +52,41 @@ export function makeScopedGaiaAuthTokenSync(options: ScopedGaiaTokenOptions): st
   return `v1:${token}`;
 }
 
-const DEFAULT_PAYLOAD: HubInfo = {
-  challenge_text: '["gaiahub","0","storage2.blockstack.org","blockstack_storage_please_sign"]',
+const DEFAULT_PAYLOAD = (read_url_prefix: string): HubInfo => ({
+  challenge_text: '["gaiahub","0","gaia-0","blockstack_storage_please_sign"]',
   latest_auth_version: 'v1',
   max_file_upload_size_megabytes: 20,
-  read_url_prefix: 'https://gaia.blockstack.org/hub/',
-};
+  read_url_prefix,
+});
 
 /**
  * Generates a gaia hub config to share with someone so they can edit a file
  */
 export async function generateGaiaHubConfig(
-  options: GenerateGaiaHubConfigOptions,
-  fetchHubInfo = false
+  {
+    gaiaHubUrl = 'https://gaia.blockstack.org',
+    privateKey,
+    associationToken,
+    scopes,
+    gaiaReadUrl = 'https://gaia.blockstack.org/hub/',
+  }: GenerateGaiaHubConfigOptions,
+  options?: {
+    fetchHubInfo?: boolean;
+    fetcher?: FetcherFn;
+  }
 ): Promise<GaiaHubConfig> {
-  const { gaiaHubUrl, privateKey, associationToken, scopes } = options;
-  let hubInfo = DEFAULT_PAYLOAD;
-
-  if (fetchHubInfo) {
-    const response = await fetchPrivate(`${gaiaHubUrl}/hub_info`);
-    hubInfo = await response.json();
+  let hubInfo = DEFAULT_PAYLOAD(gaiaReadUrl);
+
+  if (options?.fetchHubInfo) {
+    try {
+      const path = `${gaiaHubUrl}/hub_info`;
+      const fetcher = options.fetcher ?? fetchPrivate;
+      const response = await fetcher(path);
+      hubInfo = await response.json();
+    } catch (e) {
+      console.error(e);
+      console.error('Cannot fetch Gaia hub information, using default gaia hub configuration');
+    }
   }
 
   const { read_url_prefix: url_prefix, max_file_upload_size_megabytes } = hubInfo;
@@ -96,8 +112,13 @@ export async function generateGaiaHubConfig(
 
 // sync version
 export function generateGaiaHubConfigSync(options: GenerateGaiaHubConfigOptions): GaiaHubConfig {
-  const { gaiaHubUrl, privateKey, associationToken, scopes } = options;
-  const hubInfo = DEFAULT_PAYLOAD;
+  const {
+    gaiaHubUrl = 'https://gaia.blockstack.org',
+    privateKey,
+    associationToken,
+    scopes,
+  } = options;
+  const hubInfo = DEFAULT_PAYLOAD(gaiaHubUrl);
 
   const { read_url_prefix: url_prefix, max_file_upload_size_megabytes } = hubInfo;
 

packages/core/src/storage/gaia/hub.ts

@@ -1,29 +1,34 @@
 import type { GaiaHubConfig } from './types';
 import { getBlockstackErrorFromResponse } from './errors';
 import { fetchPrivate } from 'micro-stacks/common';
+import { FetcherFn } from '../common/types';
 
 interface UploadToGaiaHub {
   filename: string;
   contents: Blob | Uint8Array | ArrayBufferView | string;
   hubConfig: GaiaHubConfig;
   contentType?: string;
+  fetcher?: FetcherFn;
 }
 
 export async function uploadToGaiaHub(options: UploadToGaiaHub): Promise<any> {
-  const { filename, contents, hubConfig, contentType = 'application/octet-stream' } = options;
+  const {
+    filename,
+    contents,
+    hubConfig,
+    contentType = 'application/octet-stream',
+    fetcher = fetchPrivate,
+  } = options;
   const headers: { [key: string]: string } = {
     'Content-Type': contentType,
     Authorization: `bearer ${hubConfig.token}`,
   };
 
-  const response = await fetchPrivate(
-    `${hubConfig.server}/store/${hubConfig.address}/${filename}`,
-    {
-      method: 'POST',
-      headers,
-      body: contents,
-    }
-  );
+  const response = await fetcher(`${hubConfig.server}/store/${hubConfig.address}/${filename}`, {
+    method: 'POST',
+    headers,
+    body: contents,
+  });
 
   if (!response.ok) {
     throw await getBlockstackErrorFromResponse(
@@ -43,6 +48,6 @@ export async function uploadToGaiaHub(options: UploadToGaiaHub): Promise<any> {
  *
  * @ignore
  */
-export function getFullReadUrl(filename: string, hubConfig: GaiaHubConfig): Promise<string> {
-  return Promise.resolve(`${hubConfig.url_prefix}${hubConfig.address}/${filename}`);
+export function getFullReadUrl(filename: string, hubConfig: GaiaHubConfig): string {
+  return `${hubConfig.url_prefix}${hubConfig.address}/${filename}`;
 }

packages/core/src/storage/gaia/types.ts

@@ -46,5 +46,6 @@ export interface GenerateGaiaHubConfigOptions {
   gaiaHubUrl: string;
   privateKey: string;
   associationToken?: string;
+  gaiaReadUrl?: string;
   scopes?: GaiaAuthScope[];
 }

packages/core/src/storage/get-file/get-file.ts

@@ -3,11 +3,14 @@ import { handleSignedEncryptedContents } from './sign';
 import { getFileContents, getFileSignedUnencrypted } from './getters';
 
 import { decryptContent } from 'micro-stacks/crypto';
-import { getGlobalObject } from 'micro-stacks/common';
+import { fetchPrivate, getGlobalObject } from 'micro-stacks/common';
 
 import type { GetFileOptions } from '../common/types';
 
-export async function getFile(path: string, getFileOptions: GetFileOptions) {
+export async function getFile(
+  path: string,
+  { fetcher = fetchPrivate, ...getFileOptions }: GetFileOptions
+) {
   const options: GetFileOptions = {
     decrypt: true,
     verify: false,
@@ -29,6 +32,7 @@ export async function getFile(path: string, getFileOptions: GetFileOptions) {
     zoneFileLookupURL: options.zoneFileLookupURL,
     forceText: !!options.decrypt,
     gaiaHubConfig: options.gaiaHubConfig,
+    fetcher,
   });
   if (storedContents === null) return storedContents;