diff --git a/.github/workflows/pr_test.yml b/.github/workflows/pr_test.yml index 7c1a33d4c..1910c992b 100644 --- a/.github/workflows/pr_test.yml +++ b/.github/workflows/pr_test.yml @@ -55,6 +55,7 @@ jobs: strategy: matrix: go-version: [1.14.x] + node-version: [12] rust-toolchain: [1.46.0] platform: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.platform }} @@ -83,6 +84,9 @@ jobs: toolchain: ${{ matrix.rust-toolchain }} - name: Add wasm32-wasi Rust target run: rustup target add wasm32-wasi --toolchain ${{ matrix.rust-toolchain }} + - uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} - name: Test run: make test shell: bash diff --git a/.gitignore b/.gitignore index e81969e68..1f1bf85f4 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ RELEASE_CHANGELOG.md **/target rust-toolchain .cargo +**/node_modules # Binaries for programs and plugins *.exe diff --git a/pkg/app/run.go b/pkg/app/run.go index 14afe93ae..9af47c5ca 100644 --- a/pkg/app/run.go +++ b/pkg/app/run.go @@ -129,7 +129,7 @@ func Run(args []string, env config.Environment, file config.File, configFilePath serviceVersionLock := serviceversion.NewLockCommand(serviceVersionRoot.CmdClause, &globals) computeRoot := compute.NewRootCommand(app, &globals) - computeInit := compute.NewInitCommand(computeRoot.CmdClause, &globals) + computeInit := compute.NewInitCommand(computeRoot.CmdClause, httpClient, &globals) computeBuild := compute.NewBuildCommand(computeRoot.CmdClause, httpClient, &globals) computeDeploy := compute.NewDeployCommand(computeRoot.CmdClause, httpClient, &globals) computeUpdate := compute.NewUpdateCommand(computeRoot.CmdClause, httpClient, &globals) diff --git a/pkg/app/run_test.go b/pkg/app/run_test.go index ac3e2985f..32cbfd58b 100644 --- a/pkg/app/run_test.go +++ b/pkg/app/run_test.go @@ -274,6 +274,7 @@ COMMANDS of the --path destination -d, --description=DESCRIPTION Description of the package -a, --author=AUTHOR ... Author(s) of the package + -l, --language=LANGUAGE Language of the package -f, --from=FROM Git repository containing package template -p, --path=PATH Destination to write the new package, defaulting to the current directory diff --git a/pkg/common/exec.go b/pkg/common/exec.go new file mode 100644 index 000000000..b46f0ce21 --- /dev/null +++ b/pkg/common/exec.go @@ -0,0 +1,62 @@ +package common + +import ( + "bytes" + "fmt" + "io" + "os" + "os/exec" + "strings" + // "sync" +) + +// StreamingExec models a generic command execution that consumers can use to +// execute commands and stream their output to an io.Writer. For example +// compute commands can use this to standardize the flow control for each +// compiler toolchain. +type StreamingExec struct { + command string + args []string + env []string + verbose bool + output io.Writer +} + +// NewStreamingExec constructs a new StreamingExec instance. +func NewStreamingExec(cmd string, args, env []string, verbose bool, out io.Writer) *StreamingExec { + return &StreamingExec{ + cmd, + args, + env, + verbose, + out, + } +} + +// Exec executes the compiler command and pipes the child process stdout and +// stderr output to the supplied io.Writer, it waits for the command to exit +// cleanly or returns an error. +func (s StreamingExec) Exec() error { + // Construct the command with given arguments and environment. + // + // gosec flagged this: + // G204 (CWE-78): Subprocess launched with variable + // Disabling as the variables come from trusted sources. + /* #nosec */ + cmd := exec.Command(s.command, s.args...) + cmd.Env = append(os.Environ(), s.env...) + + // Pipe the child process stdout and stderr to our own output writer. + var stderrBuf bytes.Buffer + cmd.Stdout = s.output + cmd.Stderr = io.MultiWriter(s.output, &stderrBuf) + + if err := cmd.Run(); err != nil { + if !s.verbose && stderrBuf.Len() > 0 { + return fmt.Errorf("error during execution process:\n%s", strings.TrimSpace(stderrBuf.String())) + } + return fmt.Errorf("error during execution process") + } + + return nil +} diff --git a/pkg/common/file.go b/pkg/common/file.go index ff862fa80..258c5d261 100644 --- a/pkg/common/file.go +++ b/pkg/common/file.go @@ -80,3 +80,21 @@ func CopyFile(src, dst string) (err error) { return } + +// MakeDirectoryIfNotExists asserts whether a directory exists and makes it +// if not. Returns nil if exists or successfully made. +func MakeDirectoryIfNotExists(path string) error { + fi, err := os.Stat(path) + switch { + case err == nil && fi.IsDir(): + return nil + case err == nil && !fi.IsDir(): + return fmt.Errorf("%s already exists as a regular file", path) + case os.IsNotExist(err): + return os.MkdirAll(path, 0750) + case err != nil: + return err + } + + return nil +} diff --git a/pkg/compute/assemblyscript.go b/pkg/compute/assemblyscript.go new file mode 100644 index 000000000..58b77c8fb --- /dev/null +++ b/pkg/compute/assemblyscript.go @@ -0,0 +1,198 @@ +package compute + +import ( + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/fastly/cli/pkg/common" + "github.com/fastly/cli/pkg/errors" + "github.com/fastly/cli/pkg/text" +) + +// AssemblyScript implements Toolchain for the AssemblyScript language. +type AssemblyScript struct{} + +// NewAssemblyScript constructs a new AssemblyScript. +func NewAssemblyScript() *AssemblyScript { + return &AssemblyScript{} +} + +// Verify implements the Toolchain interface and verifies whether the +// AssemblyScript language toolchain is correctly configured on the host. +func (a AssemblyScript) Verify(out io.Writer) error { + // 1) Check `npm` is on $PATH + // + // npm is Node/AssemblyScript's toolchain installer and manager, it is + // needed to assert that the correct versions of the asc compiler and + // @fastly/as-compute package are installed. We only check whether the + // binary exists on the users $PATH and error with installation help text. + fmt.Fprintf(out, "Checking if npm is installed...\n") + + p, err := exec.LookPath("npm") + if err != nil { + return errors.RemediationError{ + Inner: fmt.Errorf("`npm` not found in $PATH"), + Remediation: fmt.Sprintf("To fix this error, install Node.js and npm by visiting:\n\n\t$ %s", text.Bold("https://nodejs.org/")), + } + } + + fmt.Fprintf(out, "Found npm at %s\n", p) + + // 2) Check package.json file exists in $PWD + // + // A valid npm package is needed for compilation and to assert whether the + // required dependencies are installed locally. Therefore, we first assert + // whether one exists in the current $PWD. + fpath, err := filepath.Abs("package.json") + if err != nil { + return fmt.Errorf("getting package.json path: %w", err) + } + + if !common.FileExists(fpath) { + return errors.RemediationError{ + Inner: fmt.Errorf("package.json not found"), + Remediation: fmt.Sprintf("To fix this error, run the following command:\n\n\t$ %s", text.Bold("npm init")), + } + } + + fmt.Fprintf(out, "Found package.json at %s\n", fpath) + + // 3) Check if `asc` is installed. + // + // asc is the AssemblyScript compiler. We first check if it exists in the + // package.json and then whether the binary exists in the npm bin directory. + fmt.Fprintf(out, "Checking if AssemblyScript is installed...\n") + if !checkPackageDependencyExists("assemblyscript") { + return errors.RemediationError{ + Inner: fmt.Errorf("`assemblyscript` not found in package.json"), + Remediation: fmt.Sprintf("To fix this error, run the following command:\n\n\t$ %s", text.Bold("npm install --save-dev assemblyscript")), + } + } + + p, err = getNpmBinPath() + if err != nil { + return errors.RemediationError{ + Inner: fmt.Errorf("could not determine npm bin path"), + Remediation: fmt.Sprintf("To fix this error, run the following command:\n\n\t$ %s", text.Bold("npm install --global npm@latest")), + } + } + + path, err := exec.LookPath(filepath.Join(p, "asc")) + if err != nil { + return fmt.Errorf("getting asc path: %w", err) + } + if !common.FileExists(path) { + return errors.RemediationError{ + Inner: fmt.Errorf("`asc` binary not found in %s", p), + Remediation: fmt.Sprintf("To fix this error, run the following command:\n\n\t$ %s", text.Bold("npm install --save-dev assemblyscript")), + } + } + + fmt.Fprintf(out, "Found asc at %s\n", path) + + return nil +} + +// Initialize implements the Toolchain interface and initializes a newly cloned +// package by installing required dependencies. +func (a AssemblyScript) Initialize(out io.Writer) error { + // 1) Check `npm` is on $PATH + // + // npm is Node/AssemblyScript's toolchain package manager, it is needed to + // install the package dependencies on initialization. We only check whether + // the binary exists on the users $PATH and error with installation help text. + fmt.Fprintf(out, "Checking if npm is installed...\n") + + p, err := exec.LookPath("npm") + if err != nil { + return errors.RemediationError{ + Inner: fmt.Errorf("`npm` not found in $PATH"), + Remediation: fmt.Sprintf("To fix this error, install Node.js and npm by visiting:\n\n\t$ %s", text.Bold("https://nodejs.org/")), + } + } + + fmt.Fprintf(out, "Found npm at %s\n", p) + + // 2) Check package.json file exists in $PWD + // + // A valid npm package manifest file is needed for the install command to + // work. Therefore, we first assert whether one exists in the current $PWD. + fpath, err := filepath.Abs("package.json") + if err != nil { + return fmt.Errorf("getting package.json path: %w", err) + } + + if !common.FileExists(fpath) { + return errors.RemediationError{ + Inner: fmt.Errorf("package.json not found"), + Remediation: fmt.Sprintf("To fix this error, run the following command:\n\n\t$ %s", text.Bold("npm init")), + } + } + + fmt.Fprintf(out, "Found package.json at %s\n", fpath) + + // Call npm install. + cmd := common.NewStreamingExec("npm", []string{"install"}, []string{}, false, out) + return cmd.Exec() +} + +// Build implements the Toolchain interface and attempts to compile the package +// AssemblyScript source to a Wasm binary. +func (a AssemblyScript) Build(out io.Writer, verbose bool) error { + // Check if bin directory exists and create if not. + pwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("getting current working directory: %w", err) + } + binDir := filepath.Join(pwd, "bin") + if err := common.MakeDirectoryIfNotExists(binDir); err != nil { + return fmt.Errorf("making bin directory: %w", err) + } + + npmdir, err := getNpmBinPath() + if err != nil { + return fmt.Errorf("getting npm path: %w", err) + } + + args := []string{ + "assembly/index.ts", + "--binaryFile", + filepath.Join(binDir, "main.wasm"), + "--optimize", + "--noAssert", + } + if verbose { + args = append(args, "--verbose") + } + + fmt.Fprintf(out, "Installing package dependencies...\n") + + // Call asc with the build arguments. + cmd := common.NewStreamingExec(filepath.Join(npmdir, "asc"), args, []string{}, verbose, out) + if err := cmd.Exec(); err != nil { + return err + } + + return nil +} + +func getNpmBinPath() (string, error) { + path, err := exec.Command("npm", "bin").Output() + if err != nil { + return "", err + } + return strings.TrimSpace(string(path)), nil +} + +func checkPackageDependencyExists(name string) bool { + // gosec flagged this: + // G204 (CWE-78): Subprocess launched with variable + // Disabling as the variables come from trusted sources. + /* #nosec */ + err := exec.Command("npm", "list", "--json", "--depth", "0", name).Run() + return err == nil +} diff --git a/pkg/compute/build.go b/pkg/compute/build.go index efeaca4d3..4b3f58db8 100644 --- a/pkg/compute/build.go +++ b/pkg/compute/build.go @@ -23,10 +23,44 @@ const IgnoreFilePath = ".fastlyignore" // Toolchain abstracts a Compute@Edge source language toolchain. type Toolchain interface { + Initialize(out io.Writer) error Verify(out io.Writer) error Build(out io.Writer, verbose bool) error } +// Language models a Compute@Edge source language. +type Language struct { + Name string + DisplayName string + StarterKits []StarterKit + SourceDirectory string + IncludeFiles []string + + Toolchain +} + +// LanguageOptions models configuration options for a Language. +type LanguageOptions struct { + Name string + DisplayName string + StarterKits []StarterKit + SourceDirectory string + IncludeFiles []string + Toolchain Toolchain +} + +// NewLanguage constructs a new Language from a LangaugeOptions. +func NewLanguage(options *LanguageOptions) *Language { + return &Language{ + options.Name, + options.DisplayName, + options.StarterKits, + options.SourceDirectory, + options.IncludeFiles, + options.Toolchain, + } +} + // BuildCommand produces a deployable artifact from files on the local disk. type BuildCommand struct { common.Base @@ -97,10 +131,22 @@ func (c *BuildCommand) Exec(in io.Reader, out io.Writer) (err error) { } name = sanitize.BaseName(name) - var toolchain Toolchain + var language *Language switch lang { + case "assemblyscript": + language = NewLanguage(&LanguageOptions{ + Name: "assemblyscript", + SourceDirectory: "assembly", + IncludeFiles: []string{"package.json"}, + Toolchain: NewAssemblyScript(), + }) case "rust": - toolchain = &Rust{c.client} + language = NewLanguage(&LanguageOptions{ + Name: "rust", + SourceDirectory: "src", + IncludeFiles: []string{"Cargo.toml"}, + Toolchain: NewRust(c.client), + }) default: return fmt.Errorf("unsupported language %s", lang) } @@ -108,7 +154,7 @@ func (c *BuildCommand) Exec(in io.Reader, out io.Writer) (err error) { if !c.force { progress.Step(fmt.Sprintf("Verifying local %s toolchain...", lang)) - err = toolchain.Verify(progress) + err = language.Verify(progress) if err != nil { return err } @@ -116,7 +162,7 @@ func (c *BuildCommand) Exec(in io.Reader, out io.Writer) (err error) { progress.Step(fmt.Sprintf("Building package using %s toolchain...", lang)) - if err := toolchain.Build(progress, c.Globals.Flag.Verbose); err != nil { + if err := language.Build(progress, c.Globals.Flag.Verbose); err != nil { return err } @@ -126,8 +172,8 @@ func (c *BuildCommand) Exec(in io.Reader, out io.Writer) (err error) { files := []string{ ManifestFilename, - "Cargo.toml", } + files = append(files, language.IncludeFiles...) ignoreFiles, err := getIgnoredFiles(IgnoreFilePath) if err != nil { @@ -141,9 +187,7 @@ func (c *BuildCommand) Exec(in io.Reader, out io.Writer) (err error) { files = append(files, binFiles...) if c.includeSrc { - // TODO(phamann): we will need to lookup the directory name based on the - // source language type when we support multiple languages. - srcFiles, err := getNonIgnoredFiles("src", ignoreFiles) + srcFiles, err := getNonIgnoredFiles(language.SourceDirectory, ignoreFiles) if err != nil { return err } diff --git a/pkg/compute/compute_integration_test.go b/pkg/compute/compute_integration_test.go index 9a448fea0..cc4eabe96 100644 --- a/pkg/compute/compute_integration_test.go +++ b/pkg/compute/compute_integration_test.go @@ -2,7 +2,6 @@ package compute_test import ( "bytes" - "crypto/rand" "errors" "fmt" "io" @@ -10,6 +9,7 @@ import ( "net/http" "net/http/httptest" "os" + "os/exec" "path/filepath" "strings" "testing" @@ -284,7 +284,20 @@ func TestInit(t *testing.T) { CreateDomainFn: createDomainOK, CreateBackendFn: createBackendOK, }, - manifestIncludes: `name = "fastly-build`, + manifestIncludes: `name = "fastly-init`, + }, + { + name: "with AssemblyScript language", + args: []string{"compute", "init", "--language", "assemblyscript"}, + configFile: config.File{Token: "123"}, + api: mock.API{ + GetTokenSelfFn: tokenOK, + GetUserFn: getUserOk, + CreateServiceFn: createServiceOK, + CreateDomainFn: createDomainOK, + CreateBackendFn: createBackendOK, + }, + manifestIncludes: `name = "fastly-init`, }, } { t.Run(testcase.name, func(t *testing.T) { @@ -346,8 +359,8 @@ func TestInit(t *testing.T) { } } -func TestBuild(t *testing.T) { - if os.Getenv("TEST_COMPUTE_BUILD") == "" { +func TestBuildRust(t *testing.T) { + if os.Getenv("TEST_COMPUTE_BUILD_RUST") == "" && os.Getenv("TEST_COMPUTE_BUILD") == "" { t.Log("skipping test") t.Skip("Set TEST_COMPUTE_BUILD to run this test") } @@ -418,7 +431,7 @@ func TestBuild(t *testing.T) { wantRemediationError: "fastly = \"^0.4.0\"", }, { - name: "success", + name: "Rust success", args: []string{"compute", "build"}, fastlyManifest: "name = \"test\"\nlanguage = \"rust\"\n", cargoLock: "[[package]]\nname = \"fastly\"\nversion = \"0.3.2\"\n\n[[package]]\nname = \"fastly-sys\"\nversion = \"0.3.2\"", @@ -436,7 +449,7 @@ func TestBuild(t *testing.T) { // Create our build environment in a temp dir. // Defer a call to clean it up. - rootdir := makeBuildEnvironment(t, testcase.fastlyManifest, testcase.cargoManifest, testcase.cargoLock) + rootdir := makeRustBuildEnvironment(t, testcase.fastlyManifest, testcase.cargoManifest, testcase.cargoLock) defer os.RemoveAll(rootdir) // Before running the test, chdir into the build environment. @@ -469,6 +482,93 @@ func TestBuild(t *testing.T) { } } +func TestBuildAssemblyScript(t *testing.T) { + if os.Getenv("TEST_COMPUTE_BUILD_ASSEMBLYSCRIPT") == "" && os.Getenv("TEST_COMPUTE_BUILD") == "" { + t.Log("skipping test") + t.Skip("Set TEST_COMPUTE_BUILD to run this test") + } + + for _, testcase := range []struct { + name string + args []string + fastlyManifest string + wantError string + wantRemediationError string + wantOutputContains string + }{ + { + name: "no fastly.toml manifest", + args: []string{"compute", "build"}, + wantError: "error reading package manifest: open fastly.toml:", // actual message differs on Windows + }, + { + name: "empty language", + args: []string{"compute", "build"}, + fastlyManifest: "name = \"test\"\n", + wantError: "language cannot be empty, please provide a language", + }, + { + name: "empty name", + args: []string{"compute", "build"}, + fastlyManifest: "language = \"assemblyscript\"\n", + wantError: "name cannot be empty, please provide a name", + }, + { + name: "unknown language", + args: []string{"compute", "build"}, + fastlyManifest: "name = \"test\"\nlanguage = \"javascript\"\n", + wantError: "unsupported language javascript", + }, + { + name: "AssemblyScript success", + args: []string{"compute", "build"}, + fastlyManifest: "name = \"test\"\nlanguage = \"assemblyscript\"\n", + wantOutputContains: "Built assemblyscript package test", + }, + } { + t.Run(testcase.name, func(t *testing.T) { + // We're going to chdir to a build environment, + // so save the PWD to return to, afterwards. + pwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + // Create our build environment in a temp dir. + // Defer a call to clean it up. + rootdir := makeAssemblyScriptBuildEnvironment(t, testcase.fastlyManifest) + defer os.RemoveAll(rootdir) + + // Before running the test, chdir into the build environment. + // When we're done, chdir back to our original location. + // This is so we can reliably copy the testdata/ fixtures. + if err := os.Chdir(rootdir); err != nil { + t.Fatal(err) + } + defer os.Chdir(pwd) + + var ( + args = testcase.args + env = config.Environment{} + file = config.File{} + appConfigFile = "/dev/null" + clientFactory = mock.APIClient(mock.API{}) + httpClient = http.DefaultClient + versioner update.Versioner = nil + in io.Reader = nil + buf bytes.Buffer + out io.Writer = common.NewSyncWriter(&buf) + ) + err = app.Run(args, env, file, appConfigFile, clientFactory, httpClient, versioner, in, out) + testutil.AssertErrorContains(t, err, testcase.wantError) + testutil.AssertRemediationErrorContains(t, err, testcase.wantRemediationError) + if testcase.wantOutputContains != "" { + testutil.AssertStringContains(t, buf.String(), testcase.wantOutputContains) + } + }) + } +} + func TestDeploy(t *testing.T) { for _, testcase := range []struct { name string @@ -878,17 +978,11 @@ func TestValidate(t *testing.T) { func makeInitEnvironment(t *testing.T, manifestContent string) (rootdir string) { t.Helper() - p := make([]byte, 8) - n, err := rand.Read(p) + rootdir, err := ioutil.TempDir("", "fastly-init-*") if err != nil { t.Fatal(err) } - rootdir = filepath.Join( - os.TempDir(), - fmt.Sprintf("fastly-build-%x", p[:n]), - ) - if err := os.MkdirAll(rootdir, 0700); err != nil { t.Fatal(err) } @@ -903,20 +997,14 @@ func makeInitEnvironment(t *testing.T, manifestContent string) (rootdir string) return rootdir } -func makeBuildEnvironment(t *testing.T, fastlyManifestContent, cargoManifestContent, cargoLockContent string) (rootdir string) { +func makeRustBuildEnvironment(t *testing.T, fastlyManifestContent, cargoManifestContent, cargoLockContent string) (rootdir string) { t.Helper() - p := make([]byte, 8) - n, err := rand.Read(p) + rootdir, err := ioutil.TempDir("", "fastly-build-*") if err != nil { t.Fatal(err) } - rootdir = filepath.Join( - os.TempDir(), - fmt.Sprintf("fastly-build-%x", p[:n]), - ) - if err := os.MkdirAll(rootdir, 0700); err != nil { t.Fatal(err) } @@ -955,19 +1043,50 @@ func makeBuildEnvironment(t *testing.T, fastlyManifestContent, cargoManifestCont return rootdir } -func makeDeployEnvironment(t *testing.T, manifestContent string) (rootdir string) { +func makeAssemblyScriptBuildEnvironment(t *testing.T, fastlyManifestContent string) (rootdir string) { t.Helper() - p := make([]byte, 8) - n, err := rand.Read(p) + rootdir, err := ioutil.TempDir("", "fastly-build-*") if err != nil { t.Fatal(err) } - rootdir = filepath.Join( - os.TempDir(), - fmt.Sprintf("fastly-deploy-%x", p[:n]), - ) + if err := os.MkdirAll(rootdir, 0700); err != nil { + t.Fatal(err) + } + + for _, filename := range [][]string{ + {"package.json"}, + {"assembly", "index.ts"}, + } { + fromFilename := filepath.Join("testdata", "build", filepath.Join(filename...)) + toFilename := filepath.Join(rootdir, filepath.Join(filename...)) + copyFile(t, fromFilename, toFilename) + } + + if fastlyManifestContent != "" { + filename := filepath.Join(rootdir, compute.ManifestFilename) + if err := ioutil.WriteFile(filename, []byte(fastlyManifestContent), 0777); err != nil { + t.Fatal(err) + } + } + + cmd := exec.Command("npm", "install") + cmd.Dir = rootdir + if err := cmd.Run(); err != nil { + t.Fatal(err) + } + + return rootdir +} + +func makeDeployEnvironment(t *testing.T, manifestContent string) (rootdir string) { + t.Helper() + + rootdir, err := ioutil.TempDir("", "fastly-deploy-*") + if err != nil { + t.Fatal(err) + } if err := os.MkdirAll(rootdir, 0700); err != nil { t.Fatal(err) diff --git a/pkg/compute/init.go b/pkg/compute/init.go index 0df77bfe1..560d5aac3 100644 --- a/pkg/compute/init.go +++ b/pkg/compute/init.go @@ -14,6 +14,7 @@ import ( "time" "github.com/dustinkirkland/golang-petname" + "github.com/fastly/cli/pkg/api" "github.com/fastly/cli/pkg/common" "github.com/fastly/cli/pkg/compute/manifest" "github.com/fastly/cli/pkg/config" @@ -24,14 +25,7 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing" ) -type template struct { - Name string - Path string -} - const ( - defaultTemplate = "https://github.com/fastly/fastly-template-rust-default.git" - defaultTemplateBranch = "0.4.0" defaultTopLevelDomain = "edgecompute.app" manageServiceBaseURL = "https://manage.fastly.com/configure/services/" ) @@ -40,38 +34,62 @@ var ( gitRepositoryRegEx = regexp.MustCompile(`((git|ssh|http(s)?)|(git@[\w\.]+))(:(//)?)([\w\.@\:/\-~]+)(\.git)(/)?`) domainNameRegEx = regexp.MustCompile(`(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]`) fastlyOrgRegEx = regexp.MustCompile(`^https:\/\/github\.com\/fastly`) - fastlyFileIgnoreListRegEx = regexp.MustCompile(`\.github|LICENSE|SECURITY\.md`) - defaultTemplates = map[int]template{ - 1: { - Name: "Starter kit", - Path: defaultTemplate, + fastlyFileIgnoreListRegEx = regexp.MustCompile(`\.github|LICENSE|SECURITY\.md|CHANGELOG\.md|screenshot\.png`) + starterKits = map[string][]StarterKit{ + "assemblyscript": { + { + Name: "Default", + Path: "https://github.com/fastly/compute-starter-kit-assemblyscript-default", + Tag: "v0.2.0", + }, + }, + "rust": { + { + Name: "Default", + Path: "https://github.com/fastly/fastly-template-rust-default.git", + Branch: "0.4.0", + }, }, } ) +// StarterKit models a Compute@Edge package template and its git location. +type StarterKit struct { + Name string + Path string + Branch string + Tag string +} + // InitCommand initializes a Compute@Edge project package on the local machine. type InitCommand struct { common.Base + client api.HTTPClient manifest manifest.Data + language string from string branch string + tag string path string domain string backend string } // NewInitCommand returns a usable command registered under the parent. -func NewInitCommand(parent common.Registerer, globals *config.Data) *InitCommand { +func NewInitCommand(parent common.Registerer, client api.HTTPClient, globals *config.Data) *InitCommand { var c InitCommand c.Globals = globals + c.client = client c.manifest.File.Read(manifest.Filename) c.CmdClause = parent.Command("init", "Initialize a new Compute@Edge package locally") c.CmdClause.Flag("service-id", "Existing service ID to use. By default, this command creates a new service").Short('s').StringVar(&c.manifest.Flag.ServiceID) c.CmdClause.Flag("name", "Name of package, defaulting to directory name of the --path destination").Short('n').StringVar(&c.manifest.File.Name) c.CmdClause.Flag("description", "Description of the package").Short('d').StringVar(&c.manifest.File.Description) c.CmdClause.Flag("author", "Author(s) of the package").Short('a').StringsVar(&c.manifest.File.Authors) + c.CmdClause.Flag("language", "Language of the package").Short('l').StringVar(&c.language) c.CmdClause.Flag("from", "Git repository containing package template").Short('f').StringVar(&c.from) c.CmdClause.Flag("branch", "Git branch name to clone from package template repository").Hidden().StringVar(&c.branch) + c.CmdClause.Flag("tag", "Git tag name to clone from package template repository").Hidden().StringVar(&c.tag) c.CmdClause.Flag("path", "Destination to write the new package, defaulting to the current directory").Short('p').StringVar(&c.path) c.CmdClause.Flag("domain", "The name of the domain associated to the package").StringVar(&c.path) c.CmdClause.Flag("backend", "A hostname, IPv4, or IPv6 address for the package backend").StringVar(&c.path) @@ -116,8 +134,24 @@ func (c *InitCommand) Exec(in io.Reader, out io.Writer) (err error) { name string description string authors []string + language *Language ) + languages := []*Language{ + NewLanguage(&LanguageOptions{ + Name: "rust", + DisplayName: "Rust", + StarterKits: starterKits["rust"], + Toolchain: NewRust(c.client), + }), + NewLanguage(&LanguageOptions{ + Name: "assemblyscript", + DisplayName: "AssemblyScript (beta)", + StarterKits: starterKits["assemblyscript"], + Toolchain: NewAssemblyScript(), + }), + } + name, _ = c.manifest.Name() description, _ = c.manifest.Description() authors, _ = c.manifest.Authors() @@ -187,22 +221,52 @@ func (c *InitCommand) Exec(in io.Reader, out io.Writer) (err error) { } } + if c.language == "" { + text.Output(out, "%s", text.Bold("Language:")) + for i, lang := range languages { + text.Output(out, "[%d] %s", i+1, lang.DisplayName) + } + option, err := text.Input(out, "Choose option: [1] ", in, validateLanguageOption(languages)) + if err != nil { + return fmt.Errorf("reading input %w", err) + } + if option == "" { + option = "1" + } + if i, err := strconv.Atoi(option); err == nil { + language = languages[i-1] + } else { + return fmt.Errorf("selecting language") + } + } else { + for _, l := range languages { + if strings.EqualFold(c.language, l.Name) { + language = l + } + } + } + if c.from == "" && !c.manifest.File.Exists() { - text.Output(out, "%s", text.Bold("Template:")) - for i, template := range defaultTemplates { - text.Output(out, "[%d] %s (%s)", i, template.Name, template.Path) + text.Output(out, "%s", text.Bold("Starter kit:")) + for i, kit := range language.StarterKits { + text.Output(out, "[%d] %s (%s)", i+1, kit.Name, kit.Path) } - template, err := text.Input(out, "Choose option or type URL: [1] ", in, validateTemplateOptionOrURL) + option, err := text.Input(out, "Choose option or type URL: [1] ", in, validateTemplateOptionOrURL(language.StarterKits)) if err != nil { return fmt.Errorf("error reading input %w", err) } - if template == "" { - template = "1" + if option == "" { + option = "1" } - if i, err := strconv.Atoi(template); err == nil { - template = defaultTemplates[i].Path + + if i, err := strconv.Atoi(option); err == nil { + template := language.StarterKits[i-1] + c.from = template.Path + c.branch = template.Branch + c.tag = template.Tag + } else { + c.from = option } - c.from = template } if c.domain == "" { @@ -321,13 +385,17 @@ func (c *InitCommand) Exec(in io.Reader, out io.Writer) (err error) { } defer os.RemoveAll(tempdir) - var ref plumbing.ReferenceName - if c.from == defaultTemplate { - ref = plumbing.NewBranchReferenceName(defaultTemplateBranch) + if c.branch != "" && c.tag != "" { + return fmt.Errorf("cannot use both git branch and tag name") } + + var ref plumbing.ReferenceName if c.branch != "" { ref = plumbing.NewBranchReferenceName(c.branch) } + if c.tag != "" { + ref = plumbing.NewTagReferenceName(c.tag) + } if _, err := git.PlainClone(tempdir, false, &git.CloneOptions{ URL: c.from, @@ -396,10 +464,20 @@ func (c *InitCommand) Exec(in io.Reader, out io.Writer) (err error) { fmt.Fprintf(progress, "Setting version in manifest to %d...\n", version) m.Version = version + fmt.Fprintf(progress, "Setting language in manifest to %s...\n", language.Name) + m.Language = language.Name + if err := m.Write(filepath.Join(c.path, ManifestFilename)); err != nil { return fmt.Errorf("error saving package manifest: %w", err) } + progress.Step("Initializing package...") + + if err := language.Initialize(progress); err != nil { + fmt.Println(err) + return fmt.Errorf("error initializing package: %w", err) + } + progress.Done() text.Break(out) @@ -474,21 +552,39 @@ func tempDir(prefix string) (abspath string, err error) { return abspath, nil } -func validateTemplateOptionOrURL(input string) error { - msg := "must be a valid option or Git URL" - if input == "" { - return nil +func validateLanguageOption(languages []*Language) func(string) error { + return func(input string) error { + errMsg := fmt.Errorf("must be a valid option") + if input == "" { + return nil + } + if option, err := strconv.Atoi(input); err == nil { + if option > len(languages) { + return errMsg + } + return nil + } + return errMsg } - if option, err := strconv.Atoi(input); err == nil { - if _, ok := defaultTemplates[option]; !ok { +} + +func validateTemplateOptionOrURL(templates []StarterKit) func(string) error { + return func(input string) error { + msg := "must be a valid option or Git URL" + if input == "" { + return nil + } + if option, err := strconv.Atoi(input); err == nil { + if option > len(templates) { + return fmt.Errorf(msg) + } + return nil + } + if !gitRepositoryRegEx.MatchString(input) { return fmt.Errorf(msg) } return nil } - if !gitRepositoryRegEx.MatchString(input) { - return fmt.Errorf(msg) - } - return nil } func validateBackend(input string) error { diff --git a/pkg/compute/rust.go b/pkg/compute/rust.go index ab0ca1074..850a337e5 100644 --- a/pkg/compute/rust.go +++ b/pkg/compute/rust.go @@ -2,7 +2,6 @@ package compute import ( "bufio" - "bytes" "encoding/json" "fmt" "io" @@ -13,7 +12,6 @@ import ( "path/filepath" "sort" "strings" - "sync" "github.com/BurntSushi/toml" "github.com/Masterminds/semver/v3" @@ -31,7 +29,7 @@ const ( WasmWasiTarget = "wasm32-wasi" ) -// CargoPackage models the package confuiguration properties of a Rust Cargo +// CargoPackage models the package configuration properties of a Rust Cargo // package which we are interested in and is embedded within CargoManifest and // CargoLock. type CargoPackage struct { @@ -40,7 +38,7 @@ type CargoPackage struct { Dependencies []CargoPackage `toml:"-" json:"dependencies"` } -// CargoManifest models the package confuiguration properties of a Rust Cargo +// CargoManifest models the package configuration properties of a Rust Cargo // manifest which we are interested in and are read from the Cargo.toml manifest // file within the $PWD of the package. type CargoManifest struct { @@ -78,11 +76,24 @@ func (m *CargoMetadata) Read() error { return nil } -// Rust is an implments Toolchain for the Rust lanaguage. +// Rust is an implements Toolchain for the Rust language. type Rust struct { client api.HTTPClient } +// NewRust constructs a new Rust. +func NewRust(client api.HTTPClient) *Rust { + return &Rust{client} +} + +// SourceDirectory implements the Toolchain interface and returns the source +// directory for Rust packages. +func (r Rust) SourceDirectory() string { return "src" } + +// IncludeFiles implements the Toolchain interface and returns a list of +// additional files to include in the package archive for Rust packages. +func (r Rust) IncludeFiles() []string { return []string{"Cargo.toml"} } + // Verify implments the Toolchain interface and verifies whether the Rust // language toolchain is correctly configured on the host. func (r Rust) Verify(out io.Writer) error { @@ -206,7 +217,7 @@ func (r Rust) Verify(out io.Writer) error { return fmt.Errorf("error fetching latest crate version: %w", err) } - // Create a semver contraint to be within the latest minor range or above. + // Create a semver constraint to be within the latest minor range or above. // TODO(phamann): Update this to major when fastly-sys hits 1.x.x. fastlySysConstraint, err := semver.NewConstraint(fmt.Sprintf("~%d.%d.0", latestFastlySys.Major(), latestFastlySys.Minor())) if err != nil { @@ -247,7 +258,11 @@ func (r Rust) Verify(out io.Writer) error { return nil } -// Build implments the Toolchain interface and attempts to compile the package +// Initialize implements the Toolchain interface and initializes a newly cloned +// package. It is a noop for Rust as the Cargo toolchain handles these steps. +func (r Rust) Initialize(out io.Writer) error { return nil } + +// Build implements the Toolchain interface and attempts to compile the package // Rust source to a Wasm binary. func (r Rust) Build(out io.Writer, verbose bool) error { // Get binary name from Cargo.toml. @@ -274,88 +289,34 @@ func (r Rust) Build(out io.Writer, verbose bool) error { if verbose { args = append(args, "--verbose") } - - // Call cargo build with Wasm Wasi target and release flags. - // gosec flagged this: - // G204 (CWE-78): Subprocess launched with variable - // Disabling as the variables come from trusted sources. - /* #nosec */ - cmd := exec.Command("cargo", args...) - // Add debuginfo RUSTFLAGS to command environment to ensure DWARF debug - // infomation (such as, source mappings) are compiled into the binary. - cmd.Env = append(os.Environ(), - `RUSTFLAGS=-C debuginfo=2`, - ) - - // Pipe the child process stdout and stderr to our own writer. - var stdoutBuf, stderrBuf bytes.Buffer - stdoutIn, _ := cmd.StdoutPipe() - stderrIn, _ := cmd.StderrPipe() - stdout := io.MultiWriter(out, &stdoutBuf) - stderr := io.MultiWriter(out, &stderrBuf) - - // Start the command. - if err := cmd.Start(); err != nil { - return fmt.Errorf("failed to start compilation process: %w", err) - } - - var errStdout, errStderr error - var wg sync.WaitGroup - wg.Add(1) + // information (such as, source mappings) are compiled into the binary. + env := append(os.Environ(), `RUSTFLAGS=-C debuginfo=2`) - go func() { - _, errStdout = io.Copy(stdout, stdoutIn) - wg.Done() - }() - - _, errStderr = io.Copy(stderr, stderrIn) - wg.Wait() - - if errStdout != nil { - return fmt.Errorf("error streaming stdout output from child process: %w", errStdout) - } - if errStderr != nil { - return fmt.Errorf("error streaming stderr output from child process: %w", errStderr) - } - - // Wait for the command to exit. - if err := cmd.Wait(); err != nil { - // If we're not in verbose mode return the bufferred stderr output - // from cargo as the error. - if !verbose && stderrBuf.Len() > 0 { - return fmt.Errorf("error during compilation process:\n%s", strings.TrimSpace(stderrBuf.String())) - } - return fmt.Errorf("error during compilation process") + // Execute the `cargo build` commands with the Wasm WASI target, release + // flags and env vars. + cmd := common.NewStreamingExec("cargo", args, env, verbose, out) + if err := cmd.Exec(); err != nil { + return err } // Get working directory. dir, err := os.Getwd() if err != nil { - return fmt.Errorf("error getting current working directory: %w", err) + return fmt.Errorf("getting current working directory: %w", err) } src := filepath.Join(dir, "target", WasmWasiTarget, "release", fmt.Sprintf("%s.wasm", binName)) dst := filepath.Join(dir, "bin", "main.wasm") // Check if bin directory exists and create if not. binDir := filepath.Join(dir, "bin") - fi, err := os.Stat(binDir) - switch { - case err == nil && fi.IsDir(): - // no problem - case err == nil && !fi.IsDir(): - return fmt.Errorf("error creating bin directory: target already exists as a regular file") - case os.IsNotExist(err): - if err := os.MkdirAll(binDir, 0750); err != nil { - return err - } - case err != nil: - return err + if err := common.MakeDirectoryIfNotExists(binDir); err != nil { + return fmt.Errorf("creating bin directory: %w", err) } err = common.CopyFile(src, dst) if err != nil { - return fmt.Errorf("error copying wasm binary: %w", err) + return fmt.Errorf("copying wasm binary: %w", err) } return nil diff --git a/pkg/compute/testdata/build/Cargo.toml b/pkg/compute/testdata/build/Cargo.toml index d2242b18a..879138e46 100644 --- a/pkg/compute/testdata/build/Cargo.toml +++ b/pkg/compute/testdata/build/Cargo.toml @@ -8,4 +8,4 @@ edition = "2018" debug = true [dependencies] -fastly = "^0.4.0" +fastly = "^0.5.0" diff --git a/pkg/compute/testdata/build/assembly/index.ts b/pkg/compute/testdata/build/assembly/index.ts new file mode 100644 index 000000000..c3e3e0955 --- /dev/null +++ b/pkg/compute/testdata/build/assembly/index.ts @@ -0,0 +1,79 @@ +import { Request, Response, Fastly } from "@fastly/as-compute"; + +// The name of a backend server associated with this service. +// +// This should be changed to match the name of your own backend. See the the +// `Hosts` section of the Fastly Wasm service UI for more information. +const BACKEND_NAME = "backend_name"; + +/// The name of a second backend associated with this service. +const OTHER_BACKEND_NAME = "other_backend_name"; + +// The entry point for your application. +// +// Use this function to define your main request handling logic. It could be +// used to route based on the request properties (such as method or path), send +// the request to a backend, make completely new requests, and/or generate +// synthetic responses. +function main(req: Request): Response { + // Make any desired changes to the client request. + req.headers().set("Host", "example.com"); + + // We can filter requests that have unexpected methods. + const VALID_METHODS = ["HEAD", "GET", "POST"]; + if (!VALID_METHODS.includes(req.method())) { + return new Response(String.UTF8.encode("This method is not allowed"), { + status: 405, + }); + } + + let method = req.method(); + let urlParts = req.url().split("//").pop().split("/"); + let host = urlParts.shift(); + let path = "/" + urlParts.join("/"); + + // If request is a `GET` to the `/` path, send a default response. + if (method == "GET" && path == "/") { + return new Response(String.UTF8.encode("Welcome to Fastly Compute@Edge!"), { + status: 200, + }); + } + + // If request is a `GET` to the `/backend` path, send to a named backend. + if (method == "GET" && path == "/backend") { + // Request handling logic could go here... + // E.g., send the request to an origin backend and then cache the + // response for one minute. + let cacheOverride = new Fastly.CacheOverride(); + cacheOverride.setTTL(60); + return Fastly.fetch(req, { + backend: BACKEND_NAME, + cacheOverride, + }).wait(); + } + + // If request is a `GET` to a path starting with `/other/`. + if (method == "GET" && path.startsWith("/other/")) { + // Send request to a different backend and don't cache response. + let cacheOverride = new Fastly.CacheOverride(); + cacheOverride.setPass(); + return Fastly.fetch(req, { + backend: OTHER_BACKEND_NAME, + cacheOverride, + }).wait(); + } + + // Catch all other requests and return a 404. + return new Response(String.UTF8.encode("The page you requested could not be found"), { + status: 200, + }); +} + +// Get the request from the client. +let req = Fastly.getClientRequest(); + +// Pass the request to the main request handler function. +let resp = main(req); + +// Send the response back to the client. +Fastly.respondWith(resp); diff --git a/pkg/compute/testdata/build/package-lock.json b/pkg/compute/testdata/build/package-lock.json new file mode 100644 index 000000000..2d530de2b --- /dev/null +++ b/pkg/compute/testdata/build/package-lock.json @@ -0,0 +1,43 @@ +{ + "name": "compute-starter-kit-assemblyscript-default", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@fastly/as-compute": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@fastly/as-compute/-/as-compute-0.1.1.tgz", + "integrity": "sha512-kuDYPQjY1o/v9HUEBkAwFYjeui6HCctj7vNGrl06r5ub5p1V8mkd6pqMn5muf6quQLmHkZZZLtKI+vLfYr9Ksw==", + "requires": { + "@fastly/as-fetch": "0.1.0" + } + }, + "@fastly/as-fetch": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@fastly/as-fetch/-/as-fetch-0.1.0.tgz", + "integrity": "sha512-CIsC8qDx0jZDcMWKxFz/IKyOdeFcvle/mnK6waZQkmeZIKAdSUuZ0AgJxT+AN2Qb5DGhtrYlkJgxkmNnSumpdA==" + }, + "assemblyscript": { + "version": "0.14.13", + "resolved": "https://registry.npmjs.org/assemblyscript/-/assemblyscript-0.14.13.tgz", + "integrity": "sha512-hV/2Zolfkzn55LrKymdsHLftkf8TXGf/ZDUooN10L9r1Zgf6yOIyIf/bgGq3A2l5lmUt669gk/Uwx2G0KCbCug==", + "dev": true, + "requires": { + "binaryen": "97.0.0-nightly.20200929", + "long": "^4.0.0" + } + }, + "binaryen": { + "version": "97.0.0-nightly.20200929", + "resolved": "https://registry.npmjs.org/binaryen/-/binaryen-97.0.0-nightly.20200929.tgz", + "integrity": "sha512-HQ7VTISqwfVOylWJAE2jIyhuO5zrxTD2Vvc0cwtXTUqfmlbZdt/Z0vxUZD+uFYHRLM0p9ddJc7RPVGsvPI2oEQ==", + "dev": true + }, + "long": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", + "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==", + "dev": true + } + } +} diff --git a/pkg/compute/testdata/build/package.json b/pkg/compute/testdata/build/package.json new file mode 100644 index 000000000..f2c175ef2 --- /dev/null +++ b/pkg/compute/testdata/build/package.json @@ -0,0 +1,23 @@ +{ + "name": "compute-starter-kit-assemblyscript-default", + "version": "1.0.0", + "description": "Default package starter kit for AssemblyScript based Compute@Edge projects", + "main": "src/index.ts", + "repository": { + "type": "git", + "url": "git+https://github.com/fastly/compute-starter-kit-assemblyscript-default.git" + }, + "keywords": [], + "author": "oss@fastly.com", + "license": "MIT", + "bugs": { + "url": "https://github.com/fastly/compute-starter-kit-assemblyscript-default/issues" + }, + "homepage": "https://github.com/fastly/compute-starter-kit-assemblyscript-default#readme", + "devDependencies": { + "assemblyscript": "^0.14.11" + }, + "dependencies": { + "@fastly/as-compute": "^0.1.1" + } +}